2024-09-14T03:29:23.680 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-14T03:29:23.703 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-14T03:29:23.797 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904541 branch: main description: upgrade/quincy-x/parallel/{0-random-distro$/{centos_9.stream_runc} 0-start 1-tasks mon_election/connectivity upgrade-sequence workload/{ec-rados-default rados_api rados_loadgenbig rbd_import_export test_rbd_api test_rbd_python}} email: ceph-qa@ceph.com first_in_suite: false job_id: '7904541' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run run randomized correctness test for rados operations on an erasure-coded pool ' - desc: 'object class functional tests ' - desc: 'generate read/write load with rados objects ranging from 1MB to 25MB ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'librbd python api tests ' name: teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: main ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(POOL_APP_NOT_ENABLED\) - OSD_DOWN - mons down - mon down - MON_DOWN - out of quorum - PG_DEGRADED - Reduced data availability - Degraded data redundancy - OSDMAP_FLAGS - mons down - mon down - MON_DOWN - out of quorum - PG_AVAILABILITY sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: extra_system_packages: - python3-pytest flavor: default sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: main sha1: 1c941a66c1b420259b8fc5ac19e792dbbf60aded owner: scheduled_teuthology@teuthology priority: 850 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - mds.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mds.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b seed: 8038 sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 sleep_before_teardown: 0 subset: 31/32 suite: upgrade suite_branch: main suite_path: /home/teuthworker/src/git.ceph.com_ceph_1c941a66c1b420259b8fc5ac19e792dbbf60aded/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 1c941a66c1b420259b8fc5ac19e792dbbf60aded targets: smithi070.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIgVj4agkIK7jEWYrGPI4JL+/K0qJzrX7iUpNBjz0f76Kf057PGb1vBvOLHdfUQZnqD0gNyKbU1zfYXWD12KFis= smithi094.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOvBnOoTO45v6WcxCYe2uYsLWW3RU1LXrbSxix33ClGpJ8hHnGOnX0E2EEdmAZL0uE/rNXAT7ck8SOdzh7OLgGY= tasks: - pexec: all: - sudo dnf remove nvme-cli -y - sudo dnf install runc nvmetcli nvme-cli -y - sudo sed -i 's/^#runtime = "crun"/runtime = "runc"/g' /usr/share/containers/containers.conf - sudo sed -i 's/runtime = "crun"/#runtime = "crun"/g' /usr/share/containers/containers.conf - install: branch: quincy exclude_packages: - ceph-volume - print: '**** done install task...' - print: '**** done start installing quincy cephadm ...' - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - print: '**** done end installing quincy cephadm ...' - print: '**** done start cephadm.shell ceph config set mgr...' - cephadm.shell: mon.a: - ceph config set mgr mgr/cephadm/use_repo_digest true --force - print: '**** done cephadm.shell ceph config set mgr...' - print: '**** done start telemetry quincy...' - workunit: clients: client.0: - test_telemetry_quincy.sh - print: '**** done end telemetry quincy...' - print: '**** done start parallel' - parallel: - workload - upgrade-sequence - print: '**** done end parallel' - print: '**** done start telemetry x...' - workunit: clients: client.0: - test_telemetry_quincy_x.sh - print: '**** done end telemetry x...' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-14_03:08:15 tube: smithi upgrade-sequence: sequential: - print: '**** done start upgrade, wait' - cephadm.shell: env: - sha1 mon.a: - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - print: '**** done end upgrade, wait...' user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 workload: full_sequential: - print: '**** done start ec-rados-default.yaml' - rados: clients: - client.0 ec_pool: true objects: 50 op_weights: append: 100 copy_from: 50 delete: 50 read: 100 rmattr: 25 rollback: 50 setattr: 25 snap_create: 50 snap_remove: 50 write: 0 ops: 4000 write_append_excl: false - print: '**** done end ec-rados-default.yaml' - print: '**** done start rados_api.yaml' - workunit: branch: quincy clients: client.0: - cls - print: '**** done end rados_api.yaml' - print: '**** done start rados_loadgenbig.yaml' - workunit: branch: quincy clients: client.0: - rados/load-gen-big.sh - print: '**** done end rados_loadgenbig.yaml' - print: '**** done start rbd_import_export.yaml' - workunit: branch: quincy clients: client.1: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done end rbd_import_export.yaml' - print: '**** done start test_rbd_api.yaml' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done end test_rbd_api.yaml' - print: '**** done start test_rbd_python.yaml' - workunit: branch: quincy clients: client.0: - rbd/test_librbd_python.sh env: RBD_FEATURES: '61' - print: '**** done end test_rbd_python.yaml' 2024-09-14T03:29:23.797 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_1c941a66c1b420259b8fc5ac19e792dbbf60aded/qa; will attempt to use it 2024-09-14T03:29:23.798 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_1c941a66c1b420259b8fc5ac19e792dbbf60aded/qa/tasks 2024-09-14T03:29:23.798 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-14T03:29:23.799 INFO:teuthology.task.internal:Checking packages... 2024-09-14T03:29:23.821 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8' 2024-09-14T03:29:23.821 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-14T03:29:23.821 INFO:teuthology.packaging:ref: None 2024-09-14T03:29:23.821 INFO:teuthology.packaging:tag: None 2024-09-14T03:29:23.821 INFO:teuthology.packaging:branch: main 2024-09-14T03:29:23.821 INFO:teuthology.packaging:sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:29:23.821 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=main 2024-09-14T03:29:24.029 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-4926.g1c941a66 2024-09-14T03:29:24.031 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-14T03:29:24.039 INFO:teuthology.task.internal:no buildpackages task found 2024-09-14T03:29:24.040 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-14T03:29:24.054 INFO:teuthology.task.internal:Saving configuration 2024-09-14T03:29:24.072 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-14T03:29:24.082 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-14T03:29:24.106 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi070.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904541', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-14 03:24:29.137954', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIgVj4agkIK7jEWYrGPI4JL+/K0qJzrX7iUpNBjz0f76Kf057PGb1vBvOLHdfUQZnqD0gNyKbU1zfYXWD12KFis='} 2024-09-14T03:29:24.133 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi094.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904541', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-14 03:24:29.139285', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOvBnOoTO45v6WcxCYe2uYsLWW3RU1LXrbSxix33ClGpJ8hHnGOnX0E2EEdmAZL0uE/rNXAT7ck8SOdzh7OLgGY='} 2024-09-14T03:29:24.133 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-14T03:29:24.146 INFO:teuthology.task.internal:roles: ubuntu@smithi070.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'mds.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-09-14T03:29:24.146 INFO:teuthology.task.internal:roles: ubuntu@smithi094.front.sepia.ceph.com - ['mon.b', 'mds.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-09-14T03:29:24.147 INFO:teuthology.run_tasks:Running task console_log... 2024-09-14T03:29:24.211 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7faaabf93ac0>, signals=[15]) 2024-09-14T03:29:24.211 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-14T03:29:24.222 INFO:teuthology.task.internal:Opening connections... 2024-09-14T03:29:24.222 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi070.front.sepia.ceph.com 2024-09-14T03:29:24.224 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi070.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:29:24.302 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi094.front.sepia.ceph.com 2024-09-14T03:29:24.303 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi094.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:29:24.380 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-14T03:29:24.389 DEBUG:teuthology.orchestra.run.smithi070:> uname -m 2024-09-14T03:29:24.408 INFO:teuthology.orchestra.run.smithi070.stdout:x86_64 2024-09-14T03:29:24.408 DEBUG:teuthology.orchestra.run.smithi070:> cat /etc/os-release 2024-09-14T03:29:24.466 INFO:teuthology.orchestra.run.smithi070.stdout:NAME="CentOS Stream" 2024-09-14T03:29:24.466 INFO:teuthology.orchestra.run.smithi070.stdout:VERSION="9" 2024-09-14T03:29:24.466 INFO:teuthology.orchestra.run.smithi070.stdout:ID="centos" 2024-09-14T03:29:24.466 INFO:teuthology.orchestra.run.smithi070.stdout:ID_LIKE="rhel fedora" 2024-09-14T03:29:24.466 INFO:teuthology.orchestra.run.smithi070.stdout:VERSION_ID="9" 2024-09-14T03:29:24.467 INFO:teuthology.orchestra.run.smithi070.stdout:PLATFORM_ID="platform:el9" 2024-09-14T03:29:24.467 INFO:teuthology.orchestra.run.smithi070.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-14T03:29:24.467 INFO:teuthology.orchestra.run.smithi070.stdout:ANSI_COLOR="0;31" 2024-09-14T03:29:24.467 INFO:teuthology.orchestra.run.smithi070.stdout:LOGO="fedora-logo-icon" 2024-09-14T03:29:24.467 INFO:teuthology.orchestra.run.smithi070.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-14T03:29:24.467 INFO:teuthology.orchestra.run.smithi070.stdout:HOME_URL="https://centos.org/" 2024-09-14T03:29:24.467 INFO:teuthology.orchestra.run.smithi070.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-14T03:29:24.467 INFO:teuthology.orchestra.run.smithi070.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-14T03:29:24.468 INFO:teuthology.orchestra.run.smithi070.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-14T03:29:24.468 INFO:teuthology.lock.ops:Updating smithi070.front.sepia.ceph.com on lock server 2024-09-14T03:29:24.492 DEBUG:teuthology.orchestra.run.smithi094:> uname -m 2024-09-14T03:29:24.510 INFO:teuthology.orchestra.run.smithi094.stdout:x86_64 2024-09-14T03:29:24.511 DEBUG:teuthology.orchestra.run.smithi094:> cat /etc/os-release 2024-09-14T03:29:24.569 INFO:teuthology.orchestra.run.smithi094.stdout:NAME="CentOS Stream" 2024-09-14T03:29:24.569 INFO:teuthology.orchestra.run.smithi094.stdout:VERSION="9" 2024-09-14T03:29:24.569 INFO:teuthology.orchestra.run.smithi094.stdout:ID="centos" 2024-09-14T03:29:24.570 INFO:teuthology.orchestra.run.smithi094.stdout:ID_LIKE="rhel fedora" 2024-09-14T03:29:24.570 INFO:teuthology.orchestra.run.smithi094.stdout:VERSION_ID="9" 2024-09-14T03:29:24.570 INFO:teuthology.orchestra.run.smithi094.stdout:PLATFORM_ID="platform:el9" 2024-09-14T03:29:24.570 INFO:teuthology.orchestra.run.smithi094.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-14T03:29:24.570 INFO:teuthology.orchestra.run.smithi094.stdout:ANSI_COLOR="0;31" 2024-09-14T03:29:24.570 INFO:teuthology.orchestra.run.smithi094.stdout:LOGO="fedora-logo-icon" 2024-09-14T03:29:24.570 INFO:teuthology.orchestra.run.smithi094.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-14T03:29:24.570 INFO:teuthology.orchestra.run.smithi094.stdout:HOME_URL="https://centos.org/" 2024-09-14T03:29:24.570 INFO:teuthology.orchestra.run.smithi094.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-14T03:29:24.570 INFO:teuthology.orchestra.run.smithi094.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-14T03:29:24.570 INFO:teuthology.orchestra.run.smithi094.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-14T03:29:24.571 INFO:teuthology.lock.ops:Updating smithi094.front.sepia.ceph.com on lock server 2024-09-14T03:29:24.600 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-14T03:29:24.609 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-14T03:29:24.622 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-14T03:29:24.622 DEBUG:teuthology.orchestra.run.smithi070:> test '!' -e /home/ubuntu/cephtest 2024-09-14T03:29:24.624 DEBUG:teuthology.orchestra.run.smithi094:> test '!' -e /home/ubuntu/cephtest 2024-09-14T03:29:24.641 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-14T03:29:24.649 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-14T03:29:24.649 DEBUG:teuthology.orchestra.run.smithi070:> test -z $(ls -A /var/lib/ceph) 2024-09-14T03:29:24.682 DEBUG:teuthology.orchestra.run.smithi094:> test -z $(ls -A /var/lib/ceph) 2024-09-14T03:29:24.720 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-14T03:29:24.775 INFO:teuthology.run_tasks:Running task kernel... 2024-09-14T03:29:24.794 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-14T03:29:24.794 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-14T03:29:24.794 DEBUG:teuthology.orchestra.run.smithi070:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-14T03:29:24.795 DEBUG:teuthology.orchestra.run.smithi094:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-14T03:29:24.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T03:29:24.812 DEBUG:teuthology.orchestra.run.smithi094:> uname -r 2024-09-14T03:29:24.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T03:29:24.813 DEBUG:teuthology.orchestra.run.smithi070:> uname -r 2024-09-14T03:29:24.869 INFO:teuthology.orchestra.run.smithi094.stdout:5.14.0-503.el9.x86_64 2024-09-14T03:29:24.870 INFO:teuthology.task.kernel:Running kernel on smithi094: 5.14.0-503.el9.x86_64 2024-09-14T03:29:24.870 DEBUG:teuthology.orchestra.run.smithi094:> sudo yum install -y kernel 2024-09-14T03:29:24.871 INFO:teuthology.orchestra.run.smithi070.stdout:5.14.0-503.el9.x86_64 2024-09-14T03:29:24.871 INFO:teuthology.task.kernel:Running kernel on smithi070: 5.14.0-503.el9.x86_64 2024-09-14T03:29:24.871 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum install -y kernel 2024-09-14T03:29:27.167 INFO:teuthology.orchestra.run.smithi094.stdout:CentOS Stream 9 - BaseOS 17 MB/s | 8.2 MB 00:00 2024-09-14T03:29:28.357 INFO:teuthology.orchestra.run.smithi070.stdout:CentOS Stream 9 - BaseOS 4.7 MB/s | 8.2 MB 00:01 2024-09-14T03:29:28.876 INFO:teuthology.orchestra.run.smithi094.stdout:CentOS Stream 9 - AppStream 33 MB/s | 20 MB 00:00 2024-09-14T03:29:30.233 INFO:teuthology.orchestra.run.smithi070.stdout:CentOS Stream 9 - AppStream 27 MB/s | 20 MB 00:00 2024-09-14T03:29:33.369 INFO:teuthology.orchestra.run.smithi094.stdout:CentOS Stream 9 - CRB 9.4 MB/s | 6.5 MB 00:00 2024-09-14T03:29:34.458 INFO:teuthology.orchestra.run.smithi070.stdout:CentOS Stream 9 - CRB 14 MB/s | 6.5 MB 00:00 2024-09-14T03:29:34.844 INFO:teuthology.orchestra.run.smithi094.stdout:CentOS Stream 9 - Extras packages 71 kB/s | 19 kB 00:00 2024-09-14T03:29:35.924 INFO:teuthology.orchestra.run.smithi094.stdout:Extra Packages for Enterprise Linux 24 MB/s | 23 MB 00:00 2024-09-14T03:29:36.433 INFO:teuthology.orchestra.run.smithi070.stdout:CentOS Stream 9 - Extras packages 27 kB/s | 19 kB 00:00 2024-09-14T03:29:37.187 INFO:teuthology.orchestra.run.smithi070.stdout:Extra Packages for Enterprise Linux 36 MB/s | 23 MB 00:00 2024-09-14T03:29:41.313 INFO:teuthology.orchestra.run.smithi094.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-09-14T03:29:42.601 INFO:teuthology.orchestra.run.smithi070.stdout:lab-extras 28 kB/s | 1.7 kB 00:00 2024-09-14T03:29:43.369 INFO:teuthology.orchestra.run.smithi094.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-14T03:29:43.369 INFO:teuthology.orchestra.run.smithi094.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-14T03:29:43.456 INFO:teuthology.orchestra.run.smithi094.stdout:Dependencies resolved. 2024-09-14T03:29:43.462 INFO:teuthology.orchestra.run.smithi094.stdout:================================================================================ 2024-09-14T03:29:43.462 INFO:teuthology.orchestra.run.smithi094.stdout: Package Architecture Version Repository Size 2024-09-14T03:29:43.462 INFO:teuthology.orchestra.run.smithi094.stdout:================================================================================ 2024-09-14T03:29:43.463 INFO:teuthology.orchestra.run.smithi094.stdout:Installing: 2024-09-14T03:29:43.463 INFO:teuthology.orchestra.run.smithi094.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-14T03:29:43.463 INFO:teuthology.orchestra.run.smithi094.stdout:Installing dependencies: 2024-09-14T03:29:43.463 INFO:teuthology.orchestra.run.smithi094.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-14T03:29:43.463 INFO:teuthology.orchestra.run.smithi094.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-14T03:29:43.463 INFO:teuthology.orchestra.run.smithi094.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-14T03:29:43.463 INFO:teuthology.orchestra.run.smithi094.stdout: 2024-09-14T03:29:43.463 INFO:teuthology.orchestra.run.smithi094.stdout:Transaction Summary 2024-09-14T03:29:43.463 INFO:teuthology.orchestra.run.smithi094.stdout:================================================================================ 2024-09-14T03:29:43.463 INFO:teuthology.orchestra.run.smithi094.stdout:Install 4 Packages 2024-09-14T03:29:43.463 INFO:teuthology.orchestra.run.smithi094.stdout: 2024-09-14T03:29:43.464 INFO:teuthology.orchestra.run.smithi094.stdout:Total download size: 79 M 2024-09-14T03:29:43.464 INFO:teuthology.orchestra.run.smithi094.stdout:Installed size: 126 M 2024-09-14T03:29:43.464 INFO:teuthology.orchestra.run.smithi094.stdout:Downloading Packages: 2024-09-14T03:29:43.564 INFO:teuthology.orchestra.run.smithi094.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 286 kB/s | 23 kB 00:00 2024-09-14T03:29:44.006 INFO:teuthology.orchestra.run.smithi094.stdout:(2/4): kernel-core-5.14.0-505.el9.x86_64.rpm 30 MB/s | 16 MB 00:00 2024-09-14T03:29:44.406 INFO:teuthology.orchestra.run.smithi094.stdout:(3/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 37 MB/s | 35 MB 00:00 2024-09-14T03:29:44.641 INFO:teuthology.orchestra.run.smithi070.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-14T03:29:44.641 INFO:teuthology.orchestra.run.smithi070.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-14T03:29:44.740 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2024-09-14T03:29:44.747 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2024-09-14T03:29:44.747 INFO:teuthology.orchestra.run.smithi070.stdout: Package Architecture Version Repository Size 2024-09-14T03:29:44.747 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2024-09-14T03:29:44.748 INFO:teuthology.orchestra.run.smithi070.stdout:Installing: 2024-09-14T03:29:44.748 INFO:teuthology.orchestra.run.smithi070.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-14T03:29:44.748 INFO:teuthology.orchestra.run.smithi070.stdout:Installing dependencies: 2024-09-14T03:29:44.748 INFO:teuthology.orchestra.run.smithi070.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-14T03:29:44.748 INFO:teuthology.orchestra.run.smithi070.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-14T03:29:44.748 INFO:teuthology.orchestra.run.smithi070.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-14T03:29:44.749 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:29:44.749 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2024-09-14T03:29:44.749 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2024-09-14T03:29:44.749 INFO:teuthology.orchestra.run.smithi070.stdout:Install 4 Packages 2024-09-14T03:29:44.749 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:29:44.751 INFO:teuthology.orchestra.run.smithi094.stdout:(4/4): kernel-modules-core-5.14.0-505.el9.x86_6 24 MB/s | 29 MB 00:01 2024-09-14T03:29:44.751 INFO:teuthology.orchestra.run.smithi094.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:29:44.751 INFO:teuthology.orchestra.run.smithi094.stdout:Total 61 MB/s | 79 MB 00:01 2024-09-14T03:29:44.752 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 79 M 2024-09-14T03:29:44.752 INFO:teuthology.orchestra.run.smithi070.stdout:Installed size: 126 M 2024-09-14T03:29:44.752 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2024-09-14T03:29:44.961 INFO:teuthology.orchestra.run.smithi070.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 253 kB/s | 23 kB 00:00 2024-09-14T03:29:45.431 INFO:teuthology.orchestra.run.smithi094.stdout:Running transaction check 2024-09-14T03:29:45.453 INFO:teuthology.orchestra.run.smithi070.stdout:(2/4): kernel-modules-core-5.14.0-505.el9.x86_6 58 MB/s | 29 MB 00:00 2024-09-14T03:29:45.653 INFO:teuthology.orchestra.run.smithi070.stdout:(3/4): kernel-core-5.14.0-505.el9.x86_64.rpm 20 MB/s | 16 MB 00:00 2024-09-14T03:29:45.794 INFO:teuthology.orchestra.run.smithi094.stdout:Transaction check succeeded. 2024-09-14T03:29:45.794 INFO:teuthology.orchestra.run.smithi094.stdout:Running transaction test 2024-09-14T03:29:46.095 INFO:teuthology.orchestra.run.smithi070.stdout:(4/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 28 MB/s | 35 MB 00:01 2024-09-14T03:29:46.095 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:29:46.096 INFO:teuthology.orchestra.run.smithi070.stdout:Total 58 MB/s | 79 MB 00:01 2024-09-14T03:29:46.752 INFO:teuthology.orchestra.run.smithi094.stdout:Transaction test succeeded. 2024-09-14T03:29:46.753 INFO:teuthology.orchestra.run.smithi094.stdout:Running transaction 2024-09-14T03:29:46.780 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2024-09-14T03:29:47.137 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2024-09-14T03:29:47.137 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2024-09-14T03:29:48.091 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2024-09-14T03:29:48.091 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2024-09-14T03:29:49.209 INFO:teuthology.orchestra.run.smithi094.stdout: Preparing : 1/1 2024-09-14T03:29:49.859 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-14T03:29:49.939 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-14T03:29:50.534 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2024-09-14T03:29:51.187 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-14T03:29:51.270 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-14T03:29:52.748 INFO:teuthology.orchestra.run.smithi094.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-14T03:29:52.873 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-14T03:29:54.063 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-14T03:29:54.198 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-14T03:29:57.713 INFO:teuthology.orchestra.run.smithi094.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-14T03:29:57.815 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:29:59.077 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-14T03:29:59.190 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:30:02.660 INFO:teuthology.orchestra.run.smithi094.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:30:04.059 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:30:41.081 INFO:teuthology.orchestra.run.smithi094.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:30:41.131 INFO:teuthology.orchestra.run.smithi094.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:30:41.489 INFO:teuthology.orchestra.run.smithi094.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:30:41.489 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-14T03:30:41.489 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-14T03:30:41.489 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-14T03:30:42.278 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:30:42.278 INFO:teuthology.orchestra.run.smithi094.stdout: 2024-09-14T03:30:42.279 INFO:teuthology.orchestra.run.smithi094.stdout:Installed: 2024-09-14T03:30:42.279 INFO:teuthology.orchestra.run.smithi094.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-14T03:30:42.279 INFO:teuthology.orchestra.run.smithi094.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-14T03:30:42.279 INFO:teuthology.orchestra.run.smithi094.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-14T03:30:42.279 INFO:teuthology.orchestra.run.smithi094.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-14T03:30:42.279 INFO:teuthology.orchestra.run.smithi094.stdout: 2024-09-14T03:30:42.279 INFO:teuthology.orchestra.run.smithi094.stdout:Complete! 2024-09-14T03:30:42.327 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:30:42.367 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:30:42.663 DEBUG:teuthology.orchestra.run.smithi094:> rpm -q kernel | sort -rV | head -n 1 2024-09-14T03:30:42.715 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:30:42.715 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-14T03:30:42.715 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-14T03:30:42.715 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-14T03:30:42.735 INFO:teuthology.orchestra.run.smithi094.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-14T03:30:42.736 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-14T03:30:42.736 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-14T03:30:42.736 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-14T03:30:42.736 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-14T03:30:42.736 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-09-14T03:30:42.736 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi094.front.sepia.ceph.com, path=None, version=distro) 2024-09-14T03:30:42.736 DEBUG:teuthology.orchestra.run.smithi094:> rpm -q kernel | sort -rV | head -n 1 2024-09-14T03:30:42.795 INFO:teuthology.orchestra.run.smithi094.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-14T03:30:42.796 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-14T03:30:42.796 DEBUG:teuthology.orchestra.run.smithi094:> sudo rpm -qi grub2-tools 2024-09-14T03:30:42.828 INFO:teuthology.orchestra.run.smithi094.stdout:Name : grub2-tools 2024-09-14T03:30:42.828 INFO:teuthology.orchestra.run.smithi094.stdout:Epoch : 1 2024-09-14T03:30:42.828 INFO:teuthology.orchestra.run.smithi094.stdout:Version : 2.06 2024-09-14T03:30:42.828 INFO:teuthology.orchestra.run.smithi094.stdout:Release : 82.el9 2024-09-14T03:30:42.828 INFO:teuthology.orchestra.run.smithi094.stdout:Architecture: x86_64 2024-09-14T03:30:42.829 INFO:teuthology.orchestra.run.smithi094.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-14T03:30:42.829 INFO:teuthology.orchestra.run.smithi094.stdout:Group : Unspecified 2024-09-14T03:30:42.829 INFO:teuthology.orchestra.run.smithi094.stdout:Size : 8274670 2024-09-14T03:30:42.829 INFO:teuthology.orchestra.run.smithi094.stdout:License : GPLv3+ 2024-09-14T03:30:42.829 INFO:teuthology.orchestra.run.smithi094.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-14T03:30:42.829 INFO:teuthology.orchestra.run.smithi094.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-14T03:30:42.829 INFO:teuthology.orchestra.run.smithi094.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-14T03:30:42.829 INFO:teuthology.orchestra.run.smithi094.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-14T03:30:42.829 INFO:teuthology.orchestra.run.smithi094.stdout:Packager : builder@centos.org 2024-09-14T03:30:42.829 INFO:teuthology.orchestra.run.smithi094.stdout:Vendor : CentOS 2024-09-14T03:30:42.829 INFO:teuthology.orchestra.run.smithi094.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-14T03:30:42.829 INFO:teuthology.orchestra.run.smithi094.stdout:Summary : Support tools for GRUB. 2024-09-14T03:30:42.829 INFO:teuthology.orchestra.run.smithi094.stdout:Description : 2024-09-14T03:30:42.830 INFO:teuthology.orchestra.run.smithi094.stdout: 2024-09-14T03:30:42.830 INFO:teuthology.orchestra.run.smithi094.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-14T03:30:42.830 INFO:teuthology.orchestra.run.smithi094.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-14T03:30:42.830 INFO:teuthology.orchestra.run.smithi094.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-14T03:30:42.830 INFO:teuthology.orchestra.run.smithi094.stdout:hardware devices. 2024-09-14T03:30:42.830 INFO:teuthology.orchestra.run.smithi094.stdout: 2024-09-14T03:30:42.830 INFO:teuthology.orchestra.run.smithi094.stdout:This subpackage provides tools for support of all platforms. 2024-09-14T03:30:42.831 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-14T03:30:42.831 INFO:teuthology.task.kernel:Updating grub on smithi094 to boot 5.14.0-505.el9.x86_64 2024-09-14T03:30:42.831 DEBUG:teuthology.orchestra.run.smithi094:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-14T03:30:43.542 INFO:teuthology.orchestra.run.smithi094.stderr:Generating grub configuration file ... 2024-09-14T03:30:43.572 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:30:43.572 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:30:43.572 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2024-09-14T03:30:43.572 INFO:teuthology.orchestra.run.smithi070.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-14T03:30:43.573 INFO:teuthology.orchestra.run.smithi070.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-14T03:30:43.573 INFO:teuthology.orchestra.run.smithi070.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-14T03:30:43.573 INFO:teuthology.orchestra.run.smithi070.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-14T03:30:43.573 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:30:43.573 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2024-09-14T03:30:43.919 DEBUG:teuthology.orchestra.run.smithi070:> rpm -q kernel | sort -rV | head -n 1 2024-09-14T03:30:44.012 INFO:teuthology.orchestra.run.smithi070.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-14T03:30:44.013 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-14T03:30:44.013 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-14T03:30:44.014 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-14T03:30:44.014 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-14T03:30:44.014 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-09-14T03:30:44.014 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi070.front.sepia.ceph.com, path=None, version=distro) 2024-09-14T03:30:44.014 DEBUG:teuthology.orchestra.run.smithi070:> rpm -q kernel | sort -rV | head -n 1 2024-09-14T03:30:44.073 INFO:teuthology.orchestra.run.smithi070.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-14T03:30:44.073 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-14T03:30:44.073 DEBUG:teuthology.orchestra.run.smithi070:> sudo rpm -qi grub2-tools 2024-09-14T03:30:44.105 INFO:teuthology.orchestra.run.smithi070.stdout:Name : grub2-tools 2024-09-14T03:30:44.105 INFO:teuthology.orchestra.run.smithi070.stdout:Epoch : 1 2024-09-14T03:30:44.105 INFO:teuthology.orchestra.run.smithi070.stdout:Version : 2.06 2024-09-14T03:30:44.105 INFO:teuthology.orchestra.run.smithi070.stdout:Release : 82.el9 2024-09-14T03:30:44.106 INFO:teuthology.orchestra.run.smithi070.stdout:Architecture: x86_64 2024-09-14T03:30:44.106 INFO:teuthology.orchestra.run.smithi070.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-14T03:30:44.106 INFO:teuthology.orchestra.run.smithi070.stdout:Group : Unspecified 2024-09-14T03:30:44.106 INFO:teuthology.orchestra.run.smithi070.stdout:Size : 8274670 2024-09-14T03:30:44.106 INFO:teuthology.orchestra.run.smithi070.stdout:License : GPLv3+ 2024-09-14T03:30:44.106 INFO:teuthology.orchestra.run.smithi070.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-14T03:30:44.106 INFO:teuthology.orchestra.run.smithi070.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-14T03:30:44.106 INFO:teuthology.orchestra.run.smithi070.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-14T03:30:44.106 INFO:teuthology.orchestra.run.smithi070.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-14T03:30:44.106 INFO:teuthology.orchestra.run.smithi070.stdout:Packager : builder@centos.org 2024-09-14T03:30:44.106 INFO:teuthology.orchestra.run.smithi070.stdout:Vendor : CentOS 2024-09-14T03:30:44.106 INFO:teuthology.orchestra.run.smithi070.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-14T03:30:44.106 INFO:teuthology.orchestra.run.smithi070.stdout:Summary : Support tools for GRUB. 2024-09-14T03:30:44.107 INFO:teuthology.orchestra.run.smithi070.stdout:Description : 2024-09-14T03:30:44.107 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:30:44.107 INFO:teuthology.orchestra.run.smithi070.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-14T03:30:44.107 INFO:teuthology.orchestra.run.smithi070.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-14T03:30:44.107 INFO:teuthology.orchestra.run.smithi070.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-14T03:30:44.107 INFO:teuthology.orchestra.run.smithi070.stdout:hardware devices. 2024-09-14T03:30:44.107 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:30:44.107 INFO:teuthology.orchestra.run.smithi070.stdout:This subpackage provides tools for support of all platforms. 2024-09-14T03:30:44.108 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-14T03:30:44.108 INFO:teuthology.task.kernel:Updating grub on smithi070 to boot 5.14.0-505.el9.x86_64 2024-09-14T03:30:44.108 DEBUG:teuthology.orchestra.run.smithi070:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-14T03:30:44.766 INFO:teuthology.orchestra.run.smithi070.stderr:Generating grub configuration file ... 2024-09-14T03:30:44.884 INFO:teuthology.orchestra.run.smithi094.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-14T03:30:44.912 INFO:teuthology.orchestra.run.smithi094.stderr:done 2024-09-14T03:30:44.916 DEBUG:teuthology.orchestra.run.smithi094:> mktemp 2024-09-14T03:30:44.931 INFO:teuthology.orchestra.run.smithi094.stdout:/tmp/tmp.Pris8lcK4e 2024-09-14T03:30:44.932 DEBUG:teuthology.orchestra.run.smithi094:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.Pris8lcK4e 2024-09-14T03:30:44.998 DEBUG:teuthology.orchestra.run.smithi094:> sudo chmod 0666 /tmp/tmp.Pris8lcK4e 2024-09-14T03:30:45.141 DEBUG:teuthology.orchestra.remote:smithi094:/tmp/tmp.Pris8lcK4e is 6KB 2024-09-14T03:30:45.189 DEBUG:teuthology.orchestra.run.smithi094:> rm -fr /tmp/tmp.Pris8lcK4e 2024-09-14T03:30:45.203 DEBUG:teuthology.orchestra.run.smithi094:> sudo /bin/ls /boot/loader/entries || true 2024-09-14T03:30:45.266 INFO:teuthology.orchestra.run.smithi094.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-14T03:30:45.267 INFO:teuthology.orchestra.run.smithi094.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-14T03:30:45.267 INFO:teuthology.orchestra.run.smithi094.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-14T03:30:45.267 INFO:teuthology.orchestra.run.smithi094.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-14T03:30:45.268 DEBUG:teuthology.orchestra.run.smithi094:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-14T03:30:45.440 DEBUG:teuthology.orchestra.run.smithi094:> sudo shutdown -r now 2024-09-14T03:30:46.102 INFO:teuthology.orchestra.run.smithi070.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-14T03:30:46.128 INFO:teuthology.orchestra.run.smithi070.stderr:done 2024-09-14T03:30:46.131 DEBUG:teuthology.orchestra.run.smithi070:> mktemp 2024-09-14T03:30:46.147 INFO:teuthology.orchestra.run.smithi070.stdout:/tmp/tmp.q7P5jtQp57 2024-09-14T03:30:46.147 DEBUG:teuthology.orchestra.run.smithi070:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.q7P5jtQp57 2024-09-14T03:30:46.242 DEBUG:teuthology.orchestra.run.smithi070:> sudo chmod 0666 /tmp/tmp.q7P5jtQp57 2024-09-14T03:30:46.342 DEBUG:teuthology.orchestra.remote:smithi070:/tmp/tmp.q7P5jtQp57 is 6KB 2024-09-14T03:30:46.393 DEBUG:teuthology.orchestra.run.smithi070:> rm -fr /tmp/tmp.q7P5jtQp57 2024-09-14T03:30:46.408 DEBUG:teuthology.orchestra.run.smithi070:> sudo /bin/ls /boot/loader/entries || true 2024-09-14T03:30:46.471 INFO:teuthology.orchestra.run.smithi070.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-14T03:30:46.471 INFO:teuthology.orchestra.run.smithi070.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-14T03:30:46.471 INFO:teuthology.orchestra.run.smithi070.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-14T03:30:46.471 INFO:teuthology.orchestra.run.smithi070.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-14T03:30:46.473 DEBUG:teuthology.orchestra.run.smithi070:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-14T03:30:46.618 DEBUG:teuthology.orchestra.run.smithi070:> sudo shutdown -r now 2024-09-14T03:30:46.687 ERROR:paramiko.transport:Socket exception: Connection reset by peer (104) 2024-09-14T03:31:15.469 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-09-14T03:31:15.470 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi094.front.sepia.ceph.com' 2024-09-14T03:31:15.471 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi094.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:31:16.621 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-09-14T03:31:16.621 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi070.front.sepia.ceph.com' 2024-09-14T03:31:16.622 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi070.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:32:15.531 DEBUG:teuthology.orchestra.remote:timed out 2024-09-14T03:32:16.625 DEBUG:teuthology.orchestra.remote:timed out 2024-09-14T03:32:24.537 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi094.front.sepia.ceph.com' 2024-09-14T03:32:24.539 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi094.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:32:24.976 DEBUG:teuthology.orchestra.run.smithi094:> true 2024-09-14T03:32:25.353 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi094.front.sepia.ceph.com' 2024-09-14T03:32:25.353 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-505.el9.x86_64"... 2024-09-14T03:32:25.353 DEBUG:teuthology.orchestra.run.smithi094:> uname -r 2024-09-14T03:32:25.368 INFO:teuthology.orchestra.run.smithi094.stdout:5.14.0-505.el9.x86_64 2024-09-14T03:32:25.368 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-14T03:32:25.368 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-14T03:32:25.368 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-14T03:32:25.626 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi070.front.sepia.ceph.com' 2024-09-14T03:32:25.627 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi070.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:32:26.071 DEBUG:teuthology.orchestra.run.smithi070:> true 2024-09-14T03:32:26.369 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-09-14T03:32:26.369 DEBUG:teuthology.orchestra.run.smithi094:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-14T03:32:26.427 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi070.front.sepia.ceph.com' 2024-09-14T03:32:26.427 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-505.el9.x86_64"... 2024-09-14T03:32:26.427 DEBUG:teuthology.orchestra.run.smithi070:> uname -r 2024-09-14T03:32:26.442 INFO:teuthology.orchestra.run.smithi070.stdout:5.14.0-505.el9.x86_64 2024-09-14T03:32:26.442 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-14T03:32:26.442 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-14T03:32:26.443 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-14T03:32:26.445 INFO:teuthology.orchestra.run.smithi094.stdout:ttyS1 2024-09-14T03:32:26.479 DEBUG:teuthology.parallel:result is None 2024-09-14T03:32:27.444 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-09-14T03:32:27.444 DEBUG:teuthology.orchestra.run.smithi070:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-14T03:32:27.523 INFO:teuthology.orchestra.run.smithi070.stdout:ttyS1 2024-09-14T03:32:27.544 DEBUG:teuthology.parallel:result is None 2024-09-14T03:32:27.544 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-14T03:32:27.551 INFO:teuthology.task.internal:Creating test directory... 2024-09-14T03:32:27.552 DEBUG:teuthology.orchestra.run.smithi070:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-14T03:32:27.554 DEBUG:teuthology.orchestra.run.smithi094:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-14T03:32:27.570 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-14T03:32:27.576 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-14T03:32:27.582 INFO:teuthology.task.internal:Creating archive directory... 2024-09-14T03:32:27.582 DEBUG:teuthology.orchestra.run.smithi070:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-14T03:32:27.610 DEBUG:teuthology.orchestra.run.smithi094:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-14T03:32:27.640 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-14T03:32:27.646 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-14T03:32:27.646 DEBUG:teuthology.orchestra.run.smithi070:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-14T03:32:27.677 DEBUG:teuthology.orchestra.run.smithi094:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-14T03:32:27.713 INFO:teuthology.orchestra.run.smithi070.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-14T03:32:27.720 INFO:teuthology.orchestra.run.smithi094.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-14T03:32:27.723 INFO:teuthology.orchestra.run.smithi070.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-14T03:32:27.729 INFO:teuthology.orchestra.run.smithi094.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-14T03:32:27.731 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-14T03:32:27.737 INFO:teuthology.task.internal:Configuring sudo... 2024-09-14T03:32:27.737 DEBUG:teuthology.orchestra.run.smithi070:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-14T03:32:27.768 DEBUG:teuthology.orchestra.run.smithi094:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-14T03:32:27.796 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-14T03:32:27.805 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-14T03:32:27.806 DEBUG:teuthology.orchestra.run.smithi070:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-14T03:32:27.832 DEBUG:teuthology.orchestra.run.smithi094:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-14T03:32:27.851 DEBUG:teuthology.orchestra.run.smithi070:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-14T03:32:27.927 DEBUG:teuthology.orchestra.run.smithi070:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-14T03:32:27.974 DEBUG:teuthology.orchestra.run.smithi070:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-14T03:32:28.057 DEBUG:teuthology.orchestra.run.smithi070:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-14T03:32:28.095 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2024-09-14T03:32:28.095 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-14T03:32:28.170 DEBUG:teuthology.orchestra.run.smithi094:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-14T03:32:28.212 DEBUG:teuthology.orchestra.run.smithi094:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-14T03:32:28.261 DEBUG:teuthology.orchestra.run.smithi094:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-14T03:32:28.343 DEBUG:teuthology.orchestra.run.smithi094:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-14T03:32:28.383 DEBUG:teuthology.orchestra.run.smithi094:> set -ex 2024-09-14T03:32:28.383 DEBUG:teuthology.orchestra.run.smithi094:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-14T03:32:28.448 DEBUG:teuthology.orchestra.run.smithi070:> sudo service rsyslog restart 2024-09-14T03:32:28.450 DEBUG:teuthology.orchestra.run.smithi094:> sudo service rsyslog restart 2024-09-14T03:32:28.506 INFO:teuthology.orchestra.run.smithi070.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-14T03:32:28.536 INFO:teuthology.orchestra.run.smithi094.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-14T03:32:28.921 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-14T03:32:28.928 INFO:teuthology.task.internal:Starting timer... 2024-09-14T03:32:28.928 INFO:teuthology.run_tasks:Running task pcp... 2024-09-14T03:32:28.939 INFO:teuthology.run_tasks:Running task selinux... 2024-09-14T03:32:28.948 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-09-14T03:32:28.948 DEBUG:teuthology.orchestra.run.smithi070:> sudo service auditd rotate 2024-09-14T03:32:29.040 INFO:teuthology.orchestra.run.smithi070.stdout:Rotating logs: 2024-09-14T03:32:29.042 DEBUG:teuthology.orchestra.run.smithi094:> sudo service auditd rotate 2024-09-14T03:32:29.138 INFO:teuthology.orchestra.run.smithi094.stdout:Rotating logs: 2024-09-14T03:32:29.140 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-14T03:32:29.140 DEBUG:teuthology.orchestra.run.smithi070:> /usr/sbin/getenforce 2024-09-14T03:32:29.161 INFO:teuthology.orchestra.run.smithi070.stdout:Permissive 2024-09-14T03:32:29.162 DEBUG:teuthology.orchestra.run.smithi094:> /usr/sbin/getenforce 2024-09-14T03:32:29.235 INFO:teuthology.orchestra.run.smithi094.stdout:Permissive 2024-09-14T03:32:29.236 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi070.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi094.front.sepia.ceph.com': 'permissive'} 2024-09-14T03:32:29.236 DEBUG:teuthology.orchestra.run.smithi070:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-14T03:32:29.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T03:32:29.262 DEBUG:teuthology.orchestra.run.smithi094:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-14T03:32:29.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T03:32:29.287 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-14T03:32:29.288 DEBUG:teuthology.orchestra.run.smithi070:> sudo /usr/sbin/setenforce permissive 2024-09-14T03:32:29.332 DEBUG:teuthology.orchestra.run.smithi094:> sudo /usr/sbin/setenforce permissive 2024-09-14T03:32:29.437 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-14T03:32:29.449 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-14T03:32:29.461 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-14T03:32:29.462 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi070.front.sepia.ceph.com,smithi094.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-14T03:36:18.763 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi070.front.sepia.ceph.com'), Remote(name='ubuntu@smithi094.front.sepia.ceph.com')] 2024-09-14T03:36:18.765 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi070.front.sepia.ceph.com' 2024-09-14T03:36:18.766 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi070.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:36:18.849 DEBUG:teuthology.orchestra.run.smithi070:> true 2024-09-14T03:36:18.949 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi070.front.sepia.ceph.com' 2024-09-14T03:36:18.950 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi094.front.sepia.ceph.com' 2024-09-14T03:36:18.950 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi094.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:36:19.029 DEBUG:teuthology.orchestra.run.smithi094:> true 2024-09-14T03:36:19.108 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi094.front.sepia.ceph.com' 2024-09-14T03:36:19.108 INFO:teuthology.run_tasks:Running task clock... 2024-09-14T03:36:19.118 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-14T03:36:19.118 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-14T03:36:19.119 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-14T03:36:19.122 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-14T03:36:19.122 DEBUG:teuthology.orchestra.run.smithi094:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-14T03:36:19.150 INFO:teuthology.orchestra.run.smithi070.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-14T03:36:19.166 INFO:teuthology.orchestra.run.smithi070.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-14T03:36:19.178 INFO:teuthology.orchestra.run.smithi094.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-14T03:36:19.191 INFO:teuthology.orchestra.run.smithi070.stderr:sudo: ntpd: command not found 2024-09-14T03:36:19.192 INFO:teuthology.orchestra.run.smithi094.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-14T03:36:19.202 INFO:teuthology.orchestra.run.smithi070.stdout:506 Cannot talk to daemon 2024-09-14T03:36:19.217 INFO:teuthology.orchestra.run.smithi094.stderr:sudo: ntpd: command not found 2024-09-14T03:36:19.217 INFO:teuthology.orchestra.run.smithi070.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-14T03:36:19.228 INFO:teuthology.orchestra.run.smithi094.stdout:506 Cannot talk to daemon 2024-09-14T03:36:19.231 INFO:teuthology.orchestra.run.smithi070.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-14T03:36:19.242 INFO:teuthology.orchestra.run.smithi094.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-14T03:36:19.256 INFO:teuthology.orchestra.run.smithi094.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-14T03:36:19.296 INFO:teuthology.orchestra.run.smithi070.stderr:bash: line 1: ntpq: command not found 2024-09-14T03:36:19.299 INFO:teuthology.orchestra.run.smithi070.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-14T03:36:19.299 INFO:teuthology.orchestra.run.smithi070.stdout:=============================================================================== 2024-09-14T03:36:19.299 INFO:teuthology.orchestra.run.smithi070.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:36:19.299 INFO:teuthology.orchestra.run.smithi070.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:36:19.299 INFO:teuthology.orchestra.run.smithi070.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:36:19.300 INFO:teuthology.orchestra.run.smithi070.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:36:19.314 INFO:teuthology.orchestra.run.smithi094.stderr:bash: line 1: ntpq: command not found 2024-09-14T03:36:19.317 INFO:teuthology.orchestra.run.smithi094.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-14T03:36:19.318 INFO:teuthology.orchestra.run.smithi094.stdout:=============================================================================== 2024-09-14T03:36:19.318 INFO:teuthology.orchestra.run.smithi094.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:36:19.318 INFO:teuthology.orchestra.run.smithi094.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:36:19.318 INFO:teuthology.orchestra.run.smithi094.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:36:19.318 INFO:teuthology.orchestra.run.smithi094.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:36:19.319 INFO:teuthology.run_tasks:Running task pexec... 2024-09-14T03:36:19.328 INFO:teuthology.task.pexec:Executing custom commands... 2024-09-14T03:36:19.329 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi070.front.sepia.ceph.com 2024-09-14T03:36:19.329 DEBUG:teuthology.orchestra.run.smithi070:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-09-14T03:36:19.329 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi094.front.sepia.ceph.com 2024-09-14T03:36:19.329 DEBUG:teuthology.orchestra.run.smithi094:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-09-14T03:36:19.686 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2024-09-14T03:36:19.699 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2024-09-14T03:36:19.700 INFO:teuthology.orchestra.run.smithi070.stdout: Package Architecture Version Repository Size 2024-09-14T03:36:19.700 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2024-09-14T03:36:19.700 INFO:teuthology.orchestra.run.smithi070.stdout:Removing: 2024-09-14T03:36:19.700 INFO:teuthology.orchestra.run.smithi070.stdout: nvme-cli x86_64 2.9.1-6.el9 @baseos 5.3 M 2024-09-14T03:36:19.700 INFO:teuthology.orchestra.run.smithi070.stdout:Removing unused dependencies: 2024-09-14T03:36:19.700 INFO:teuthology.orchestra.run.smithi070.stdout: libnvme x86_64 1.9-3.el9 @baseos 273 k 2024-09-14T03:36:19.700 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:36:19.700 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2024-09-14T03:36:19.700 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2024-09-14T03:36:19.700 INFO:teuthology.orchestra.run.smithi070.stdout:Remove 2 Packages 2024-09-14T03:36:19.700 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:36:19.700 INFO:teuthology.orchestra.run.smithi070.stdout:Freed space: 5.5 M 2024-09-14T03:36:19.700 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2024-09-14T03:36:19.704 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2024-09-14T03:36:19.704 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2024-09-14T03:36:19.710 INFO:teuthology.orchestra.run.smithi094.stdout:Dependencies resolved. 2024-09-14T03:36:19.712 INFO:teuthology.orchestra.run.smithi094.stdout:================================================================================ 2024-09-14T03:36:19.712 INFO:teuthology.orchestra.run.smithi094.stdout: Package Architecture Version Repository Size 2024-09-14T03:36:19.712 INFO:teuthology.orchestra.run.smithi094.stdout:================================================================================ 2024-09-14T03:36:19.712 INFO:teuthology.orchestra.run.smithi094.stdout:Removing: 2024-09-14T03:36:19.712 INFO:teuthology.orchestra.run.smithi094.stdout: nvme-cli x86_64 2.9.1-6.el9 @baseos 5.3 M 2024-09-14T03:36:19.712 INFO:teuthology.orchestra.run.smithi094.stdout:Removing unused dependencies: 2024-09-14T03:36:19.712 INFO:teuthology.orchestra.run.smithi094.stdout: libnvme x86_64 1.9-3.el9 @baseos 273 k 2024-09-14T03:36:19.712 INFO:teuthology.orchestra.run.smithi094.stdout: 2024-09-14T03:36:19.712 INFO:teuthology.orchestra.run.smithi094.stdout:Transaction Summary 2024-09-14T03:36:19.713 INFO:teuthology.orchestra.run.smithi094.stdout:================================================================================ 2024-09-14T03:36:19.713 INFO:teuthology.orchestra.run.smithi094.stdout:Remove 2 Packages 2024-09-14T03:36:19.713 INFO:teuthology.orchestra.run.smithi094.stdout: 2024-09-14T03:36:19.713 INFO:teuthology.orchestra.run.smithi094.stdout:Freed space: 5.5 M 2024-09-14T03:36:19.713 INFO:teuthology.orchestra.run.smithi094.stdout:Running transaction check 2024-09-14T03:36:19.716 INFO:teuthology.orchestra.run.smithi094.stdout:Transaction check succeeded. 2024-09-14T03:36:19.716 INFO:teuthology.orchestra.run.smithi094.stdout:Running transaction test 2024-09-14T03:36:19.766 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2024-09-14T03:36:19.766 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2024-09-14T03:36:19.792 INFO:teuthology.orchestra.run.smithi094.stdout:Transaction test succeeded. 2024-09-14T03:36:19.793 INFO:teuthology.orchestra.run.smithi094.stdout:Running transaction 2024-09-14T03:36:19.916 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2024-09-14T03:36:19.945 INFO:teuthology.orchestra.run.smithi094.stdout: Preparing : 1/1 2024-09-14T03:36:19.992 INFO:teuthology.orchestra.run.smithi070.stdout: Erasing : nvme-cli-2.9.1-6.el9.x86_64 1/2 2024-09-14T03:36:20.024 INFO:teuthology.orchestra.run.smithi094.stdout: Erasing : nvme-cli-2.9.1-6.el9.x86_64 1/2 2024-09-14T03:36:20.048 INFO:teuthology.orchestra.run.smithi070.stdout: Erasing : libnvme-1.9-3.el9.x86_64 2/2 2024-09-14T03:36:20.076 INFO:teuthology.orchestra.run.smithi094.stdout: Erasing : libnvme-1.9-3.el9.x86_64 2/2 2024-09-14T03:36:20.549 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: libnvme-1.9-3.el9.x86_64 2/2 2024-09-14T03:36:20.549 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/2 2024-09-14T03:36:20.581 INFO:teuthology.orchestra.run.smithi094.stdout: Running scriptlet: libnvme-1.9-3.el9.x86_64 2/2 2024-09-14T03:36:20.582 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/2 2024-09-14T03:36:20.817 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/2 2024-09-14T03:36:20.817 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:36:20.817 INFO:teuthology.orchestra.run.smithi070.stdout:Removed: 2024-09-14T03:36:20.817 INFO:teuthology.orchestra.run.smithi070.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-14T03:36:20.817 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:36:20.817 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2024-09-14T03:36:20.837 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/2 2024-09-14T03:36:20.838 INFO:teuthology.orchestra.run.smithi094.stdout: 2024-09-14T03:36:20.838 INFO:teuthology.orchestra.run.smithi094.stdout:Removed: 2024-09-14T03:36:20.838 INFO:teuthology.orchestra.run.smithi094.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-14T03:36:20.838 INFO:teuthology.orchestra.run.smithi094.stdout: 2024-09-14T03:36:20.838 INFO:teuthology.orchestra.run.smithi094.stdout:Complete! 2024-09-14T03:36:21.634 INFO:teuthology.orchestra.run.smithi094.stdout:Last metadata expiration check: 0:00:45 ago on Sat 14 Sep 2024 03:35:36 AM UTC. 2024-09-14T03:36:21.644 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:00:54 ago on Sat 14 Sep 2024 03:35:27 AM UTC. 2024-09-14T03:36:21.770 INFO:teuthology.orchestra.run.smithi094.stdout:Dependencies resolved. 2024-09-14T03:36:21.771 INFO:teuthology.orchestra.run.smithi094.stdout:================================================================================ 2024-09-14T03:36:21.771 INFO:teuthology.orchestra.run.smithi094.stdout: Package Arch Version Repository Size 2024-09-14T03:36:21.771 INFO:teuthology.orchestra.run.smithi094.stdout:================================================================================ 2024-09-14T03:36:21.771 INFO:teuthology.orchestra.run.smithi094.stdout:Installing: 2024-09-14T03:36:21.771 INFO:teuthology.orchestra.run.smithi094.stdout: nvme-cli x86_64 2.9.1-6.el9 baseos 906 k 2024-09-14T03:36:21.771 INFO:teuthology.orchestra.run.smithi094.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-09-14T03:36:21.771 INFO:teuthology.orchestra.run.smithi094.stdout: runc x86_64 4:1.1.13-4.el9 appstream 3.2 M 2024-09-14T03:36:21.771 INFO:teuthology.orchestra.run.smithi094.stdout:Installing dependencies: 2024-09-14T03:36:21.771 INFO:teuthology.orchestra.run.smithi094.stdout: libnvme x86_64 1.9-3.el9 baseos 106 k 2024-09-14T03:36:21.771 INFO:teuthology.orchestra.run.smithi094.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-09-14T03:36:21.771 INFO:teuthology.orchestra.run.smithi094.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-09-14T03:36:21.771 INFO:teuthology.orchestra.run.smithi094.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-09-14T03:36:21.772 INFO:teuthology.orchestra.run.smithi094.stdout: 2024-09-14T03:36:21.772 INFO:teuthology.orchestra.run.smithi094.stdout:Transaction Summary 2024-09-14T03:36:21.772 INFO:teuthology.orchestra.run.smithi094.stdout:================================================================================ 2024-09-14T03:36:21.772 INFO:teuthology.orchestra.run.smithi094.stdout:Install 7 Packages 2024-09-14T03:36:21.772 INFO:teuthology.orchestra.run.smithi094.stdout: 2024-09-14T03:36:21.783 INFO:teuthology.orchestra.run.smithi094.stdout:Total download size: 5.2 M 2024-09-14T03:36:21.783 INFO:teuthology.orchestra.run.smithi094.stdout:Installed size: 19 M 2024-09-14T03:36:21.783 INFO:teuthology.orchestra.run.smithi094.stdout:Downloading Packages: 2024-09-14T03:36:21.789 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2024-09-14T03:36:21.790 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2024-09-14T03:36:21.790 INFO:teuthology.orchestra.run.smithi070.stdout: Package Arch Version Repository Size 2024-09-14T03:36:21.790 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2024-09-14T03:36:21.790 INFO:teuthology.orchestra.run.smithi070.stdout:Installing: 2024-09-14T03:36:21.790 INFO:teuthology.orchestra.run.smithi070.stdout: nvme-cli x86_64 2.9.1-6.el9 baseos 906 k 2024-09-14T03:36:21.790 INFO:teuthology.orchestra.run.smithi070.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-09-14T03:36:21.790 INFO:teuthology.orchestra.run.smithi070.stdout: runc x86_64 4:1.1.13-4.el9 appstream 3.2 M 2024-09-14T03:36:21.790 INFO:teuthology.orchestra.run.smithi070.stdout:Installing dependencies: 2024-09-14T03:36:21.790 INFO:teuthology.orchestra.run.smithi070.stdout: libnvme x86_64 1.9-3.el9 baseos 106 k 2024-09-14T03:36:21.790 INFO:teuthology.orchestra.run.smithi070.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-09-14T03:36:21.790 INFO:teuthology.orchestra.run.smithi070.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-09-14T03:36:21.790 INFO:teuthology.orchestra.run.smithi070.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-09-14T03:36:21.790 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:36:21.791 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2024-09-14T03:36:21.791 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2024-09-14T03:36:21.791 INFO:teuthology.orchestra.run.smithi070.stdout:Install 7 Packages 2024-09-14T03:36:21.791 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:36:21.797 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 5.2 M 2024-09-14T03:36:21.797 INFO:teuthology.orchestra.run.smithi070.stdout:Installed size: 19 M 2024-09-14T03:36:21.797 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2024-09-14T03:36:21.971 INFO:teuthology.orchestra.run.smithi070.stdout:(1/7): nvmetcli-0.7-3.el9.noarch.rpm 426 kB/s | 44 kB 00:00 2024-09-14T03:36:21.997 INFO:teuthology.orchestra.run.smithi070.stdout:(2/7): libnvme-1.9-3.el9.x86_64.rpm 818 kB/s | 106 kB 00:00 2024-09-14T03:36:22.030 INFO:teuthology.orchestra.run.smithi070.stdout:(3/7): python3-configshell-1.1.30-1.el9.noarch. 1.2 MB/s | 72 kB 00:00 2024-09-14T03:36:22.055 INFO:teuthology.orchestra.run.smithi070.stdout:(4/7): python3-kmod-0.9-32.el9.x86_64.rpm 1.4 MB/s | 84 kB 00:00 2024-09-14T03:36:22.081 INFO:teuthology.orchestra.run.smithi070.stdout:(5/7): nvme-cli-2.9.1-6.el9.x86_64.rpm 4.1 MB/s | 906 kB 00:00 2024-09-14T03:36:22.139 INFO:teuthology.orchestra.run.smithi070.stdout:(6/7): python3-urwid-2.1.2-4.el9.x86_64.rpm 7.5 MB/s | 837 kB 00:00 2024-09-14T03:36:22.225 INFO:teuthology.orchestra.run.smithi094.stdout:(1/7): nvmetcli-0.7-3.el9.noarch.rpm 317 kB/s | 44 kB 00:00 2024-09-14T03:36:22.251 INFO:teuthology.orchestra.run.smithi094.stdout:(2/7): libnvme-1.9-3.el9.x86_64.rpm 640 kB/s | 106 kB 00:00 2024-09-14T03:36:22.301 INFO:teuthology.orchestra.run.smithi094.stdout:(3/7): python3-kmod-0.9-32.el9.x86_64.rpm 1.6 MB/s | 84 kB 00:00 2024-09-14T03:36:22.326 INFO:teuthology.orchestra.run.smithi094.stdout:(4/7): python3-configshell-1.1.30-1.el9.noarch. 712 kB/s | 72 kB 00:00 2024-09-14T03:36:22.339 INFO:teuthology.orchestra.run.smithi070.stdout:(7/7): runc-1.1.13-4.el9.x86_64.rpm 11 MB/s | 3.2 MB 00:00 2024-09-14T03:36:22.339 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:36:22.339 INFO:teuthology.orchestra.run.smithi070.stdout:Total 9.5 MB/s | 5.2 MB 00:00 2024-09-14T03:36:22.393 INFO:teuthology.orchestra.run.smithi094.stdout:(5/7): nvme-cli-2.9.1-6.el9.x86_64.rpm 2.9 MB/s | 906 kB 00:00 2024-09-14T03:36:22.449 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2024-09-14T03:36:22.463 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2024-09-14T03:36:22.464 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2024-09-14T03:36:22.477 INFO:teuthology.orchestra.run.smithi094.stdout:(6/7): python3-urwid-2.1.2-4.el9.x86_64.rpm 4.7 MB/s | 837 kB 00:00 2024-09-14T03:36:22.797 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2024-09-14T03:36:22.798 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2024-09-14T03:36:22.919 INFO:teuthology.orchestra.run.smithi094.stdout:(7/7): runc-1.1.13-4.el9.x86_64.rpm 5.3 MB/s | 3.2 MB 00:00 2024-09-14T03:36:22.919 INFO:teuthology.orchestra.run.smithi094.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:36:22.919 INFO:teuthology.orchestra.run.smithi094.stdout:Total 4.6 MB/s | 5.2 MB 00:01 2024-09-14T03:36:23.029 INFO:teuthology.orchestra.run.smithi094.stdout:Running transaction check 2024-09-14T03:36:23.044 INFO:teuthology.orchestra.run.smithi094.stdout:Transaction check succeeded. 2024-09-14T03:36:23.044 INFO:teuthology.orchestra.run.smithi094.stdout:Running transaction test 2024-09-14T03:36:23.279 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2024-09-14T03:36:23.337 INFO:teuthology.orchestra.run.smithi094.stdout:Transaction test succeeded. 2024-09-14T03:36:23.337 INFO:teuthology.orchestra.run.smithi094.stdout:Running transaction 2024-09-14T03:36:23.352 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 1/7 2024-09-14T03:36:23.403 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 2/7 2024-09-14T03:36:23.440 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 3/7 2024-09-14T03:36:23.653 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : libnvme-1.9-3.el9.x86_64 4/7 2024-09-14T03:36:23.688 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-14T03:36:23.837 INFO:teuthology.orchestra.run.smithi094.stdout: Preparing : 1/1 2024-09-14T03:36:23.929 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 1/7 2024-09-14T03:36:23.964 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 2/7 2024-09-14T03:36:23.993 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 3/7 2024-09-14T03:36:24.140 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-14T03:36:24.175 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : nvmetcli-0.7-3.el9.noarch 6/7 2024-09-14T03:36:24.207 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : libnvme-1.9-3.el9.x86_64 4/7 2024-09-14T03:36:24.233 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-14T03:36:24.281 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 6/7 2024-09-14T03:36:24.344 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-14T03:36:24.656 INFO:teuthology.orchestra.run.smithi094.stdout: Running scriptlet: nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-14T03:36:24.686 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : nvmetcli-0.7-3.el9.noarch 6/7 2024-09-14T03:36:24.793 INFO:teuthology.orchestra.run.smithi094.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 6/7 2024-09-14T03:36:24.856 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-14T03:36:25.329 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-14T03:36:25.329 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/7 2024-09-14T03:36:25.329 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/7 2024-09-14T03:36:25.329 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 3/7 2024-09-14T03:36:25.330 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 4/7 2024-09-14T03:36:25.330 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 5/7 2024-09-14T03:36:25.330 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 6/7 2024-09-14T03:36:25.730 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-14T03:36:25.730 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:36:25.730 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2024-09-14T03:36:25.730 INFO:teuthology.orchestra.run.smithi070.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-14T03:36:25.730 INFO:teuthology.orchestra.run.smithi070.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-09-14T03:36:25.730 INFO:teuthology.orchestra.run.smithi070.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-09-14T03:36:25.730 INFO:teuthology.orchestra.run.smithi070.stdout: runc-4:1.1.13-4.el9.x86_64 2024-09-14T03:36:25.730 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:36:25.731 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2024-09-14T03:36:25.820 INFO:teuthology.orchestra.run.smithi094.stdout: Running scriptlet: runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-14T03:36:25.821 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/7 2024-09-14T03:36:25.821 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/7 2024-09-14T03:36:25.821 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 3/7 2024-09-14T03:36:25.821 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 4/7 2024-09-14T03:36:25.821 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 5/7 2024-09-14T03:36:25.821 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 6/7 2024-09-14T03:36:26.110 DEBUG:teuthology.parallel:result is None 2024-09-14T03:36:26.170 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-14T03:36:26.170 INFO:teuthology.orchestra.run.smithi094.stdout: 2024-09-14T03:36:26.170 INFO:teuthology.orchestra.run.smithi094.stdout:Installed: 2024-09-14T03:36:26.170 INFO:teuthology.orchestra.run.smithi094.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-14T03:36:26.170 INFO:teuthology.orchestra.run.smithi094.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-09-14T03:36:26.170 INFO:teuthology.orchestra.run.smithi094.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-09-14T03:36:26.170 INFO:teuthology.orchestra.run.smithi094.stdout: runc-4:1.1.13-4.el9.x86_64 2024-09-14T03:36:26.170 INFO:teuthology.orchestra.run.smithi094.stdout: 2024-09-14T03:36:26.171 INFO:teuthology.orchestra.run.smithi094.stdout:Complete! 2024-09-14T03:36:26.519 DEBUG:teuthology.parallel:result is None 2024-09-14T03:36:26.519 INFO:teuthology.run_tasks:Running task install... 2024-09-14T03:36:26.528 DEBUG:teuthology.task.install:project ceph 2024-09-14T03:36:26.528 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'extra_system_packages': ['python3-pytest'], 'flavor': 'default', 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8'}} 2024-09-14T03:36:26.528 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'extra_system_packages': ['python3-pytest'], 'flavor': 'default', 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8'} 2024-09-14T03:36:26.528 INFO:teuthology.task.install:Using flavor: default 2024-09-14T03:36:26.535 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-09-14T03:36:26.535 INFO:teuthology.task.install:extra packages: [] 2024-09-14T03:36:26.535 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': ['python3-pytest'], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8', 'tag': None, 'wait_for_package': False} 2024-09-14T03:36:26.535 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-14T03:36:26.536 INFO:teuthology.packaging:ref: None 2024-09-14T03:36:26.536 INFO:teuthology.packaging:tag: None 2024-09-14T03:36:26.536 INFO:teuthology.packaging:branch: quincy 2024-09-14T03:36:26.536 INFO:teuthology.packaging:sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:36:26.536 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-09-14T03:36:26.538 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': ['python3-pytest'], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8', 'tag': None, 'wait_for_package': False} 2024-09-14T03:36:26.538 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-14T03:36:26.538 INFO:teuthology.packaging:ref: None 2024-09-14T03:36:26.538 INFO:teuthology.packaging:tag: None 2024-09-14T03:36:26.538 INFO:teuthology.packaging:branch: quincy 2024-09-14T03:36:26.538 INFO:teuthology.packaging:sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:36:26.539 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-09-14T03:36:26.740 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/quincy/eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b/centos/9/flavors/default/ 2024-09-14T03:36:26.741 INFO:teuthology.task.install.rpm:Package version is 17.2.7-1629.geb16b97b 2024-09-14T03:36:26.765 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/quincy/eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b/centos/9/flavors/default/ 2024-09-14T03:36:26.766 INFO:teuthology.task.install.rpm:Package version is 17.2.7-1629.geb16b97b 2024-09-14T03:36:26.855 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/quincy/eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-14T03:36:26.856 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2024-09-14T03:36:26.856 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-14T03:36:26.883 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, python3-pytest, python3-pytest, python3-pytest on remote rpm x86_64 2024-09-14T03:36:26.883 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-14T03:36:26.883 INFO:teuthology.packaging:ref: None 2024-09-14T03:36:26.883 INFO:teuthology.packaging:tag: None 2024-09-14T03:36:26.883 INFO:teuthology.packaging:branch: quincy 2024-09-14T03:36:26.883 INFO:teuthology.packaging:sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:36:26.884 DEBUG:teuthology.orchestra.run.smithi070:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-14T03:36:26.886 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/quincy/eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-14T03:36:26.886 DEBUG:teuthology.orchestra.run.smithi094:> set -ex 2024-09-14T03:36:26.886 DEBUG:teuthology.orchestra.run.smithi094:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-14T03:36:26.913 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, python3-pytest, python3-pytest, python3-pytest, python3-pytest on remote rpm x86_64 2024-09-14T03:36:26.914 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-14T03:36:26.914 INFO:teuthology.packaging:ref: None 2024-09-14T03:36:26.914 INFO:teuthology.packaging:tag: None 2024-09-14T03:36:26.914 INFO:teuthology.packaging:branch: quincy 2024-09-14T03:36:26.914 INFO:teuthology.packaging:sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:36:26.914 DEBUG:teuthology.orchestra.run.smithi094:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-14T03:36:26.949 DEBUG:teuthology.orchestra.run.smithi070:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-14T03:36:26.995 DEBUG:teuthology.orchestra.run.smithi094:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-14T03:36:27.025 DEBUG:teuthology.orchestra.run.smithi070:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-14T03:36:27.072 DEBUG:teuthology.orchestra.run.smithi094:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-14T03:36:27.089 INFO:teuthology.orchestra.run.smithi070.stdout:check_obsoletes = 1 2024-09-14T03:36:27.091 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum clean all 2024-09-14T03:36:27.137 INFO:teuthology.orchestra.run.smithi094.stdout:check_obsoletes = 1 2024-09-14T03:36:27.139 DEBUG:teuthology.orchestra.run.smithi094:> sudo yum clean all 2024-09-14T03:36:27.322 INFO:teuthology.orchestra.run.smithi070.stdout:45 files removed 2024-09-14T03:36:27.351 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-pytest python3-pytest python3-pytest python3-pytest 2024-09-14T03:36:27.361 INFO:teuthology.orchestra.run.smithi094.stdout:45 files removed 2024-09-14T03:36:27.388 DEBUG:teuthology.orchestra.run.smithi094:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-pytest python3-pytest python3-pytest python3-pytest 2024-09-14T03:36:27.881 INFO:teuthology.orchestra.run.smithi070.stdout:ceph packages for x86_64 312 kB/s | 77 kB 00:00 2024-09-14T03:36:27.944 INFO:teuthology.orchestra.run.smithi094.stdout:ceph packages for x86_64 286 kB/s | 77 kB 00:00 2024-09-14T03:36:28.165 INFO:teuthology.orchestra.run.smithi070.stdout:ceph noarch packages 45 kB/s | 11 kB 00:00 2024-09-14T03:36:28.228 INFO:teuthology.orchestra.run.smithi094.stdout:ceph noarch packages 45 kB/s | 11 kB 00:00 2024-09-14T03:36:28.432 INFO:teuthology.orchestra.run.smithi070.stdout:ceph source packages 7.7 kB/s | 1.8 kB 00:00 2024-09-14T03:36:28.503 INFO:teuthology.orchestra.run.smithi094.stdout:ceph source packages 7.5 kB/s | 1.8 kB 00:00 2024-09-14T03:36:29.212 INFO:teuthology.orchestra.run.smithi094.stdout:CentOS Stream 9 - BaseOS 12 MB/s | 8.2 MB 00:00 2024-09-14T03:36:31.122 INFO:teuthology.orchestra.run.smithi094.stdout:CentOS Stream 9 - AppStream 25 MB/s | 20 MB 00:00 2024-09-14T03:36:32.574 INFO:teuthology.orchestra.run.smithi070.stdout:CentOS Stream 9 - BaseOS 2.0 MB/s | 8.2 MB 00:04 2024-09-14T03:36:34.483 INFO:teuthology.orchestra.run.smithi070.stdout:CentOS Stream 9 - AppStream 25 MB/s | 20 MB 00:00 2024-09-14T03:36:35.581 INFO:teuthology.orchestra.run.smithi094.stdout:CentOS Stream 9 - CRB 10 MB/s | 6.5 MB 00:00 2024-09-14T03:36:37.149 INFO:teuthology.orchestra.run.smithi094.stdout:CentOS Stream 9 - Extras packages 84 kB/s | 19 kB 00:00 2024-09-14T03:36:38.120 INFO:teuthology.orchestra.run.smithi094.stdout:Extra Packages for Enterprise Linux 27 MB/s | 23 MB 00:00 2024-09-14T03:36:38.759 INFO:teuthology.orchestra.run.smithi070.stdout:CentOS Stream 9 - CRB 13 MB/s | 6.5 MB 00:00 2024-09-14T03:36:40.376 INFO:teuthology.orchestra.run.smithi070.stdout:CentOS Stream 9 - Extras packages 76 kB/s | 19 kB 00:00 2024-09-14T03:36:41.372 INFO:teuthology.orchestra.run.smithi070.stdout:Extra Packages for Enterprise Linux 26 MB/s | 23 MB 00:00 2024-09-14T03:36:43.575 INFO:teuthology.orchestra.run.smithi094.stdout:lab-extras 41 kB/s | 1.7 kB 00:00 2024-09-14T03:36:45.960 INFO:teuthology.orchestra.run.smithi094.stdout:Package librados2-2:16.2.4-5.el9.x86_64 is already installed. 2024-09-14T03:36:45.960 INFO:teuthology.orchestra.run.smithi094.stdout:Package librbd1-2:16.2.4-5.el9.x86_64 is already installed. 2024-09-14T03:36:46.028 INFO:teuthology.orchestra.run.smithi094.stdout:Dependencies resolved. 2024-09-14T03:36:46.035 INFO:teuthology.orchestra.run.smithi094.stdout:======================================================================================= 2024-09-14T03:36:46.035 INFO:teuthology.orchestra.run.smithi094.stdout: Package Arch Version Repository Size 2024-09-14T03:36:46.035 INFO:teuthology.orchestra.run.smithi094.stdout:======================================================================================= 2024-09-14T03:36:46.035 INFO:teuthology.orchestra.run.smithi094.stdout:Installing: 2024-09-14T03:36:46.035 INFO:teuthology.orchestra.run.smithi094.stdout: ceph x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 6.5 k 2024-09-14T03:36:46.035 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-base x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 5.0 M 2024-09-14T03:36:46.035 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-fuse x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 813 k 2024-09-14T03:36:46.035 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 137 k 2024-09-14T03:36:46.036 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-mgr x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 1.4 M 2024-09-14T03:36:46.036 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-mgr-cephadm noarch 2:17.2.7-1629.geb16b97b.el9 ceph-noarch 124 k 2024-09-14T03:36:46.036 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-mgr-dashboard noarch 2:17.2.7-1629.geb16b97b.el9 ceph-noarch 3.4 M 2024-09-14T03:36:46.036 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-1629.geb16b97b.el9 ceph-noarch 7.4 M 2024-09-14T03:36:46.036 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-mgr-rook noarch 2:17.2.7-1629.geb16b97b.el9 ceph-noarch 47 k 2024-09-14T03:36:46.036 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-radosgw x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 14 M 2024-09-14T03:36:46.036 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-test x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 48 M 2024-09-14T03:36:46.036 INFO:teuthology.orchestra.run.smithi094.stdout: cephadm noarch 2:17.2.7-1629.geb16b97b.el9 ceph-noarch 83 k 2024-09-14T03:36:46.036 INFO:teuthology.orchestra.run.smithi094.stdout: libcephfs-devel x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 25 k 2024-09-14T03:36:46.036 INFO:teuthology.orchestra.run.smithi094.stdout: libcephfs2 x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 636 k 2024-09-14T03:36:46.036 INFO:teuthology.orchestra.run.smithi094.stdout: librados-devel x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 126 k 2024-09-14T03:36:46.037 INFO:teuthology.orchestra.run.smithi094.stdout: python3-cephfs x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 157 k 2024-09-14T03:36:46.037 INFO:teuthology.orchestra.run.smithi094.stdout: python3-pytest noarch 6.2.2-7.el9 appstream 519 k 2024-09-14T03:36:46.037 INFO:teuthology.orchestra.run.smithi094.stdout: python3-rados x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 320 k 2024-09-14T03:36:46.037 INFO:teuthology.orchestra.run.smithi094.stdout: python3-rbd x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 296 k 2024-09-14T03:36:46.037 INFO:teuthology.orchestra.run.smithi094.stdout: python3-rgw x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 99 k 2024-09-14T03:36:46.037 INFO:teuthology.orchestra.run.smithi094.stdout: rbd-fuse x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 82 k 2024-09-14T03:36:46.037 INFO:teuthology.orchestra.run.smithi094.stdout: rbd-mirror x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 2.9 M 2024-09-14T03:36:46.037 INFO:teuthology.orchestra.run.smithi094.stdout: rbd-nbd x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 165 k 2024-09-14T03:36:46.037 INFO:teuthology.orchestra.run.smithi094.stdout:Upgrading: 2024-09-14T03:36:46.037 INFO:teuthology.orchestra.run.smithi094.stdout: librados2 x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 3.1 M 2024-09-14T03:36:46.037 INFO:teuthology.orchestra.run.smithi094.stdout: librbd1 x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 2.9 M 2024-09-14T03:36:46.037 INFO:teuthology.orchestra.run.smithi094.stdout:Installing dependencies: 2024-09-14T03:36:46.037 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-common x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 23 M 2024-09-14T03:36:46.038 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-grafana-dashboards noarch 2:17.2.7-1629.geb16b97b.el9 ceph-noarch 24 k 2024-09-14T03:36:46.038 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-mds x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 2.1 M 2024-09-14T03:36:46.038 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-mgr-modules-core noarch 2:17.2.7-1629.geb16b97b.el9 ceph-noarch 240 k 2024-09-14T03:36:46.038 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-mon x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 3.9 M 2024-09-14T03:36:46.038 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-osd x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 15 M 2024-09-14T03:36:46.038 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-prometheus-alerts noarch 2:17.2.7-1629.geb16b97b.el9 ceph-noarch 15 k 2024-09-14T03:36:46.038 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-selinux x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 25 k 2024-09-14T03:36:46.038 INFO:teuthology.orchestra.run.smithi094.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-09-14T03:36:46.038 INFO:teuthology.orchestra.run.smithi094.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-09-14T03:36:46.038 INFO:teuthology.orchestra.run.smithi094.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-09-14T03:36:46.038 INFO:teuthology.orchestra.run.smithi094.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-09-14T03:36:46.038 INFO:teuthology.orchestra.run.smithi094.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2024-09-14T03:36:46.038 INFO:teuthology.orchestra.run.smithi094.stdout: libcephsqlite x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 163 k 2024-09-14T03:36:46.039 INFO:teuthology.orchestra.run.smithi094.stdout: libgfortran x86_64 11.5.0-2.el9 baseos 801 k 2024-09-14T03:36:46.039 INFO:teuthology.orchestra.run.smithi094.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-09-14T03:36:46.039 INFO:teuthology.orchestra.run.smithi094.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-09-14T03:36:46.039 INFO:teuthology.orchestra.run.smithi094.stdout: libquadmath x86_64 11.5.0-2.el9 baseos 191 k 2024-09-14T03:36:46.039 INFO:teuthology.orchestra.run.smithi094.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-09-14T03:36:46.039 INFO:teuthology.orchestra.run.smithi094.stdout: libradosstriper1 x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 458 k 2024-09-14T03:36:46.039 INFO:teuthology.orchestra.run.smithi094.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-09-14T03:36:46.039 INFO:teuthology.orchestra.run.smithi094.stdout: librgw2 x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 4.9 M 2024-09-14T03:36:46.039 INFO:teuthology.orchestra.run.smithi094.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-09-14T03:36:46.039 INFO:teuthology.orchestra.run.smithi094.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-09-14T03:36:46.039 INFO:teuthology.orchestra.run.smithi094.stdout: openblas x86_64 0.3.26-2.el9 appstream 38 k 2024-09-14T03:36:46.039 INFO:teuthology.orchestra.run.smithi094.stdout: openblas-openmp x86_64 0.3.26-2.el9 appstream 4.9 M 2024-09-14T03:36:46.039 INFO:teuthology.orchestra.run.smithi094.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-09-14T03:36:46.040 INFO:teuthology.orchestra.run.smithi094.stdout: python3-attrs noarch 20.3.0-7.el9 appstream 87 k 2024-09-14T03:36:46.040 INFO:teuthology.orchestra.run.smithi094.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-09-14T03:36:46.040 INFO:teuthology.orchestra.run.smithi094.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-09-14T03:36:46.040 INFO:teuthology.orchestra.run.smithi094.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-09-14T03:36:46.040 INFO:teuthology.orchestra.run.smithi094.stdout: python3-ceph-argparse x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 45 k 2024-09-14T03:36:46.040 INFO:teuthology.orchestra.run.smithi094.stdout: python3-ceph-common x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 120 k 2024-09-14T03:36:46.040 INFO:teuthology.orchestra.run.smithi094.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-09-14T03:36:46.040 INFO:teuthology.orchestra.run.smithi094.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-09-14T03:36:46.040 INFO:teuthology.orchestra.run.smithi094.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-09-14T03:36:46.040 INFO:teuthology.orchestra.run.smithi094.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-09-14T03:36:46.040 INFO:teuthology.orchestra.run.smithi094.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-09-14T03:36:46.040 INFO:teuthology.orchestra.run.smithi094.stdout: python3-devel x86_64 3.9.19-8.el9 appstream 245 k 2024-09-14T03:36:46.040 INFO:teuthology.orchestra.run.smithi094.stdout: python3-google-auth noarch 1:2.34.0-1.el9 epel 220 k 2024-09-14T03:36:46.041 INFO:teuthology.orchestra.run.smithi094.stdout: python3-iniconfig noarch 1.1.1-7.el9 appstream 17 k 2024-09-14T03:36:46.041 INFO:teuthology.orchestra.run.smithi094.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-09-14T03:36:46.041 INFO:teuthology.orchestra.run.smithi094.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-09-14T03:36:46.041 INFO:teuthology.orchestra.run.smithi094.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-09-14T03:36:46.041 INFO:teuthology.orchestra.run.smithi094.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-09-14T03:36:46.041 INFO:teuthology.orchestra.run.smithi094.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-09-14T03:36:46.041 INFO:teuthology.orchestra.run.smithi094.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-09-14T03:36:46.041 INFO:teuthology.orchestra.run.smithi094.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-09-14T03:36:46.041 INFO:teuthology.orchestra.run.smithi094.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-09-14T03:36:46.041 INFO:teuthology.orchestra.run.smithi094.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-09-14T03:36:46.041 INFO:teuthology.orchestra.run.smithi094.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-09-14T03:36:46.041 INFO:teuthology.orchestra.run.smithi094.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-09-14T03:36:46.042 INFO:teuthology.orchestra.run.smithi094.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-09-14T03:36:46.042 INFO:teuthology.orchestra.run.smithi094.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-09-14T03:36:46.042 INFO:teuthology.orchestra.run.smithi094.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-09-14T03:36:46.042 INFO:teuthology.orchestra.run.smithi094.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-09-14T03:36:46.042 INFO:teuthology.orchestra.run.smithi094.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-09-14T03:36:46.042 INFO:teuthology.orchestra.run.smithi094.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-09-14T03:36:46.042 INFO:teuthology.orchestra.run.smithi094.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-09-14T03:36:46.042 INFO:teuthology.orchestra.run.smithi094.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-09-14T03:36:46.042 INFO:teuthology.orchestra.run.smithi094.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-09-14T03:36:46.043 INFO:teuthology.orchestra.run.smithi094.stdout: python3-pluggy noarch 0.13.1-7.el9 appstream 41 k 2024-09-14T03:36:46.043 INFO:teuthology.orchestra.run.smithi094.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-09-14T03:36:46.043 INFO:teuthology.orchestra.run.smithi094.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-09-14T03:36:46.043 INFO:teuthology.orchestra.run.smithi094.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-09-14T03:36:46.043 INFO:teuthology.orchestra.run.smithi094.stdout: python3-py noarch 1.10.0-6.el9 appstream 477 k 2024-09-14T03:36:46.043 INFO:teuthology.orchestra.run.smithi094.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-09-14T03:36:46.043 INFO:teuthology.orchestra.run.smithi094.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-09-14T03:36:46.043 INFO:teuthology.orchestra.run.smithi094.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-09-14T03:36:46.043 INFO:teuthology.orchestra.run.smithi094.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-09-14T03:36:46.043 INFO:teuthology.orchestra.run.smithi094.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-09-14T03:36:46.043 INFO:teuthology.orchestra.run.smithi094.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-09-14T03:36:46.043 INFO:teuthology.orchestra.run.smithi094.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-09-14T03:36:46.044 INFO:teuthology.orchestra.run.smithi094.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-09-14T03:36:46.044 INFO:teuthology.orchestra.run.smithi094.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-09-14T03:36:46.044 INFO:teuthology.orchestra.run.smithi094.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-09-14T03:36:46.044 INFO:teuthology.orchestra.run.smithi094.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-09-14T03:36:46.044 INFO:teuthology.orchestra.run.smithi094.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-09-14T03:36:46.044 INFO:teuthology.orchestra.run.smithi094.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-09-14T03:36:46.044 INFO:teuthology.orchestra.run.smithi094.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-09-14T03:36:46.044 INFO:teuthology.orchestra.run.smithi094.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-09-14T03:36:46.044 INFO:teuthology.orchestra.run.smithi094.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-09-14T03:36:46.044 INFO:teuthology.orchestra.run.smithi094.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-09-14T03:36:46.044 INFO:teuthology.orchestra.run.smithi094.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-09-14T03:36:46.044 INFO:teuthology.orchestra.run.smithi094.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-09-14T03:36:46.045 INFO:teuthology.orchestra.run.smithi094.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-09-14T03:36:46.045 INFO:teuthology.orchestra.run.smithi094.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-09-14T03:36:46.045 INFO:teuthology.orchestra.run.smithi094.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-09-14T03:36:46.045 INFO:teuthology.orchestra.run.smithi094.stdout:Installing weak dependencies: 2024-09-14T03:36:46.045 INFO:teuthology.orchestra.run.smithi094.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-09-14T03:36:46.045 INFO:teuthology.orchestra.run.smithi094.stdout: 2024-09-14T03:36:46.045 INFO:teuthology.orchestra.run.smithi094.stdout:Transaction Summary 2024-09-14T03:36:46.045 INFO:teuthology.orchestra.run.smithi094.stdout:======================================================================================= 2024-09-14T03:36:46.045 INFO:teuthology.orchestra.run.smithi094.stdout:Install 112 Packages 2024-09-14T03:36:46.045 INFO:teuthology.orchestra.run.smithi094.stdout:Upgrade 2 Packages 2024-09-14T03:36:46.045 INFO:teuthology.orchestra.run.smithi094.stdout: 2024-09-14T03:36:46.046 INFO:teuthology.orchestra.run.smithi094.stdout:Total download size: 190 M 2024-09-14T03:36:46.046 INFO:teuthology.orchestra.run.smithi094.stdout:Downloading Packages: 2024-09-14T03:36:46.669 INFO:teuthology.orchestra.run.smithi094.stdout:(1/114): ceph-17.2.7-1629.geb16b97b.el9.x86_64. 45 kB/s | 6.5 kB 00:00 2024-09-14T03:36:46.819 INFO:teuthology.orchestra.run.smithi070.stdout:lab-extras 46 kB/s | 1.7 kB 00:00 2024-09-14T03:36:46.969 INFO:teuthology.orchestra.run.smithi094.stdout:(2/114): ceph-base-17.2.7-1629.geb16b97b.el9.x8 11 MB/s | 5.0 MB 00:00 2024-09-14T03:36:47.027 INFO:teuthology.orchestra.run.smithi094.stdout:(3/114): ceph-fuse-17.2.7-1629.geb16b97b.el9.x8 2.2 MB/s | 813 kB 00:00 2024-09-14T03:36:47.053 INFO:teuthology.orchestra.run.smithi094.stdout:(4/114): ceph-immutable-object-cache-17.2.7-162 1.6 MB/s | 137 kB 00:00 2024-09-14T03:36:47.145 INFO:teuthology.orchestra.run.smithi094.stdout:(5/114): ceph-mgr-17.2.7-1629.geb16b97b.el9.x86 15 MB/s | 1.4 MB 00:00 2024-09-14T03:36:47.403 INFO:teuthology.orchestra.run.smithi094.stdout:(6/114): ceph-mon-17.2.7-1629.geb16b97b.el9.x86 15 MB/s | 3.9 MB 00:00 2024-09-14T03:36:47.645 INFO:teuthology.orchestra.run.smithi094.stdout:(7/114): ceph-common-17.2.7-1629.geb16b97b.el9. 20 MB/s | 23 MB 00:01 2024-09-14T03:36:47.704 INFO:teuthology.orchestra.run.smithi094.stdout:(8/114): ceph-mds-17.2.7-1629.geb16b97b.el9.x86 3.1 MB/s | 2.1 MB 00:00 2024-09-14T03:36:47.787 INFO:teuthology.orchestra.run.smithi094.stdout:(9/114): ceph-selinux-17.2.7-1629.geb16b97b.el9 303 kB/s | 25 kB 00:00 2024-09-14T03:36:48.179 INFO:teuthology.orchestra.run.smithi094.stdout:(10/114): ceph-radosgw-17.2.7-1629.geb16b97b.el 27 MB/s | 14 MB 00:00 2024-09-14T03:36:48.229 INFO:teuthology.orchestra.run.smithi094.stdout:(11/114): libcephfs-devel-17.2.7-1629.geb16b97b 502 kB/s | 25 kB 00:00 2024-09-14T03:36:48.297 INFO:teuthology.orchestra.run.smithi094.stdout:(12/114): libcephfs2-17.2.7-1629.geb16b97b.el9. 9.3 MB/s | 636 kB 00:00 2024-09-14T03:36:48.355 INFO:teuthology.orchestra.run.smithi094.stdout:(13/114): libcephsqlite-17.2.7-1629.geb16b97b.e 2.7 MB/s | 163 kB 00:00 2024-09-14T03:36:48.414 INFO:teuthology.orchestra.run.smithi094.stdout:(14/114): librados-devel-17.2.7-1629.geb16b97b. 2.1 MB/s | 126 kB 00:00 2024-09-14T03:36:48.489 INFO:teuthology.orchestra.run.smithi094.stdout:(15/114): libradosstriper1-17.2.7-1629.geb16b97 6.0 MB/s | 458 kB 00:00 2024-09-14T03:36:48.656 INFO:teuthology.orchestra.run.smithi094.stdout:(16/114): ceph-osd-17.2.7-1629.geb16b97b.el9.x8 12 MB/s | 15 MB 00:01 2024-09-14T03:36:48.707 INFO:teuthology.orchestra.run.smithi094.stdout:(17/114): python3-ceph-argparse-17.2.7-1629.geb 907 kB/s | 45 kB 00:00 2024-09-14T03:36:48.782 INFO:teuthology.orchestra.run.smithi094.stdout:(18/114): librgw2-17.2.7-1629.geb16b97b.el9.x86 17 MB/s | 4.9 MB 00:00 2024-09-14T03:36:48.807 INFO:teuthology.orchestra.run.smithi094.stdout:(19/114): python3-ceph-common-17.2.7-1629.geb16 1.2 MB/s | 120 kB 00:00 2024-09-14T03:36:48.832 INFO:teuthology.orchestra.run.smithi094.stdout:(20/114): python3-cephfs-17.2.7-1629.geb16b97b. 3.1 MB/s | 157 kB 00:00 2024-09-14T03:36:48.866 INFO:teuthology.orchestra.run.smithi094.stdout:(21/114): python3-rados-17.2.7-1629.geb16b97b.e 5.3 MB/s | 320 kB 00:00 2024-09-14T03:36:48.900 INFO:teuthology.orchestra.run.smithi094.stdout:(22/114): python3-rbd-17.2.7-1629.geb16b97b.el9 4.3 MB/s | 296 kB 00:00 2024-09-14T03:36:48.925 INFO:teuthology.orchestra.run.smithi094.stdout:(23/114): python3-rgw-17.2.7-1629.geb16b97b.el9 1.7 MB/s | 99 kB 00:00 2024-09-14T03:36:48.950 INFO:teuthology.orchestra.run.smithi094.stdout:(24/114): rbd-fuse-17.2.7-1629.geb16b97b.el9.x8 1.6 MB/s | 82 kB 00:00 2024-09-14T03:36:49.001 INFO:teuthology.orchestra.run.smithi094.stdout:(25/114): rbd-nbd-17.2.7-1629.geb16b97b.el9.x86 3.2 MB/s | 165 kB 00:00 2024-09-14T03:36:49.059 INFO:teuthology.orchestra.run.smithi094.stdout:(26/114): ceph-grafana-dashboards-17.2.7-1629.g 420 kB/s | 24 kB 00:00 2024-09-14T03:36:49.118 INFO:teuthology.orchestra.run.smithi094.stdout:(27/114): rbd-mirror-17.2.7-1629.geb16b97b.el9. 15 MB/s | 2.9 MB 00:00 2024-09-14T03:36:49.143 INFO:teuthology.orchestra.run.smithi094.stdout:(28/114): ceph-mgr-cephadm-17.2.7-1629.geb16b97 1.5 MB/s | 124 kB 00:00 2024-09-14T03:36:49.152 INFO:teuthology.orchestra.run.smithi070.stdout:Package librados2-2:16.2.4-5.el9.x86_64 is already installed. 2024-09-14T03:36:49.153 INFO:teuthology.orchestra.run.smithi070.stdout:Package librbd1-2:16.2.4-5.el9.x86_64 is already installed. 2024-09-14T03:36:49.217 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2024-09-14T03:36:49.224 INFO:teuthology.orchestra.run.smithi070.stdout:======================================================================================= 2024-09-14T03:36:49.224 INFO:teuthology.orchestra.run.smithi070.stdout: Package Arch Version Repository Size 2024-09-14T03:36:49.224 INFO:teuthology.orchestra.run.smithi070.stdout:======================================================================================= 2024-09-14T03:36:49.224 INFO:teuthology.orchestra.run.smithi070.stdout:Installing: 2024-09-14T03:36:49.224 INFO:teuthology.orchestra.run.smithi070.stdout: ceph x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 6.5 k 2024-09-14T03:36:49.225 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-base x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 5.0 M 2024-09-14T03:36:49.225 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-fuse x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 813 k 2024-09-14T03:36:49.225 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 137 k 2024-09-14T03:36:49.225 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 1.4 M 2024-09-14T03:36:49.225 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr-cephadm noarch 2:17.2.7-1629.geb16b97b.el9 ceph-noarch 124 k 2024-09-14T03:36:49.225 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr-dashboard noarch 2:17.2.7-1629.geb16b97b.el9 ceph-noarch 3.4 M 2024-09-14T03:36:49.225 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-1629.geb16b97b.el9 ceph-noarch 7.4 M 2024-09-14T03:36:49.225 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr-rook noarch 2:17.2.7-1629.geb16b97b.el9 ceph-noarch 47 k 2024-09-14T03:36:49.225 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-radosgw x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 14 M 2024-09-14T03:36:49.225 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-test x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 48 M 2024-09-14T03:36:49.225 INFO:teuthology.orchestra.run.smithi070.stdout: cephadm noarch 2:17.2.7-1629.geb16b97b.el9 ceph-noarch 83 k 2024-09-14T03:36:49.225 INFO:teuthology.orchestra.run.smithi070.stdout: libcephfs-devel x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 25 k 2024-09-14T03:36:49.225 INFO:teuthology.orchestra.run.smithi070.stdout: libcephfs2 x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 636 k 2024-09-14T03:36:49.226 INFO:teuthology.orchestra.run.smithi070.stdout: librados-devel x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 126 k 2024-09-14T03:36:49.226 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cephfs x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 157 k 2024-09-14T03:36:49.226 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pytest noarch 6.2.2-7.el9 appstream 519 k 2024-09-14T03:36:49.226 INFO:teuthology.orchestra.run.smithi070.stdout: python3-rados x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 320 k 2024-09-14T03:36:49.226 INFO:teuthology.orchestra.run.smithi070.stdout: python3-rbd x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 296 k 2024-09-14T03:36:49.226 INFO:teuthology.orchestra.run.smithi070.stdout: python3-rgw x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 99 k 2024-09-14T03:36:49.226 INFO:teuthology.orchestra.run.smithi070.stdout: rbd-fuse x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 82 k 2024-09-14T03:36:49.226 INFO:teuthology.orchestra.run.smithi070.stdout: rbd-mirror x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 2.9 M 2024-09-14T03:36:49.226 INFO:teuthology.orchestra.run.smithi070.stdout: rbd-nbd x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 165 k 2024-09-14T03:36:49.226 INFO:teuthology.orchestra.run.smithi070.stdout:Upgrading: 2024-09-14T03:36:49.226 INFO:teuthology.orchestra.run.smithi070.stdout: librados2 x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 3.1 M 2024-09-14T03:36:49.226 INFO:teuthology.orchestra.run.smithi070.stdout: librbd1 x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 2.9 M 2024-09-14T03:36:49.226 INFO:teuthology.orchestra.run.smithi070.stdout:Installing dependencies: 2024-09-14T03:36:49.227 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-common x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 23 M 2024-09-14T03:36:49.227 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-grafana-dashboards noarch 2:17.2.7-1629.geb16b97b.el9 ceph-noarch 24 k 2024-09-14T03:36:49.227 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mds x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 2.1 M 2024-09-14T03:36:49.227 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr-modules-core noarch 2:17.2.7-1629.geb16b97b.el9 ceph-noarch 240 k 2024-09-14T03:36:49.227 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mon x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 3.9 M 2024-09-14T03:36:49.227 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-osd x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 15 M 2024-09-14T03:36:49.227 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-prometheus-alerts noarch 2:17.2.7-1629.geb16b97b.el9 ceph-noarch 15 k 2024-09-14T03:36:49.227 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-selinux x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 25 k 2024-09-14T03:36:49.227 INFO:teuthology.orchestra.run.smithi070.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-09-14T03:36:49.228 INFO:teuthology.orchestra.run.smithi070.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-09-14T03:36:49.228 INFO:teuthology.orchestra.run.smithi070.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-09-14T03:36:49.228 INFO:teuthology.orchestra.run.smithi070.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-09-14T03:36:49.228 INFO:teuthology.orchestra.run.smithi070.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2024-09-14T03:36:49.228 INFO:teuthology.orchestra.run.smithi070.stdout: libcephsqlite x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 163 k 2024-09-14T03:36:49.228 INFO:teuthology.orchestra.run.smithi070.stdout: libgfortran x86_64 11.5.0-2.el9 baseos 801 k 2024-09-14T03:36:49.228 INFO:teuthology.orchestra.run.smithi070.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-09-14T03:36:49.228 INFO:teuthology.orchestra.run.smithi070.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-09-14T03:36:49.229 INFO:teuthology.orchestra.run.smithi070.stdout: libquadmath x86_64 11.5.0-2.el9 baseos 191 k 2024-09-14T03:36:49.229 INFO:teuthology.orchestra.run.smithi070.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-09-14T03:36:49.229 INFO:teuthology.orchestra.run.smithi070.stdout: libradosstriper1 x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 458 k 2024-09-14T03:36:49.229 INFO:teuthology.orchestra.run.smithi070.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-09-14T03:36:49.229 INFO:teuthology.orchestra.run.smithi070.stdout: librgw2 x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 4.9 M 2024-09-14T03:36:49.229 INFO:teuthology.orchestra.run.smithi070.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-09-14T03:36:49.229 INFO:teuthology.orchestra.run.smithi070.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-09-14T03:36:49.229 INFO:teuthology.orchestra.run.smithi070.stdout: openblas x86_64 0.3.26-2.el9 appstream 38 k 2024-09-14T03:36:49.229 INFO:teuthology.orchestra.run.smithi070.stdout: openblas-openmp x86_64 0.3.26-2.el9 appstream 4.9 M 2024-09-14T03:36:49.229 INFO:teuthology.orchestra.run.smithi070.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-09-14T03:36:49.229 INFO:teuthology.orchestra.run.smithi070.stdout: python3-attrs noarch 20.3.0-7.el9 appstream 87 k 2024-09-14T03:36:49.229 INFO:teuthology.orchestra.run.smithi070.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-09-14T03:36:49.229 INFO:teuthology.orchestra.run.smithi070.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-09-14T03:36:49.230 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-09-14T03:36:49.230 INFO:teuthology.orchestra.run.smithi070.stdout: python3-ceph-argparse x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 45 k 2024-09-14T03:36:49.230 INFO:teuthology.orchestra.run.smithi070.stdout: python3-ceph-common x86_64 2:17.2.7-1629.geb16b97b.el9 ceph 120 k 2024-09-14T03:36:49.230 INFO:teuthology.orchestra.run.smithi070.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-09-14T03:36:49.230 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-09-14T03:36:49.230 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-09-14T03:36:49.230 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-09-14T03:36:49.230 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-09-14T03:36:49.230 INFO:teuthology.orchestra.run.smithi070.stdout: python3-devel x86_64 3.9.19-8.el9 appstream 245 k 2024-09-14T03:36:49.230 INFO:teuthology.orchestra.run.smithi070.stdout: python3-google-auth noarch 1:2.34.0-1.el9 epel 220 k 2024-09-14T03:36:49.230 INFO:teuthology.orchestra.run.smithi070.stdout: python3-iniconfig noarch 1.1.1-7.el9 appstream 17 k 2024-09-14T03:36:49.230 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-09-14T03:36:49.230 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-09-14T03:36:49.231 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-09-14T03:36:49.231 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-09-14T03:36:49.231 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-09-14T03:36:49.231 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-09-14T03:36:49.231 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-09-14T03:36:49.231 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-09-14T03:36:49.231 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-09-14T03:36:49.231 INFO:teuthology.orchestra.run.smithi070.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-09-14T03:36:49.231 INFO:teuthology.orchestra.run.smithi070.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-09-14T03:36:49.231 INFO:teuthology.orchestra.run.smithi070.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-09-14T03:36:49.231 INFO:teuthology.orchestra.run.smithi070.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-09-14T03:36:49.231 INFO:teuthology.orchestra.run.smithi070.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-09-14T03:36:49.231 INFO:teuthology.orchestra.run.smithi070.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-09-14T03:36:49.232 INFO:teuthology.orchestra.run.smithi070.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-09-14T03:36:49.232 INFO:teuthology.orchestra.run.smithi070.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-09-14T03:36:49.232 INFO:teuthology.orchestra.run.smithi070.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-09-14T03:36:49.232 INFO:teuthology.orchestra.run.smithi070.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-09-14T03:36:49.232 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-09-14T03:36:49.232 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pluggy noarch 0.13.1-7.el9 appstream 41 k 2024-09-14T03:36:49.232 INFO:teuthology.orchestra.run.smithi070.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-09-14T03:36:49.232 INFO:teuthology.orchestra.run.smithi070.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-09-14T03:36:49.232 INFO:teuthology.orchestra.run.smithi070.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-09-14T03:36:49.232 INFO:teuthology.orchestra.run.smithi070.stdout: python3-py noarch 1.10.0-6.el9 appstream 477 k 2024-09-14T03:36:49.232 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-09-14T03:36:49.232 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-09-14T03:36:49.232 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-09-14T03:36:49.233 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-09-14T03:36:49.233 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-09-14T03:36:49.233 INFO:teuthology.orchestra.run.smithi070.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-09-14T03:36:49.233 INFO:teuthology.orchestra.run.smithi070.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-09-14T03:36:49.233 INFO:teuthology.orchestra.run.smithi070.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-09-14T03:36:49.233 INFO:teuthology.orchestra.run.smithi070.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-09-14T03:36:49.233 INFO:teuthology.orchestra.run.smithi070.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-09-14T03:36:49.233 INFO:teuthology.orchestra.run.smithi070.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-09-14T03:36:49.233 INFO:teuthology.orchestra.run.smithi070.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-09-14T03:36:49.233 INFO:teuthology.orchestra.run.smithi070.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-09-14T03:36:49.233 INFO:teuthology.orchestra.run.smithi070.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-09-14T03:36:49.233 INFO:teuthology.orchestra.run.smithi070.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-09-14T03:36:49.233 INFO:teuthology.orchestra.run.smithi070.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-09-14T03:36:49.233 INFO:teuthology.orchestra.run.smithi070.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-09-14T03:36:49.234 INFO:teuthology.orchestra.run.smithi070.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-09-14T03:36:49.234 INFO:teuthology.orchestra.run.smithi070.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-09-14T03:36:49.234 INFO:teuthology.orchestra.run.smithi070.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-09-14T03:36:49.234 INFO:teuthology.orchestra.run.smithi070.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-09-14T03:36:49.234 INFO:teuthology.orchestra.run.smithi070.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-09-14T03:36:49.234 INFO:teuthology.orchestra.run.smithi070.stdout:Installing weak dependencies: 2024-09-14T03:36:49.234 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-09-14T03:36:49.234 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:36:49.234 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2024-09-14T03:36:49.234 INFO:teuthology.orchestra.run.smithi070.stdout:======================================================================================= 2024-09-14T03:36:49.234 INFO:teuthology.orchestra.run.smithi070.stdout:Install 112 Packages 2024-09-14T03:36:49.234 INFO:teuthology.orchestra.run.smithi070.stdout:Upgrade 2 Packages 2024-09-14T03:36:49.234 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:36:49.235 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 190 M 2024-09-14T03:36:49.235 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2024-09-14T03:36:49.435 INFO:teuthology.orchestra.run.smithi094.stdout:(29/114): ceph-mgr-dashboard-17.2.7-1629.geb16b 11 MB/s | 3.4 MB 00:00 2024-09-14T03:36:49.518 INFO:teuthology.orchestra.run.smithi094.stdout:(30/114): ceph-mgr-diskprediction-local-17.2.7- 20 MB/s | 7.4 MB 00:00 2024-09-14T03:36:49.530 INFO:teuthology.orchestra.run.smithi070.stdout:(1/114): ceph-17.2.7-1629.geb16b97b.el9.x86_64. 44 kB/s | 6.5 kB 00:00 2024-09-14T03:36:49.544 INFO:teuthology.orchestra.run.smithi094.stdout:(31/114): ceph-mgr-modules-core-17.2.7-1629.geb 2.2 MB/s | 240 kB 00:00 2024-09-14T03:36:49.561 INFO:teuthology.orchestra.run.smithi094.stdout:(32/114): ceph-mgr-rook-17.2.7-1629.geb16b97b.e 1.1 MB/s | 47 kB 00:00 2024-09-14T03:36:49.578 INFO:teuthology.orchestra.run.smithi094.stdout:(33/114): ceph-prometheus-alerts-17.2.7-1629.ge 434 kB/s | 15 kB 00:00 2024-09-14T03:36:49.612 INFO:teuthology.orchestra.run.smithi094.stdout:(34/114): cephadm-17.2.7-1629.geb16b97b.el9.noa 1.6 MB/s | 83 kB 00:00 2024-09-14T03:36:49.720 INFO:teuthology.orchestra.run.smithi094.stdout:(35/114): fuse-2.9.9-16.el9.x86_64.rpm 562 kB/s | 80 kB 00:00 2024-09-14T03:36:49.753 INFO:teuthology.orchestra.run.smithi070.stdout:(2/114): ceph-fuse-17.2.7-1629.geb16b97b.el9.x8 3.6 MB/s | 813 kB 00:00 2024-09-14T03:36:49.820 INFO:teuthology.orchestra.run.smithi094.stdout:(36/114): libquadmath-11.5.0-2.el9.x86_64.rpm 1.9 MB/s | 191 kB 00:00 2024-09-14T03:36:49.821 INFO:teuthology.orchestra.run.smithi070.stdout:(3/114): ceph-immutable-object-cache-17.2.7-162 2.0 MB/s | 137 kB 00:00 2024-09-14T03:36:49.904 INFO:teuthology.orchestra.run.smithi094.stdout:(37/114): libgfortran-11.5.0-2.el9.x86_64.rpm 2.7 MB/s | 801 kB 00:00 2024-09-14T03:36:49.996 INFO:teuthology.orchestra.run.smithi070.stdout:(4/114): ceph-base-17.2.7-1629.geb16b97b.el9.x8 8.1 MB/s | 5.0 MB 00:00 2024-09-14T03:36:50.038 INFO:teuthology.orchestra.run.smithi094.stdout:(38/114): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.1 MB/s | 253 kB 00:00 2024-09-14T03:36:50.088 INFO:teuthology.orchestra.run.smithi094.stdout:(39/114): python3-ply-3.11-14.el9.noarch.rpm 2.1 MB/s | 106 kB 00:00 2024-09-14T03:36:50.104 INFO:teuthology.orchestra.run.smithi070.stdout:(5/114): ceph-mds-17.2.7-1629.geb16b97b.el9.x86 7.3 MB/s | 2.1 MB 00:00 2024-09-14T03:36:50.138 INFO:teuthology.orchestra.run.smithi070.stdout:(6/114): ceph-mgr-17.2.7-1629.geb16b97b.el9.x86 9.9 MB/s | 1.4 MB 00:00 2024-09-14T03:36:50.139 INFO:teuthology.orchestra.run.smithi094.stdout:(40/114): python3-pycparser-2.20-6.el9.noarch.r 2.6 MB/s | 135 kB 00:00 2024-09-14T03:36:50.172 INFO:teuthology.orchestra.run.smithi094.stdout:(41/114): python3-cryptography-36.0.1-4.el9.x86 4.6 MB/s | 1.2 MB 00:00 2024-09-14T03:36:50.200 INFO:teuthology.orchestra.run.smithi094.stdout:(42/114): python3-requests-2.25.1-8.el9.noarch. 2.1 MB/s | 125 kB 00:00 2024-09-14T03:36:50.256 INFO:teuthology.orchestra.run.smithi094.stdout:(43/114): python3-urllib3-1.26.5-6.el9.noarch.r 2.5 MB/s | 215 kB 00:00 2024-09-14T03:36:50.438 INFO:teuthology.orchestra.run.smithi070.stdout:(7/114): ceph-mon-17.2.7-1629.geb16b97b.el9.x86 12 MB/s | 3.9 MB 00:00 2024-09-14T03:36:50.732 INFO:teuthology.orchestra.run.smithi094.stdout:(44/114): ceph-test-17.2.7-1629.geb16b97b.el9.x 16 MB/s | 48 MB 00:02 2024-09-14T03:36:50.757 INFO:teuthology.orchestra.run.smithi094.stdout:(45/114): flexiblas-3.0.4-8.el9.x86_64.rpm 58 kB/s | 33 kB 00:00 2024-09-14T03:36:50.849 INFO:teuthology.orchestra.run.smithi094.stdout:(46/114): flexiblas-openblas-openmp-3.0.4-8.el9 156 kB/s | 18 kB 00:00 2024-09-14T03:36:50.924 INFO:teuthology.orchestra.run.smithi094.stdout:(47/114): libpmemobj-1.12.1-1.el9.x86_64.rpm 960 kB/s | 160 kB 00:00 2024-09-14T03:36:50.938 INFO:teuthology.orchestra.run.smithi070.stdout:(8/114): ceph-osd-17.2.7-1629.geb16b97b.el9.x86 19 MB/s | 15 MB 00:00 2024-09-14T03:36:50.949 INFO:teuthology.orchestra.run.smithi094.stdout:(48/114): librabbitmq-0.11.0-7.el9.x86_64.rpm 451 kB/s | 45 kB 00:00 2024-09-14T03:36:50.997 INFO:teuthology.orchestra.run.smithi070.stdout:(9/114): ceph-selinux-17.2.7-1629.geb16b97b.el9 432 kB/s | 25 kB 00:00 2024-09-14T03:36:51.091 INFO:teuthology.orchestra.run.smithi094.stdout:(49/114): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.0 MB/s | 292 kB 00:00 2024-09-14T03:36:51.153 INFO:teuthology.orchestra.run.smithi094.stdout:(50/114): librdkafka-1.6.1-102.el9.x86_64.rpm 2.8 MB/s | 662 kB 00:00 2024-09-14T03:36:51.179 INFO:teuthology.orchestra.run.smithi094.stdout:(51/114): openblas-0.3.26-2.el9.x86_64.rpm 441 kB/s | 38 kB 00:00 2024-09-14T03:36:51.237 INFO:teuthology.orchestra.run.smithi094.stdout:(52/114): python3-attrs-20.3.0-7.el9.noarch.rpm 1.5 MB/s | 87 kB 00:00 2024-09-14T03:36:51.373 INFO:teuthology.orchestra.run.smithi070.stdout:(10/114): ceph-common-17.2.7-1629.geb16b97b.el9 11 MB/s | 23 MB 00:01 2024-09-14T03:36:51.523 INFO:teuthology.orchestra.run.smithi070.stdout:(11/114): ceph-radosgw-17.2.7-1629.geb16b97b.el 13 MB/s | 14 MB 00:01 2024-09-14T03:36:51.548 INFO:teuthology.orchestra.run.smithi070.stdout:(12/114): libcephfs-devel-17.2.7-1629.geb16b97b 143 kB/s | 25 kB 00:00 2024-09-14T03:36:51.582 INFO:teuthology.orchestra.run.smithi070.stdout:(13/114): libcephfs2-17.2.7-1629.geb16b97b.el9. 11 MB/s | 636 kB 00:00 2024-09-14T03:36:51.607 INFO:teuthology.orchestra.run.smithi070.stdout:(14/114): libcephsqlite-17.2.7-1629.geb16b97b.e 2.7 MB/s | 163 kB 00:00 2024-09-14T03:36:51.632 INFO:teuthology.orchestra.run.smithi070.stdout:(15/114): librados-devel-17.2.7-1629.geb16b97b. 2.5 MB/s | 126 kB 00:00 2024-09-14T03:36:51.666 INFO:teuthology.orchestra.run.smithi070.stdout:(16/114): libradosstriper1-17.2.7-1629.geb16b97 7.6 MB/s | 458 kB 00:00 2024-09-14T03:36:51.716 INFO:teuthology.orchestra.run.smithi070.stdout:(17/114): python3-ceph-argparse-17.2.7-1629.geb 907 kB/s | 45 kB 00:00 2024-09-14T03:36:51.800 INFO:teuthology.orchestra.run.smithi070.stdout:(18/114): python3-ceph-common-17.2.7-1629.geb16 1.4 MB/s | 120 kB 00:00 2024-09-14T03:36:51.904 INFO:teuthology.orchestra.run.smithi094.stdout:(53/114): flexiblas-netlib-3.0.4-8.el9.x86_64.r 1.8 MB/s | 3.0 MB 00:01 2024-09-14T03:36:51.917 INFO:teuthology.orchestra.run.smithi070.stdout:(19/114): librgw2-17.2.7-1629.geb16b97b.el9.x86 17 MB/s | 4.9 MB 00:00 2024-09-14T03:36:51.942 INFO:teuthology.orchestra.run.smithi070.stdout:(20/114): python3-cephfs-17.2.7-1629.geb16b97b. 1.1 MB/s | 157 kB 00:00 2024-09-14T03:36:51.967 INFO:teuthology.orchestra.run.smithi070.stdout:(21/114): python3-rados-17.2.7-1629.geb16b97b.e 6.3 MB/s | 320 kB 00:00 2024-09-14T03:36:51.993 INFO:teuthology.orchestra.run.smithi070.stdout:(22/114): python3-rbd-17.2.7-1629.geb16b97b.el9 5.7 MB/s | 296 kB 00:00 2024-09-14T03:36:52.018 INFO:teuthology.orchestra.run.smithi070.stdout:(23/114): python3-rgw-17.2.7-1629.geb16b97b.el9 1.9 MB/s | 99 kB 00:00 2024-09-14T03:36:52.043 INFO:teuthology.orchestra.run.smithi070.stdout:(24/114): rbd-fuse-17.2.7-1629.geb16b97b.el9.x8 1.6 MB/s | 82 kB 00:00 2024-09-14T03:36:52.054 INFO:teuthology.orchestra.run.smithi094.stdout:(54/114): python3-devel-3.9.19-8.el9.x86_64.rpm 1.6 MB/s | 245 kB 00:00 2024-09-14T03:36:52.102 INFO:teuthology.orchestra.run.smithi070.stdout:(25/114): rbd-nbd-17.2.7-1629.geb16b97b.el9.x86 2.8 MB/s | 165 kB 00:00 2024-09-14T03:36:52.121 INFO:teuthology.orchestra.run.smithi094.stdout:(55/114): python3-iniconfig-1.1.1-7.el9.noarch. 262 kB/s | 17 kB 00:00 2024-09-14T03:36:52.169 INFO:teuthology.orchestra.run.smithi070.stdout:(26/114): rbd-mirror-17.2.7-1629.geb16b97b.el9. 19 MB/s | 2.9 MB 00:00 2024-09-14T03:36:52.194 INFO:teuthology.orchestra.run.smithi070.stdout:(27/114): ceph-grafana-dashboards-17.2.7-1629.g 266 kB/s | 24 kB 00:00 2024-09-14T03:36:52.219 INFO:teuthology.orchestra.run.smithi070.stdout:(28/114): ceph-mgr-cephadm-17.2.7-1629.geb16b97 2.4 MB/s | 124 kB 00:00 2024-09-14T03:36:52.247 INFO:teuthology.orchestra.run.smithi094.stdout:(56/114): python3-jinja2-2.11.3-6.el9.noarch.rp 1.9 MB/s | 248 kB 00:00 2024-09-14T03:36:52.330 INFO:teuthology.orchestra.run.smithi094.stdout:(57/114): openblas-openmp-0.3.26-2.el9.x86_64.r 4.2 MB/s | 4.9 MB 00:01 2024-09-14T03:36:52.356 INFO:teuthology.orchestra.run.smithi094.stdout:(58/114): python3-jsonpatch-1.21-16.el9.noarch. 236 kB/s | 26 kB 00:00 2024-09-14T03:36:52.381 INFO:teuthology.orchestra.run.smithi094.stdout:(59/114): python3-jsonpointer-2.0-4.el9.noarch. 383 kB/s | 19 kB 00:00 2024-09-14T03:36:52.465 INFO:teuthology.orchestra.run.smithi094.stdout:(60/114): python3-babel-2.9.1-2.el9.noarch.rpm 4.9 MB/s | 6.0 MB 00:01 2024-09-14T03:36:52.490 INFO:teuthology.orchestra.run.smithi094.stdout:(61/114): python3-markupsafe-1.1.1-12.el9.x86_6 319 kB/s | 35 kB 00:00 2024-09-14T03:36:52.495 INFO:teuthology.orchestra.run.smithi070.stdout:(29/114): ceph-mgr-dashboard-17.2.7-1629.geb16b 11 MB/s | 3.4 MB 00:00 2024-09-14T03:36:52.515 INFO:teuthology.orchestra.run.smithi094.stdout:(62/114): python3-mako-1.1.4-6.el9.noarch.rpm 1.1 MB/s | 172 kB 00:00 2024-09-14T03:36:52.553 INFO:teuthology.orchestra.run.smithi070.stdout:(30/114): ceph-mgr-modules-core-17.2.7-1629.geb 4.0 MB/s | 240 kB 00:00 2024-09-14T03:36:52.557 INFO:teuthology.orchestra.run.smithi094.stdout:(63/114): python3-numpy-f2py-1.20.1-5.el9.x86_6 3.7 MB/s | 250 kB 00:00 2024-09-14T03:36:52.583 INFO:teuthology.orchestra.run.smithi094.stdout:(64/114): python3-oauthlib-3.1.1-5.el9.noarch.r 3.2 MB/s | 222 kB 00:00 2024-09-14T03:36:52.658 INFO:teuthology.orchestra.run.smithi094.stdout:(65/114): python3-numpy-1.20.1-5.el9.x86_64.rpm 26 MB/s | 5.1 MB 00:00 2024-09-14T03:36:52.662 INFO:teuthology.orchestra.run.smithi070.stdout:(31/114): ceph-mgr-diskprediction-local-17.2.7- 17 MB/s | 7.4 MB 00:00 2024-09-14T03:36:52.679 INFO:teuthology.orchestra.run.smithi070.stdout:(32/114): ceph-mgr-rook-17.2.7-1629.geb16b97b.e 379 kB/s | 47 kB 00:00 2024-09-14T03:36:52.683 INFO:teuthology.orchestra.run.smithi094.stdout:(66/114): python3-pluggy-0.13.1-7.el9.noarch.rp 413 kB/s | 41 kB 00:00 2024-09-14T03:36:52.704 INFO:teuthology.orchestra.run.smithi070.stdout:(33/114): ceph-prometheus-alerts-17.2.7-1629.ge 351 kB/s | 15 kB 00:00 2024-09-14T03:36:52.708 INFO:teuthology.orchestra.run.smithi094.stdout:(67/114): python3-packaging-20.9-5.el9.noarch.r 512 kB/s | 77 kB 00:00 2024-09-14T03:36:52.734 INFO:teuthology.orchestra.run.smithi094.stdout:(68/114): python3-prettytable-0.7.2-27.el9.noar 560 kB/s | 42 kB 00:00 2024-09-14T03:36:52.738 INFO:teuthology.orchestra.run.smithi070.stdout:(34/114): cephadm-17.2.7-1629.geb16b97b.el9.noa 1.4 MB/s | 83 kB 00:00 2024-09-14T03:36:52.767 INFO:teuthology.orchestra.run.smithi094.stdout:(69/114): python3-py-1.10.0-6.el9.noarch.rpm 5.6 MB/s | 477 kB 00:00 2024-09-14T03:36:52.793 INFO:teuthology.orchestra.run.smithi094.stdout:(70/114): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.9 MB/s | 159 kB 00:00 2024-09-14T03:36:52.818 INFO:teuthology.orchestra.run.smithi094.stdout:(71/114): python3-pyasn1-modules-0.4.8-6.el9.no 3.3 MB/s | 279 kB 00:00 2024-09-14T03:36:52.838 INFO:teuthology.orchestra.run.smithi070.stdout:(35/114): fuse-2.9.9-16.el9.x86_64.rpm 597 kB/s | 80 kB 00:00 2024-09-14T03:36:52.853 INFO:teuthology.orchestra.run.smithi094.stdout:(72/114): python3-pytest-6.2.2-7.el9.noarch.rpm 6.0 MB/s | 519 kB 00:00 2024-09-14T03:36:52.869 INFO:teuthology.orchestra.run.smithi094.stdout:(73/114): python3-pytz-2021.1-5.el9.noarch.rpm 674 kB/s | 51 kB 00:00 2024-09-14T03:36:52.885 INFO:teuthology.orchestra.run.smithi094.stdout:(74/114): python3-requests-oauthlib-1.3.0-12.el 800 kB/s | 54 kB 00:00 2024-09-14T03:36:52.936 INFO:teuthology.orchestra.run.smithi094.stdout:(75/114): python3-toml-0.10.2-6.el9.noarch.rpm 833 kB/s | 42 kB 00:00 2024-09-14T03:36:52.969 INFO:teuthology.orchestra.run.smithi094.stdout:(76/114): socat-1.7.4.1-6.el9.x86_64.rpm 3.6 MB/s | 304 kB 00:00 2024-09-14T03:36:52.986 INFO:teuthology.orchestra.run.smithi094.stdout:(77/114): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.2 MB/s | 64 kB 00:00 2024-09-14T03:36:53.044 INFO:teuthology.orchestra.run.smithi094.stdout:(78/114): gperftools-libs-2.9.1-3.el9.x86_64.rp 4.0 MB/s | 308 kB 00:00 2024-09-14T03:36:53.061 INFO:teuthology.orchestra.run.smithi094.stdout:(79/114): liboath-2.6.7-2.el9.x86_64.rpm 647 kB/s | 49 kB 00:00 2024-09-14T03:36:53.087 INFO:teuthology.orchestra.run.smithi094.stdout:(80/114): libunwind-1.6.2-1.el9.x86_64.rpm 1.6 MB/s | 67 kB 00:00 2024-09-14T03:36:53.104 INFO:teuthology.orchestra.run.smithi094.stdout:(81/114): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 2.6 MB/s | 43 kB 00:00 2024-09-14T03:36:53.121 INFO:teuthology.orchestra.run.smithi094.stdout:(82/114): python3-cachetools-4.2.4-1.el9.noarch 1.9 MB/s | 32 kB 00:00 2024-09-14T03:36:53.146 INFO:teuthology.orchestra.run.smithi094.stdout:(83/114): python3-certifi-2023.05.07-4.el9.noar 563 kB/s | 14 kB 00:00 2024-09-14T03:36:53.171 INFO:teuthology.orchestra.run.smithi094.stdout:(84/114): python3-asyncssh-2.13.2-5.el9.noarch. 4.9 MB/s | 548 kB 00:00 2024-09-14T03:36:53.197 INFO:teuthology.orchestra.run.smithi094.stdout:(85/114): python3-cheroot-8.6.0-4.el9.noarch.rp 3.3 MB/s | 172 kB 00:00 2024-09-14T03:36:53.238 INFO:teuthology.orchestra.run.smithi094.stdout:(86/114): python3-google-auth-2.34.0-1.el9.noar 5.2 MB/s | 220 kB 00:00 2024-09-14T03:36:53.272 INFO:teuthology.orchestra.run.smithi094.stdout:(87/114): python3-cherrypy-18.6.1-2.el9.noarch. 3.5 MB/s | 358 kB 00:00 2024-09-14T03:36:53.273 INFO:teuthology.orchestra.run.smithi070.stdout:(36/114): ceph-test-17.2.7-1629.geb16b97b.el9.x 21 MB/s | 48 MB 00:02 2024-09-14T03:36:53.297 INFO:teuthology.orchestra.run.smithi070.stdout:(37/114): libquadmath-11.5.0-2.el9.x86_64.rpm 417 kB/s | 191 kB 00:00 2024-09-14T03:36:53.298 INFO:teuthology.orchestra.run.smithi094.stdout:(88/114): python3-jaraco-8.2.1-3.el9.noarch.rpm 182 kB/s | 11 kB 00:00 2024-09-14T03:36:53.323 INFO:teuthology.orchestra.run.smithi094.stdout:(89/114): python3-jaraco-classes-3.2.1-5.el9.no 350 kB/s | 18 kB 00:00 2024-09-14T03:36:53.348 INFO:teuthology.orchestra.run.smithi094.stdout:(90/114): python3-jaraco-collections-3.0.0-8.el 459 kB/s | 23 kB 00:00 2024-09-14T03:36:53.374 INFO:teuthology.orchestra.run.smithi094.stdout:(91/114): python3-jaraco-functools-3.5.0-2.el9. 384 kB/s | 19 kB 00:00 2024-09-14T03:36:53.399 INFO:teuthology.orchestra.run.smithi094.stdout:(92/114): python3-jaraco-text-3.2.0-6.el9.noarc 391 kB/s | 20 kB 00:00 2024-09-14T03:36:53.406 INFO:teuthology.orchestra.run.smithi070.stdout:(38/114): python3-cryptography-36.0.1-4.el9.x86 11 MB/s | 1.2 MB 00:00 2024-09-14T03:36:53.424 INFO:teuthology.orchestra.run.smithi094.stdout:(93/114): python3-jwt+crypto-2.4.0-1.el9.noarch 180 kB/s | 9.0 kB 00:00 2024-09-14T03:36:53.431 INFO:teuthology.orchestra.run.smithi070.stdout:(39/114): libgfortran-11.5.0-2.el9.x86_64.rpm 1.1 MB/s | 801 kB 00:00 2024-09-14T03:36:53.450 INFO:teuthology.orchestra.run.smithi094.stdout:(94/114): python3-jwt-2.4.0-1.el9.noarch.rpm 812 kB/s | 41 kB 00:00 2024-09-14T03:36:53.473 INFO:teuthology.orchestra.run.smithi070.stdout:(40/114): python3-ply-3.11-14.el9.noarch.rpm 1.6 MB/s | 106 kB 00:00 2024-09-14T03:36:53.475 INFO:teuthology.orchestra.run.smithi094.stdout:(95/114): python3-logutils-0.3.5-21.el9.noarch. 1.8 MB/s | 46 kB 00:00 2024-09-14T03:36:53.500 INFO:teuthology.orchestra.run.smithi094.stdout:(96/114): python3-more-itertools-8.12.0-2.el9.n 3.1 MB/s | 79 kB 00:00 2024-09-14T03:36:53.501 INFO:teuthology.orchestra.run.smithi070.stdout:(41/114): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.1 MB/s | 253 kB 00:00 2024-09-14T03:36:53.525 INFO:teuthology.orchestra.run.smithi094.stdout:(97/114): python3-natsort-7.1.1-5.el9.noarch.rp 2.3 MB/s | 58 kB 00:00 2024-09-14T03:36:53.526 INFO:teuthology.orchestra.run.smithi070.stdout:(42/114): python3-pycparser-2.20-6.el9.noarch.r 1.4 MB/s | 135 kB 00:00 2024-09-14T03:36:53.552 INFO:teuthology.orchestra.run.smithi070.stdout:(43/114): python3-requests-2.25.1-8.el9.noarch. 1.6 MB/s | 125 kB 00:00 2024-09-14T03:36:53.559 INFO:teuthology.orchestra.run.smithi094.stdout:(98/114): python3-pecan-1.4.2-3.el9.noarch.rpm 8.0 MB/s | 272 kB 00:00 2024-09-14T03:36:53.593 INFO:teuthology.orchestra.run.smithi094.stdout:(99/114): python3-kubernetes-26.1.0-2.el9.noarc 6.1 MB/s | 1.0 MB 00:00 2024-09-14T03:36:53.594 INFO:teuthology.orchestra.run.smithi070.stdout:(44/114): flexiblas-3.0.4-8.el9.x86_64.rpm 488 kB/s | 33 kB 00:00 2024-09-14T03:36:53.618 INFO:teuthology.orchestra.run.smithi094.stdout:(100/114): python3-portend-3.1.0-2.el9.noarch.r 279 kB/s | 16 kB 00:00 2024-09-14T03:36:53.619 INFO:teuthology.orchestra.run.smithi070.stdout:(45/114): python3-urllib3-1.26.5-6.el9.noarch.r 1.8 MB/s | 215 kB 00:00 2024-09-14T03:36:53.635 INFO:teuthology.orchestra.run.smithi094.stdout:(101/114): python3-repoze-lru-0.7-16.el9.noarch 1.8 MB/s | 31 kB 00:00 2024-09-14T03:36:53.660 INFO:teuthology.orchestra.run.smithi094.stdout:(102/114): python3-pyOpenSSL-21.0.0-1.el9.noarc 1.3 MB/s | 90 kB 00:00 2024-09-14T03:36:53.669 INFO:teuthology.orchestra.run.smithi070.stdout:(46/114): flexiblas-openblas-openmp-3.0.4-8.el9 242 kB/s | 18 kB 00:00 2024-09-14T03:36:53.686 INFO:teuthology.orchestra.run.smithi094.stdout:(103/114): python3-rsa-4.9-2.el9.noarch.rpm 2.3 MB/s | 59 kB 00:00 2024-09-14T03:36:53.728 INFO:teuthology.orchestra.run.smithi070.stdout:(47/114): flexiblas-netlib-3.0.4-8.el9.x86_64.r 17 MB/s | 3.0 MB 00:00 2024-09-14T03:36:53.736 INFO:teuthology.orchestra.run.smithi094.stdout:(104/114): python3-routes-2.5.1-5.el9.noarch.rp 1.8 MB/s | 188 kB 00:00 2024-09-14T03:36:53.753 INFO:teuthology.orchestra.run.smithi094.stdout:(105/114): python3-tempora-5.0.0-2.el9.noarch.r 535 kB/s | 36 kB 00:00 2024-09-14T03:36:53.753 INFO:teuthology.orchestra.run.smithi070.stdout:(48/114): librabbitmq-0.11.0-7.el9.x86_64.rpm 541 kB/s | 45 kB 00:00 2024-09-14T03:36:53.778 INFO:teuthology.orchestra.run.smithi094.stdout:(106/114): python3-typing-extensions-4.12.2-2.e 1.7 MB/s | 74 kB 00:00 2024-09-14T03:36:53.779 INFO:teuthology.orchestra.run.smithi070.stdout:(49/114): libpmemobj-1.12.1-1.el9.x86_64.rpm 1.0 MB/s | 160 kB 00:00 2024-09-14T03:36:53.803 INFO:teuthology.orchestra.run.smithi094.stdout:(107/114): python3-webob-1.8.8-2.el9.noarch.rpm 4.5 MB/s | 230 kB 00:00 2024-09-14T03:36:53.829 INFO:teuthology.orchestra.run.smithi094.stdout:(108/114): python3-websocket-client-1.2.3-2.el9 1.7 MB/s | 90 kB 00:00 2024-09-14T03:36:53.837 INFO:teuthology.orchestra.run.smithi070.stdout:(50/114): librdkafka-1.6.1-102.el9.x86_64.rpm 5.9 MB/s | 662 kB 00:00 2024-09-14T03:36:53.846 INFO:teuthology.orchestra.run.smithi094.stdout:(109/114): python3-zc-lockfile-2.0-10.el9.noarc 1.2 MB/s | 20 kB 00:00 2024-09-14T03:36:53.862 INFO:teuthology.orchestra.run.smithi070.stdout:(51/114): openblas-0.3.26-2.el9.x86_64.rpm 460 kB/s | 38 kB 00:00 2024-09-14T03:36:53.888 INFO:teuthology.orchestra.run.smithi070.stdout:(52/114): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.1 MB/s | 292 kB 00:00 2024-09-14T03:36:53.980 INFO:teuthology.orchestra.run.smithi070.stdout:(53/114): openblas-openmp-0.3.26-2.el9.x86_64.r 35 MB/s | 4.9 MB 00:00 2024-09-14T03:36:54.005 INFO:teuthology.orchestra.run.smithi094.stdout:(110/114): python3-scipy-1.6.2-8.el9.x86_64.rpm 14 MB/s | 16 MB 00:01 2024-09-14T03:36:54.022 INFO:teuthology.orchestra.run.smithi070.stdout:(54/114): python3-attrs-20.3.0-7.el9.noarch.rpm 549 kB/s | 87 kB 00:00 2024-09-14T03:36:54.030 INFO:teuthology.orchestra.run.smithi094.stdout:(111/114): python3-werkzeug-2.0.3-3.el9.1.noarc 1.8 MB/s | 427 kB 00:00 2024-09-14T03:36:54.064 INFO:teuthology.orchestra.run.smithi070.stdout:(55/114): python3-devel-3.9.19-8.el9.x86_64.rpm 2.9 MB/s | 245 kB 00:00 2024-09-14T03:36:54.088 INFO:teuthology.orchestra.run.smithi094.stdout:(112/114): thrift-0.15.0-2.el9.x86_64.rpm 6.5 MB/s | 1.6 MB 00:00 2024-09-14T03:36:54.106 INFO:teuthology.orchestra.run.smithi070.stdout:(56/114): python3-iniconfig-1.1.1-7.el9.noarch. 209 kB/s | 17 kB 00:00 2024-09-14T03:36:54.139 INFO:teuthology.orchestra.run.smithi070.stdout:(57/114): python3-jinja2-2.11.3-6.el9.noarch.rp 3.2 MB/s | 248 kB 00:00 2024-09-14T03:36:54.190 INFO:teuthology.orchestra.run.smithi070.stdout:(58/114): python3-jsonpatch-1.21-16.el9.noarch. 307 kB/s | 26 kB 00:00 2024-09-14T03:36:54.215 INFO:teuthology.orchestra.run.smithi070.stdout:(59/114): python3-jsonpointer-2.0-4.el9.noarch. 254 kB/s | 19 kB 00:00 2024-09-14T03:36:54.247 INFO:teuthology.orchestra.run.smithi094.stdout:(113/114): librados2-17.2.7-1629.geb16b97b.el9. 13 MB/s | 3.1 MB 00:00 2024-09-14T03:36:54.286 INFO:teuthology.orchestra.run.smithi070.stdout:(60/114): python3-markupsafe-1.1.1-12.el9.x86_6 521 kB/s | 35 kB 00:00 2024-09-14T03:36:54.314 INFO:teuthology.orchestra.run.smithi094.stdout:(114/114): librbd1-17.2.7-1629.geb16b97b.el9.x8 10 MB/s | 2.9 MB 00:00 2024-09-14T03:36:54.315 INFO:teuthology.orchestra.run.smithi070.stdout:(61/114): python3-mako-1.1.4-6.el9.noarch.rpm 1.3 MB/s | 172 kB 00:00 2024-09-14T03:36:54.318 INFO:teuthology.orchestra.run.smithi094.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:36:54.319 INFO:teuthology.orchestra.run.smithi094.stdout:Total 23 MB/s | 190 MB 00:08 2024-09-14T03:36:54.399 INFO:teuthology.orchestra.run.smithi070.stdout:(62/114): python3-babel-2.9.1-2.el9.noarch.rpm 12 MB/s | 6.0 MB 00:00 2024-09-14T03:36:54.483 INFO:teuthology.orchestra.run.smithi070.stdout:(63/114): python3-numpy-1.20.1-5.el9.x86_64.rpm 25 MB/s | 5.1 MB 00:00 2024-09-14T03:36:54.508 INFO:teuthology.orchestra.run.smithi070.stdout:(64/114): python3-numpy-f2py-1.20.1-5.el9.x86_6 1.3 MB/s | 250 kB 00:00 2024-09-14T03:36:54.533 INFO:teuthology.orchestra.run.smithi070.stdout:(65/114): python3-oauthlib-3.1.1-5.el9.noarch.r 1.6 MB/s | 222 kB 00:00 2024-09-14T03:36:54.567 INFO:teuthology.orchestra.run.smithi070.stdout:(66/114): python3-packaging-20.9-5.el9.noarch.r 923 kB/s | 77 kB 00:00 2024-09-14T03:36:54.592 INFO:teuthology.orchestra.run.smithi070.stdout:(67/114): python3-pluggy-0.13.1-7.el9.noarch.rp 495 kB/s | 41 kB 00:00 2024-09-14T03:36:54.617 INFO:teuthology.orchestra.run.smithi070.stdout:(68/114): python3-prettytable-0.7.2-27.el9.noar 503 kB/s | 42 kB 00:00 2024-09-14T03:36:54.651 INFO:teuthology.orchestra.run.smithi070.stdout:(69/114): python3-py-1.10.0-6.el9.noarch.rpm 5.5 MB/s | 477 kB 00:00 2024-09-14T03:36:54.693 INFO:teuthology.orchestra.run.smithi070.stdout:(70/114): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.5 MB/s | 159 kB 00:00 2024-09-14T03:36:54.718 INFO:teuthology.orchestra.run.smithi070.stdout:(71/114): python3-pyasn1-modules-0.4.8-6.el9.no 2.7 MB/s | 279 kB 00:00 2024-09-14T03:36:54.744 INFO:teuthology.orchestra.run.smithi070.stdout:(72/114): python3-pytest-6.2.2-7.el9.noarch.rpm 5.5 MB/s | 519 kB 00:00 2024-09-14T03:36:54.769 INFO:teuthology.orchestra.run.smithi070.stdout:(73/114): python3-pytz-2021.1-5.el9.noarch.rpm 676 kB/s | 51 kB 00:00 2024-09-14T03:36:54.819 INFO:teuthology.orchestra.run.smithi070.stdout:(74/114): python3-requests-oauthlib-1.3.0-12.el 533 kB/s | 54 kB 00:00 2024-09-14T03:36:54.844 INFO:teuthology.orchestra.run.smithi070.stdout:(75/114): python3-toml-0.10.2-6.el9.noarch.rpm 554 kB/s | 42 kB 00:00 2024-09-14T03:36:54.895 INFO:teuthology.orchestra.run.smithi070.stdout:(76/114): socat-1.7.4.1-6.el9.x86_64.rpm 4.0 MB/s | 304 kB 00:00 2024-09-14T03:36:54.928 INFO:teuthology.orchestra.run.smithi070.stdout:(77/114): xmlstarlet-1.6.1-20.el9.x86_64.rpm 763 kB/s | 64 kB 00:00 2024-09-14T03:36:54.979 INFO:teuthology.orchestra.run.smithi070.stdout:(78/114): gperftools-libs-2.9.1-3.el9.x86_64.rp 3.6 MB/s | 308 kB 00:00 2024-09-14T03:36:55.004 INFO:teuthology.orchestra.run.smithi070.stdout:(79/114): libunwind-1.6.2-1.el9.x86_64.rpm 2.9 MB/s | 67 kB 00:00 2024-09-14T03:36:55.029 INFO:teuthology.orchestra.run.smithi070.stdout:(80/114): liboath-2.6.7-2.el9.x86_64.rpm 493 kB/s | 49 kB 00:00 2024-09-14T03:36:55.054 INFO:teuthology.orchestra.run.smithi070.stdout:(81/114): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 1.7 MB/s | 43 kB 00:00 2024-09-14T03:36:55.078 INFO:teuthology.orchestra.run.smithi094.stdout:Running transaction check 2024-09-14T03:36:55.088 INFO:teuthology.orchestra.run.smithi070.stdout:(82/114): python3-asyncssh-2.13.2-5.el9.noarch. 6.4 MB/s | 548 kB 00:00 2024-09-14T03:36:55.122 INFO:teuthology.orchestra.run.smithi070.stdout:(83/114): python3-cachetools-4.2.4-1.el9.noarch 481 kB/s | 32 kB 00:00 2024-09-14T03:36:55.147 INFO:teuthology.orchestra.run.smithi070.stdout:(84/114): python3-certifi-2023.05.07-4.el9.noar 240 kB/s | 14 kB 00:00 2024-09-14T03:36:55.171 INFO:teuthology.orchestra.run.smithi094.stdout:Transaction check succeeded. 2024-09-14T03:36:55.171 INFO:teuthology.orchestra.run.smithi094.stdout:Running transaction test 2024-09-14T03:36:55.181 INFO:teuthology.orchestra.run.smithi070.stdout:(85/114): python3-cheroot-8.6.0-4.el9.noarch.rp 2.9 MB/s | 172 kB 00:00 2024-09-14T03:36:55.206 INFO:teuthology.orchestra.run.smithi070.stdout:(86/114): python3-cherrypy-18.6.1-2.el9.noarch. 6.0 MB/s | 358 kB 00:00 2024-09-14T03:36:55.231 INFO:teuthology.orchestra.run.smithi070.stdout:(87/114): python3-jaraco-8.2.1-3.el9.noarch.rpm 431 kB/s | 11 kB 00:00 2024-09-14T03:36:55.257 INFO:teuthology.orchestra.run.smithi070.stdout:(88/114): python3-jaraco-classes-3.2.1-5.el9.no 709 kB/s | 18 kB 00:00 2024-09-14T03:36:55.282 INFO:teuthology.orchestra.run.smithi070.stdout:(89/114): python3-google-auth-2.34.0-1.el9.noar 2.1 MB/s | 220 kB 00:00 2024-09-14T03:36:55.299 INFO:teuthology.orchestra.run.smithi070.stdout:(90/114): python3-jaraco-collections-3.0.0-8.el 552 kB/s | 23 kB 00:00 2024-09-14T03:36:55.316 INFO:teuthology.orchestra.run.smithi070.stdout:(91/114): python3-jaraco-functools-3.5.0-2.el9. 581 kB/s | 19 kB 00:00 2024-09-14T03:36:55.333 INFO:teuthology.orchestra.run.smithi070.stdout:(92/114): python3-jaraco-text-3.2.0-6.el9.noarc 588 kB/s | 20 kB 00:00 2024-09-14T03:36:55.350 INFO:teuthology.orchestra.run.smithi070.stdout:(93/114): python3-jwt+crypto-2.4.0-1.el9.noarch 269 kB/s | 9.0 kB 00:00 2024-09-14T03:36:55.367 INFO:teuthology.orchestra.run.smithi070.stdout:(94/114): python3-jwt-2.4.0-1.el9.noarch.rpm 1.2 MB/s | 41 kB 00:00 2024-09-14T03:36:55.384 INFO:teuthology.orchestra.run.smithi070.stdout:(95/114): python3-logutils-0.3.5-21.el9.noarch. 2.7 MB/s | 46 kB 00:00 2024-09-14T03:36:55.417 INFO:teuthology.orchestra.run.smithi070.stdout:(96/114): python3-more-itertools-8.12.0-2.el9.n 2.3 MB/s | 79 kB 00:00 2024-09-14T03:36:55.443 INFO:teuthology.orchestra.run.smithi070.stdout:(97/114): python3-natsort-7.1.1-5.el9.noarch.rp 2.3 MB/s | 58 kB 00:00 2024-09-14T03:36:55.476 INFO:teuthology.orchestra.run.smithi070.stdout:(98/114): python3-kubernetes-26.1.0-2.el9.noarc 8.1 MB/s | 1.0 MB 00:00 2024-09-14T03:36:55.643 INFO:teuthology.orchestra.run.smithi070.stdout:(99/114): python3-scipy-1.6.2-8.el9.x86_64.rpm 18 MB/s | 16 MB 00:00 2024-09-14T03:36:55.669 INFO:teuthology.orchestra.run.smithi070.stdout:(100/114): python3-portend-3.1.0-2.el9.noarch.r 85 kB/s | 16 kB 00:00 2024-09-14T03:36:55.702 INFO:teuthology.orchestra.run.smithi070.stdout:(101/114): python3-pecan-1.4.2-3.el9.noarch.rpm 1.0 MB/s | 272 kB 00:00 2024-09-14T03:36:55.728 INFO:teuthology.orchestra.run.smithi070.stdout:(102/114): python3-repoze-lru-0.7-16.el9.noarch 603 kB/s | 31 kB 00:00 2024-09-14T03:36:55.753 INFO:teuthology.orchestra.run.smithi070.stdout:(103/114): python3-pyOpenSSL-21.0.0-1.el9.noarc 826 kB/s | 90 kB 00:00 2024-09-14T03:36:55.778 INFO:teuthology.orchestra.run.smithi070.stdout:(104/114): python3-rsa-4.9-2.el9.noarch.rpm 1.2 MB/s | 59 kB 00:00 2024-09-14T03:36:55.804 INFO:teuthology.orchestra.run.smithi070.stdout:(105/114): python3-tempora-5.0.0-2.el9.noarch.r 713 kB/s | 36 kB 00:00 2024-09-14T03:36:55.829 INFO:teuthology.orchestra.run.smithi070.stdout:(106/114): python3-routes-2.5.1-5.el9.noarch.rp 1.5 MB/s | 188 kB 00:00 2024-09-14T03:36:55.854 INFO:teuthology.orchestra.run.smithi070.stdout:(107/114): python3-typing-extensions-4.12.2-2.e 988 kB/s | 74 kB 00:00 2024-09-14T03:36:55.879 INFO:teuthology.orchestra.run.smithi070.stdout:(108/114): python3-websocket-client-1.2.3-2.el9 1.7 MB/s | 90 kB 00:00 2024-09-14T03:36:55.905 INFO:teuthology.orchestra.run.smithi070.stdout:(109/114): python3-webob-1.8.8-2.el9.noarch.rpm 2.2 MB/s | 230 kB 00:00 2024-09-14T03:36:55.930 INFO:teuthology.orchestra.run.smithi070.stdout:(110/114): python3-zc-lockfile-2.0-10.el9.noarc 397 kB/s | 20 kB 00:00 2024-09-14T03:36:55.964 INFO:teuthology.orchestra.run.smithi070.stdout:(111/114): python3-werkzeug-2.0.3-3.el9.1.noarc 3.8 MB/s | 427 kB 00:00 2024-09-14T03:36:56.039 INFO:teuthology.orchestra.run.smithi070.stdout:(112/114): thrift-0.15.0-2.el9.x86_64.rpm 12 MB/s | 1.6 MB 00:00 2024-09-14T03:36:56.423 INFO:teuthology.orchestra.run.smithi070.stdout:(113/114): librbd1-17.2.7-1629.geb16b97b.el9.x8 6.4 MB/s | 2.9 MB 00:00 2024-09-14T03:36:56.473 INFO:teuthology.orchestra.run.smithi070.stdout:(114/114): librados2-17.2.7-1629.geb16b97b.el9. 5.7 MB/s | 3.1 MB 00:00 2024-09-14T03:36:56.478 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:36:56.479 INFO:teuthology.orchestra.run.smithi070.stdout:Total 26 MB/s | 190 MB 00:07 2024-09-14T03:36:56.809 INFO:teuthology.orchestra.run.smithi094.stdout:Transaction test succeeded. 2024-09-14T03:36:56.809 INFO:teuthology.orchestra.run.smithi094.stdout:Running transaction 2024-09-14T03:36:57.237 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2024-09-14T03:36:57.330 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2024-09-14T03:36:57.330 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2024-09-14T03:36:58.994 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2024-09-14T03:36:58.994 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2024-09-14T03:37:01.437 INFO:teuthology.orchestra.run.smithi094.stdout: Preparing : 1/1 2024-09-14T03:37:01.517 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : thrift-0.15.0-2.el9.x86_64 1/116 2024-09-14T03:37:01.542 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 2/116 2024-09-14T03:37:01.596 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 3/116 2024-09-14T03:37:01.906 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 4/116 2024-09-14T03:37:02.034 INFO:teuthology.orchestra.run.smithi094.stdout: Upgrading : librados2-2:17.2.7-1629.geb16b97b.el9.x86_64 5/116 2024-09-14T03:37:02.131 INFO:teuthology.orchestra.run.smithi094.stdout: Running scriptlet: librados2-2:17.2.7-1629.geb16b97b.el9.x86_64 5/116 2024-09-14T03:37:02.161 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : libcephfs2-2:17.2.7-1629.geb16b97b.el9.x86_64 6/116 2024-09-14T03:37:02.243 INFO:teuthology.orchestra.run.smithi094.stdout: Running scriptlet: libcephfs2-2:17.2.7-1629.geb16b97b.el9.x86_64 6/116 2024-09-14T03:37:02.294 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-rados-2:17.2.7-1629.geb16b97b.el9.x86_64 7/116 2024-09-14T03:37:02.323 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 8/116 2024-09-14T03:37:02.768 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 9/116 2024-09-14T03:37:02.796 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : librgw2-2:17.2.7-1629.geb16b97b.el9.x86_64 10/116 2024-09-14T03:37:02.871 INFO:teuthology.orchestra.run.smithi094.stdout: Running scriptlet: librgw2-2:17.2.7-1629.geb16b97b.el9.x86_64 10/116 2024-09-14T03:37:02.898 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : libcephsqlite-2:17.2.7-1629.geb16b97b.el9.x86_64 11/116 2024-09-14T03:37:02.997 INFO:teuthology.orchestra.run.smithi094.stdout: Running scriptlet: libcephsqlite-2:17.2.7-1629.geb16b97b.el9.x86_64 11/116 2024-09-14T03:37:03.023 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : libradosstriper1-2:17.2.7-1629.geb16b97b.el9.x86 12/116 2024-09-14T03:37:03.143 INFO:teuthology.orchestra.run.smithi094.stdout: Running scriptlet: libradosstriper1-2:17.2.7-1629.geb16b97b.el9.x86 12/116 2024-09-14T03:37:03.178 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 13/116 2024-09-14T03:37:03.210 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : liboath-2.6.7-2.el9.x86_64 14/116 2024-09-14T03:37:03.240 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 15/116 2024-09-14T03:37:03.280 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 16/116 2024-09-14T03:37:03.346 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 17/116 2024-09-14T03:37:03.390 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 18/116 2024-09-14T03:37:03.430 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-packaging-20.9-5.el9.noarch 19/116 2024-09-14T03:37:03.457 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 20/116 2024-09-14T03:37:03.527 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 21/116 2024-09-14T03:37:03.581 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 22/116 2024-09-14T03:37:03.617 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 23/116 2024-09-14T03:37:03.620 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2024-09-14T03:37:03.655 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : libquadmath-11.5.0-2.el9.x86_64 24/116 2024-09-14T03:37:03.689 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : libgfortran-11.5.0-2.el9.x86_64 25/116 2024-09-14T03:37:03.691 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : thrift-0.15.0-2.el9.x86_64 1/116 2024-09-14T03:37:03.717 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 2/116 2024-09-14T03:37:03.724 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-ceph-argparse-2:17.2.7-1629.geb16b97b.el 26/116 2024-09-14T03:37:03.769 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 3/116 2024-09-14T03:37:03.799 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-cephfs-2:17.2.7-1629.geb16b97b.el9.x86_6 27/116 2024-09-14T03:37:03.931 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 28/116 2024-09-14T03:37:03.984 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 29/116 2024-09-14T03:37:04.069 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 4/116 2024-09-14T03:37:04.168 INFO:teuthology.orchestra.run.smithi070.stdout: Upgrading : librados2-2:17.2.7-1629.geb16b97b.el9.x86_64 5/116 2024-09-14T03:37:04.264 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: librados2-2:17.2.7-1629.geb16b97b.el9.x86_64 5/116 2024-09-14T03:37:04.293 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : libcephfs2-2:17.2.7-1629.geb16b97b.el9.x86_64 6/116 2024-09-14T03:37:04.386 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: libcephfs2-2:17.2.7-1629.geb16b97b.el9.x86_64 6/116 2024-09-14T03:37:04.436 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-rados-2:17.2.7-1629.geb16b97b.el9.x86_64 7/116 2024-09-14T03:37:04.465 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 8/116 2024-09-14T03:37:04.506 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-rsa-4.9-2.el9.noarch 30/116 2024-09-14T03:37:04.589 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 31/116 2024-09-14T03:37:04.626 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 32/116 2024-09-14T03:37:04.669 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 33/116 2024-09-14T03:37:04.704 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 34/116 2024-09-14T03:37:04.752 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 35/116 2024-09-14T03:37:04.788 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-rgw-2:17.2.7-1629.geb16b97b.el9.x86_64 36/116 2024-09-14T03:37:04.825 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : librados-devel-2:17.2.7-1629.geb16b97b.el9.x86_6 37/116 2024-09-14T03:37:04.856 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 38/116 2024-09-14T03:37:04.909 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 39/116 2024-09-14T03:37:04.910 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 9/116 2024-09-14T03:37:04.938 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : librgw2-2:17.2.7-1629.geb16b97b.el9.x86_64 10/116 2024-09-14T03:37:04.964 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 40/116 2024-09-14T03:37:04.996 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: librgw2-2:17.2.7-1629.geb16b97b.el9.x86_64 10/116 2024-09-14T03:37:05.022 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : libcephsqlite-2:17.2.7-1629.geb16b97b.el9.x86_64 11/116 2024-09-14T03:37:05.036 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 41/116 2024-09-14T03:37:05.069 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 42/116 2024-09-14T03:37:05.105 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 43/116 2024-09-14T03:37:05.114 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: libcephsqlite-2:17.2.7-1629.geb16b97b.el9.x86_64 11/116 2024-09-14T03:37:05.140 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : libradosstriper1-2:17.2.7-1629.geb16b97b.el9.x86 12/116 2024-09-14T03:37:05.151 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 44/116 2024-09-14T03:37:05.197 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 45/116 2024-09-14T03:37:05.239 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 46/116 2024-09-14T03:37:05.251 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: libradosstriper1-2:17.2.7-1629.geb16b97b.el9.x86 12/116 2024-09-14T03:37:05.286 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 13/116 2024-09-14T03:37:05.318 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : liboath-2.6.7-2.el9.x86_64 14/116 2024-09-14T03:37:05.348 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 15/116 2024-09-14T03:37:05.375 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 47/116 2024-09-14T03:37:05.389 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 16/116 2024-09-14T03:37:05.424 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 48/116 2024-09-14T03:37:05.454 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 17/116 2024-09-14T03:37:05.465 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 49/116 2024-09-14T03:37:05.499 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 18/116 2024-09-14T03:37:05.539 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-packaging-20.9-5.el9.noarch 19/116 2024-09-14T03:37:05.567 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 20/116 2024-09-14T03:37:05.567 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 50/116 2024-09-14T03:37:05.600 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-google-auth-1:2.34.0-1.el9.noarch 51/116 2024-09-14T03:37:05.636 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 21/116 2024-09-14T03:37:05.646 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 52/116 2024-09-14T03:37:05.677 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 53/116 2024-09-14T03:37:05.688 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 22/116 2024-09-14T03:37:05.723 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 23/116 2024-09-14T03:37:05.740 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 54/116 2024-09-14T03:37:05.763 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : libquadmath-11.5.0-2.el9.x86_64 24/116 2024-09-14T03:37:05.798 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : libgfortran-11.5.0-2.el9.x86_64 25/116 2024-09-14T03:37:05.841 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-ceph-argparse-2:17.2.7-1629.geb16b97b.el 26/116 2024-09-14T03:37:05.860 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 55/116 2024-09-14T03:37:05.897 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-py-1.10.0-6.el9.noarch 56/116 2024-09-14T03:37:05.915 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-cephfs-2:17.2.7-1629.geb16b97b.el9.x86_6 27/116 2024-09-14T03:37:05.936 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 57/116 2024-09-14T03:37:06.033 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-pluggy-0.13.1-7.el9.noarch 58/116 2024-09-14T03:37:06.075 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 28/116 2024-09-14T03:37:06.081 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 59/116 2024-09-14T03:37:06.144 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 29/116 2024-09-14T03:37:06.676 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-rsa-4.9-2.el9.noarch 30/116 2024-09-14T03:37:06.756 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 31/116 2024-09-14T03:37:06.794 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 32/116 2024-09-14T03:37:06.808 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 60/116 2024-09-14T03:37:06.820 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 33/116 2024-09-14T03:37:06.855 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 34/116 2024-09-14T03:37:06.893 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 35/116 2024-09-14T03:37:06.929 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-rgw-2:17.2.7-1629.geb16b97b.el9.x86_64 36/116 2024-09-14T03:37:06.967 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : librados-devel-2:17.2.7-1629.geb16b97b.el9.x86_6 37/116 2024-09-14T03:37:06.998 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 38/116 2024-09-14T03:37:07.034 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 39/116 2024-09-14T03:37:07.042 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 61/116 2024-09-14T03:37:07.075 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 62/116 2024-09-14T03:37:07.089 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 40/116 2024-09-14T03:37:07.109 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 63/116 2024-09-14T03:37:07.163 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 41/116 2024-09-14T03:37:07.194 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 42/116 2024-09-14T03:37:07.215 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-iniconfig-1.1.1-7.el9.noarch 64/116 2024-09-14T03:37:07.230 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 43/116 2024-09-14T03:37:07.267 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-devel-3.9.19-8.el9.x86_64 65/116 2024-09-14T03:37:07.277 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 44/116 2024-09-14T03:37:07.301 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-attrs-20.3.0-7.el9.noarch 66/116 2024-09-14T03:37:07.322 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 45/116 2024-09-14T03:37:07.365 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 46/116 2024-09-14T03:37:07.485 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : openblas-0.3.26-2.el9.x86_64 67/116 2024-09-14T03:37:07.504 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 47/116 2024-09-14T03:37:07.533 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : openblas-openmp-0.3.26-2.el9.x86_64 68/116 2024-09-14T03:37:07.541 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 48/116 2024-09-14T03:37:07.583 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 49/116 2024-09-14T03:37:07.610 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 69/116 2024-09-14T03:37:07.686 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 50/116 2024-09-14T03:37:07.725 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-google-auth-1:2.34.0-1.el9.noarch 51/116 2024-09-14T03:37:07.772 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 52/116 2024-09-14T03:37:07.811 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 53/116 2024-09-14T03:37:07.856 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 54/116 2024-09-14T03:37:07.981 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 55/116 2024-09-14T03:37:08.025 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-py-1.10.0-6.el9.noarch 56/116 2024-09-14T03:37:08.064 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 57/116 2024-09-14T03:37:08.162 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-pluggy-0.13.1-7.el9.noarch 58/116 2024-09-14T03:37:08.218 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 59/116 2024-09-14T03:37:08.257 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 70/116 2024-09-14T03:37:08.372 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 71/116 2024-09-14T03:37:08.939 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 60/116 2024-09-14T03:37:08.990 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 61/116 2024-09-14T03:37:09.024 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 62/116 2024-09-14T03:37:09.058 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 63/116 2024-09-14T03:37:09.157 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-iniconfig-1.1.1-7.el9.noarch 64/116 2024-09-14T03:37:09.216 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-devel-3.9.19-8.el9.x86_64 65/116 2024-09-14T03:37:09.251 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-attrs-20.3.0-7.el9.noarch 66/116 2024-09-14T03:37:09.435 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : openblas-0.3.26-2.el9.x86_64 67/116 2024-09-14T03:37:09.466 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : openblas-openmp-0.3.26-2.el9.x86_64 68/116 2024-09-14T03:37:09.646 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 72/116 2024-09-14T03:37:09.713 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 69/116 2024-09-14T03:37:09.753 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 73/116 2024-09-14T03:37:10.047 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 74/116 2024-09-14T03:37:10.090 INFO:teuthology.orchestra.run.smithi094.stdout: Upgrading : librbd1-2:17.2.7-1629.geb16b97b.el9.x86_64 75/116 2024-09-14T03:37:10.181 INFO:teuthology.orchestra.run.smithi094.stdout: Running scriptlet: librbd1-2:17.2.7-1629.geb16b97b.el9.x86_64 75/116 2024-09-14T03:37:10.220 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-rbd-2:17.2.7-1629.geb16b97b.el9.x86_64 76/116 2024-09-14T03:37:10.269 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-ply-3.11-14.el9.noarch 77/116 2024-09-14T03:37:10.324 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 78/116 2024-09-14T03:37:10.374 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 70/116 2024-09-14T03:37:10.488 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 71/116 2024-09-14T03:37:10.509 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 79/116 2024-09-14T03:37:10.558 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 80/116 2024-09-14T03:37:10.636 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 81/116 2024-09-14T03:37:10.734 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 82/116 2024-09-14T03:37:10.888 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 83/116 2024-09-14T03:37:10.929 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 84/116 2024-09-14T03:37:10.950 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 85/116 2024-09-14T03:37:10.992 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 86/116 2024-09-14T03:37:11.024 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 87/116 2024-09-14T03:37:11.052 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : fuse-2.9.9-16.el9.x86_64 88/116 2024-09-14T03:37:11.488 INFO:teuthology.orchestra.run.smithi094.stdout: Running scriptlet: cephadm-2:17.2.7-1629.geb16b97b.el9.noarch 89/116 2024-09-14T03:37:11.516 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : cephadm-2:17.2.7-1629.geb16b97b.el9.noarch 89/116 2024-09-14T03:37:11.554 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-1629.geb16b97b.e 90/116 2024-09-14T03:37:11.628 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-1629.geb16b97b. 91/116 2024-09-14T03:37:11.658 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-ceph-common-2:17.2.7-1629.geb16b97b.el9. 92/116 2024-09-14T03:37:11.763 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 72/116 2024-09-14T03:37:11.858 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 73/116 2024-09-14T03:37:12.150 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 74/116 2024-09-14T03:37:12.187 INFO:teuthology.orchestra.run.smithi070.stdout: Upgrading : librbd1-2:17.2.7-1629.geb16b97b.el9.x86_64 75/116 2024-09-14T03:37:12.276 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: librbd1-2:17.2.7-1629.geb16b97b.el9.x86_64 75/116 2024-09-14T03:37:12.317 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-rbd-2:17.2.7-1629.geb16b97b.el9.x86_64 76/116 2024-09-14T03:37:12.365 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-ply-3.11-14.el9.noarch 77/116 2024-09-14T03:37:12.421 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 78/116 2024-09-14T03:37:12.606 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 79/116 2024-09-14T03:37:12.654 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 80/116 2024-09-14T03:37:12.724 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 81/116 2024-09-14T03:37:12.812 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 82/116 2024-09-14T03:37:12.942 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 83/116 2024-09-14T03:37:12.992 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 84/116 2024-09-14T03:37:13.021 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 85/116 2024-09-14T03:37:13.055 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 86/116 2024-09-14T03:37:13.087 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 87/116 2024-09-14T03:37:13.115 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : fuse-2.9.9-16.el9.x86_64 88/116 2024-09-14T03:37:13.551 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: cephadm-2:17.2.7-1629.geb16b97b.el9.noarch 89/116 2024-09-14T03:37:13.579 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : cephadm-2:17.2.7-1629.geb16b97b.el9.noarch 89/116 2024-09-14T03:37:13.609 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-1629.geb16b97b.e 90/116 2024-09-14T03:37:13.684 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-1629.geb16b97b. 91/116 2024-09-14T03:37:13.712 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-ceph-common-2:17.2.7-1629.geb16b97b.el9. 92/116 2024-09-14T03:37:14.074 INFO:teuthology.orchestra.run.smithi094.stdout: Running scriptlet: ceph-common-2:17.2.7-1629.geb16b97b.el9.x86_64 93/116 2024-09-14T03:37:14.289 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : ceph-common-2:17.2.7-1629.geb16b97b.el9.x86_64 93/116 2024-09-14T03:37:14.810 INFO:teuthology.orchestra.run.smithi094.stdout: Running scriptlet: ceph-common-2:17.2.7-1629.geb16b97b.el9.x86_64 93/116 2024-09-14T03:37:14.851 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : ceph-base-2:17.2.7-1629.geb16b97b.el9.x86_64 94/116 2024-09-14T03:37:14.928 INFO:teuthology.orchestra.run.smithi094.stdout: Running scriptlet: ceph-base-2:17.2.7-1629.geb16b97b.el9.x86_64 94/116 2024-09-14T03:37:14.928 INFO:teuthology.orchestra.run.smithi094.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-09-14T03:37:14.928 INFO:teuthology.orchestra.run.smithi094.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-09-14T03:37:14.928 INFO:teuthology.orchestra.run.smithi094.stdout: 2024-09-14T03:37:14.978 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : ceph-selinux-2:17.2.7-1629.geb16b97b.el9.x86_64 95/116 2024-09-14T03:37:16.163 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-common-2:17.2.7-1629.geb16b97b.el9.x86_64 93/116 2024-09-14T03:37:16.210 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-common-2:17.2.7-1629.geb16b97b.el9.x86_64 93/116 2024-09-14T03:37:16.737 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-common-2:17.2.7-1629.geb16b97b.el9.x86_64 93/116 2024-09-14T03:37:16.780 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-base-2:17.2.7-1629.geb16b97b.el9.x86_64 94/116 2024-09-14T03:37:16.884 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-base-2:17.2.7-1629.geb16b97b.el9.x86_64 94/116 2024-09-14T03:37:16.884 INFO:teuthology.orchestra.run.smithi070.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-09-14T03:37:16.884 INFO:teuthology.orchestra.run.smithi070.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-09-14T03:37:16.884 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:37:16.934 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-selinux-2:17.2.7-1629.geb16b97b.el9.x86_64 95/116 2024-09-14T03:37:26.717 INFO:teuthology.orchestra.run.smithi094.stdout: Running scriptlet: ceph-selinux-2:17.2.7-1629.geb16b97b.el9.x86_64 95/116 2024-09-14T03:37:26.718 INFO:teuthology.orchestra.run.smithi094.stdout:skipping the directory /sys 2024-09-14T03:37:26.718 INFO:teuthology.orchestra.run.smithi094.stdout:skipping the directory /proc 2024-09-14T03:37:26.718 INFO:teuthology.orchestra.run.smithi094.stdout:skipping the directory /mnt 2024-09-14T03:37:26.718 INFO:teuthology.orchestra.run.smithi094.stdout:skipping the directory /var/tmp 2024-09-14T03:37:26.718 INFO:teuthology.orchestra.run.smithi094.stdout:skipping the directory /home 2024-09-14T03:37:26.718 INFO:teuthology.orchestra.run.smithi094.stdout:skipping the directory /root 2024-09-14T03:37:26.718 INFO:teuthology.orchestra.run.smithi094.stdout:skipping the directory /tmp 2024-09-14T03:37:26.718 INFO:teuthology.orchestra.run.smithi094.stdout: 2024-09-14T03:37:26.798 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-1629.geb16b97b.el9.noa 96/116 2024-09-14T03:37:27.818 INFO:teuthology.orchestra.run.smithi094.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-1629.geb16b97b.el9.noa 96/116 2024-09-14T03:37:27.861 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-1629.geb16b97b.el9.n 97/116 2024-09-14T03:37:28.681 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-selinux-2:17.2.7-1629.geb16b97b.el9.x86_64 95/116 2024-09-14T03:37:28.681 INFO:teuthology.orchestra.run.smithi070.stdout:skipping the directory /sys 2024-09-14T03:37:28.681 INFO:teuthology.orchestra.run.smithi070.stdout:skipping the directory /proc 2024-09-14T03:37:28.681 INFO:teuthology.orchestra.run.smithi070.stdout:skipping the directory /mnt 2024-09-14T03:37:28.681 INFO:teuthology.orchestra.run.smithi070.stdout:skipping the directory /var/tmp 2024-09-14T03:37:28.682 INFO:teuthology.orchestra.run.smithi070.stdout:skipping the directory /home 2024-09-14T03:37:28.682 INFO:teuthology.orchestra.run.smithi070.stdout:skipping the directory /root 2024-09-14T03:37:28.682 INFO:teuthology.orchestra.run.smithi070.stdout:skipping the directory /tmp 2024-09-14T03:37:28.682 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:37:28.752 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-1629.geb16b97b.el9.noa 96/116 2024-09-14T03:37:28.827 INFO:teuthology.orchestra.run.smithi094.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-1629.geb16b97b.el9.n 97/116 2024-09-14T03:37:28.862 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-1629.geb1 98/116 2024-09-14T03:37:28.962 INFO:teuthology.orchestra.run.smithi094.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-1629.geb1 98/116 2024-09-14T03:37:29.115 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-1629.geb16b97b.el 99/116 2024-09-14T03:37:29.156 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : ceph-mgr-2:17.2.7-1629.geb16b97b.el9.x86_64 100/116 2024-09-14T03:37:29.180 INFO:teuthology.orchestra.run.smithi094.stdout: Running scriptlet: ceph-mgr-2:17.2.7-1629.geb16b97b.el9.x86_64 100/116 2024-09-14T03:37:29.180 INFO:teuthology.orchestra.run.smithi094.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:37:29.180 INFO:teuthology.orchestra.run.smithi094.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-09-14T03:37:29.180 INFO:teuthology.orchestra.run.smithi094.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-14T03:37:29.181 INFO:teuthology.orchestra.run.smithi094.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-14T03:37:29.181 INFO:teuthology.orchestra.run.smithi094.stdout: 2024-09-14T03:37:29.223 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : ceph-mgr-rook-2:17.2.7-1629.geb16b97b.el9.noarch 101/116 2024-09-14T03:37:29.406 INFO:teuthology.orchestra.run.smithi094.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-1629.geb16b97b.el9.noarch 101/116 2024-09-14T03:37:29.432 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : ceph-mds-2:17.2.7-1629.geb16b97b.el9.x86_64 102/116 2024-09-14T03:37:29.455 INFO:teuthology.orchestra.run.smithi094.stdout: Running scriptlet: ceph-mds-2:17.2.7-1629.geb16b97b.el9.x86_64 102/116 2024-09-14T03:37:29.456 INFO:teuthology.orchestra.run.smithi094.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:37:29.456 INFO:teuthology.orchestra.run.smithi094.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-09-14T03:37:29.456 INFO:teuthology.orchestra.run.smithi094.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-14T03:37:29.456 INFO:teuthology.orchestra.run.smithi094.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-14T03:37:29.456 INFO:teuthology.orchestra.run.smithi094.stdout: 2024-09-14T03:37:29.725 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-1629.geb16b97b.el9.noa 96/116 2024-09-14T03:37:29.754 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-1629.geb16b97b.el9.n 97/116 2024-09-14T03:37:29.833 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : ceph-mon-2:17.2.7-1629.geb16b97b.el9.x86_64 103/116 2024-09-14T03:37:29.856 INFO:teuthology.orchestra.run.smithi094.stdout: Running scriptlet: ceph-mon-2:17.2.7-1629.geb16b97b.el9.x86_64 103/116 2024-09-14T03:37:29.856 INFO:teuthology.orchestra.run.smithi094.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:37:29.856 INFO:teuthology.orchestra.run.smithi094.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-09-14T03:37:29.856 INFO:teuthology.orchestra.run.smithi094.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-14T03:37:29.857 INFO:teuthology.orchestra.run.smithi094.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-14T03:37:29.857 INFO:teuthology.orchestra.run.smithi094.stdout: 2024-09-14T03:37:30.713 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-1629.geb16b97b.el9.n 97/116 2024-09-14T03:37:30.737 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-1629.geb1 98/116 2024-09-14T03:37:30.838 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-1629.geb1 98/116 2024-09-14T03:37:31.000 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-1629.geb16b97b.el 99/116 2024-09-14T03:37:31.032 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-mgr-2:17.2.7-1629.geb16b97b.el9.x86_64 100/116 2024-09-14T03:37:31.056 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-mgr-2:17.2.7-1629.geb16b97b.el9.x86_64 100/116 2024-09-14T03:37:31.056 INFO:teuthology.orchestra.run.smithi070.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:37:31.056 INFO:teuthology.orchestra.run.smithi070.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-09-14T03:37:31.056 INFO:teuthology.orchestra.run.smithi070.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-14T03:37:31.056 INFO:teuthology.orchestra.run.smithi070.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-14T03:37:31.056 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:37:31.099 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-mgr-rook-2:17.2.7-1629.geb16b97b.el9.noarch 101/116 2024-09-14T03:37:31.175 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : ceph-osd-2:17.2.7-1629.geb16b97b.el9.x86_64 104/116 2024-09-14T03:37:31.202 INFO:teuthology.orchestra.run.smithi094.stdout: Running scriptlet: ceph-osd-2:17.2.7-1629.geb16b97b.el9.x86_64 104/116 2024-09-14T03:37:31.202 INFO:teuthology.orchestra.run.smithi094.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:37:31.202 INFO:teuthology.orchestra.run.smithi094.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-09-14T03:37:31.202 INFO:teuthology.orchestra.run.smithi094.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-14T03:37:31.202 INFO:teuthology.orchestra.run.smithi094.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-14T03:37:31.202 INFO:teuthology.orchestra.run.smithi094.stdout: 2024-09-14T03:37:31.283 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-1629.geb16b97b.el9.noarch 101/116 2024-09-14T03:37:31.316 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-mds-2:17.2.7-1629.geb16b97b.el9.x86_64 102/116 2024-09-14T03:37:31.339 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-mds-2:17.2.7-1629.geb16b97b.el9.x86_64 102/116 2024-09-14T03:37:31.339 INFO:teuthology.orchestra.run.smithi070.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:37:31.340 INFO:teuthology.orchestra.run.smithi070.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-09-14T03:37:31.340 INFO:teuthology.orchestra.run.smithi070.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-14T03:37:31.340 INFO:teuthology.orchestra.run.smithi070.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-14T03:37:31.340 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:37:31.717 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-mon-2:17.2.7-1629.geb16b97b.el9.x86_64 103/116 2024-09-14T03:37:31.740 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-mon-2:17.2.7-1629.geb16b97b.el9.x86_64 103/116 2024-09-14T03:37:31.740 INFO:teuthology.orchestra.run.smithi070.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:37:31.741 INFO:teuthology.orchestra.run.smithi070.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-09-14T03:37:31.741 INFO:teuthology.orchestra.run.smithi070.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-14T03:37:31.741 INFO:teuthology.orchestra.run.smithi070.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-14T03:37:31.741 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:37:32.452 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : ceph-2:17.2.7-1629.geb16b97b.el9.x86_64 105/116 2024-09-14T03:37:32.485 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : ceph-radosgw-2:17.2.7-1629.geb16b97b.el9.x86_64 106/116 2024-09-14T03:37:32.563 INFO:teuthology.orchestra.run.smithi094.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-1629.geb16b97b.el9.x86_64 106/116 2024-09-14T03:37:32.563 INFO:teuthology.orchestra.run.smithi094.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:37:32.563 INFO:teuthology.orchestra.run.smithi094.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-09-14T03:37:32.563 INFO:teuthology.orchestra.run.smithi094.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-14T03:37:32.563 INFO:teuthology.orchestra.run.smithi094.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-14T03:37:32.564 INFO:teuthology.orchestra.run.smithi094.stdout: 2024-09-14T03:37:32.603 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-1629.geb16b 107/116 2024-09-14T03:37:32.626 INFO:teuthology.orchestra.run.smithi094.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-1629.geb16b 107/116 2024-09-14T03:37:32.626 INFO:teuthology.orchestra.run.smithi094.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:37:32.627 INFO:teuthology.orchestra.run.smithi094.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-09-14T03:37:32.627 INFO:teuthology.orchestra.run.smithi094.stdout: 2024-09-14T03:37:32.912 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : rbd-mirror-2:17.2.7-1629.geb16b97b.el9.x86_64 108/116 2024-09-14T03:37:32.936 INFO:teuthology.orchestra.run.smithi094.stdout: Running scriptlet: rbd-mirror-2:17.2.7-1629.geb16b97b.el9.x86_64 108/116 2024-09-14T03:37:32.936 INFO:teuthology.orchestra.run.smithi094.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:37:32.936 INFO:teuthology.orchestra.run.smithi094.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-09-14T03:37:32.936 INFO:teuthology.orchestra.run.smithi094.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-14T03:37:32.937 INFO:teuthology.orchestra.run.smithi094.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-14T03:37:32.937 INFO:teuthology.orchestra.run.smithi094.stdout: 2024-09-14T03:37:33.051 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-osd-2:17.2.7-1629.geb16b97b.el9.x86_64 104/116 2024-09-14T03:37:33.078 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-osd-2:17.2.7-1629.geb16b97b.el9.x86_64 104/116 2024-09-14T03:37:33.078 INFO:teuthology.orchestra.run.smithi070.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:37:33.078 INFO:teuthology.orchestra.run.smithi070.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-09-14T03:37:33.078 INFO:teuthology.orchestra.run.smithi070.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-14T03:37:33.078 INFO:teuthology.orchestra.run.smithi070.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-14T03:37:33.079 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:37:34.317 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-2:17.2.7-1629.geb16b97b.el9.x86_64 105/116 2024-09-14T03:37:34.344 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-radosgw-2:17.2.7-1629.geb16b97b.el9.x86_64 106/116 2024-09-14T03:37:34.422 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-1629.geb16b97b.el9.x86_64 106/116 2024-09-14T03:37:34.422 INFO:teuthology.orchestra.run.smithi070.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:37:34.422 INFO:teuthology.orchestra.run.smithi070.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-09-14T03:37:34.422 INFO:teuthology.orchestra.run.smithi070.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-14T03:37:34.423 INFO:teuthology.orchestra.run.smithi070.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-14T03:37:34.423 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:37:34.462 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-1629.geb16b 107/116 2024-09-14T03:37:34.485 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-1629.geb16b 107/116 2024-09-14T03:37:34.485 INFO:teuthology.orchestra.run.smithi070.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:37:34.485 INFO:teuthology.orchestra.run.smithi070.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-09-14T03:37:34.485 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:37:34.771 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : rbd-mirror-2:17.2.7-1629.geb16b97b.el9.x86_64 108/116 2024-09-14T03:37:34.794 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: rbd-mirror-2:17.2.7-1629.geb16b97b.el9.x86_64 108/116 2024-09-14T03:37:34.794 INFO:teuthology.orchestra.run.smithi070.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:37:34.794 INFO:teuthology.orchestra.run.smithi070.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-09-14T03:37:34.795 INFO:teuthology.orchestra.run.smithi070.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-14T03:37:34.795 INFO:teuthology.orchestra.run.smithi070.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-14T03:37:34.795 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:37:37.385 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : ceph-test-2:17.2.7-1629.geb16b97b.el9.x86_64 109/116 2024-09-14T03:37:37.425 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : ceph-fuse-2:17.2.7-1629.geb16b97b.el9.x86_64 110/116 2024-09-14T03:37:37.465 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : rbd-fuse-2:17.2.7-1629.geb16b97b.el9.x86_64 111/116 2024-09-14T03:37:37.584 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : rbd-nbd-2:17.2.7-1629.geb16b97b.el9.x86_64 112/116 2024-09-14T03:37:37.623 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : python3-pytest-6.2.2-7.el9.noarch 113/116 2024-09-14T03:37:37.646 INFO:teuthology.orchestra.run.smithi094.stdout: Installing : libcephfs-devel-2:17.2.7-1629.geb16b97b.el9.x86_ 114/116 2024-09-14T03:37:37.646 INFO:teuthology.orchestra.run.smithi094.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 115/116 2024-09-14T03:37:37.730 INFO:teuthology.orchestra.run.smithi094.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 115/116 2024-09-14T03:37:37.730 INFO:teuthology.orchestra.run.smithi094.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 116/116 2024-09-14T03:37:39.243 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-test-2:17.2.7-1629.geb16b97b.el9.x86_64 109/116 2024-09-14T03:37:39.283 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-fuse-2:17.2.7-1629.geb16b97b.el9.x86_64 110/116 2024-09-14T03:37:39.323 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : rbd-fuse-2:17.2.7-1629.geb16b97b.el9.x86_64 111/116 2024-09-14T03:37:39.440 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : rbd-nbd-2:17.2.7-1629.geb16b97b.el9.x86_64 112/116 2024-09-14T03:37:39.472 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-pytest-6.2.2-7.el9.noarch 113/116 2024-09-14T03:37:39.495 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : libcephfs-devel-2:17.2.7-1629.geb16b97b.el9.x86_ 114/116 2024-09-14T03:37:39.496 INFO:teuthology.orchestra.run.smithi070.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 115/116 2024-09-14T03:37:39.596 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 115/116 2024-09-14T03:37:39.596 INFO:teuthology.orchestra.run.smithi070.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 116/116 2024-09-14T03:37:42.733 INFO:teuthology.orchestra.run.smithi094.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 116/116 2024-09-14T03:37:42.733 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : ceph-2:17.2.7-1629.geb16b97b.el9.x86_64 1/116 2024-09-14T03:37:42.733 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : ceph-base-2:17.2.7-1629.geb16b97b.el9.x86_64 2/116 2024-09-14T03:37:42.733 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : ceph-common-2:17.2.7-1629.geb16b97b.el9.x86_64 3/116 2024-09-14T03:37:42.733 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : ceph-fuse-2:17.2.7-1629.geb16b97b.el9.x86_64 4/116 2024-09-14T03:37:42.734 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-1629.geb16b 5/116 2024-09-14T03:37:42.734 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : ceph-mds-2:17.2.7-1629.geb16b97b.el9.x86_64 6/116 2024-09-14T03:37:42.734 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : ceph-mgr-2:17.2.7-1629.geb16b97b.el9.x86_64 7/116 2024-09-14T03:37:42.734 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : ceph-mon-2:17.2.7-1629.geb16b97b.el9.x86_64 8/116 2024-09-14T03:37:42.734 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : ceph-osd-2:17.2.7-1629.geb16b97b.el9.x86_64 9/116 2024-09-14T03:37:42.734 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : ceph-radosgw-2:17.2.7-1629.geb16b97b.el9.x86_64 10/116 2024-09-14T03:37:42.734 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : ceph-selinux-2:17.2.7-1629.geb16b97b.el9.x86_64 11/116 2024-09-14T03:37:42.734 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : ceph-test-2:17.2.7-1629.geb16b97b.el9.x86_64 12/116 2024-09-14T03:37:42.734 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : libcephfs-devel-2:17.2.7-1629.geb16b97b.el9.x86_ 13/116 2024-09-14T03:37:42.734 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : libcephfs2-2:17.2.7-1629.geb16b97b.el9.x86_64 14/116 2024-09-14T03:37:42.734 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : libcephsqlite-2:17.2.7-1629.geb16b97b.el9.x86_64 15/116 2024-09-14T03:37:42.734 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : librados-devel-2:17.2.7-1629.geb16b97b.el9.x86_6 16/116 2024-09-14T03:37:42.735 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : libradosstriper1-2:17.2.7-1629.geb16b97b.el9.x86 17/116 2024-09-14T03:37:42.735 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : librgw2-2:17.2.7-1629.geb16b97b.el9.x86_64 18/116 2024-09-14T03:37:42.735 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-ceph-argparse-2:17.2.7-1629.geb16b97b.el 19/116 2024-09-14T03:37:42.735 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-ceph-common-2:17.2.7-1629.geb16b97b.el9. 20/116 2024-09-14T03:37:42.735 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-cephfs-2:17.2.7-1629.geb16b97b.el9.x86_6 21/116 2024-09-14T03:37:42.735 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-rados-2:17.2.7-1629.geb16b97b.el9.x86_64 22/116 2024-09-14T03:37:42.735 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-rbd-2:17.2.7-1629.geb16b97b.el9.x86_64 23/116 2024-09-14T03:37:42.735 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-rgw-2:17.2.7-1629.geb16b97b.el9.x86_64 24/116 2024-09-14T03:37:42.735 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : rbd-fuse-2:17.2.7-1629.geb16b97b.el9.x86_64 25/116 2024-09-14T03:37:42.735 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : rbd-mirror-2:17.2.7-1629.geb16b97b.el9.x86_64 26/116 2024-09-14T03:37:42.735 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : rbd-nbd-2:17.2.7-1629.geb16b97b.el9.x86_64 27/116 2024-09-14T03:37:42.743 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-1629.geb16b97b. 28/116 2024-09-14T03:37:42.743 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-1629.geb16b97b.el9.noa 29/116 2024-09-14T03:37:42.744 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-1629.geb16b97b.el9.n 30/116 2024-09-14T03:37:42.744 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-1629.geb1 31/116 2024-09-14T03:37:42.744 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-1629.geb16b97b.el 32/116 2024-09-14T03:37:42.744 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : ceph-mgr-rook-2:17.2.7-1629.geb16b97b.el9.noarch 33/116 2024-09-14T03:37:42.744 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-1629.geb16b97b.e 34/116 2024-09-14T03:37:42.744 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : cephadm-2:17.2.7-1629.geb16b97b.el9.noarch 35/116 2024-09-14T03:37:42.744 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 36/116 2024-09-14T03:37:42.744 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : libgfortran-11.5.0-2.el9.x86_64 37/116 2024-09-14T03:37:42.745 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : libquadmath-11.5.0-2.el9.x86_64 38/116 2024-09-14T03:37:42.745 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 39/116 2024-09-14T03:37:42.745 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 40/116 2024-09-14T03:37:42.745 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-ply-3.11-14.el9.noarch 41/116 2024-09-14T03:37:42.745 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 42/116 2024-09-14T03:37:42.745 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 43/116 2024-09-14T03:37:42.745 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 44/116 2024-09-14T03:37:42.745 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 45/116 2024-09-14T03:37:42.745 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 46/116 2024-09-14T03:37:42.745 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 47/116 2024-09-14T03:37:42.745 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 48/116 2024-09-14T03:37:42.745 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 49/116 2024-09-14T03:37:42.745 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 50/116 2024-09-14T03:37:42.746 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 51/116 2024-09-14T03:37:42.746 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : openblas-0.3.26-2.el9.x86_64 52/116 2024-09-14T03:37:42.746 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : openblas-openmp-0.3.26-2.el9.x86_64 53/116 2024-09-14T03:37:42.746 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-attrs-20.3.0-7.el9.noarch 54/116 2024-09-14T03:37:42.746 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 55/116 2024-09-14T03:37:42.746 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-devel-3.9.19-8.el9.x86_64 56/116 2024-09-14T03:37:42.746 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-iniconfig-1.1.1-7.el9.noarch 57/116 2024-09-14T03:37:42.746 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 58/116 2024-09-14T03:37:42.746 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 59/116 2024-09-14T03:37:42.746 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 60/116 2024-09-14T03:37:42.746 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 61/116 2024-09-14T03:37:42.746 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 62/116 2024-09-14T03:37:42.746 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 63/116 2024-09-14T03:37:42.746 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 64/116 2024-09-14T03:37:42.747 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 65/116 2024-09-14T03:37:42.747 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 66/116 2024-09-14T03:37:42.747 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-pluggy-0.13.1-7.el9.noarch 67/116 2024-09-14T03:37:42.747 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 68/116 2024-09-14T03:37:42.747 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-py-1.10.0-6.el9.noarch 69/116 2024-09-14T03:37:42.747 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 70/116 2024-09-14T03:37:42.747 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 71/116 2024-09-14T03:37:42.747 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-pytest-6.2.2-7.el9.noarch 72/116 2024-09-14T03:37:42.747 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 73/116 2024-09-14T03:37:42.747 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 74/116 2024-09-14T03:37:42.747 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 75/116 2024-09-14T03:37:42.747 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 76/116 2024-09-14T03:37:42.747 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 77/116 2024-09-14T03:37:42.748 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 78/116 2024-09-14T03:37:42.748 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 79/116 2024-09-14T03:37:42.748 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 80/116 2024-09-14T03:37:42.748 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 81/116 2024-09-14T03:37:42.748 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 82/116 2024-09-14T03:37:42.748 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 83/116 2024-09-14T03:37:42.748 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 84/116 2024-09-14T03:37:42.748 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 85/116 2024-09-14T03:37:42.748 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 86/116 2024-09-14T03:37:42.748 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 87/116 2024-09-14T03:37:42.748 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-google-auth-1:2.34.0-1.el9.noarch 88/116 2024-09-14T03:37:42.748 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 89/116 2024-09-14T03:37:42.748 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 90/116 2024-09-14T03:37:42.749 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 91/116 2024-09-14T03:37:42.749 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 92/116 2024-09-14T03:37:42.749 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 93/116 2024-09-14T03:37:42.749 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 94/116 2024-09-14T03:37:42.749 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 95/116 2024-09-14T03:37:42.749 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 96/116 2024-09-14T03:37:42.749 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 97/116 2024-09-14T03:37:42.749 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 98/116 2024-09-14T03:37:42.749 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 99/116 2024-09-14T03:37:42.749 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 100/116 2024-09-14T03:37:42.749 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 101/116 2024-09-14T03:37:42.749 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 102/116 2024-09-14T03:37:42.750 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 103/116 2024-09-14T03:37:42.750 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 104/116 2024-09-14T03:37:42.750 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 105/116 2024-09-14T03:37:42.750 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 106/116 2024-09-14T03:37:42.750 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 107/116 2024-09-14T03:37:42.750 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 108/116 2024-09-14T03:37:42.750 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 109/116 2024-09-14T03:37:42.750 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 110/116 2024-09-14T03:37:42.750 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 111/116 2024-09-14T03:37:42.750 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 112/116 2024-09-14T03:37:42.750 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : librados2-2:17.2.7-1629.geb16b97b.el9.x86_64 113/116 2024-09-14T03:37:42.750 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 114/116 2024-09-14T03:37:42.750 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : librbd1-2:17.2.7-1629.geb16b97b.el9.x86_64 115/116 2024-09-14T03:37:44.501 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 116/116 2024-09-14T03:37:44.502 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-2:17.2.7-1629.geb16b97b.el9.x86_64 1/116 2024-09-14T03:37:44.502 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-base-2:17.2.7-1629.geb16b97b.el9.x86_64 2/116 2024-09-14T03:37:44.502 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-common-2:17.2.7-1629.geb16b97b.el9.x86_64 3/116 2024-09-14T03:37:44.502 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-fuse-2:17.2.7-1629.geb16b97b.el9.x86_64 4/116 2024-09-14T03:37:44.502 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-1629.geb16b 5/116 2024-09-14T03:37:44.502 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-mds-2:17.2.7-1629.geb16b97b.el9.x86_64 6/116 2024-09-14T03:37:44.502 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-mgr-2:17.2.7-1629.geb16b97b.el9.x86_64 7/116 2024-09-14T03:37:44.502 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-mon-2:17.2.7-1629.geb16b97b.el9.x86_64 8/116 2024-09-14T03:37:44.502 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-osd-2:17.2.7-1629.geb16b97b.el9.x86_64 9/116 2024-09-14T03:37:44.503 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-radosgw-2:17.2.7-1629.geb16b97b.el9.x86_64 10/116 2024-09-14T03:37:44.503 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-selinux-2:17.2.7-1629.geb16b97b.el9.x86_64 11/116 2024-09-14T03:37:44.503 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-test-2:17.2.7-1629.geb16b97b.el9.x86_64 12/116 2024-09-14T03:37:44.503 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : libcephfs-devel-2:17.2.7-1629.geb16b97b.el9.x86_ 13/116 2024-09-14T03:37:44.503 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : libcephfs2-2:17.2.7-1629.geb16b97b.el9.x86_64 14/116 2024-09-14T03:37:44.503 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : libcephsqlite-2:17.2.7-1629.geb16b97b.el9.x86_64 15/116 2024-09-14T03:37:44.503 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : librados-devel-2:17.2.7-1629.geb16b97b.el9.x86_6 16/116 2024-09-14T03:37:44.503 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : libradosstriper1-2:17.2.7-1629.geb16b97b.el9.x86 17/116 2024-09-14T03:37:44.504 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : librgw2-2:17.2.7-1629.geb16b97b.el9.x86_64 18/116 2024-09-14T03:37:44.504 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-ceph-argparse-2:17.2.7-1629.geb16b97b.el 19/116 2024-09-14T03:37:44.504 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-ceph-common-2:17.2.7-1629.geb16b97b.el9. 20/116 2024-09-14T03:37:44.504 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-cephfs-2:17.2.7-1629.geb16b97b.el9.x86_6 21/116 2024-09-14T03:37:44.504 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-rados-2:17.2.7-1629.geb16b97b.el9.x86_64 22/116 2024-09-14T03:37:44.504 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-rbd-2:17.2.7-1629.geb16b97b.el9.x86_64 23/116 2024-09-14T03:37:44.504 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-rgw-2:17.2.7-1629.geb16b97b.el9.x86_64 24/116 2024-09-14T03:37:44.504 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : rbd-fuse-2:17.2.7-1629.geb16b97b.el9.x86_64 25/116 2024-09-14T03:37:44.504 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : rbd-mirror-2:17.2.7-1629.geb16b97b.el9.x86_64 26/116 2024-09-14T03:37:44.505 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : rbd-nbd-2:17.2.7-1629.geb16b97b.el9.x86_64 27/116 2024-09-14T03:37:44.505 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-1629.geb16b97b. 28/116 2024-09-14T03:37:44.505 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-1629.geb16b97b.el9.noa 29/116 2024-09-14T03:37:44.505 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-1629.geb16b97b.el9.n 30/116 2024-09-14T03:37:44.505 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-1629.geb1 31/116 2024-09-14T03:37:44.505 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-1629.geb16b97b.el 32/116 2024-09-14T03:37:44.505 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-mgr-rook-2:17.2.7-1629.geb16b97b.el9.noarch 33/116 2024-09-14T03:37:44.505 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-1629.geb16b97b.e 34/116 2024-09-14T03:37:44.505 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : cephadm-2:17.2.7-1629.geb16b97b.el9.noarch 35/116 2024-09-14T03:37:44.506 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 36/116 2024-09-14T03:37:44.506 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : libgfortran-11.5.0-2.el9.x86_64 37/116 2024-09-14T03:37:44.506 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : libquadmath-11.5.0-2.el9.x86_64 38/116 2024-09-14T03:37:44.506 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 39/116 2024-09-14T03:37:44.506 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 40/116 2024-09-14T03:37:44.506 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-ply-3.11-14.el9.noarch 41/116 2024-09-14T03:37:44.506 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 42/116 2024-09-14T03:37:44.506 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 43/116 2024-09-14T03:37:44.507 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 44/116 2024-09-14T03:37:44.507 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 45/116 2024-09-14T03:37:44.507 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 46/116 2024-09-14T03:37:44.507 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 47/116 2024-09-14T03:37:44.507 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 48/116 2024-09-14T03:37:44.507 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 49/116 2024-09-14T03:37:44.508 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 50/116 2024-09-14T03:37:44.508 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 51/116 2024-09-14T03:37:44.508 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : openblas-0.3.26-2.el9.x86_64 52/116 2024-09-14T03:37:44.508 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : openblas-openmp-0.3.26-2.el9.x86_64 53/116 2024-09-14T03:37:44.508 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-attrs-20.3.0-7.el9.noarch 54/116 2024-09-14T03:37:44.508 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 55/116 2024-09-14T03:37:44.508 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-devel-3.9.19-8.el9.x86_64 56/116 2024-09-14T03:37:44.508 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-iniconfig-1.1.1-7.el9.noarch 57/116 2024-09-14T03:37:44.508 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 58/116 2024-09-14T03:37:44.509 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 59/116 2024-09-14T03:37:44.509 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 60/116 2024-09-14T03:37:44.509 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 61/116 2024-09-14T03:37:44.509 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 62/116 2024-09-14T03:37:44.509 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 63/116 2024-09-14T03:37:44.509 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 64/116 2024-09-14T03:37:44.509 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 65/116 2024-09-14T03:37:44.509 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 66/116 2024-09-14T03:37:44.510 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-pluggy-0.13.1-7.el9.noarch 67/116 2024-09-14T03:37:44.510 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 68/116 2024-09-14T03:37:44.510 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-py-1.10.0-6.el9.noarch 69/116 2024-09-14T03:37:44.510 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 70/116 2024-09-14T03:37:44.510 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 71/116 2024-09-14T03:37:44.510 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-pytest-6.2.2-7.el9.noarch 72/116 2024-09-14T03:37:44.510 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 73/116 2024-09-14T03:37:44.510 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 74/116 2024-09-14T03:37:44.510 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 75/116 2024-09-14T03:37:44.511 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 76/116 2024-09-14T03:37:44.511 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 77/116 2024-09-14T03:37:44.511 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 78/116 2024-09-14T03:37:44.511 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 79/116 2024-09-14T03:37:44.511 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 80/116 2024-09-14T03:37:44.511 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 81/116 2024-09-14T03:37:44.511 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 82/116 2024-09-14T03:37:44.512 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 83/116 2024-09-14T03:37:44.513 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 84/116 2024-09-14T03:37:44.513 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 85/116 2024-09-14T03:37:44.513 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 86/116 2024-09-14T03:37:44.513 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 87/116 2024-09-14T03:37:44.513 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-google-auth-1:2.34.0-1.el9.noarch 88/116 2024-09-14T03:37:44.513 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 89/116 2024-09-14T03:37:44.513 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 90/116 2024-09-14T03:37:44.513 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 91/116 2024-09-14T03:37:44.513 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 92/116 2024-09-14T03:37:44.513 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 93/116 2024-09-14T03:37:44.513 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 94/116 2024-09-14T03:37:44.513 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 95/116 2024-09-14T03:37:44.514 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 96/116 2024-09-14T03:37:44.514 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 97/116 2024-09-14T03:37:44.514 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 98/116 2024-09-14T03:37:44.514 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 99/116 2024-09-14T03:37:44.514 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 100/116 2024-09-14T03:37:44.514 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 101/116 2024-09-14T03:37:44.514 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 102/116 2024-09-14T03:37:44.514 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 103/116 2024-09-14T03:37:44.514 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 104/116 2024-09-14T03:37:44.514 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 105/116 2024-09-14T03:37:44.514 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 106/116 2024-09-14T03:37:44.514 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 107/116 2024-09-14T03:37:44.515 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 108/116 2024-09-14T03:37:44.515 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 109/116 2024-09-14T03:37:44.515 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 110/116 2024-09-14T03:37:44.515 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 111/116 2024-09-14T03:37:44.515 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 112/116 2024-09-14T03:37:44.515 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : librados2-2:17.2.7-1629.geb16b97b.el9.x86_64 113/116 2024-09-14T03:37:44.515 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 114/116 2024-09-14T03:37:44.515 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : librbd1-2:17.2.7-1629.geb16b97b.el9.x86_64 115/116 2024-09-14T03:37:44.698 INFO:teuthology.orchestra.run.smithi094.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 116/116 2024-09-14T03:37:44.698 INFO:teuthology.orchestra.run.smithi094.stdout: 2024-09-14T03:37:44.698 INFO:teuthology.orchestra.run.smithi094.stdout:Upgraded: 2024-09-14T03:37:44.698 INFO:teuthology.orchestra.run.smithi094.stdout: librados2-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:44.698 INFO:teuthology.orchestra.run.smithi094.stdout: librbd1-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:44.698 INFO:teuthology.orchestra.run.smithi094.stdout:Installed: 2024-09-14T03:37:44.698 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:44.698 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-base-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:44.698 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-common-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:44.698 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-fuse-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:44.698 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-grafana-dashboards-2:17.2.7-1629.geb16b97b.el9.noarch 2024-09-14T03:37:44.699 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-immutable-object-cache-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:44.699 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-mds-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:44.699 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-mgr-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:44.699 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-mgr-cephadm-2:17.2.7-1629.geb16b97b.el9.noarch 2024-09-14T03:37:44.699 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-mgr-dashboard-2:17.2.7-1629.geb16b97b.el9.noarch 2024-09-14T03:37:44.699 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-mgr-diskprediction-local-2:17.2.7-1629.geb16b97b.el9.noarch 2024-09-14T03:37:44.699 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-mgr-modules-core-2:17.2.7-1629.geb16b97b.el9.noarch 2024-09-14T03:37:44.699 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-mgr-rook-2:17.2.7-1629.geb16b97b.el9.noarch 2024-09-14T03:37:44.699 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-mon-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:44.699 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-osd-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:44.699 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-prometheus-alerts-2:17.2.7-1629.geb16b97b.el9.noarch 2024-09-14T03:37:44.699 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-radosgw-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:44.699 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-selinux-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:44.699 INFO:teuthology.orchestra.run.smithi094.stdout: ceph-test-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:44.700 INFO:teuthology.orchestra.run.smithi094.stdout: cephadm-2:17.2.7-1629.geb16b97b.el9.noarch 2024-09-14T03:37:44.700 INFO:teuthology.orchestra.run.smithi094.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-09-14T03:37:44.700 INFO:teuthology.orchestra.run.smithi094.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-09-14T03:37:44.700 INFO:teuthology.orchestra.run.smithi094.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-09-14T03:37:44.700 INFO:teuthology.orchestra.run.smithi094.stdout: fuse-2.9.9-16.el9.x86_64 2024-09-14T03:37:44.700 INFO:teuthology.orchestra.run.smithi094.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2024-09-14T03:37:44.700 INFO:teuthology.orchestra.run.smithi094.stdout: libcephfs-devel-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:44.700 INFO:teuthology.orchestra.run.smithi094.stdout: libcephfs2-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:44.700 INFO:teuthology.orchestra.run.smithi094.stdout: libcephsqlite-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:44.700 INFO:teuthology.orchestra.run.smithi094.stdout: libgfortran-11.5.0-2.el9.x86_64 2024-09-14T03:37:44.700 INFO:teuthology.orchestra.run.smithi094.stdout: liboath-2.6.7-2.el9.x86_64 2024-09-14T03:37:44.701 INFO:teuthology.orchestra.run.smithi094.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-09-14T03:37:44.701 INFO:teuthology.orchestra.run.smithi094.stdout: libquadmath-11.5.0-2.el9.x86_64 2024-09-14T03:37:44.701 INFO:teuthology.orchestra.run.smithi094.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-09-14T03:37:44.701 INFO:teuthology.orchestra.run.smithi094.stdout: librados-devel-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:44.701 INFO:teuthology.orchestra.run.smithi094.stdout: libradosstriper1-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:44.701 INFO:teuthology.orchestra.run.smithi094.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-09-14T03:37:44.701 INFO:teuthology.orchestra.run.smithi094.stdout: librgw2-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:44.701 INFO:teuthology.orchestra.run.smithi094.stdout: libunwind-1.6.2-1.el9.x86_64 2024-09-14T03:37:44.701 INFO:teuthology.orchestra.run.smithi094.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-09-14T03:37:44.701 INFO:teuthology.orchestra.run.smithi094.stdout: openblas-0.3.26-2.el9.x86_64 2024-09-14T03:37:44.701 INFO:teuthology.orchestra.run.smithi094.stdout: openblas-openmp-0.3.26-2.el9.x86_64 2024-09-14T03:37:44.701 INFO:teuthology.orchestra.run.smithi094.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-09-14T03:37:44.701 INFO:teuthology.orchestra.run.smithi094.stdout: python3-attrs-20.3.0-7.el9.noarch 2024-09-14T03:37:44.702 INFO:teuthology.orchestra.run.smithi094.stdout: python3-babel-2.9.1-2.el9.noarch 2024-09-14T03:37:44.702 INFO:teuthology.orchestra.run.smithi094.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-09-14T03:37:44.702 INFO:teuthology.orchestra.run.smithi094.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-09-14T03:37:44.702 INFO:teuthology.orchestra.run.smithi094.stdout: python3-ceph-argparse-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:44.702 INFO:teuthology.orchestra.run.smithi094.stdout: python3-ceph-common-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:44.702 INFO:teuthology.orchestra.run.smithi094.stdout: python3-cephfs-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:44.702 INFO:teuthology.orchestra.run.smithi094.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-09-14T03:37:44.702 INFO:teuthology.orchestra.run.smithi094.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-09-14T03:37:44.702 INFO:teuthology.orchestra.run.smithi094.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-09-14T03:37:44.702 INFO:teuthology.orchestra.run.smithi094.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-09-14T03:37:44.702 INFO:teuthology.orchestra.run.smithi094.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-09-14T03:37:44.702 INFO:teuthology.orchestra.run.smithi094.stdout: python3-devel-3.9.19-8.el9.x86_64 2024-09-14T03:37:44.702 INFO:teuthology.orchestra.run.smithi094.stdout: python3-google-auth-1:2.34.0-1.el9.noarch 2024-09-14T03:37:44.703 INFO:teuthology.orchestra.run.smithi094.stdout: python3-iniconfig-1.1.1-7.el9.noarch 2024-09-14T03:37:44.703 INFO:teuthology.orchestra.run.smithi094.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-09-14T03:37:44.703 INFO:teuthology.orchestra.run.smithi094.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-09-14T03:37:44.703 INFO:teuthology.orchestra.run.smithi094.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-09-14T03:37:44.703 INFO:teuthology.orchestra.run.smithi094.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-09-14T03:37:44.703 INFO:teuthology.orchestra.run.smithi094.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-09-14T03:37:44.703 INFO:teuthology.orchestra.run.smithi094.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-09-14T03:37:44.703 INFO:teuthology.orchestra.run.smithi094.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-09-14T03:37:44.703 INFO:teuthology.orchestra.run.smithi094.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-09-14T03:37:44.703 INFO:teuthology.orchestra.run.smithi094.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-09-14T03:37:44.703 INFO:teuthology.orchestra.run.smithi094.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-09-14T03:37:44.703 INFO:teuthology.orchestra.run.smithi094.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-09-14T03:37:44.704 INFO:teuthology.orchestra.run.smithi094.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-09-14T03:37:44.704 INFO:teuthology.orchestra.run.smithi094.stdout: python3-mako-1.1.4-6.el9.noarch 2024-09-14T03:37:44.704 INFO:teuthology.orchestra.run.smithi094.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-09-14T03:37:44.704 INFO:teuthology.orchestra.run.smithi094.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-09-14T03:37:44.704 INFO:teuthology.orchestra.run.smithi094.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-09-14T03:37:44.704 INFO:teuthology.orchestra.run.smithi094.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-09-14T03:37:44.704 INFO:teuthology.orchestra.run.smithi094.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-09-14T03:37:44.704 INFO:teuthology.orchestra.run.smithi094.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-09-14T03:37:44.704 INFO:teuthology.orchestra.run.smithi094.stdout: python3-packaging-20.9-5.el9.noarch 2024-09-14T03:37:44.704 INFO:teuthology.orchestra.run.smithi094.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-09-14T03:37:44.704 INFO:teuthology.orchestra.run.smithi094.stdout: python3-pluggy-0.13.1-7.el9.noarch 2024-09-14T03:37:44.704 INFO:teuthology.orchestra.run.smithi094.stdout: python3-ply-3.11-14.el9.noarch 2024-09-14T03:37:44.704 INFO:teuthology.orchestra.run.smithi094.stdout: python3-portend-3.1.0-2.el9.noarch 2024-09-14T03:37:44.704 INFO:teuthology.orchestra.run.smithi094.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-09-14T03:37:44.705 INFO:teuthology.orchestra.run.smithi094.stdout: python3-py-1.10.0-6.el9.noarch 2024-09-14T03:37:44.705 INFO:teuthology.orchestra.run.smithi094.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-09-14T03:37:44.705 INFO:teuthology.orchestra.run.smithi094.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-09-14T03:37:44.705 INFO:teuthology.orchestra.run.smithi094.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-09-14T03:37:44.705 INFO:teuthology.orchestra.run.smithi094.stdout: python3-pycparser-2.20-6.el9.noarch 2024-09-14T03:37:44.705 INFO:teuthology.orchestra.run.smithi094.stdout: python3-pytest-6.2.2-7.el9.noarch 2024-09-14T03:37:44.705 INFO:teuthology.orchestra.run.smithi094.stdout: python3-pytz-2021.1-5.el9.noarch 2024-09-14T03:37:44.705 INFO:teuthology.orchestra.run.smithi094.stdout: python3-rados-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:44.705 INFO:teuthology.orchestra.run.smithi094.stdout: python3-rbd-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:44.705 INFO:teuthology.orchestra.run.smithi094.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-09-14T03:37:44.705 INFO:teuthology.orchestra.run.smithi094.stdout: python3-requests-2.25.1-8.el9.noarch 2024-09-14T03:37:44.705 INFO:teuthology.orchestra.run.smithi094.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-09-14T03:37:44.705 INFO:teuthology.orchestra.run.smithi094.stdout: python3-rgw-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:44.706 INFO:teuthology.orchestra.run.smithi094.stdout: python3-routes-2.5.1-5.el9.noarch 2024-09-14T03:37:44.706 INFO:teuthology.orchestra.run.smithi094.stdout: python3-rsa-4.9-2.el9.noarch 2024-09-14T03:37:44.706 INFO:teuthology.orchestra.run.smithi094.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-09-14T03:37:44.706 INFO:teuthology.orchestra.run.smithi094.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-09-14T03:37:44.706 INFO:teuthology.orchestra.run.smithi094.stdout: python3-toml-0.10.2-6.el9.noarch 2024-09-14T03:37:44.706 INFO:teuthology.orchestra.run.smithi094.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-09-14T03:37:44.706 INFO:teuthology.orchestra.run.smithi094.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-09-14T03:37:44.706 INFO:teuthology.orchestra.run.smithi094.stdout: python3-webob-1.8.8-2.el9.noarch 2024-09-14T03:37:44.706 INFO:teuthology.orchestra.run.smithi094.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-09-14T03:37:44.706 INFO:teuthology.orchestra.run.smithi094.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-09-14T03:37:44.706 INFO:teuthology.orchestra.run.smithi094.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-09-14T03:37:44.706 INFO:teuthology.orchestra.run.smithi094.stdout: rbd-fuse-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:44.706 INFO:teuthology.orchestra.run.smithi094.stdout: rbd-mirror-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:44.706 INFO:teuthology.orchestra.run.smithi094.stdout: rbd-nbd-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:44.707 INFO:teuthology.orchestra.run.smithi094.stdout: socat-1.7.4.1-6.el9.x86_64 2024-09-14T03:37:44.707 INFO:teuthology.orchestra.run.smithi094.stdout: thrift-0.15.0-2.el9.x86_64 2024-09-14T03:37:44.707 INFO:teuthology.orchestra.run.smithi094.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-09-14T03:37:44.707 INFO:teuthology.orchestra.run.smithi094.stdout: 2024-09-14T03:37:44.707 INFO:teuthology.orchestra.run.smithi094.stdout:Complete! 2024-09-14T03:37:45.031 DEBUG:teuthology.parallel:result is None 2024-09-14T03:37:46.703 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 116/116 2024-09-14T03:37:46.704 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:37:46.704 INFO:teuthology.orchestra.run.smithi070.stdout:Upgraded: 2024-09-14T03:37:46.704 INFO:teuthology.orchestra.run.smithi070.stdout: librados2-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:46.704 INFO:teuthology.orchestra.run.smithi070.stdout: librbd1-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:46.704 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2024-09-14T03:37:46.704 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:46.704 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-base-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:46.704 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-common-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:46.704 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-fuse-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:46.704 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-grafana-dashboards-2:17.2.7-1629.geb16b97b.el9.noarch 2024-09-14T03:37:46.704 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-immutable-object-cache-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:46.705 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mds-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:46.705 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:46.705 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr-cephadm-2:17.2.7-1629.geb16b97b.el9.noarch 2024-09-14T03:37:46.705 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr-dashboard-2:17.2.7-1629.geb16b97b.el9.noarch 2024-09-14T03:37:46.705 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr-diskprediction-local-2:17.2.7-1629.geb16b97b.el9.noarch 2024-09-14T03:37:46.705 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr-modules-core-2:17.2.7-1629.geb16b97b.el9.noarch 2024-09-14T03:37:46.705 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr-rook-2:17.2.7-1629.geb16b97b.el9.noarch 2024-09-14T03:37:46.705 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mon-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:46.705 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-osd-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:46.705 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-prometheus-alerts-2:17.2.7-1629.geb16b97b.el9.noarch 2024-09-14T03:37:46.705 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-radosgw-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:46.705 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-selinux-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:46.706 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-test-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:46.706 INFO:teuthology.orchestra.run.smithi070.stdout: cephadm-2:17.2.7-1629.geb16b97b.el9.noarch 2024-09-14T03:37:46.706 INFO:teuthology.orchestra.run.smithi070.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-09-14T03:37:46.706 INFO:teuthology.orchestra.run.smithi070.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-09-14T03:37:46.706 INFO:teuthology.orchestra.run.smithi070.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-09-14T03:37:46.706 INFO:teuthology.orchestra.run.smithi070.stdout: fuse-2.9.9-16.el9.x86_64 2024-09-14T03:37:46.706 INFO:teuthology.orchestra.run.smithi070.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2024-09-14T03:37:46.706 INFO:teuthology.orchestra.run.smithi070.stdout: libcephfs-devel-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:46.706 INFO:teuthology.orchestra.run.smithi070.stdout: libcephfs2-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:46.706 INFO:teuthology.orchestra.run.smithi070.stdout: libcephsqlite-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:46.706 INFO:teuthology.orchestra.run.smithi070.stdout: libgfortran-11.5.0-2.el9.x86_64 2024-09-14T03:37:46.706 INFO:teuthology.orchestra.run.smithi070.stdout: liboath-2.6.7-2.el9.x86_64 2024-09-14T03:37:46.707 INFO:teuthology.orchestra.run.smithi070.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-09-14T03:37:46.707 INFO:teuthology.orchestra.run.smithi070.stdout: libquadmath-11.5.0-2.el9.x86_64 2024-09-14T03:37:46.707 INFO:teuthology.orchestra.run.smithi070.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-09-14T03:37:46.707 INFO:teuthology.orchestra.run.smithi070.stdout: librados-devel-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:46.707 INFO:teuthology.orchestra.run.smithi070.stdout: libradosstriper1-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:46.707 INFO:teuthology.orchestra.run.smithi070.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-09-14T03:37:46.707 INFO:teuthology.orchestra.run.smithi070.stdout: librgw2-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:46.707 INFO:teuthology.orchestra.run.smithi070.stdout: libunwind-1.6.2-1.el9.x86_64 2024-09-14T03:37:46.707 INFO:teuthology.orchestra.run.smithi070.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-09-14T03:37:46.707 INFO:teuthology.orchestra.run.smithi070.stdout: openblas-0.3.26-2.el9.x86_64 2024-09-14T03:37:46.707 INFO:teuthology.orchestra.run.smithi070.stdout: openblas-openmp-0.3.26-2.el9.x86_64 2024-09-14T03:37:46.707 INFO:teuthology.orchestra.run.smithi070.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-09-14T03:37:46.708 INFO:teuthology.orchestra.run.smithi070.stdout: python3-attrs-20.3.0-7.el9.noarch 2024-09-14T03:37:46.708 INFO:teuthology.orchestra.run.smithi070.stdout: python3-babel-2.9.1-2.el9.noarch 2024-09-14T03:37:46.708 INFO:teuthology.orchestra.run.smithi070.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-09-14T03:37:46.708 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-09-14T03:37:46.708 INFO:teuthology.orchestra.run.smithi070.stdout: python3-ceph-argparse-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:46.708 INFO:teuthology.orchestra.run.smithi070.stdout: python3-ceph-common-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:46.708 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cephfs-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:46.708 INFO:teuthology.orchestra.run.smithi070.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-09-14T03:37:46.708 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-09-14T03:37:46.708 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-09-14T03:37:46.708 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-09-14T03:37:46.708 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-09-14T03:37:46.709 INFO:teuthology.orchestra.run.smithi070.stdout: python3-devel-3.9.19-8.el9.x86_64 2024-09-14T03:37:46.709 INFO:teuthology.orchestra.run.smithi070.stdout: python3-google-auth-1:2.34.0-1.el9.noarch 2024-09-14T03:37:46.709 INFO:teuthology.orchestra.run.smithi070.stdout: python3-iniconfig-1.1.1-7.el9.noarch 2024-09-14T03:37:46.709 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-09-14T03:37:46.709 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-09-14T03:37:46.709 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-09-14T03:37:46.709 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-09-14T03:37:46.709 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-09-14T03:37:46.709 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-09-14T03:37:46.709 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-09-14T03:37:46.709 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-09-14T03:37:46.709 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-09-14T03:37:46.709 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-09-14T03:37:46.710 INFO:teuthology.orchestra.run.smithi070.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-09-14T03:37:46.710 INFO:teuthology.orchestra.run.smithi070.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-09-14T03:37:46.710 INFO:teuthology.orchestra.run.smithi070.stdout: python3-mako-1.1.4-6.el9.noarch 2024-09-14T03:37:46.710 INFO:teuthology.orchestra.run.smithi070.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-09-14T03:37:46.710 INFO:teuthology.orchestra.run.smithi070.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-09-14T03:37:46.710 INFO:teuthology.orchestra.run.smithi070.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-09-14T03:37:46.710 INFO:teuthology.orchestra.run.smithi070.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-09-14T03:37:46.710 INFO:teuthology.orchestra.run.smithi070.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-09-14T03:37:46.710 INFO:teuthology.orchestra.run.smithi070.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-09-14T03:37:46.710 INFO:teuthology.orchestra.run.smithi070.stdout: python3-packaging-20.9-5.el9.noarch 2024-09-14T03:37:46.710 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-09-14T03:37:46.710 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pluggy-0.13.1-7.el9.noarch 2024-09-14T03:37:46.710 INFO:teuthology.orchestra.run.smithi070.stdout: python3-ply-3.11-14.el9.noarch 2024-09-14T03:37:46.711 INFO:teuthology.orchestra.run.smithi070.stdout: python3-portend-3.1.0-2.el9.noarch 2024-09-14T03:37:46.711 INFO:teuthology.orchestra.run.smithi070.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-09-14T03:37:46.711 INFO:teuthology.orchestra.run.smithi070.stdout: python3-py-1.10.0-6.el9.noarch 2024-09-14T03:37:46.711 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-09-14T03:37:46.711 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-09-14T03:37:46.711 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-09-14T03:37:46.711 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pycparser-2.20-6.el9.noarch 2024-09-14T03:37:46.711 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pytest-6.2.2-7.el9.noarch 2024-09-14T03:37:46.711 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pytz-2021.1-5.el9.noarch 2024-09-14T03:37:46.711 INFO:teuthology.orchestra.run.smithi070.stdout: python3-rados-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:46.712 INFO:teuthology.orchestra.run.smithi070.stdout: python3-rbd-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:46.712 INFO:teuthology.orchestra.run.smithi070.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-09-14T03:37:46.712 INFO:teuthology.orchestra.run.smithi070.stdout: python3-requests-2.25.1-8.el9.noarch 2024-09-14T03:37:46.712 INFO:teuthology.orchestra.run.smithi070.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-09-14T03:37:46.712 INFO:teuthology.orchestra.run.smithi070.stdout: python3-rgw-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:46.712 INFO:teuthology.orchestra.run.smithi070.stdout: python3-routes-2.5.1-5.el9.noarch 2024-09-14T03:37:46.712 INFO:teuthology.orchestra.run.smithi070.stdout: python3-rsa-4.9-2.el9.noarch 2024-09-14T03:37:46.712 INFO:teuthology.orchestra.run.smithi070.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-09-14T03:37:46.712 INFO:teuthology.orchestra.run.smithi070.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-09-14T03:37:46.712 INFO:teuthology.orchestra.run.smithi070.stdout: python3-toml-0.10.2-6.el9.noarch 2024-09-14T03:37:46.712 INFO:teuthology.orchestra.run.smithi070.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-09-14T03:37:46.712 INFO:teuthology.orchestra.run.smithi070.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-09-14T03:37:46.713 INFO:teuthology.orchestra.run.smithi070.stdout: python3-webob-1.8.8-2.el9.noarch 2024-09-14T03:37:46.713 INFO:teuthology.orchestra.run.smithi070.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-09-14T03:37:46.713 INFO:teuthology.orchestra.run.smithi070.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-09-14T03:37:46.713 INFO:teuthology.orchestra.run.smithi070.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-09-14T03:37:46.713 INFO:teuthology.orchestra.run.smithi070.stdout: rbd-fuse-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:46.713 INFO:teuthology.orchestra.run.smithi070.stdout: rbd-mirror-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:46.713 INFO:teuthology.orchestra.run.smithi070.stdout: rbd-nbd-2:17.2.7-1629.geb16b97b.el9.x86_64 2024-09-14T03:37:46.713 INFO:teuthology.orchestra.run.smithi070.stdout: socat-1.7.4.1-6.el9.x86_64 2024-09-14T03:37:46.713 INFO:teuthology.orchestra.run.smithi070.stdout: thrift-0.15.0-2.el9.x86_64 2024-09-14T03:37:46.714 INFO:teuthology.orchestra.run.smithi070.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-09-14T03:37:46.714 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:37:46.714 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2024-09-14T03:37:47.013 DEBUG:teuthology.parallel:result is None 2024-09-14T03:37:47.013 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-14T03:37:47.014 INFO:teuthology.packaging:ref: None 2024-09-14T03:37:47.014 INFO:teuthology.packaging:tag: None 2024-09-14T03:37:47.014 INFO:teuthology.packaging:branch: quincy 2024-09-14T03:37:47.014 INFO:teuthology.packaging:sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:37:47.014 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-09-14T03:37:47.219 DEBUG:teuthology.orchestra.run.smithi070:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-14T03:37:47.241 INFO:teuthology.orchestra.run.smithi070.stdout:17.2.7-1629.geb16b97b.el9 2024-09-14T03:37:47.242 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-1629.geb16b97b.el9 2024-09-14T03:37:47.242 INFO:teuthology.task.install:The correct ceph version 17.2.7-1629.geb16b97b is installed. 2024-09-14T03:37:47.243 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-14T03:37:47.244 INFO:teuthology.packaging:ref: None 2024-09-14T03:37:47.244 INFO:teuthology.packaging:tag: None 2024-09-14T03:37:47.244 INFO:teuthology.packaging:branch: quincy 2024-09-14T03:37:47.244 INFO:teuthology.packaging:sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:37:47.244 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-09-14T03:37:47.442 DEBUG:teuthology.orchestra.run.smithi094:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-14T03:37:47.469 INFO:teuthology.orchestra.run.smithi094.stdout:17.2.7-1629.geb16b97b.el9 2024-09-14T03:37:47.469 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-1629.geb16b97b.el9 2024-09-14T03:37:47.469 INFO:teuthology.task.install:The correct ceph version 17.2.7-1629.geb16b97b is installed. 2024-09-14T03:37:47.471 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-09-14T03:37:47.471 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2024-09-14T03:37:47.471 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-14T03:37:47.498 DEBUG:teuthology.orchestra.run.smithi094:> set -ex 2024-09-14T03:37:47.498 DEBUG:teuthology.orchestra.run.smithi094:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-14T03:37:47.541 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-09-14T03:37:47.541 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2024-09-14T03:37:47.541 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd of=/usr/bin/daemon-helper 2024-09-14T03:37:47.566 DEBUG:teuthology.orchestra.run.smithi070:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-14T03:37:47.630 DEBUG:teuthology.orchestra.run.smithi094:> set -ex 2024-09-14T03:37:47.630 DEBUG:teuthology.orchestra.run.smithi094:> sudo dd of=/usr/bin/daemon-helper 2024-09-14T03:37:47.659 DEBUG:teuthology.orchestra.run.smithi094:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-14T03:37:47.726 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-09-14T03:37:47.726 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2024-09-14T03:37:47.726 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-14T03:37:47.751 DEBUG:teuthology.orchestra.run.smithi070:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-14T03:37:47.816 DEBUG:teuthology.orchestra.run.smithi094:> set -ex 2024-09-14T03:37:47.816 DEBUG:teuthology.orchestra.run.smithi094:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-14T03:37:47.843 DEBUG:teuthology.orchestra.run.smithi094:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-14T03:37:47.909 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-09-14T03:37:47.909 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2024-09-14T03:37:47.909 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd of=/usr/bin/stdin-killer 2024-09-14T03:37:47.933 DEBUG:teuthology.orchestra.run.smithi070:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-14T03:37:47.999 DEBUG:teuthology.orchestra.run.smithi094:> set -ex 2024-09-14T03:37:47.999 DEBUG:teuthology.orchestra.run.smithi094:> sudo dd of=/usr/bin/stdin-killer 2024-09-14T03:37:48.026 DEBUG:teuthology.orchestra.run.smithi094:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-14T03:37:48.091 INFO:teuthology.run_tasks:Running task print... 2024-09-14T03:37:48.100 INFO:teuthology.task.print:**** done install task... 2024-09-14T03:37:48.100 INFO:teuthology.run_tasks:Running task print... 2024-09-14T03:37:48.107 INFO:teuthology.task.print:**** done start installing quincy cephadm ... 2024-09-14T03:37:48.107 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-14T03:37:48.204 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(POOL_APP_NOT_ENABLED\\)', 'OSD_DOWN', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'OSDMAP_FLAGS', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_AVAILABILITY'], 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8'} 2024-09-14T03:37:48.204 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2024-09-14T03:37:48.204 INFO:tasks.cephadm:Cluster fsid is b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:37:48.205 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-14T03:37:48.205 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.70', 'mon.c': '[v2:172.21.15.70:3301,v1:172.21.15.70:6790]', 'mon.b': '172.21.15.94'} 2024-09-14T03:37:48.205 INFO:tasks.cephadm:First mon is mon.a on smithi070 2024-09-14T03:37:48.205 INFO:tasks.cephadm:First mgr is y 2024-09-14T03:37:48.205 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-14T03:37:48.205 DEBUG:teuthology.orchestra.run.smithi070:> sudo hostname $(hostname -s) 2024-09-14T03:37:48.231 DEBUG:teuthology.orchestra.run.smithi094:> sudo hostname $(hostname -s) 2024-09-14T03:37:48.257 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2024-09-14T03:37:48.258 DEBUG:teuthology.orchestra.run.smithi070:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-14T03:37:48.448 INFO:teuthology.orchestra.run.smithi070.stdout:-rw-r--r--. 1 ubuntu ubuntu 378585 Sep 14 03:37 /home/ubuntu/cephtest/cephadm 2024-09-14T03:37:48.449 DEBUG:teuthology.orchestra.run.smithi094:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-14T03:37:48.582 INFO:teuthology.orchestra.run.smithi094.stdout:-rw-r--r--. 1 ubuntu ubuntu 378585 Sep 14 03:37 /home/ubuntu/cephtest/cephadm 2024-09-14T03:37:48.582 DEBUG:teuthology.orchestra.run.smithi070:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-14T03:37:48.599 DEBUG:teuthology.orchestra.run.smithi094:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-14T03:37:48.625 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2024-09-14T03:37:48.626 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-09-14T03:37:48.641 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-09-14T03:37:50.277 INFO:teuthology.orchestra.run.smithi070.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-14T03:37:50.348 INFO:teuthology.orchestra.run.smithi094.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-14T03:38:17.362 INFO:teuthology.orchestra.run.smithi094.stdout:{ 2024-09-14T03:38:17.362 INFO:teuthology.orchestra.run.smithi094.stdout: "ceph_version": "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)", 2024-09-14T03:38:17.362 INFO:teuthology.orchestra.run.smithi094.stdout: "image_id": "ab51d1bebd1200e205d11a014c20a7f2cf49d932afcbeabdc1c9fc887c00f446", 2024-09-14T03:38:17.363 INFO:teuthology.orchestra.run.smithi094.stdout: "repo_digests": [ 2024-09-14T03:38:17.363 INFO:teuthology.orchestra.run.smithi094.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6" 2024-09-14T03:38:17.363 INFO:teuthology.orchestra.run.smithi094.stdout: ] 2024-09-14T03:38:17.363 INFO:teuthology.orchestra.run.smithi094.stdout:} 2024-09-14T03:38:20.324 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:38:20.324 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph_version": "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)", 2024-09-14T03:38:20.325 INFO:teuthology.orchestra.run.smithi070.stdout: "image_id": "ab51d1bebd1200e205d11a014c20a7f2cf49d932afcbeabdc1c9fc887c00f446", 2024-09-14T03:38:20.325 INFO:teuthology.orchestra.run.smithi070.stdout: "repo_digests": [ 2024-09-14T03:38:20.325 INFO:teuthology.orchestra.run.smithi070.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6" 2024-09-14T03:38:20.325 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2024-09-14T03:38:20.325 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:38:20.343 DEBUG:teuthology.orchestra.run.smithi070:> sudo mkdir -p /etc/ceph 2024-09-14T03:38:20.386 DEBUG:teuthology.orchestra.run.smithi094:> sudo mkdir -p /etc/ceph 2024-09-14T03:38:20.421 DEBUG:teuthology.orchestra.run.smithi070:> sudo chmod 777 /etc/ceph 2024-09-14T03:38:20.450 DEBUG:teuthology.orchestra.run.smithi094:> sudo chmod 777 /etc/ceph 2024-09-14T03:38:20.490 INFO:tasks.cephadm:Writing seed config... 2024-09-14T03:38:20.491 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-09-14T03:38:20.491 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-09-14T03:38:20.491 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-14T03:38:20.491 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-14T03:38:20.491 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-14T03:38:20.492 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2024-09-14T03:38:20.492 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-14T03:38:20.492 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-14T03:38:20.492 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-14T03:38:20.492 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-14T03:38:20.492 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-14T03:38:20.493 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2024-09-14T03:38:20.493 DEBUG:teuthology.orchestra.run.smithi070:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-14T03:38:20.508 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = b6380d42-724a-11ef-bceb-c7b262605968 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-14T03:38:20.508 DEBUG:teuthology.orchestra.run.smithi070:mon.a> sudo journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@mon.a.service 2024-09-14T03:38:20.550 DEBUG:teuthology.orchestra.run.smithi070:mgr.y> sudo journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@mgr.y.service 2024-09-14T03:38:20.593 INFO:tasks.cephadm:Bootstrapping... 2024-09-14T03:38:20.594 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid b6380d42-724a-11ef-bceb-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.70 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-14T03:38:20.798 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:38:20.799 INFO:teuthology.orchestra.run.smithi070.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', 'b6380d42-724a-11ef-bceb-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.70', '--skip-admin-label'] 2024-09-14T03:38:20.834 INFO:teuthology.orchestra.run.smithi070.stdout:/bin/podman: stdout 5.2.2 2024-09-14T03:38:20.835 INFO:teuthology.orchestra.run.smithi070.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-09-14T03:38:20.835 INFO:teuthology.orchestra.run.smithi070.stdout:Verifying podman|docker is present... 2024-09-14T03:38:20.857 INFO:teuthology.orchestra.run.smithi070.stdout:/bin/podman: stdout 5.2.2 2024-09-14T03:38:20.857 INFO:teuthology.orchestra.run.smithi070.stdout:Verifying lvm2 is present... 2024-09-14T03:38:20.857 INFO:teuthology.orchestra.run.smithi070.stdout:Verifying time synchronization is in place... 2024-09-14T03:38:20.865 INFO:teuthology.orchestra.run.smithi070.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-09-14T03:38:20.865 INFO:teuthology.orchestra.run.smithi070.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-09-14T03:38:20.872 INFO:teuthology.orchestra.run.smithi070.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-09-14T03:38:20.872 INFO:teuthology.orchestra.run.smithi070.stdout:systemctl: stdout inactive 2024-09-14T03:38:20.879 INFO:teuthology.orchestra.run.smithi070.stdout:systemctl: stdout enabled 2024-09-14T03:38:20.885 INFO:teuthology.orchestra.run.smithi070.stdout:systemctl: stdout active 2024-09-14T03:38:20.886 INFO:teuthology.orchestra.run.smithi070.stdout:Unit chronyd.service is enabled and running 2024-09-14T03:38:20.886 INFO:teuthology.orchestra.run.smithi070.stdout:Repeating the final host check... 2024-09-14T03:38:20.908 INFO:teuthology.orchestra.run.smithi070.stdout:/bin/podman: stdout 5.2.2 2024-09-14T03:38:20.908 INFO:teuthology.orchestra.run.smithi070.stdout:podman (/bin/podman) version 5.2.2 is present 2024-09-14T03:38:20.908 INFO:teuthology.orchestra.run.smithi070.stdout:systemctl is present 2024-09-14T03:38:20.908 INFO:teuthology.orchestra.run.smithi070.stdout:lvcreate is present 2024-09-14T03:38:20.915 INFO:teuthology.orchestra.run.smithi070.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-09-14T03:38:20.915 INFO:teuthology.orchestra.run.smithi070.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-09-14T03:38:20.921 INFO:teuthology.orchestra.run.smithi070.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-09-14T03:38:20.921 INFO:teuthology.orchestra.run.smithi070.stdout:systemctl: stdout inactive 2024-09-14T03:38:20.928 INFO:teuthology.orchestra.run.smithi070.stdout:systemctl: stdout enabled 2024-09-14T03:38:20.935 INFO:teuthology.orchestra.run.smithi070.stdout:systemctl: stdout active 2024-09-14T03:38:20.935 INFO:teuthology.orchestra.run.smithi070.stdout:Unit chronyd.service is enabled and running 2024-09-14T03:38:20.935 INFO:teuthology.orchestra.run.smithi070.stdout:Host looks OK 2024-09-14T03:38:20.935 INFO:teuthology.orchestra.run.smithi070.stdout:Cluster fsid: b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:38:20.936 INFO:teuthology.orchestra.run.smithi070.stdout:Acquiring lock 139977221575584 on /run/cephadm/b6380d42-724a-11ef-bceb-c7b262605968.lock 2024-09-14T03:38:20.936 INFO:teuthology.orchestra.run.smithi070.stdout:Lock 139977221575584 acquired on /run/cephadm/b6380d42-724a-11ef-bceb-c7b262605968.lock 2024-09-14T03:38:20.936 INFO:teuthology.orchestra.run.smithi070.stdout:Verifying IP 172.21.15.70 port 3300 ... 2024-09-14T03:38:20.936 INFO:teuthology.orchestra.run.smithi070.stdout:Verifying IP 172.21.15.70 port 6789 ... 2024-09-14T03:38:20.936 INFO:teuthology.orchestra.run.smithi070.stdout:Base mon IP(s) is [172.21.15.70:3300, 172.21.15.70:6789], mon addrv is [v2:172.21.15.70:3300,v1:172.21.15.70:6789] 2024-09-14T03:38:20.940 INFO:teuthology.orchestra.run.smithi070.stdout:/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.70 metric 100 2024-09-14T03:38:20.940 INFO:teuthology.orchestra.run.smithi070.stdout:/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.70 metric 100 2024-09-14T03:38:20.943 INFO:teuthology.orchestra.run.smithi070.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-09-14T03:38:20.943 INFO:teuthology.orchestra.run.smithi070.stdout:/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2024-09-14T03:38:20.943 INFO:teuthology.orchestra.run.smithi070.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2024-09-14T03:38:20.946 INFO:teuthology.orchestra.run.smithi070.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-14T03:38:20.946 INFO:teuthology.orchestra.run.smithi070.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-09-14T03:38:20.946 INFO:teuthology.orchestra.run.smithi070.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-09-14T03:38:20.946 INFO:teuthology.orchestra.run.smithi070.stdout:/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-09-14T03:38:20.946 INFO:teuthology.orchestra.run.smithi070.stdout:/sbin/ip: stdout inet6 fe80::ec4:7aff:fe8f:d3e7/64 scope link noprefixroute 2024-09-14T03:38:20.946 INFO:teuthology.orchestra.run.smithi070.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-09-14T03:38:20.947 INFO:teuthology.orchestra.run.smithi070.stdout:Mon IP `172.21.15.70` is in CIDR network `172.21.0.0/20` 2024-09-14T03:38:20.947 INFO:teuthology.orchestra.run.smithi070.stdout:Mon IP `172.21.15.70` is in CIDR network `172.21.0.0/20` 2024-09-14T03:38:20.947 INFO:teuthology.orchestra.run.smithi070.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-09-14T03:38:20.947 INFO:teuthology.orchestra.run.smithi070.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-14T03:38:20.948 INFO:teuthology.orchestra.run.smithi070.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-14T03:38:21.383 INFO:teuthology.orchestra.run.smithi070.stdout:/bin/podman: stdout ab51d1bebd1200e205d11a014c20a7f2cf49d932afcbeabdc1c9fc887c00f446 2024-09-14T03:38:21.383 INFO:teuthology.orchestra.run.smithi070.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-14T03:38:21.383 INFO:teuthology.orchestra.run.smithi070.stdout:/bin/podman: stderr Getting image source signatures 2024-09-14T03:38:21.383 INFO:teuthology.orchestra.run.smithi070.stdout:/bin/podman: stderr Copying blob sha256:9b559dbc30e0607f8b5bc9e421c2d3b7f5b89b8ae7a0518eaf6d2b64276fd6bc 2024-09-14T03:38:21.383 INFO:teuthology.orchestra.run.smithi070.stdout:/bin/podman: stderr Copying blob sha256:e8b54c863393b7a8216a71737771b73b16a8e43ec7acf927c7faa175c255e551 2024-09-14T03:38:21.383 INFO:teuthology.orchestra.run.smithi070.stdout:/bin/podman: stderr Copying config sha256:ab51d1bebd1200e205d11a014c20a7f2cf49d932afcbeabdc1c9fc887c00f446 2024-09-14T03:38:21.383 INFO:teuthology.orchestra.run.smithi070.stdout:/bin/podman: stderr Writing manifest to image destination 2024-09-14T03:38:22.278 INFO:teuthology.orchestra.run.smithi070.stdout:ceph: stdout ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable) 2024-09-14T03:38:22.279 INFO:teuthology.orchestra.run.smithi070.stdout:Ceph version: ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable) 2024-09-14T03:38:22.279 INFO:teuthology.orchestra.run.smithi070.stdout:Extracting ceph user uid/gid from container image... 2024-09-14T03:38:23.200 INFO:teuthology.orchestra.run.smithi070.stdout:stat: stdout 167 167 2024-09-14T03:38:23.200 INFO:teuthology.orchestra.run.smithi070.stdout:Creating initial keys... 2024-09-14T03:38:24.064 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph-authtool: stdout AQAvBeVm79sxHhAAUG7prvxgYs6EdVd8nQSCcg== 2024-09-14T03:38:25.003 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph-authtool: stdout AQAwBeVmJ2mIGBAAeyFg+2c32Zd8W3AP6IuYjg== 2024-09-14T03:38:25.942 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph-authtool: stdout AQAxBeVmxcPnERAAaMhIIpPU3ztdmps59a8G1Q== 2024-09-14T03:38:25.942 INFO:teuthology.orchestra.run.smithi070.stdout:Creating initial monmap... 2024-09-14T03:38:26.817 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-14T03:38:26.817 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2024-09-14T03:38:26.817 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:38:26.817 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-14T03:38:26.818 INFO:teuthology.orchestra.run.smithi070.stdout:monmaptool for a [v2:172.21.15.70:3300,v1:172.21.15.70:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-14T03:38:26.818 INFO:teuthology.orchestra.run.smithi070.stdout:setting min_mon_release = octopus 2024-09-14T03:38:26.818 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/monmaptool: set fsid to b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:38:26.818 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-14T03:38:26.818 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:38:26.818 INFO:teuthology.orchestra.run.smithi070.stdout:Creating mon... 2024-09-14T03:38:27.781 INFO:teuthology.orchestra.run.smithi070.stdout:create mon.a on 2024-09-14T03:38:27.991 INFO:teuthology.orchestra.run.smithi070.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-09-14T03:38:28.136 INFO:teuthology.orchestra.run.smithi070.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-14T03:38:28.287 INFO:teuthology.orchestra.run.smithi070.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-b6380d42-724a-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-b6380d42-724a-11ef-bceb-c7b262605968.target. 2024-09-14T03:38:28.287 INFO:teuthology.orchestra.run.smithi070.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-b6380d42-724a-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-b6380d42-724a-11ef-bceb-c7b262605968.target. 2024-09-14T03:38:28.567 INFO:teuthology.orchestra.run.smithi070.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-b6380d42-724a-11ef-bceb-c7b262605968@mon.a 2024-09-14T03:38:28.567 INFO:teuthology.orchestra.run.smithi070.stdout:systemctl: stderr Failed to reset failed state of unit ceph-b6380d42-724a-11ef-bceb-c7b262605968@mon.a.service: Unit ceph-b6380d42-724a-11ef-bceb-c7b262605968@mon.a.service not loaded. 2024-09-14T03:38:28.911 INFO:teuthology.orchestra.run.smithi070.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-b6380d42-724a-11ef-bceb-c7b262605968.target.wants/ceph-b6380d42-724a-11ef-bceb-c7b262605968@mon.a.service → /etc/systemd/system/ceph-b6380d42-724a-11ef-bceb-c7b262605968@.service. 2024-09-14T03:38:29.245 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:29 smithi070 podman[26984]: 2024-09-14 03:38:29.034976132 +0000 UTC m=+0.015617572 image pull ab51d1bebd1200e205d11a014c20a7f2cf49d932afcbeabdc1c9fc887c00f446 quay.ceph.io/ceph-ci/ceph:quincy 2024-09-14T03:38:29.442 INFO:teuthology.orchestra.run.smithi070.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-14T03:38:29.442 INFO:teuthology.orchestra.run.smithi070.stdout:systemctl: stdout disabled 2024-09-14T03:38:29.449 INFO:teuthology.orchestra.run.smithi070.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-14T03:38:29.449 INFO:teuthology.orchestra.run.smithi070.stdout:systemctl: stdout inactive 2024-09-14T03:38:29.449 INFO:teuthology.orchestra.run.smithi070.stdout:firewalld.service is not enabled 2024-09-14T03:38:29.449 INFO:teuthology.orchestra.run.smithi070.stdout:Not possible to enable service . firewalld.service is not available 2024-09-14T03:38:29.450 INFO:teuthology.orchestra.run.smithi070.stdout:Waiting for mon to start... 2024-09-14T03:38:29.450 INFO:teuthology.orchestra.run.smithi070.stdout:Waiting for mon... 2024-09-14T03:38:29.686 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:29 smithi070 podman[26984]: 2024-09-14 03:38:29.369592842 +0000 UTC m=+0.350234275 container init 7a4f301592710a04fa5c66556635c61b344ca03d3811ea3def93685848c95d0e (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-a, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=quincy-eb16b97, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:38:29.687 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:29 smithi070 podman[26984]: 2024-09-14 03:38:29.374209162 +0000 UTC m=+0.354850599 container start 7a4f301592710a04fa5c66556635c61b344ca03d3811ea3def93685848c95d0e (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-a, RELEASE=quincy-eb16b97, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-14T03:38:29.687 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:29 smithi070 ceph-mon[27022]: mkfs b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:38:29.687 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:29 smithi070 ceph-mon[27022]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-14T03:38:29.687 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:29 smithi070 bash[26984]: 7a4f301592710a04fa5c66556635c61b344ca03d3811ea3def93685848c95d0e 2024-09-14T03:38:29.687 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:29 smithi070 systemd[1]: Started Ceph mon.a for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:38:30.509 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:30 smithi070 ceph-mon[27022]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-14T03:38:30.510 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:30 smithi070 ceph-mon[27022]: monmap e1: 1 mons at {a=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-14T03:38:30.510 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:30 smithi070 ceph-mon[27022]: fsmap 2024-09-14T03:38:30.510 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:30 smithi070 ceph-mon[27022]: osdmap e1: 0 total, 0 up, 0 in 2024-09-14T03:38:30.510 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:30 smithi070 ceph-mon[27022]: mgrmap e1: no daemons active 2024-09-14T03:38:30.510 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:30 smithi070 ceph-mon[27022]: from='client.? 172.21.15.70:0/4160483016' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-14T03:38:30.513 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout cluster: 2024-09-14T03:38:30.513 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout id: b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:38:30.513 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-09-14T03:38:30.513 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout 2024-09-14T03:38:30.514 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout services: 2024-09-14T03:38:30.514 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.540017s) 2024-09-14T03:38:30.514 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-09-14T03:38:30.514 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-09-14T03:38:30.514 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout 2024-09-14T03:38:30.514 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout data: 2024-09-14T03:38:30.514 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-09-14T03:38:30.514 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-09-14T03:38:30.514 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-09-14T03:38:30.514 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout pgs: 2024-09-14T03:38:30.514 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout 2024-09-14T03:38:30.514 INFO:teuthology.orchestra.run.smithi070.stdout:mon is available 2024-09-14T03:38:30.514 INFO:teuthology.orchestra.run.smithi070.stdout:Assimilating anything we can from ceph.conf... 2024-09-14T03:38:31.590 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:31 smithi070 ceph-mon[27022]: from='client.? 172.21.15.70:0/3959608995' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-14T03:38:31.590 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:31 smithi070 ceph-mon[27022]: from='client.? 172.21.15.70:0/3959608995' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-14T03:38:31.594 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout 2024-09-14T03:38:31.594 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout [global] 2024-09-14T03:38:31.594 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout fsid = b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:38:31.594 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout mon_cluster_log_file_level = debug 2024-09-14T03:38:31.594 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.70:3300,v1:172.21.15.70:6789] 2024-09-14T03:38:31.594 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-09-14T03:38:31.595 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-09-14T03:38:31.595 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-09-14T03:38:31.595 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-09-14T03:38:31.595 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout 2024-09-14T03:38:31.595 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout [mgr] 2024-09-14T03:38:31.595 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-09-14T03:38:31.595 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout 2024-09-14T03:38:31.595 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout [osd] 2024-09-14T03:38:31.595 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-09-14T03:38:31.595 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-09-14T03:38:31.595 INFO:teuthology.orchestra.run.smithi070.stdout:Generating new minimal ceph.conf... 2024-09-14T03:38:32.590 INFO:teuthology.orchestra.run.smithi070.stdout:Restarting the monitor... 2024-09-14T03:38:32.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:32 smithi070 ceph-mon[27022]: from='client.? 172.21.15.70:0/534611717' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:38:32.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:32 smithi070 systemd[1]: Stopping Ceph mon.a for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:38:33.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:32 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-a[26999]: 2024-09-14T03:38:32.874+0000 7fce7a0e5640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:38:33.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:32 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-a[26999]: 2024-09-14T03:38:32.874+0000 7fce7a0e5640 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-09-14T03:38:33.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:32 smithi070 podman[27341]: 2024-09-14 03:38:32.926377545 +0000 UTC m=+0.177274994 container died 7a4f301592710a04fa5c66556635c61b344ca03d3811ea3def93685848c95d0e (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-a, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=quincy-eb16b97, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:38:33.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:33 smithi070 podman[27341]: 2024-09-14 03:38:33.297349357 +0000 UTC m=+0.548246805 container cleanup 7a4f301592710a04fa5c66556635c61b344ca03d3811ea3def93685848c95d0e (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-a, ceph=True, maintainer=Guillaume Abrioux , RELEASE=quincy-eb16b97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-14T03:38:33.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:33 smithi070 bash[27341]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-a 2024-09-14T03:38:33.916 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:33 smithi070 podman[27358]: 2024-09-14 03:38:33.573268433 +0000 UTC m=+0.637738689 container remove 7a4f301592710a04fa5c66556635c61b344ca03d3811ea3def93685848c95d0e (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-a, RELEASE=quincy-eb16b97, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-14T03:38:33.916 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:33 smithi070 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@mon.a.service: Deactivated successfully. 2024-09-14T03:38:33.916 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:33 smithi070 systemd[1]: Stopped Ceph mon.a for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:38:33.916 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:33 smithi070 systemd[1]: Starting Ceph mon.a for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:38:34.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:33 smithi070 podman[27456]: 2024-09-14 03:38:33.915663298 +0000 UTC m=+0.106888029 container create 12d19db91f54d5f441d45c517e4322d8db7d960f73065890c8753bb54579fc25 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-a, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-eb16b97) 2024-09-14T03:38:34.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:33 smithi070 podman[27456]: 2024-09-14 03:38:33.824599933 +0000 UTC m=+0.015824673 image pull ab51d1bebd1200e205d11a014c20a7f2cf49d932afcbeabdc1c9fc887c00f446 quay.ceph.io/ceph-ci/ceph:quincy 2024-09-14T03:38:35.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 podman[27456]: 2024-09-14 03:38:35.362556781 +0000 UTC m=+1.553781519 container init 12d19db91f54d5f441d45c517e4322d8db7d960f73065890c8753bb54579fc25 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-a, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=quincy-eb16b97, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-14T03:38:35.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 podman[27456]: 2024-09-14 03:38:35.369298992 +0000 UTC m=+1.560523725 container start 12d19db91f54d5f441d45c517e4322d8db7d960f73065890c8753bb54579fc25 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-a, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, RELEASE=quincy-eb16b97, org.label-schema.schema-version=1.0) 2024-09-14T03:38:35.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: set uid:gid to 167:167 (ceph:ceph) 2024-09-14T03:38:35.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable), process ceph-mon, pid 7 2024-09-14T03:38:35.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: pidfile_write: ignore empty --pid-file 2024-09-14T03:38:35.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: load: jerasure load: lrc 2024-09-14T03:38:35.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: RocksDB version: 6.15.5 2024-09-14T03:38:35.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-14T03:38:35.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Compile date Sep 13 2024 2024-09-14T03:38:35.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: DB SUMMARY 2024-09-14T03:38:35.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: DB Session ID: MSD8K7NW3KK8V9S59QGK 2024-09-14T03:38:35.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: CURRENT file: CURRENT 2024-09-14T03:38:35.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: IDENTITY file: IDENTITY 2024-09-14T03:38:35.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-14T03:38:35.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-09-14T03:38:35.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 83684 ; 2024-09-14T03:38:35.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.error_if_exists: 0 2024-09-14T03:38:35.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.create_if_missing: 0 2024-09-14T03:38:35.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.paranoid_checks: 1 2024-09-14T03:38:35.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-14T03:38:35.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.env: 0x560c1d6a0440 2024-09-14T03:38:35.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.fs: Posix File System 2024-09-14T03:38:35.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.info_log: 0x560c1f120ec0 2024-09-14T03:38:35.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.max_file_opening_threads: 16 2024-09-14T03:38:35.789 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.statistics: (nil) 2024-09-14T03:38:35.789 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.use_fsync: 0 2024-09-14T03:38:35.789 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.max_log_file_size: 0 2024-09-14T03:38:35.789 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-14T03:38:35.789 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-14T03:38:35.789 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.keep_log_file_num: 1000 2024-09-14T03:38:35.789 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.recycle_log_file_num: 0 2024-09-14T03:38:35.789 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.allow_fallocate: 1 2024-09-14T03:38:35.789 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.allow_mmap_reads: 0 2024-09-14T03:38:35.789 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.allow_mmap_writes: 0 2024-09-14T03:38:35.789 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.use_direct_reads: 0 2024-09-14T03:38:35.789 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-14T03:38:35.789 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.create_missing_column_families: 0 2024-09-14T03:38:35.790 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.db_log_dir: 2024-09-14T03:38:35.790 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-09-14T03:38:35.790 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-14T03:38:35.790 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-14T03:38:35.790 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-14T03:38:35.790 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-14T03:38:35.790 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-14T03:38:35.790 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-14T03:38:35.790 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.advise_random_on_open: 1 2024-09-14T03:38:35.790 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.db_write_buffer_size: 0 2024-09-14T03:38:35.790 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.write_buffer_manager: 0x560c1f1de7e0 2024-09-14T03:38:35.790 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-14T03:38:35.790 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-14T03:38:35.791 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-14T03:38:35.791 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-14T03:38:35.791 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.rate_limiter: (nil) 2024-09-14T03:38:35.791 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-14T03:38:35.791 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.wal_recovery_mode: 2 2024-09-14T03:38:35.791 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.enable_thread_tracking: 0 2024-09-14T03:38:35.791 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.enable_pipelined_write: 0 2024-09-14T03:38:35.791 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.unordered_write: 0 2024-09-14T03:38:35.791 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-14T03:38:35.791 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-14T03:38:35.791 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-14T03:38:35.791 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-14T03:38:35.792 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.row_cache: None 2024-09-14T03:38:35.792 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.wal_filter: None 2024-09-14T03:38:35.792 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-14T03:38:35.792 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.allow_ingest_behind: 0 2024-09-14T03:38:35.792 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.preserve_deletes: 0 2024-09-14T03:38:35.793 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.two_write_queues: 0 2024-09-14T03:38:35.793 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.manual_wal_flush: 0 2024-09-14T03:38:35.793 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.atomic_flush: 0 2024-09-14T03:38:35.793 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-14T03:38:35.794 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-14T03:38:35.794 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-14T03:38:35.794 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.log_readahead_size: 0 2024-09-14T03:38:35.794 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-14T03:38:35.794 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.best_efforts_recovery: 0 2024-09-14T03:38:35.794 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-14T03:38:35.794 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-14T03:38:35.794 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.allow_data_in_errors: 0 2024-09-14T03:38:35.794 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.db_host_id: __hostname__ 2024-09-14T03:38:35.794 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.max_background_jobs: 2 2024-09-14T03:38:35.794 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.max_background_compactions: -1 2024-09-14T03:38:35.794 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.max_subcompactions: 1 2024-09-14T03:38:35.794 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-14T03:38:35.795 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-14T03:38:35.795 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-14T03:38:35.795 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.max_total_wal_size: 0 2024-09-14T03:38:35.795 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-14T03:38:35.795 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-14T03:38:35.795 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-14T03:38:35.796 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-14T03:38:35.796 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.max_open_files: -1 2024-09-14T03:38:35.796 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.bytes_per_sync: 0 2024-09-14T03:38:35.796 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-14T03:38:35.796 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-14T03:38:35.796 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.compaction_readahead_size: 0 2024-09-14T03:38:35.796 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.max_background_flushes: -1 2024-09-14T03:38:35.796 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Compression algorithms supported: 2024-09-14T03:38:35.796 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: kZSTD supported: 0 2024-09-14T03:38:35.796 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: kXpressCompression supported: 0 2024-09-14T03:38:35.797 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: kBZip2Compression supported: 0 2024-09-14T03:38:35.797 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-14T03:38:35.797 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: kLZ4Compression supported: 1 2024-09-14T03:38:35.797 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: kZlibCompression supported: 1 2024-09-14T03:38:35.797 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: kLZ4HCCompression supported: 1 2024-09-14T03:38:35.797 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: kSnappyCompression supported: 1 2024-09-14T03:38:35.797 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-14T03:38:35.797 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-09-14T03:38:35.797 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-09-14T03:38:35.797 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-14T03:38:35.797 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.merge_operator: 2024-09-14T03:38:35.797 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.compaction_filter: None 2024-09-14T03:38:35.797 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.compaction_filter_factory: None 2024-09-14T03:38:35.798 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.sst_partitioner_factory: None 2024-09-14T03:38:35.798 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-14T03:38:35.798 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-14T03:38:35.798 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x560c1f0c7a38) 2024-09-14T03:38:35.798 INFO:journalctl@ceph.mon.a.smithi070.stdout: cache_index_and_filter_blocks: 1 2024-09-14T03:38:35.798 INFO:journalctl@ceph.mon.a.smithi070.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-14T03:38:35.798 INFO:journalctl@ceph.mon.a.smithi070.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-14T03:38:35.798 INFO:journalctl@ceph.mon.a.smithi070.stdout: pin_top_level_index_and_filter: 1 2024-09-14T03:38:35.798 INFO:journalctl@ceph.mon.a.smithi070.stdout: index_type: 0 2024-09-14T03:38:35.798 INFO:journalctl@ceph.mon.a.smithi070.stdout: data_block_index_type: 0 2024-09-14T03:38:35.798 INFO:journalctl@ceph.mon.a.smithi070.stdout: index_shortening: 1 2024-09-14T03:38:35.798 INFO:journalctl@ceph.mon.a.smithi070.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-14T03:38:35.798 INFO:journalctl@ceph.mon.a.smithi070.stdout: hash_index_allow_collision: 1 2024-09-14T03:38:35.799 INFO:journalctl@ceph.mon.a.smithi070.stdout: checksum: 1 2024-09-14T03:38:35.799 INFO:journalctl@ceph.mon.a.smithi070.stdout: no_block_cache: 0 2024-09-14T03:38:35.799 INFO:journalctl@ceph.mon.a.smithi070.stdout: block_cache: 0x560c1f0fd1f0 2024-09-14T03:38:35.799 INFO:journalctl@ceph.mon.a.smithi070.stdout: block_cache_name: BinnedLRUCache 2024-09-14T03:38:35.799 INFO:journalctl@ceph.mon.a.smithi070.stdout: block_cache_options: 2024-09-14T03:38:35.799 INFO:journalctl@ceph.mon.a.smithi070.stdout: capacity : 536870912 2024-09-14T03:38:35.799 INFO:journalctl@ceph.mon.a.smithi070.stdout: num_shard_bits : 4 2024-09-14T03:38:35.799 INFO:journalctl@ceph.mon.a.smithi070.stdout: strict_capacity_limit : 0 2024-09-14T03:38:35.799 INFO:journalctl@ceph.mon.a.smithi070.stdout: high_pri_pool_ratio: 0.000 2024-09-14T03:38:35.799 INFO:journalctl@ceph.mon.a.smithi070.stdout: block_cache_compressed: (nil) 2024-09-14T03:38:35.799 INFO:journalctl@ceph.mon.a.smithi070.stdout: persistent_cache: (nil) 2024-09-14T03:38:35.799 INFO:journalctl@ceph.mon.a.smithi070.stdout: block_size: 4096 2024-09-14T03:38:35.799 INFO:journalctl@ceph.mon.a.smithi070.stdout: block_size_deviation: 10 2024-09-14T03:38:35.800 INFO:journalctl@ceph.mon.a.smithi070.stdout: block_restart_interval: 16 2024-09-14T03:38:35.800 INFO:journalctl@ceph.mon.a.smithi070.stdout: index_block_restart_interval: 1 2024-09-14T03:38:35.800 INFO:journalctl@ceph.mon.a.smithi070.stdout: metadata_block_size: 4096 2024-09-14T03:38:35.800 INFO:journalctl@ceph.mon.a.smithi070.stdout: partition_filters: 0 2024-09-14T03:38:35.800 INFO:journalctl@ceph.mon.a.smithi070.stdout: use_delta_encoding: 1 2024-09-14T03:38:35.800 INFO:journalctl@ceph.mon.a.smithi070.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-14T03:38:35.800 INFO:journalctl@ceph.mon.a.smithi070.stdout: whole_key_filtering: 1 2024-09-14T03:38:35.800 INFO:journalctl@ceph.mon.a.smithi070.stdout: verify_compression: 0 2024-09-14T03:38:35.800 INFO:journalctl@ceph.mon.a.smithi070.stdout: read_amp_bytes_per_bit: 0 2024-09-14T03:38:35.800 INFO:journalctl@ceph.mon.a.smithi070.stdout: format_version: 4 2024-09-14T03:38:35.800 INFO:journalctl@ceph.mon.a.smithi070.stdout: enable_index_compression: 1 2024-09-14T03:38:35.800 INFO:journalctl@ceph.mon.a.smithi070.stdout: block_align: 0 2024-09-14T03:38:35.800 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.write_buffer_size: 33554432 2024-09-14T03:38:35.801 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.max_write_buffer_number: 2 2024-09-14T03:38:35.801 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.compression: NoCompression 2024-09-14T03:38:35.801 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.bottommost_compression: Disabled 2024-09-14T03:38:35.801 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.prefix_extractor: nullptr 2024-09-14T03:38:35.801 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-14T03:38:35.801 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.num_levels: 7 2024-09-14T03:38:35.801 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-14T03:38:35.801 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-14T03:38:35.801 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-14T03:38:35.802 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-14T03:38:35.802 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-14T03:38:35.802 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-14T03:38:35.802 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-14T03:38:35.802 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:38:35.802 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-14T03:38:35.802 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-14T03:38:35.802 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-14T03:38:35.802 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.compression_opts.level: 32767 2024-09-14T03:38:35.802 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.compression_opts.strategy: 0 2024-09-14T03:38:35.802 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-14T03:38:35.802 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:38:35.802 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-14T03:38:35.803 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.compression_opts.enabled: false 2024-09-14T03:38:35.803 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-14T03:38:35.803 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-14T03:38:35.803 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-14T03:38:35.803 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.target_file_size_base: 67108864 2024-09-14T03:38:35.803 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-14T03:38:35.803 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-14T03:38:35.803 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-14T03:38:35.803 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-14T03:38:35.803 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-14T03:38:35.804 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-14T03:38:35.804 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-14T03:38:35.804 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-14T03:38:35.804 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-14T03:38:35.804 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-14T03:38:35.804 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-14T03:38:35.804 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-14T03:38:35.804 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-14T03:38:35.805 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.arena_block_size: 4194304 2024-09-14T03:38:35.805 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-14T03:38:35.805 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-14T03:38:35.805 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-14T03:38:35.805 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.disable_auto_compactions: 0 2024-09-14T03:38:35.805 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-14T03:38:35.806 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-14T03:38:35.806 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-14T03:38:35.806 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-14T03:38:35.806 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-14T03:38:35.806 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-14T03:38:35.806 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-14T03:38:35.806 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-14T03:38:35.806 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-14T03:38:35.806 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-14T03:38:35.807 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-14T03:38:35.807 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.inplace_update_support: 0 2024-09-14T03:38:35.807 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-14T03:38:35.807 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-14T03:38:35.807 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-14T03:38:35.807 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-14T03:38:35.807 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.bloom_locality: 0 2024-09-14T03:38:35.807 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.max_successive_merges: 0 2024-09-14T03:38:35.807 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-14T03:38:35.807 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.paranoid_file_checks: 0 2024-09-14T03:38:35.807 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.force_consistency_checks: 1 2024-09-14T03:38:35.807 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.report_bg_io_stats: 0 2024-09-14T03:38:35.807 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.ttl: 2592000 2024-09-14T03:38:35.808 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-14T03:38:35.808 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.enable_blob_files: false 2024-09-14T03:38:35.808 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.min_blob_size: 0 2024-09-14T03:38:35.808 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.blob_file_size: 268435456 2024-09-14T03:38:35.808 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-14T03:38:35.808 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-14T03:38:35.808 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-14T03:38:35.808 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-14T03:38:35.808 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2024-09-14T03:38:35.808 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2024-09-14T03:38:35.808 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285115395289, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-14T03:38:35.808 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2024-09-14T03:38:35.809 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-14T03:38:35.809 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285115396081, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80257, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78499, "index_size": 220, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10130, "raw_average_key_size": 47, "raw_value_size": 72712, "raw_average_value_size": 341, "num_data_blocks": 10, "num_entries": 213, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726285115, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "29b140d3-74bf-45c9-9731-c0c64f5f6ae9", "db_session_id": "MSD8K7NW3KK8V9S59QGK"}} 2024-09-14T03:38:35.809 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2024-09-14T03:38:35.809 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285115396457, "job": 1, "event": "recovery_finished"} 2024-09-14T03:38:35.809 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-14T03:38:35.809 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x560c1f130700 2024-09-14T03:38:35.809 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: DB pointer 0x560c1f100000 2024-09-14T03:38:35.809 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-09-14T03:38:35.809 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-14T03:38:35.809 INFO:journalctl@ceph.mon.a.smithi070.stdout: ** DB Stats ** 2024-09-14T03:38:35.809 INFO:journalctl@ceph.mon.a.smithi070.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:38:35.809 INFO:journalctl@ceph.mon.a.smithi070.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-14T03:38:35.810 INFO:journalctl@ceph.mon.a.smithi070.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:38:35.810 INFO:journalctl@ceph.mon.a.smithi070.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:38:35.810 INFO:journalctl@ceph.mon.a.smithi070.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-14T03:38:35.810 INFO:journalctl@ceph.mon.a.smithi070.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-14T03:38:35.810 INFO:journalctl@ceph.mon.a.smithi070.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:38:35.810 INFO:journalctl@ceph.mon.a.smithi070.stdout: 2024-09-14T03:38:35.810 INFO:journalctl@ceph.mon.a.smithi070.stdout: ** Compaction Stats [default] ** 2024-09-14T03:38:35.810 INFO:journalctl@ceph.mon.a.smithi070.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-14T03:38:35.810 INFO:journalctl@ceph.mon.a.smithi070.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-14T03:38:35.810 INFO:journalctl@ceph.mon.a.smithi070.stdout: L0 2/0 80.16 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 139.4 0.00 0.00 1 0.001 0 0 2024-09-14T03:38:35.810 INFO:journalctl@ceph.mon.a.smithi070.stdout: Sum 2/0 80.16 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 139.4 0.00 0.00 1 0.001 0 0 2024-09-14T03:38:35.810 INFO:journalctl@ceph.mon.a.smithi070.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 139.4 0.00 0.00 1 0.001 0 0 2024-09-14T03:38:35.810 INFO:journalctl@ceph.mon.a.smithi070.stdout: 2024-09-14T03:38:35.811 INFO:journalctl@ceph.mon.a.smithi070.stdout: ** Compaction Stats [default] ** 2024-09-14T03:38:35.811 INFO:journalctl@ceph.mon.a.smithi070.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-14T03:38:35.811 INFO:journalctl@ceph.mon.a.smithi070.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-14T03:38:35.811 INFO:journalctl@ceph.mon.a.smithi070.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 139.4 0.00 0.00 1 0.001 0 0 2024-09-14T03:38:35.811 INFO:journalctl@ceph.mon.a.smithi070.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:38:35.811 INFO:journalctl@ceph.mon.a.smithi070.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-14T03:38:35.811 INFO:journalctl@ceph.mon.a.smithi070.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-14T03:38:35.811 INFO:journalctl@ceph.mon.a.smithi070.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-14T03:38:35.811 INFO:journalctl@ceph.mon.a.smithi070.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-14T03:38:35.811 INFO:journalctl@ceph.mon.a.smithi070.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-14T03:38:35.811 INFO:journalctl@ceph.mon.a.smithi070.stdout: Cumulative compaction: 0.00 GB write, 22.26 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:38:35.811 INFO:journalctl@ceph.mon.a.smithi070.stdout: Interval compaction: 0.00 GB write, 22.26 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:38:35.811 INFO:journalctl@ceph.mon.a.smithi070.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-14T03:38:35.811 INFO:journalctl@ceph.mon.a.smithi070.stdout: 2024-09-14T03:38:35.812 INFO:journalctl@ceph.mon.a.smithi070.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-14T03:38:35.812 INFO:journalctl@ceph.mon.a.smithi070.stdout: 2024-09-14T03:38:35.812 INFO:journalctl@ceph.mon.a.smithi070.stdout: ** Compaction Stats [default] ** 2024-09-14T03:38:35.812 INFO:journalctl@ceph.mon.a.smithi070.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-14T03:38:35.812 INFO:journalctl@ceph.mon.a.smithi070.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-14T03:38:35.812 INFO:journalctl@ceph.mon.a.smithi070.stdout: L0 2/0 80.16 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 139.4 0.00 0.00 1 0.001 0 0 2024-09-14T03:38:35.812 INFO:journalctl@ceph.mon.a.smithi070.stdout: Sum 2/0 80.16 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 139.4 0.00 0.00 1 0.001 0 0 2024-09-14T03:38:35.812 INFO:journalctl@ceph.mon.a.smithi070.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-14T03:38:35.812 INFO:journalctl@ceph.mon.a.smithi070.stdout: 2024-09-14T03:38:35.812 INFO:journalctl@ceph.mon.a.smithi070.stdout: ** Compaction Stats [default] ** 2024-09-14T03:38:35.812 INFO:journalctl@ceph.mon.a.smithi070.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-14T03:38:35.812 INFO:journalctl@ceph.mon.a.smithi070.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-14T03:38:35.812 INFO:journalctl@ceph.mon.a.smithi070.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 139.4 0.00 0.00 1 0.001 0 0 2024-09-14T03:38:35.813 INFO:journalctl@ceph.mon.a.smithi070.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:38:35.813 INFO:journalctl@ceph.mon.a.smithi070.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-14T03:38:35.813 INFO:journalctl@ceph.mon.a.smithi070.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-14T03:38:35.813 INFO:journalctl@ceph.mon.a.smithi070.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-14T03:38:35.813 INFO:journalctl@ceph.mon.a.smithi070.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-14T03:38:35.813 INFO:journalctl@ceph.mon.a.smithi070.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-14T03:38:35.813 INFO:journalctl@ceph.mon.a.smithi070.stdout: Cumulative compaction: 0.00 GB write, 22.05 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:38:35.813 INFO:journalctl@ceph.mon.a.smithi070.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:38:35.813 INFO:journalctl@ceph.mon.a.smithi070.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-14T03:38:35.814 INFO:journalctl@ceph.mon.a.smithi070.stdout: 2024-09-14T03:38:35.814 INFO:journalctl@ceph.mon.a.smithi070.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-14T03:38:35.814 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: starting mon.a rank 0 at public addrs [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] at bind addrs [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:38:35.814 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: mon.a@-1(???) e1 preinit fsid b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:38:35.814 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: mon.a@-1(???).mds e1 new map 2024-09-14T03:38:35.814 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: mon.a@-1(???).mds e1 print_map 2024-09-14T03:38:35.814 INFO:journalctl@ceph.mon.a.smithi070.stdout: e1 2024-09-14T03:38:35.814 INFO:journalctl@ceph.mon.a.smithi070.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-14T03:38:35.815 INFO:journalctl@ceph.mon.a.smithi070.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-14T03:38:35.815 INFO:journalctl@ceph.mon.a.smithi070.stdout: legacy client fscid: -1 2024-09-14T03:38:35.815 INFO:journalctl@ceph.mon.a.smithi070.stdout: 2024-09-14T03:38:35.815 INFO:journalctl@ceph.mon.a.smithi070.stdout: No filesystems configured 2024-09-14T03:38:35.815 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-14T03:38:35.815 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-14T03:38:35.815 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-14T03:38:35.815 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-14T03:38:35.815 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:35 smithi070 ceph-mon[27495]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-14T03:38:36.476 INFO:teuthology.orchestra.run.smithi070.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-09-14T03:38:36.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:36 smithi070 ceph-mon[27495]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-14T03:38:36.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:36 smithi070 ceph-mon[27495]: monmap e1: 1 mons at {a=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-14T03:38:36.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:36 smithi070 ceph-mon[27495]: fsmap 2024-09-14T03:38:36.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:36 smithi070 ceph-mon[27495]: osdmap e1: 0 total, 0 up, 0 in 2024-09-14T03:38:36.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:36 smithi070 ceph-mon[27495]: mgrmap e1: no daemons active 2024-09-14T03:38:36.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:36 smithi070 bash[27456]: 12d19db91f54d5f441d45c517e4322d8db7d960f73065890c8753bb54579fc25 2024-09-14T03:38:36.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:36 smithi070 systemd[1]: Started Ceph mon.a for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:38:40.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:40 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/3416142154' entity='client.admin' 2024-09-14T03:38:45.848 INFO:teuthology.orchestra.run.smithi070.stdout:Wrote config to /etc/ceph/ceph.conf 2024-09-14T03:38:45.848 INFO:teuthology.orchestra.run.smithi070.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-14T03:38:45.848 INFO:teuthology.orchestra.run.smithi070.stdout:Creating mgr... 2024-09-14T03:38:45.849 INFO:teuthology.orchestra.run.smithi070.stdout:Verifying port 0.0.0.0:9283 ... 2024-09-14T03:38:45.849 INFO:teuthology.orchestra.run.smithi070.stdout:Verifying port 0.0.0.0:8765 ... 2024-09-14T03:38:46.029 INFO:teuthology.orchestra.run.smithi070.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-b6380d42-724a-11ef-bceb-c7b262605968@mgr.y 2024-09-14T03:38:46.029 INFO:teuthology.orchestra.run.smithi070.stdout:systemctl: stderr Failed to reset failed state of unit ceph-b6380d42-724a-11ef-bceb-c7b262605968@mgr.y.service: Unit ceph-b6380d42-724a-11ef-bceb-c7b262605968@mgr.y.service not loaded. 2024-09-14T03:38:46.174 INFO:teuthology.orchestra.run.smithi070.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-b6380d42-724a-11ef-bceb-c7b262605968.target.wants/ceph-b6380d42-724a-11ef-bceb-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-b6380d42-724a-11ef-bceb-c7b262605968@.service. 2024-09-14T03:38:46.193 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:46 smithi070 systemd[1]: Starting Ceph mgr.y for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:38:46.492 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:46 smithi070 podman[27732]: 2024-09-14 03:38:46.304336859 +0000 UTC m=+0.015600514 image pull ab51d1bebd1200e205d11a014c20a7f2cf49d932afcbeabdc1c9fc887c00f446 quay.ceph.io/ceph-ci/ceph:quincy 2024-09-14T03:38:46.492 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:46 smithi070 podman[27732]: 2024-09-14 03:38:46.402290599 +0000 UTC m=+0.113554245 container create dc71810017b9104ef94e066b0c4569085c459823e4fdae447c9d42888fc4a016 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=quincy-eb16b97, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-14T03:38:46.719 INFO:teuthology.orchestra.run.smithi070.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-14T03:38:46.720 INFO:teuthology.orchestra.run.smithi070.stdout:systemctl: stdout disabled 2024-09-14T03:38:46.727 INFO:teuthology.orchestra.run.smithi070.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-14T03:38:46.727 INFO:teuthology.orchestra.run.smithi070.stdout:systemctl: stdout inactive 2024-09-14T03:38:46.727 INFO:teuthology.orchestra.run.smithi070.stdout:firewalld.service is not enabled 2024-09-14T03:38:46.727 INFO:teuthology.orchestra.run.smithi070.stdout:Not possible to enable service . firewalld.service is not available 2024-09-14T03:38:46.735 INFO:teuthology.orchestra.run.smithi070.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-14T03:38:46.736 INFO:teuthology.orchestra.run.smithi070.stdout:systemctl: stdout disabled 2024-09-14T03:38:46.744 INFO:teuthology.orchestra.run.smithi070.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-14T03:38:46.744 INFO:teuthology.orchestra.run.smithi070.stdout:systemctl: stdout inactive 2024-09-14T03:38:46.744 INFO:teuthology.orchestra.run.smithi070.stdout:firewalld.service is not enabled 2024-09-14T03:38:46.744 INFO:teuthology.orchestra.run.smithi070.stdout:Not possible to open ports <[9283, 8765]>. firewalld.service is not available 2024-09-14T03:38:46.744 INFO:teuthology.orchestra.run.smithi070.stdout:Waiting for mgr to start... 2024-09-14T03:38:46.744 INFO:teuthology.orchestra.run.smithi070.stdout:Waiting for mgr... 2024-09-14T03:38:46.869 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:46 smithi070 podman[27732]: 2024-09-14 03:38:46.596604125 +0000 UTC m=+0.307867775 container init dc71810017b9104ef94e066b0c4569085c459823e4fdae447c9d42888fc4a016 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y, maintainer=Guillaume Abrioux , RELEASE=quincy-eb16b97, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-14T03:38:46.869 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:46 smithi070 podman[27732]: 2024-09-14 03:38:46.601481671 +0000 UTC m=+0.312745316 container start dc71810017b9104ef94e066b0c4569085c459823e4fdae447c9d42888fc4a016 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=quincy-eb16b97, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-14T03:38:46.869 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:46 smithi070 bash[27732]: dc71810017b9104ef94e066b0c4569085c459823e4fdae447c9d42888fc4a016 2024-09-14T03:38:46.869 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:46 smithi070 systemd[1]: Started Ceph mgr.y for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:38:46.869 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:46 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:46.713+0000 7fee64394ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-14T03:38:46.869 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:46 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:46.775+0000 7fee64394ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-14T03:38:46.869 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:46 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:46.839+0000 7fee64394ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-14T03:38:47.121 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:46 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:46.966+0000 7fee64394ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:38:47.121 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:47 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:47.119+0000 7fee64394ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-14T03:38:47.394 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:47 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/2250040174' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-14T03:38:47.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:47 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:47.393+0000 7fee64394ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-14T03:38:47.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:47 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:47.454+0000 7fee64394ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-14T03:38:47.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:47 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:47.616+0000 7fee64394ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-14T03:38:47.880 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout 2024-09-14T03:38:47.886 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout { 2024-09-14T03:38:47.886 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "fsid": "b6380d42-724a-11ef-bceb-c7b262605968", 2024-09-14T03:38:47.886 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "health": { 2024-09-14T03:38:47.887 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-14T03:38:47.887 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-14T03:38:47.887 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-14T03:38:47.887 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:38:47.887 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-14T03:38:47.887 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-14T03:38:47.887 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout 0 2024-09-14T03:38:47.887 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout ], 2024-09-14T03:38:47.887 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-14T03:38:47.887 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "a" 2024-09-14T03:38:47.887 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout ], 2024-09-14T03:38:47.887 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "quorum_age": 11, 2024-09-14T03:38:47.887 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-14T03:38:47.888 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:38:47.888 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-09-14T03:38:47.888 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-14T03:38:47.888 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:38:47.888 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-14T03:38:47.888 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:38:47.888 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-14T03:38:47.888 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-14T03:38:47.888 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-14T03:38:47.888 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-14T03:38:47.888 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-14T03:38:47.888 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-14T03:38:47.888 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:38:47.889 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-14T03:38:47.889 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-14T03:38:47.889 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-14T03:38:47.889 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-14T03:38:47.889 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-14T03:38:47.889 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-14T03:38:47.889 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-14T03:38:47.889 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-14T03:38:47.889 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-14T03:38:47.890 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:38:47.890 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-14T03:38:47.890 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:38:47.890 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-14T03:38:47.890 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-14T03:38:47.890 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:38:47.890 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-14T03:38:47.890 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "available": false, 2024-09-14T03:38:47.890 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-14T03:38:47.890 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-14T03:38:47.890 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-14T03:38:47.890 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-14T03:38:47.890 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "restful" 2024-09-14T03:38:47.890 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout ], 2024-09-14T03:38:47.891 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-14T03:38:47.891 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:38:47.891 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-14T03:38:47.891 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:38:47.891 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "modified": "2024-09-14T03:38:29.407181+0000", 2024-09-14T03:38:47.891 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-14T03:38:47.891 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:38:47.891 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-14T03:38:47.891 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout } 2024-09-14T03:38:47.891 INFO:teuthology.orchestra.run.smithi070.stdout:mgr not available, waiting (1/15)... 2024-09-14T03:38:48.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:48.456+0000 7fee64394ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-14T03:38:48.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:48.519+0000 7fee64394ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-14T03:38:49.499 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:49 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:49.215+0000 7fee64394ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-14T03:38:49.500 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:49 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:49.323+0000 7fee64394ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-14T03:38:49.500 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:49 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:49.440+0000 7fee64394ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:38:49.754 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:49 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:49.498+0000 7fee64394ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-14T03:38:49.754 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:49 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:49.627+0000 7fee64394ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-14T03:38:49.754 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:49 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:49.694+0000 7fee64394ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-14T03:38:49.755 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:49 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:49.752+0000 7fee64394ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-14T03:38:50.015 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:49 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:49.890+0000 7fee64394ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-14T03:38:50.016 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:49 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:49.960+0000 7fee64394ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-14T03:38:50.286 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:50 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:50.028+0000 7fee64394ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-14T03:38:50.286 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:50 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:50.113+0000 7fee64394ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-14T03:38:50.286 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:50 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:50.174+0000 7fee64394ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-14T03:38:50.286 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:50 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:50.236+0000 7fee64394ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-14T03:38:50.775 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:50 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/1153183544' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-14T03:38:51.035 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:51 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:51.010+0000 7fee64394ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-14T03:38:51.798 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:51 smithi070 ceph-mon[27495]: Activating manager daemon y 2024-09-14T03:38:51.798 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:51 smithi070 ceph-mon[27495]: mgrmap e2: y(active, starting, since 0.00301784s) 2024-09-14T03:38:51.798 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:51 smithi070 ceph-mon[27495]: from='mgr.14100 172.21.15.70:0/771846502' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:38:51.798 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:51 smithi070 ceph-mon[27495]: from='mgr.14100 172.21.15.70:0/771846502' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:38:51.798 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:51 smithi070 ceph-mon[27495]: from='mgr.14100 172.21.15.70:0/771846502' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:38:51.799 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:51 smithi070 ceph-mon[27495]: from='mgr.14100 172.21.15.70:0/771846502' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:38:51.799 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:51 smithi070 ceph-mon[27495]: from='mgr.14100 172.21.15.70:0/771846502' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:38:51.799 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:51 smithi070 ceph-mon[27495]: Manager daemon y is now available 2024-09-14T03:38:51.799 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:51 smithi070 ceph-mon[27495]: from='mgr.14100 172.21.15.70:0/771846502' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:38:51.799 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:51 smithi070 ceph-mon[27495]: from='mgr.14100 172.21.15.70:0/771846502' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:38:51.799 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:51 smithi070 ceph-mon[27495]: from='mgr.14100 172.21.15.70:0/771846502' entity='mgr.y' 2024-09-14T03:38:51.799 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:51 smithi070 ceph-mon[27495]: from='mgr.14100 172.21.15.70:0/771846502' entity='mgr.y' 2024-09-14T03:38:51.799 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:51 smithi070 ceph-mon[27495]: from='mgr.14100 172.21.15.70:0/771846502' entity='mgr.y' 2024-09-14T03:38:51.803 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout 2024-09-14T03:38:51.803 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout { 2024-09-14T03:38:51.803 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "fsid": "b6380d42-724a-11ef-bceb-c7b262605968", 2024-09-14T03:38:51.803 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "health": { 2024-09-14T03:38:51.804 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-14T03:38:51.804 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-14T03:38:51.804 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-14T03:38:51.804 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:38:51.804 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-14T03:38:51.804 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-14T03:38:51.804 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout 0 2024-09-14T03:38:51.804 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout ], 2024-09-14T03:38:51.804 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-14T03:38:51.804 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "a" 2024-09-14T03:38:51.804 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout ], 2024-09-14T03:38:51.805 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "quorum_age": 15, 2024-09-14T03:38:51.805 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-14T03:38:51.805 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:38:51.805 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-09-14T03:38:51.805 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-14T03:38:51.805 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:38:51.805 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-14T03:38:51.805 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:38:51.805 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-14T03:38:51.805 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-14T03:38:51.805 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-14T03:38:51.805 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-14T03:38:51.809 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-14T03:38:51.809 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-14T03:38:51.809 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:38:51.809 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-14T03:38:51.809 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-14T03:38:51.809 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-14T03:38:51.809 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-14T03:38:51.809 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-14T03:38:51.809 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-14T03:38:51.809 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-14T03:38:51.809 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-14T03:38:51.810 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-14T03:38:51.810 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:38:51.810 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-14T03:38:51.810 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:38:51.810 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-14T03:38:51.810 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-14T03:38:51.810 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:38:51.810 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-14T03:38:51.810 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "available": false, 2024-09-14T03:38:51.810 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-14T03:38:51.810 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-14T03:38:51.810 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-14T03:38:51.810 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-14T03:38:51.811 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "restful" 2024-09-14T03:38:51.811 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout ], 2024-09-14T03:38:51.811 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-14T03:38:51.811 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:38:51.811 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-14T03:38:51.811 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:38:51.811 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "modified": "2024-09-14T03:38:29.407181+0000", 2024-09-14T03:38:51.811 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-14T03:38:51.811 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:38:51.811 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-14T03:38:51.811 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout } 2024-09-14T03:38:51.811 INFO:teuthology.orchestra.run.smithi070.stdout:mgr not available, waiting (2/15)... 2024-09-14T03:38:53.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:53 smithi070 ceph-mon[27495]: mgrmap e3: y(active, since 1.00709s) 2024-09-14T03:38:54.190 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:54 smithi070 ceph-mon[27495]: mgrmap e4: y(active, since 2s) 2024-09-14T03:38:55.120 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:55 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/1943518512' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-14T03:38:55.135 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout 2024-09-14T03:38:55.135 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout { 2024-09-14T03:38:55.135 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "fsid": "b6380d42-724a-11ef-bceb-c7b262605968", 2024-09-14T03:38:55.136 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "health": { 2024-09-14T03:38:55.136 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-14T03:38:55.136 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-14T03:38:55.136 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-14T03:38:55.136 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:38:55.137 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-14T03:38:55.137 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-14T03:38:55.137 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout 0 2024-09-14T03:38:55.137 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout ], 2024-09-14T03:38:55.137 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-14T03:38:55.137 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "a" 2024-09-14T03:38:55.138 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout ], 2024-09-14T03:38:55.138 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "quorum_age": 19, 2024-09-14T03:38:55.138 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-14T03:38:55.138 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:38:55.138 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-09-14T03:38:55.138 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-14T03:38:55.138 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:38:55.138 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-14T03:38:55.139 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:38:55.139 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-14T03:38:55.139 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-14T03:38:55.139 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-14T03:38:55.139 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-14T03:38:55.139 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-14T03:38:55.139 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-14T03:38:55.140 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:38:55.140 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-14T03:38:55.140 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-14T03:38:55.140 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-14T03:38:55.140 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-14T03:38:55.140 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-14T03:38:55.140 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-14T03:38:55.141 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-14T03:38:55.141 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-14T03:38:55.141 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-14T03:38:55.141 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:38:55.141 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-14T03:38:55.141 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:38:55.141 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-14T03:38:55.141 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-14T03:38:55.141 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:38:55.142 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-14T03:38:55.142 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-14T03:38:55.142 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-14T03:38:55.142 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-14T03:38:55.142 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-14T03:38:55.142 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-14T03:38:55.142 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "restful" 2024-09-14T03:38:55.142 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout ], 2024-09-14T03:38:55.142 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-14T03:38:55.142 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:38:55.142 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-14T03:38:55.142 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:38:55.142 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "modified": "2024-09-14T03:38:29.407181+0000", 2024-09-14T03:38:55.142 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-14T03:38:55.143 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:38:55.143 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-14T03:38:55.143 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout } 2024-09-14T03:38:55.143 INFO:teuthology.orchestra.run.smithi070.stdout:mgr is available 2024-09-14T03:38:56.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:56 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/2110772079' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-14T03:38:56.499 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout 2024-09-14T03:38:56.499 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout [global] 2024-09-14T03:38:56.500 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout fsid = b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:38:56.500 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout mon_cluster_log_file_level = debug 2024-09-14T03:38:56.500 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-09-14T03:38:56.500 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-09-14T03:38:56.500 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-09-14T03:38:56.500 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-09-14T03:38:56.501 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout 2024-09-14T03:38:56.501 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout [mgr] 2024-09-14T03:38:56.501 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-09-14T03:38:56.501 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout 2024-09-14T03:38:56.501 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout [osd] 2024-09-14T03:38:56.501 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-09-14T03:38:56.502 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-09-14T03:38:56.502 INFO:teuthology.orchestra.run.smithi070.stdout:Enabling cephadm module... 2024-09-14T03:38:57.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:57 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/1051382134' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-14T03:38:57.536 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:57 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: ignoring --setuser ceph since I am not root 2024-09-14T03:38:57.536 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:57 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: ignoring --setgroup ceph since I am not root 2024-09-14T03:38:57.536 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:57 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:57.346+0000 7facf23c9ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-14T03:38:57.536 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:57 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:57.400+0000 7facf23c9ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-14T03:38:57.536 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:57 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:57.465+0000 7facf23c9ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-14T03:38:57.909 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:57 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:57.594+0000 7facf23c9ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:38:57.909 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:57 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:57.731+0000 7facf23c9ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-14T03:38:58.169 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:57 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:57.992+0000 7facf23c9ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-14T03:38:58.169 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:58 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:58.057+0000 7facf23c9ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-14T03:38:58.456 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:58 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:58.225+0000 7facf23c9ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-14T03:38:58.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:58 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/1051382134' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-14T03:38:58.457 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:58 smithi070 ceph-mon[27495]: mgrmap e5: y(active, since 6s) 2024-09-14T03:38:58.967 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout { 2024-09-14T03:38:58.967 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-09-14T03:38:58.967 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-14T03:38:58.967 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-09-14T03:38:58.967 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-09-14T03:38:58.967 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout } 2024-09-14T03:38:58.967 INFO:teuthology.orchestra.run.smithi070.stdout:Waiting for the mgr to restart... 2024-09-14T03:38:58.967 INFO:teuthology.orchestra.run.smithi070.stdout:Waiting for mgr epoch 5... 2024-09-14T03:38:59.400 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:38:59 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/2659317352' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-14T03:38:59.400 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:59 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:59.085+0000 7facf23c9ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-14T03:38:59.400 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:59 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:59.151+0000 7facf23c9ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-14T03:39:00.147 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:59 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:59.859+0000 7facf23c9ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-14T03:39:00.147 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:38:59 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:38:59.968+0000 7facf23c9ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-14T03:39:00.147 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:00 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:00.087+0000 7facf23c9ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:39:00.147 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:00 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:00.145+0000 7facf23c9ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-14T03:39:00.400 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:00 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:00.272+0000 7facf23c9ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-14T03:39:00.400 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:00 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:00.341+0000 7facf23c9ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-14T03:39:00.400 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:00 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:00.398+0000 7facf23c9ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-14T03:39:00.658 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:00 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:00.531+0000 7facf23c9ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-14T03:39:00.658 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:00 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:00.593+0000 7facf23c9ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-14T03:39:00.658 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:00 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:00.656+0000 7facf23c9ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-14T03:39:01.035 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:00 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:00.736+0000 7facf23c9ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-14T03:39:01.036 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:00 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:00.792+0000 7facf23c9ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-14T03:39:01.036 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:00 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:00.852+0000 7facf23c9ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-14T03:39:02.035 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:01 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:01.587+0000 7facf23c9ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-14T03:39:02.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:01 smithi070 ceph-mon[27495]: Active manager daemon y restarted 2024-09-14T03:39:02.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:01 smithi070 ceph-mon[27495]: Activating manager daemon y 2024-09-14T03:39:02.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:01 smithi070 ceph-mon[27495]: osdmap e2: 0 total, 0 up, 0 in 2024-09-14T03:39:02.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:01 smithi070 ceph-mon[27495]: mgrmap e6: y(active, starting, since 0.0034643s) 2024-09-14T03:39:02.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:01 smithi070 ceph-mon[27495]: from='mgr.14118 172.21.15.70:0/3200588793' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:39:02.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:01 smithi070 ceph-mon[27495]: from='mgr.14118 172.21.15.70:0/3200588793' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:39:02.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:01 smithi070 ceph-mon[27495]: from='mgr.14118 172.21.15.70:0/3200588793' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:39:02.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:01 smithi070 ceph-mon[27495]: from='mgr.14118 172.21.15.70:0/3200588793' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:39:02.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:01 smithi070 ceph-mon[27495]: from='mgr.14118 172.21.15.70:0/3200588793' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:39:02.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:01 smithi070 ceph-mon[27495]: Manager daemon y is now available 2024-09-14T03:39:02.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:01 smithi070 ceph-mon[27495]: from='mgr.14118 172.21.15.70:0/3200588793' entity='mgr.y' 2024-09-14T03:39:02.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:01 smithi070 ceph-mon[27495]: from='mgr.14118 172.21.15.70:0/3200588793' entity='mgr.y' 2024-09-14T03:39:02.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:01 smithi070 ceph-mon[27495]: from='mgr.14118 172.21.15.70:0/3200588793' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:02.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:01 smithi070 ceph-mon[27495]: from='mgr.14118 172.21.15.70:0/3200588793' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:02.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:01 smithi070 ceph-mon[27495]: from='mgr.14118 172.21.15.70:0/3200588793' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:39:02.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:01 smithi070 ceph-mon[27495]: from='mgr.14118 172.21.15.70:0/3200588793' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:39:02.903 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:02 smithi070 ceph-mon[27495]: Found migration_current of "None". Setting to last migration. 2024-09-14T03:39:02.903 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:02 smithi070 ceph-mon[27495]: mgrmap e7: y(active, since 1.00532s) 2024-09-14T03:39:03.173 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout { 2024-09-14T03:39:03.173 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-09-14T03:39:03.173 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "initialized": true 2024-09-14T03:39:03.173 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout } 2024-09-14T03:39:03.173 INFO:teuthology.orchestra.run.smithi070.stdout:mgr epoch 5 is available 2024-09-14T03:39:03.173 INFO:teuthology.orchestra.run.smithi070.stdout:Setting orchestrator backend to cephadm... 2024-09-14T03:39:03.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:03 smithi070 ceph-mon[27495]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-14T03:39:03.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:03 smithi070 ceph-mon[27495]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-14T03:39:03.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:03 smithi070 ceph-mon[27495]: from='mgr.14118 172.21.15.70:0/3200588793' entity='mgr.y' 2024-09-14T03:39:03.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:03 smithi070 ceph-mon[27495]: from='mgr.14118 172.21.15.70:0/3200588793' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:03.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:03.478+0000 7faca47b8640 -1 log_channel(cephadm) log [ERR] : [14/Sep/2024:03:39:03] ENGINE Error in HTTPServer.serve 2024-09-14T03:39:03.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: Traceback (most recent call last): 2024-09-14T03:39:03.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:39:03.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: self._connections.run(self.expiration_interval) 2024-09-14T03:39:03.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:39:03.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: self._run(expiration_interval) 2024-09-14T03:39:03.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:39:03.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:39:03.787 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:39:03.787 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:39:03.787 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:39:03.787 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: s = self.context.wrap_socket( 2024-09-14T03:39:03.787 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:39:03.787 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: return self.sslsocket_class._create( 2024-09-14T03:39:03.787 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:39:03.787 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: self.do_handshake() 2024-09-14T03:39:03.787 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:39:03.787 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: self._sslobj.do_handshake() 2024-09-14T03:39:03.787 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:39:03.787 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:05.011 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:04 smithi070 ceph-mon[27495]: [14/Sep/2024:03:39:03] ENGINE Bus STARTING 2024-09-14T03:39:05.011 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:04 smithi070 ceph-mon[27495]: [14/Sep/2024:03:39:03] ENGINE Serving on https://172.21.15.70:7150 2024-09-14T03:39:05.011 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:04 smithi070 ceph-mon[27495]: [14/Sep/2024:03:39:03] ENGINE Bus STARTED 2024-09-14T03:39:05.011 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:04 smithi070 ceph-mon[27495]: [14/Sep/2024:03:39:03] ENGINE Error in HTTPServer.serve 2024-09-14T03:39:05.011 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-09-14T03:39:05.011 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:39:05.011 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:39:05.012 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:39:05.012 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._run(expiration_interval) 2024-09-14T03:39:05.012 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:39:05.012 INFO:journalctl@ceph.mon.a.smithi070.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:39:05.012 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:39:05.012 INFO:journalctl@ceph.mon.a.smithi070.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:39:05.012 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:39:05.012 INFO:journalctl@ceph.mon.a.smithi070.stdout: s = self.context.wrap_socket( 2024-09-14T03:39:05.012 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:39:05.012 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.sslsocket_class._create( 2024-09-14T03:39:05.012 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:39:05.012 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.do_handshake() 2024-09-14T03:39:05.013 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:39:05.013 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._sslobj.do_handshake() 2024-09-14T03:39:05.013 INFO:journalctl@ceph.mon.a.smithi070.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:39:05.013 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:04 smithi070 ceph-mon[27495]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:05.013 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:04 smithi070 ceph-mon[27495]: from='mgr.14118 172.21.15.70:0/3200588793' entity='mgr.y' 2024-09-14T03:39:05.013 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:04 smithi070 ceph-mon[27495]: from='mgr.14118 172.21.15.70:0/3200588793' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:05.013 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:04 smithi070 ceph-mon[27495]: mgrmap e8: y(active, since 2s) 2024-09-14T03:39:05.700 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout value unchanged 2024-09-14T03:39:05.700 INFO:teuthology.orchestra.run.smithi070.stdout:Generating ssh key... 2024-09-14T03:39:06.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:05 smithi070 ceph-mon[27495]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:06.687 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: Generating public/private rsa key pair. 2024-09-14T03:39:06.687 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: Your identification has been saved in /tmp/tmp8182xa33/key 2024-09-14T03:39:06.687 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: Your public key has been saved in /tmp/tmp8182xa33/key.pub 2024-09-14T03:39:06.687 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: The key fingerprint is: 2024-09-14T03:39:06.687 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: SHA256:yynCze32SaCYyNY5f680+51wz+OmFELlEwGYhQk5A14 ceph-b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:39:06.687 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: The key's randomart image is: 2024-09-14T03:39:06.687 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: +---[RSA 3072]----+ 2024-09-14T03:39:06.687 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: | ..Eo *o.+. | 2024-09-14T03:39:06.687 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: | . .+ = o . | 2024-09-14T03:39:06.688 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: | . o . o | 2024-09-14T03:39:06.688 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: | . . | 2024-09-14T03:39:06.688 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: | S . . | 2024-09-14T03:39:06.688 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: | . + * + + . . | 2024-09-14T03:39:06.688 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: | + O = B o o | 2024-09-14T03:39:06.688 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: | . + +o+ * +o | 2024-09-14T03:39:06.688 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: | .o+== +++. | 2024-09-14T03:39:06.688 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: +----[SHA256]-----+ 2024-09-14T03:39:07.565 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:07 smithi070 ceph-mon[27495]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:07.565 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:07 smithi070 ceph-mon[27495]: Generating ssh key... 2024-09-14T03:39:07.565 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:07 smithi070 ceph-mon[27495]: from='mgr.14118 172.21.15.70:0/3200588793' entity='mgr.y' 2024-09-14T03:39:07.565 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:07 smithi070 ceph-mon[27495]: from='mgr.14118 172.21.15.70:0/3200588793' entity='mgr.y' 2024-09-14T03:39:08.112 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDApLbiUzUSVrNCNSwufo2gGcwYNnu2KICjs2B9aUBNcXrNXE0ocV0xeoZf2VE55Ecvv+VNzbskbyoB0u9yWft7v+M5pFnSSp2G/+6sk21ZJCv4m4A9kMahMx2llTMvPWVMa/Fb/nzRu2xabYyaEwrNi6ZlLekfIPtqZyZztKIM1g5sMGbdBQkTfM8D/OMTszp15khe424YNs7dcexNRO4nK1LQkWtglrvwNM+XE/du/wg7kGMMvbByRdP9ev5pq+tdXVQAq6+yv8NcoChc0fXPVyj95gPpXKOHgMamIk4fIZc6uAAgkTDy1J8JM7SZGb9MrE45IMhToWoCRCoMS7aw6sgrdqws5xk3+HJINBI9sfgcOmAtw4FzfcVXfS/EBRJs73mU/7l+qS4qxBCPI/79Wj7GoIIXayyDS76Ce9NHa3Egx3iTj4/li4NLKq3QLk2P+xw2a9GvdYZeFgVIN3IkifJOHpdfsZYdsWvoExyAO3uwLvIgTlrVSlDuXQ4TmnE= ceph-b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:39:08.112 INFO:teuthology.orchestra.run.smithi070.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-14T03:39:08.112 INFO:teuthology.orchestra.run.smithi070.stdout:Adding key to root@localhost authorized_keys... 2024-09-14T03:39:08.116 INFO:teuthology.orchestra.run.smithi070.stdout:Adding host smithi070... 2024-09-14T03:39:08.620 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:08 smithi070 ceph-mon[27495]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:09.520 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:09 smithi070 ceph-mon[27495]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi070", "addr": "172.21.15.70", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:10.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:10 smithi070 ceph-mon[27495]: Deploying cephadm binary to smithi070 2024-09-14T03:39:12.002 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout Added host 'smithi070' with addr '172.21.15.70' 2024-09-14T03:39:12.002 INFO:teuthology.orchestra.run.smithi070.stdout:Deploying unmanaged mon service... 2024-09-14T03:39:12.642 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:12 smithi070 ceph-mon[27495]: from='mgr.14118 172.21.15.70:0/3200588793' entity='mgr.y' 2024-09-14T03:39:12.642 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:12 smithi070 ceph-mon[27495]: Added host smithi070 2024-09-14T03:39:12.642 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:12 smithi070 ceph-mon[27495]: from='mgr.14118 172.21.15.70:0/3200588793' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:13.444 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-09-14T03:39:13.444 INFO:teuthology.orchestra.run.smithi070.stdout:Deploying unmanaged mgr service... 2024-09-14T03:39:13.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:13 smithi070 ceph-mon[27495]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:13.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:13 smithi070 ceph-mon[27495]: Saving service mon spec with placement count:5 2024-09-14T03:39:13.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:13 smithi070 ceph-mon[27495]: from='mgr.14118 172.21.15.70:0/3200588793' entity='mgr.y' 2024-09-14T03:39:14.978 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-09-14T03:39:15.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:15 smithi070 ceph-mon[27495]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:15.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:15 smithi070 ceph-mon[27495]: Saving service mgr spec with placement count:2 2024-09-14T03:39:15.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:15 smithi070 ceph-mon[27495]: from='mgr.14118 172.21.15.70:0/3200588793' entity='mgr.y' 2024-09-14T03:39:16.871 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:16 smithi070 ceph-mon[27495]: from='mgr.14118 172.21.15.70:0/3200588793' entity='mgr.y' 2024-09-14T03:39:16.872 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:16 smithi070 ceph-mon[27495]: from='mgr.14118 172.21.15.70:0/3200588793' entity='mgr.y' 2024-09-14T03:39:16.872 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:16 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/4207487836' entity='client.admin' 2024-09-14T03:39:18.518 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:18 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/2768708264' entity='client.admin' 2024-09-14T03:39:18.518 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:18 smithi070 ceph-mon[27495]: from='mgr.14118 172.21.15.70:0/3200588793' entity='mgr.y' 2024-09-14T03:39:18.520 INFO:teuthology.orchestra.run.smithi070.stdout:Enabling the dashboard module... 2024-09-14T03:39:20.074 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:20 smithi070 ceph-mon[27495]: from='mgr.14118 172.21.15.70:0/3200588793' entity='mgr.y' 2024-09-14T03:39:20.074 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:20 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/3938073567' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-14T03:39:20.074 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:20 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: ignoring --setuser ceph since I am not root 2024-09-14T03:39:20.412 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:20 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: ignoring --setgroup ceph since I am not root 2024-09-14T03:39:20.412 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:20 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:20.167+0000 7f80900a4ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-14T03:39:20.412 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:20 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:20.223+0000 7f80900a4ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-14T03:39:20.412 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:20 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:20.290+0000 7f80900a4ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-14T03:39:20.695 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:20 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:20.418+0000 7f80900a4ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:39:20.695 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:20 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:20.562+0000 7f80900a4ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-14T03:39:20.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:20 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:20.829+0000 7f80900a4ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-14T03:39:20.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:20 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:20.889+0000 7f80900a4ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-14T03:39:21.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:21 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/3938073567' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-14T03:39:21.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:21 smithi070 ceph-mon[27495]: mgrmap e9: y(active, since 18s) 2024-09-14T03:39:21.286 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:21 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:21.062+0000 7f80900a4ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-14T03:39:22.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:22 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/1082539688' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-14T03:39:22.286 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:21 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:21.927+0000 7f80900a4ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-14T03:39:22.286 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:21 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:21.992+0000 7f80900a4ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-14T03:39:22.348 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout { 2024-09-14T03:39:22.348 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-09-14T03:39:22.348 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-14T03:39:22.348 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-09-14T03:39:22.348 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-09-14T03:39:22.349 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout } 2024-09-14T03:39:22.349 INFO:teuthology.orchestra.run.smithi070.stdout:Waiting for the mgr to restart... 2024-09-14T03:39:22.349 INFO:teuthology.orchestra.run.smithi070.stdout:Waiting for mgr epoch 9... 2024-09-14T03:39:22.956 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:22 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:22.723+0000 7f80900a4ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-14T03:39:22.957 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:22 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:22.837+0000 7f80900a4ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-14T03:39:23.207 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:22 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:22.955+0000 7f80900a4ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:39:23.207 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:23 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:23.014+0000 7f80900a4ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-14T03:39:23.208 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:23 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:23.139+0000 7f80900a4ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-14T03:39:23.208 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:23 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:23.206+0000 7f80900a4ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-14T03:39:23.459 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:23 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:23.263+0000 7f80900a4ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-14T03:39:23.459 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:23 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:23.395+0000 7f80900a4ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-14T03:39:23.460 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:23 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:23.458+0000 7f80900a4ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-14T03:39:23.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:23 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:23.522+0000 7f80900a4ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-14T03:39:23.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:23 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:23.601+0000 7f80900a4ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-14T03:39:23.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:23 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:23.657+0000 7f80900a4ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-14T03:39:23.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:23 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:23.713+0000 7f80900a4ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-14T03:39:24.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:24 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:24.446+0000 7f80900a4ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-14T03:39:24.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:24 smithi070 ceph-mon[27495]: Active manager daemon y restarted 2024-09-14T03:39:24.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:24 smithi070 ceph-mon[27495]: Activating manager daemon y 2024-09-14T03:39:24.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:24 smithi070 ceph-mon[27495]: osdmap e3: 0 total, 0 up, 0 in 2024-09-14T03:39:24.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:24 smithi070 ceph-mon[27495]: mgrmap e10: y(active, starting, since 0.00316188s) 2024-09-14T03:39:24.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:24 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:39:24.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:24 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:39:24.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:24 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:39:24.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:24 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:39:24.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:24 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:39:24.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:24 smithi070 ceph-mon[27495]: Manager daemon y is now available 2024-09-14T03:39:24.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:24 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:24.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:24 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:39:24.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:24 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:39:26.029 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:25 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:25.810+0000 7f8040c02640 -1 log_channel(cephadm) log [ERR] : [14/Sep/2024:03:39:25] ENGINE Error in HTTPServer.serve 2024-09-14T03:39:26.029 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:25 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: Traceback (most recent call last): 2024-09-14T03:39:26.029 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:25 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:39:26.030 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:25 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: self._connections.run(self.expiration_interval) 2024-09-14T03:39:26.030 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:25 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:39:26.030 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:25 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: self._run(expiration_interval) 2024-09-14T03:39:26.030 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:25 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:39:26.030 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:25 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:39:26.030 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:25 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:39:26.030 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:25 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:39:26.030 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:25 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:39:26.030 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:25 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: s = self.context.wrap_socket( 2024-09-14T03:39:26.030 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:25 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:39:26.030 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:25 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: return self.sslsocket_class._create( 2024-09-14T03:39:26.030 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:25 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:39:26.030 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:25 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: self.do_handshake() 2024-09-14T03:39:26.031 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:25 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:39:26.031 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:25 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: self._sslobj.do_handshake() 2024-09-14T03:39:26.031 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:25 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:39:26.031 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:39:25 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:39:26.315 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout { 2024-09-14T03:39:26.316 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-09-14T03:39:26.316 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout "initialized": true 2024-09-14T03:39:26.316 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout } 2024-09-14T03:39:26.316 INFO:teuthology.orchestra.run.smithi070.stdout:mgr epoch 9 is available 2024-09-14T03:39:26.316 INFO:teuthology.orchestra.run.smithi070.stdout:Generating a dashboard self-signed certificate... 2024-09-14T03:39:26.597 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:26 smithi070 ceph-mon[27495]: mgrmap e11: y(active, since 1.00537s) 2024-09-14T03:39:26.598 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:26 smithi070 ceph-mon[27495]: [14/Sep/2024:03:39:25] ENGINE Bus STARTING 2024-09-14T03:39:26.598 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:26 smithi070 ceph-mon[27495]: [14/Sep/2024:03:39:25] ENGINE Serving on https://172.21.15.70:7150 2024-09-14T03:39:26.598 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:26 smithi070 ceph-mon[27495]: [14/Sep/2024:03:39:25] ENGINE Bus STARTED 2024-09-14T03:39:26.598 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:26 smithi070 ceph-mon[27495]: [14/Sep/2024:03:39:25] ENGINE Error in HTTPServer.serve 2024-09-14T03:39:26.598 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-09-14T03:39:26.598 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:39:26.598 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:39:26.598 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:39:26.598 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._run(expiration_interval) 2024-09-14T03:39:26.598 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:39:26.598 INFO:journalctl@ceph.mon.a.smithi070.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:39:26.598 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:39:26.598 INFO:journalctl@ceph.mon.a.smithi070.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:39:26.599 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:39:26.599 INFO:journalctl@ceph.mon.a.smithi070.stdout: s = self.context.wrap_socket( 2024-09-14T03:39:26.599 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:39:26.599 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.sslsocket_class._create( 2024-09-14T03:39:26.599 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:39:26.599 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.do_handshake() 2024-09-14T03:39:26.599 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:39:26.599 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._sslobj.do_handshake() 2024-09-14T03:39:26.599 INFO:journalctl@ceph.mon.a.smithi070.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:39:26.599 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:26 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:39:26.599 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:26 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:39:27.696 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-09-14T03:39:27.697 INFO:teuthology.orchestra.run.smithi070.stdout:Creating initial admin user... 2024-09-14T03:39:27.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:27 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:39:27.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:27 smithi070 ceph-mon[27495]: mgrmap e12: y(active, since 2s) 2024-09-14T03:39:27.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:27 smithi070 ceph-mon[27495]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:27.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:27 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:39:27.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:27 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:39:29.479 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:29 smithi070 ceph-mon[27495]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:29.784 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$NHwLWwTB3SVqKQpns7V.9eC0d/U1.H6BWr93NI6wNrYtEPMiSQ1hK", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726285169, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-14T03:39:29.785 INFO:teuthology.orchestra.run.smithi070.stdout:Fetching dashboard port number... 2024-09-14T03:39:30.379 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:30 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:39:30.965 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stdout 8443 2024-09-14T03:39:30.975 INFO:teuthology.orchestra.run.smithi070.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-14T03:39:30.976 INFO:teuthology.orchestra.run.smithi070.stdout:systemctl: stdout disabled 2024-09-14T03:39:30.983 INFO:teuthology.orchestra.run.smithi070.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-14T03:39:30.983 INFO:teuthology.orchestra.run.smithi070.stdout:systemctl: stdout inactive 2024-09-14T03:39:30.983 INFO:teuthology.orchestra.run.smithi070.stdout:firewalld.service is not enabled 2024-09-14T03:39:30.983 INFO:teuthology.orchestra.run.smithi070.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-14T03:39:30.984 INFO:teuthology.orchestra.run.smithi070.stdout:Ceph Dashboard is now available at: 2024-09-14T03:39:30.985 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:39:30.985 INFO:teuthology.orchestra.run.smithi070.stdout: URL: https://smithi070.front.sepia.ceph.com:8443/ 2024-09-14T03:39:30.985 INFO:teuthology.orchestra.run.smithi070.stdout: User: admin 2024-09-14T03:39:30.985 INFO:teuthology.orchestra.run.smithi070.stdout: Password: 7w06opishr 2024-09-14T03:39:30.985 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:39:30.985 INFO:teuthology.orchestra.run.smithi070.stdout:Saving cluster configuration to /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config directory 2024-09-14T03:39:30.986 INFO:teuthology.orchestra.run.smithi070.stdout:Enabling autotune for osd_memory_target 2024-09-14T03:39:31.216 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:31 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/1796378085' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-14T03:39:32.227 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:32 smithi070 ceph-mon[27495]: mgrmap e13: y(active, since 6s) 2024-09-14T03:39:33.496 INFO:teuthology.orchestra.run.smithi070.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-09-14T03:39:33.496 INFO:teuthology.orchestra.run.smithi070.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-09-14T03:39:33.496 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:39:33.496 INFO:teuthology.orchestra.run.smithi070.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-14T03:39:33.497 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:39:33.497 INFO:teuthology.orchestra.run.smithi070.stdout:Or, if you are only running a single cluster on this host: 2024-09-14T03:39:33.497 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:39:33.497 INFO:teuthology.orchestra.run.smithi070.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-09-14T03:39:33.497 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:39:33.497 INFO:teuthology.orchestra.run.smithi070.stdout:Please consider enabling telemetry to help improve Ceph: 2024-09-14T03:39:33.497 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:39:33.497 INFO:teuthology.orchestra.run.smithi070.stdout: ceph telemetry on 2024-09-14T03:39:33.497 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:39:33.498 INFO:teuthology.orchestra.run.smithi070.stdout:For more information see: 2024-09-14T03:39:33.498 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:39:33.498 INFO:teuthology.orchestra.run.smithi070.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-14T03:39:33.498 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:39:33.498 INFO:teuthology.orchestra.run.smithi070.stdout:Bootstrap complete. 2024-09-14T03:39:33.532 INFO:tasks.cephadm:Fetching config... 2024-09-14T03:39:33.532 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2024-09-14T03:39:33.532 DEBUG:teuthology.orchestra.run.smithi070:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-14T03:39:33.549 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-14T03:39:33.549 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2024-09-14T03:39:33.549 DEBUG:teuthology.orchestra.run.smithi070:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-14T03:39:33.608 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-14T03:39:33.608 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2024-09-14T03:39:33.608 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd if=/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/keyring of=/dev/stdout 2024-09-14T03:39:33.674 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-14T03:39:33.674 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2024-09-14T03:39:33.674 DEBUG:teuthology.orchestra.run.smithi070:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-14T03:39:33.730 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-14T03:39:33.730 DEBUG:teuthology.orchestra.run.smithi070:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDApLbiUzUSVrNCNSwufo2gGcwYNnu2KICjs2B9aUBNcXrNXE0ocV0xeoZf2VE55Ecvv+VNzbskbyoB0u9yWft7v+M5pFnSSp2G/+6sk21ZJCv4m4A9kMahMx2llTMvPWVMa/Fb/nzRu2xabYyaEwrNi6ZlLekfIPtqZyZztKIM1g5sMGbdBQkTfM8D/OMTszp15khe424YNs7dcexNRO4nK1LQkWtglrvwNM+XE/du/wg7kGMMvbByRdP9ev5pq+tdXVQAq6+yv8NcoChc0fXPVyj95gPpXKOHgMamIk4fIZc6uAAgkTDy1J8JM7SZGb9MrE45IMhToWoCRCoMS7aw6sgrdqws5xk3+HJINBI9sfgcOmAtw4FzfcVXfS/EBRJs73mU/7l+qS4qxBCPI/79Wj7GoIIXayyDS76Ce9NHa3Egx3iTj4/li4NLKq3QLk2P+xw2a9GvdYZeFgVIN3IkifJOHpdfsZYdsWvoExyAO3uwLvIgTlrVSlDuXQ4TmnE= ceph-b6380d42-724a-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-14T03:39:33.805 INFO:teuthology.orchestra.run.smithi070.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDApLbiUzUSVrNCNSwufo2gGcwYNnu2KICjs2B9aUBNcXrNXE0ocV0xeoZf2VE55Ecvv+VNzbskbyoB0u9yWft7v+M5pFnSSp2G/+6sk21ZJCv4m4A9kMahMx2llTMvPWVMa/Fb/nzRu2xabYyaEwrNi6ZlLekfIPtqZyZztKIM1g5sMGbdBQkTfM8D/OMTszp15khe424YNs7dcexNRO4nK1LQkWtglrvwNM+XE/du/wg7kGMMvbByRdP9ev5pq+tdXVQAq6+yv8NcoChc0fXPVyj95gPpXKOHgMamIk4fIZc6uAAgkTDy1J8JM7SZGb9MrE45IMhToWoCRCoMS7aw6sgrdqws5xk3+HJINBI9sfgcOmAtw4FzfcVXfS/EBRJs73mU/7l+qS4qxBCPI/79Wj7GoIIXayyDS76Ce9NHa3Egx3iTj4/li4NLKq3QLk2P+xw2a9GvdYZeFgVIN3IkifJOHpdfsZYdsWvoExyAO3uwLvIgTlrVSlDuXQ4TmnE= ceph-b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:39:33.815 DEBUG:teuthology.orchestra.run.smithi094:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDApLbiUzUSVrNCNSwufo2gGcwYNnu2KICjs2B9aUBNcXrNXE0ocV0xeoZf2VE55Ecvv+VNzbskbyoB0u9yWft7v+M5pFnSSp2G/+6sk21ZJCv4m4A9kMahMx2llTMvPWVMa/Fb/nzRu2xabYyaEwrNi6ZlLekfIPtqZyZztKIM1g5sMGbdBQkTfM8D/OMTszp15khe424YNs7dcexNRO4nK1LQkWtglrvwNM+XE/du/wg7kGMMvbByRdP9ev5pq+tdXVQAq6+yv8NcoChc0fXPVyj95gPpXKOHgMamIk4fIZc6uAAgkTDy1J8JM7SZGb9MrE45IMhToWoCRCoMS7aw6sgrdqws5xk3+HJINBI9sfgcOmAtw4FzfcVXfS/EBRJs73mU/7l+qS4qxBCPI/79Wj7GoIIXayyDS76Ce9NHa3Egx3iTj4/li4NLKq3QLk2P+xw2a9GvdYZeFgVIN3IkifJOHpdfsZYdsWvoExyAO3uwLvIgTlrVSlDuXQ4TmnE= ceph-b6380d42-724a-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-14T03:39:33.875 INFO:teuthology.orchestra.run.smithi094.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDApLbiUzUSVrNCNSwufo2gGcwYNnu2KICjs2B9aUBNcXrNXE0ocV0xeoZf2VE55Ecvv+VNzbskbyoB0u9yWft7v+M5pFnSSp2G/+6sk21ZJCv4m4A9kMahMx2llTMvPWVMa/Fb/nzRu2xabYyaEwrNi6ZlLekfIPtqZyZztKIM1g5sMGbdBQkTfM8D/OMTszp15khe424YNs7dcexNRO4nK1LQkWtglrvwNM+XE/du/wg7kGMMvbByRdP9ev5pq+tdXVQAq6+yv8NcoChc0fXPVyj95gPpXKOHgMamIk4fIZc6uAAgkTDy1J8JM7SZGb9MrE45IMhToWoCRCoMS7aw6sgrdqws5xk3+HJINBI9sfgcOmAtw4FzfcVXfS/EBRJs73mU/7l+qS4qxBCPI/79Wj7GoIIXayyDS76Ce9NHa3Egx3iTj4/li4NLKq3QLk2P+xw2a9GvdYZeFgVIN3IkifJOHpdfsZYdsWvoExyAO3uwLvIgTlrVSlDuXQ4TmnE= ceph-b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:39:33.888 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-14T03:39:34.070 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:33 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/2104299735' entity='client.admin' 2024-09-14T03:39:34.131 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:39:36.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:36 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:39:36.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:36 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:39:36.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:36 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:36.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:36 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:39:36.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:36 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:36.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:36 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:39:37.531 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-14T03:39:37.531 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-14T03:39:37.778 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:37 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/3453909352' entity='client.admin' 2024-09-14T03:39:37.778 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:37 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:37.778 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:37 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:39:38.199 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:39:40.693 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi094 2024-09-14T03:39:40.694 DEBUG:teuthology.orchestra.run.smithi094:> set -ex 2024-09-14T03:39:40.694 DEBUG:teuthology.orchestra.run.smithi094:> dd of=/etc/ceph/ceph.conf 2024-09-14T03:39:40.713 DEBUG:teuthology.orchestra.run.smithi094:> set -ex 2024-09-14T03:39:40.713 DEBUG:teuthology.orchestra.run.smithi094:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:39:40.772 INFO:tasks.cephadm:Adding host smithi094 to orchestrator... 2024-09-14T03:39:40.772 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph orch host add smithi094 2024-09-14T03:39:41.013 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:39:41.226 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:40 smithi070 ceph-mon[27495]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:41.226 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:40 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:39:41.226 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:40 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:41.226 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:40 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:41.226 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:40 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:39:41.226 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:40 smithi070 ceph-mon[27495]: Updating smithi070:/etc/ceph/ceph.conf 2024-09-14T03:39:41.226 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:40 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:39:41.226 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:40 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:39:41.226 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:40 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:39:42.090 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:41 smithi070 ceph-mon[27495]: Updating smithi070:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:39:42.090 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:41 smithi070 ceph-mon[27495]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:39:42.090 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:41 smithi070 ceph-mon[27495]: Updating smithi070:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:39:43.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:43 smithi070 ceph-mon[27495]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi094", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:44.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:44 smithi070 ceph-mon[27495]: Deploying cephadm binary to smithi094 2024-09-14T03:39:45.402 INFO:teuthology.orchestra.run.smithi070.stdout:Added host 'smithi094' with addr '172.21.15.94' 2024-09-14T03:39:45.712 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:45 smithi070 ceph-mon[27495]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:39:45.712 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:45 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:39:45.712 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:45 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:46.006 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph orch host ls --format=json 2024-09-14T03:39:46.245 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:39:47.010 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:46 smithi070 ceph-mon[27495]: Added host smithi094 2024-09-14T03:39:47.010 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:46 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:39:47.010 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:46 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:39:47.938 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:39:47.938 INFO:teuthology.orchestra.run.smithi070.stdout:[{"addr": "172.21.15.70", "hostname": "smithi070", "labels": [], "status": ""}, {"addr": "172.21.15.94", "hostname": "smithi094", "labels": [], "status": ""}] 2024-09-14T03:39:47.938 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:47 smithi070 ceph-mon[27495]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:39:47.938 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:47 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:39:48.489 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-14T03:39:48.489 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph osd crush tunables default 2024-09-14T03:39:48.725 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:39:48.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:48 smithi070 ceph-mon[27495]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:39:49.922 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:49 smithi070 ceph-mon[27495]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:39:49.922 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:49 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:39:49.922 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:49 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:39:49.922 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:49 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:39:49.923 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:49 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:39:49.923 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:49 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi094", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:39:49.923 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:49 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:49.923 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:49 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:39:51.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:50 smithi070 ceph-mon[27495]: Updating smithi094:/etc/ceph/ceph.conf 2024-09-14T03:39:51.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:50 smithi070 ceph-mon[27495]: Updating smithi094:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:39:51.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:50 smithi070 ceph-mon[27495]: Updating smithi094:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:39:51.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:50 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:39:51.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:50 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:39:51.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:50 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:39:51.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:50 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/434194745' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-14T03:39:51.307 INFO:teuthology.orchestra.run.smithi070.stderr:adjusted tunables profile to default 2024-09-14T03:39:51.921 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:51 smithi070 ceph-mon[27495]: Updating smithi094:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:39:51.921 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:51 smithi070 ceph-mon[27495]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:39:51.921 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:51 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/434194745' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-14T03:39:51.921 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:51 smithi070 ceph-mon[27495]: osdmap e4: 0 total, 0 up, 0 in 2024-09-14T03:39:51.945 INFO:tasks.cephadm:Adding mon.a on smithi070 2024-09-14T03:39:51.945 INFO:tasks.cephadm:Adding mon.c on smithi070 2024-09-14T03:39:51.945 INFO:tasks.cephadm:Adding mon.b on smithi094 2024-09-14T03:39:51.945 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph orch apply mon '3;smithi070:172.21.15.70=a;smithi070:[v2:172.21.15.70:3301,v1:172.21.15.70:6790]=c;smithi094:172.21.15.94=b' 2024-09-14T03:39:52.189 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:39:53.838 INFO:teuthology.orchestra.run.smithi094.stdout:Scheduled mon update... 2024-09-14T03:39:54.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:53 smithi070 ceph-mon[27495]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:39:54.419 DEBUG:teuthology.orchestra.run.smithi070:mon.c> sudo journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@mon.c.service 2024-09-14T03:39:54.423 DEBUG:teuthology.orchestra.run.smithi094:mon.b> sudo journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@mon.b.service 2024-09-14T03:39:54.426 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-14T03:39:54.426 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-14T03:39:55.216 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:39:55.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:54 smithi070 ceph-mon[27495]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi070:172.21.15.70=a;smithi070:[v2:172.21.15.70:3301,v1:172.21.15.70:6790]=c;smithi094:172.21.15.94=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:39:55.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:54 smithi070 ceph-mon[27495]: Saving service mon spec with placement smithi070:172.21.15.70=a;smithi070:[v2:172.21.15.70:3301,v1:172.21.15.70:6790]=c;smithi094:172.21.15.94=b;count:3 2024-09-14T03:39:55.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:54 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:39:55.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:54 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:39:55.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:54 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:55.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:54 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:39:55.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:54 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:39:55.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:54 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:39:55.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:54 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:39:55.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:54 smithi070 ceph-mon[27495]: Deploying daemon mon.b on smithi094 2024-09-14T03:39:56.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:55 smithi070 ceph-mon[27495]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:39:57.409 INFO:teuthology.orchestra.run.smithi094.stdout: 2024-09-14T03:39:57.409 INFO:teuthology.orchestra.run.smithi094.stdout:{"epoch":1,"fsid":"b6380d42-724a-11ef-bceb-c7b262605968","modified":"2024-09-14T03:38:26.272229Z","created":"2024-09-14T03:38:26.272229Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:3300","nonce":0},{"type":"v1","addr":"172.21.15.70:6789","nonce":0}]},"addr":"172.21.15.70:6789/0","public_addr":"172.21.15.70:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-14T03:39:57.409 INFO:teuthology.orchestra.run.smithi094.stderr:dumped monmap epoch 1 2024-09-14T03:39:58.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:57 smithi070 ceph-mon[27495]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:39:58.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:57 smithi070 ceph-mon[27495]: from='client.? 172.21.15.94:0/238013610' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-14T03:39:59.138 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-14T03:39:59.138 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-14T03:39:59.385 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:40:00.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:39:59 smithi070 ceph-mon[27495]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:00.396 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: starting mon.b rank -1 at public addrs [v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0] at bind addrs [v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:40:00.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: mon.b@-1(???) e0 preinit fsid b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:40:00.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: mon.b@-1(synchronizing).mds e1 new map 2024-09-14T03:40:00.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: mon.b@-1(synchronizing).mds e1 print_map 2024-09-14T03:40:00.778 INFO:journalctl@ceph.mon.b.smithi094.stdout: e1 2024-09-14T03:40:00.778 INFO:journalctl@ceph.mon.b.smithi094.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-14T03:40:00.778 INFO:journalctl@ceph.mon.b.smithi094.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-14T03:40:00.778 INFO:journalctl@ceph.mon.b.smithi094.stdout: legacy client fscid: -1 2024-09-14T03:40:00.778 INFO:journalctl@ceph.mon.b.smithi094.stdout: 2024-09-14T03:40:00.778 INFO:journalctl@ceph.mon.b.smithi094.stdout: No filesystems configured 2024-09-14T03:40:00.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-14T03:40:00.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-14T03:40:00.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-14T03:40:00.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-14T03:40:00.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-14T03:40:00.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-14T03:40:00.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-14T03:40:00.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-14T03:40:00.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-14T03:40:00.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-14T03:40:00.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:40:00.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:00.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:00.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:00.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:00.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:00.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi094", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:40:00.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:00.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:00.780 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: Updating smithi094:/etc/ceph/ceph.conf 2024-09-14T03:40:00.780 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: Updating smithi094:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:40:00.780 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: Updating smithi094:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:40:00.780 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:00.780 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:00.780 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:00.780 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/434194745' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-14T03:40:00.780 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: Updating smithi094:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:40:00.780 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:00.780 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/434194745' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-14T03:40:00.780 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: osdmap e4: 0 total, 0 up, 0 in 2024-09-14T03:40:00.780 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:00.781 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi070:172.21.15.70=a;smithi070:[v2:172.21.15.70:3301,v1:172.21.15.70:6790]=c;smithi094:172.21.15.94=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:00.781 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: Saving service mon spec with placement smithi070:172.21.15.70=a;smithi070:[v2:172.21.15.70:3301,v1:172.21.15.70:6790]=c;smithi094:172.21.15.94=b;count:3 2024-09-14T03:40:00.781 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:00.781 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:00.781 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:00.781 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:00.781 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:00.781 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:40:00.781 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:00.781 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: Deploying daemon mon.b on smithi094 2024-09-14T03:40:00.781 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:00.781 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:00.781 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: from='client.? 172.21.15.94:0/238013610' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-14T03:40:00.781 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:00.782 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 ceph-mon[31053]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-09-14T03:40:00.782 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 bash[30996]: 90ea2e16490969668a54b75d50384cc1565b48a36a16b2f8f08aacaf4191b8a6 2024-09-14T03:40:00.782 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:00 smithi094 systemd[1]: Started Ceph mon.b for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:40:05.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:05 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:05.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:05 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:05.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:05 smithi094 ceph-mon[31053]: mon.a calling monitor election 2024-09-14T03:40:05.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:05 smithi094 ceph-mon[31053]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:05.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:05 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:05.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:05 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:05.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:05 smithi094 ceph-mon[31053]: mon.b calling monitor election 2024-09-14T03:40:05.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:05 smithi094 ceph-mon[31053]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:05.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:05 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:05.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:05 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:05.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:05 smithi094 ceph-mon[31053]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:05.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:05 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:05.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:05 smithi094 ceph-mon[31053]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-14T03:40:05.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:05 smithi094 ceph-mon[31053]: monmap e2: 2 mons at {a=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],b=[v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-14T03:40:05.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:05 smithi094 ceph-mon[31053]: fsmap 2024-09-14T03:40:05.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:05 smithi094 ceph-mon[31053]: osdmap e4: 0 total, 0 up, 0 in 2024-09-14T03:40:05.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:05 smithi094 ceph-mon[31053]: mgrmap e13: y(active, since 40s) 2024-09-14T03:40:05.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:05 smithi094 ceph-mon[31053]: overall HEALTH_OK 2024-09-14T03:40:05.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:05 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:05.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:05 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:05.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:05 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:05.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:05 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:40:05.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:05 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:05.789 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:05 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:05.789 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:05 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:05.790 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:05 smithi070 ceph-mon[27495]: mon.a calling monitor election 2024-09-14T03:40:05.790 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:05 smithi070 ceph-mon[27495]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:05.790 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:05 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:05.790 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:05 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:05.790 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:05 smithi070 ceph-mon[27495]: mon.b calling monitor election 2024-09-14T03:40:05.790 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:05 smithi070 ceph-mon[27495]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:05.790 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:05 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:05.790 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:05 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:05.790 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:05 smithi070 ceph-mon[27495]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:05.790 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:05 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:05.790 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:05 smithi070 ceph-mon[27495]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-14T03:40:05.791 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:05 smithi070 ceph-mon[27495]: monmap e2: 2 mons at {a=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],b=[v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-14T03:40:05.791 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:05 smithi070 ceph-mon[27495]: fsmap 2024-09-14T03:40:05.791 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:05 smithi070 ceph-mon[27495]: osdmap e4: 0 total, 0 up, 0 in 2024-09-14T03:40:05.791 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:05 smithi070 ceph-mon[27495]: mgrmap e13: y(active, since 40s) 2024-09-14T03:40:05.791 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:05 smithi070 ceph-mon[27495]: overall HEALTH_OK 2024-09-14T03:40:05.791 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:05 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:05.791 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:05 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:05.791 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:05 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:05.791 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:05 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:40:05.791 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:05 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:06.064 INFO:teuthology.orchestra.run.smithi094.stdout: 2024-09-14T03:40:06.065 INFO:teuthology.orchestra.run.smithi094.stdout:{"epoch":2,"fsid":"b6380d42-724a-11ef-bceb-c7b262605968","modified":"2024-09-14T03:40:00.416216Z","created":"2024-09-14T03:38:26.272229Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:3300","nonce":0},{"type":"v1","addr":"172.21.15.70:6789","nonce":0}]},"addr":"172.21.15.70:6789/0","public_addr":"172.21.15.70:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:3300","nonce":0},{"type":"v1","addr":"172.21.15.94:6789","nonce":0}]},"addr":"172.21.15.94:6789/0","public_addr":"172.21.15.94:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-14T03:40:06.065 INFO:teuthology.orchestra.run.smithi094.stderr:dumped monmap epoch 2 2024-09-14T03:40:06.640 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:06 smithi094 ceph-mon[31053]: Deploying daemon mon.c on smithi070 2024-09-14T03:40:06.640 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:06 smithi094 ceph-mon[31053]: from='client.? 172.21.15.94:0/3495285269' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-14T03:40:06.641 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:06 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:06.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:06 smithi070 ceph-mon[27495]: Deploying daemon mon.c on smithi070 2024-09-14T03:40:06.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:06 smithi070 ceph-mon[27495]: from='client.? 172.21.15.94:0/3495285269' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-14T03:40:06.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:06 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:07.666 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-14T03:40:07.667 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-14T03:40:07.764 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:07 smithi070 ceph-mon[27495]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:07.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:07 smithi094 ceph-mon[31053]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:07.925 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:40:08.713 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 podman[39250]: 2024-09-14 03:40:08.512349436 +0000 UTC m=+0.015436703 image pull ab51d1bebd1200e205d11a014c20a7f2cf49d932afcbeabdc1c9fc887c00f446 quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6 2024-09-14T03:40:09.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 podman[39250]: 2024-09-14 03:40:08.800631091 +0000 UTC m=+0.303718346 container init 44ee80c29ee0962628f2ccebe8975350c32d044147afb5468ed661736e2ec58f (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-c, org.label-schema.vendor=CentOS, RELEASE=quincy-eb16b97, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-14T03:40:09.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 podman[39250]: 2024-09-14 03:40:08.805542003 +0000 UTC m=+0.308629265 container start 44ee80c29ee0962628f2ccebe8975350c32d044147afb5468ed661736e2ec58f (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-c, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-eb16b97, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-14T03:40:09.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: set uid:gid to 167:167 (ceph:ceph) 2024-09-14T03:40:09.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable), process ceph-mon, pid 7 2024-09-14T03:40:09.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: pidfile_write: ignore empty --pid-file 2024-09-14T03:40:09.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: load: jerasure load: lrc 2024-09-14T03:40:09.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: RocksDB version: 6.15.5 2024-09-14T03:40:09.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-14T03:40:09.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Compile date Sep 13 2024 2024-09-14T03:40:09.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: DB SUMMARY 2024-09-14T03:40:09.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: DB Session ID: ZJDM5N1LZG7916N1MXUV 2024-09-14T03:40:09.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: CURRENT file: CURRENT 2024-09-14T03:40:09.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: IDENTITY file: IDENTITY 2024-09-14T03:40:09.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-09-14T03:40:09.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-09-14T03:40:09.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2024-09-14T03:40:09.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.error_if_exists: 0 2024-09-14T03:40:09.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.create_if_missing: 0 2024-09-14T03:40:09.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.paranoid_checks: 1 2024-09-14T03:40:09.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-14T03:40:09.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.env: 0x558686b9d440 2024-09-14T03:40:09.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.fs: Posix File System 2024-09-14T03:40:09.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.info_log: 0x558687fe0ec0 2024-09-14T03:40:09.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.max_file_opening_threads: 16 2024-09-14T03:40:09.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.statistics: (nil) 2024-09-14T03:40:09.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.use_fsync: 0 2024-09-14T03:40:09.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.max_log_file_size: 0 2024-09-14T03:40:09.290 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-14T03:40:09.290 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-14T03:40:09.290 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.keep_log_file_num: 1000 2024-09-14T03:40:09.290 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.recycle_log_file_num: 0 2024-09-14T03:40:09.290 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.allow_fallocate: 1 2024-09-14T03:40:09.290 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.allow_mmap_reads: 0 2024-09-14T03:40:09.290 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.allow_mmap_writes: 0 2024-09-14T03:40:09.290 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.use_direct_reads: 0 2024-09-14T03:40:09.290 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-14T03:40:09.290 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.create_missing_column_families: 0 2024-09-14T03:40:09.291 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.db_log_dir: 2024-09-14T03:40:09.291 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-09-14T03:40:09.291 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-14T03:40:09.291 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-14T03:40:09.291 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-14T03:40:09.291 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-14T03:40:09.291 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-14T03:40:09.291 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-14T03:40:09.291 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.advise_random_on_open: 1 2024-09-14T03:40:09.291 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.db_write_buffer_size: 0 2024-09-14T03:40:09.291 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.write_buffer_manager: 0x55868809e7b0 2024-09-14T03:40:09.291 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-14T03:40:09.291 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-14T03:40:09.292 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-14T03:40:09.292 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-14T03:40:09.292 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.rate_limiter: (nil) 2024-09-14T03:40:09.292 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-14T03:40:09.292 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.wal_recovery_mode: 2 2024-09-14T03:40:09.292 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.enable_thread_tracking: 0 2024-09-14T03:40:09.292 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.enable_pipelined_write: 0 2024-09-14T03:40:09.292 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.unordered_write: 0 2024-09-14T03:40:09.292 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-14T03:40:09.292 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-14T03:40:09.292 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-14T03:40:09.292 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-14T03:40:09.292 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.row_cache: None 2024-09-14T03:40:09.293 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.wal_filter: None 2024-09-14T03:40:09.293 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-14T03:40:09.293 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.allow_ingest_behind: 0 2024-09-14T03:40:09.293 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.preserve_deletes: 0 2024-09-14T03:40:09.293 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.two_write_queues: 0 2024-09-14T03:40:09.293 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.manual_wal_flush: 0 2024-09-14T03:40:09.293 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.atomic_flush: 0 2024-09-14T03:40:09.293 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-14T03:40:09.293 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-14T03:40:09.293 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-14T03:40:09.293 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.log_readahead_size: 0 2024-09-14T03:40:09.293 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-14T03:40:09.293 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.best_efforts_recovery: 0 2024-09-14T03:40:09.293 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-14T03:40:09.293 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-14T03:40:09.294 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.allow_data_in_errors: 0 2024-09-14T03:40:09.294 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.db_host_id: __hostname__ 2024-09-14T03:40:09.294 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.max_background_jobs: 2 2024-09-14T03:40:09.294 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.max_background_compactions: -1 2024-09-14T03:40:09.294 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.max_subcompactions: 1 2024-09-14T03:40:09.294 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-14T03:40:09.294 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-14T03:40:09.294 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-14T03:40:09.294 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.max_total_wal_size: 0 2024-09-14T03:40:09.294 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-14T03:40:09.294 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-14T03:40:09.294 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-14T03:40:09.294 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-14T03:40:09.294 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.max_open_files: -1 2024-09-14T03:40:09.295 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.bytes_per_sync: 0 2024-09-14T03:40:09.295 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-14T03:40:09.295 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-14T03:40:09.295 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.compaction_readahead_size: 0 2024-09-14T03:40:09.295 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.max_background_flushes: -1 2024-09-14T03:40:09.295 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Compression algorithms supported: 2024-09-14T03:40:09.295 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: kZSTD supported: 0 2024-09-14T03:40:09.295 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: kXpressCompression supported: 0 2024-09-14T03:40:09.295 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: kBZip2Compression supported: 0 2024-09-14T03:40:09.295 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-14T03:40:09.295 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: kLZ4Compression supported: 1 2024-09-14T03:40:09.295 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: kZlibCompression supported: 1 2024-09-14T03:40:09.295 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: kLZ4HCCompression supported: 1 2024-09-14T03:40:09.295 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: kSnappyCompression supported: 1 2024-09-14T03:40:09.296 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-14T03:40:09.296 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-09-14T03:40:09.296 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-09-14T03:40:09.296 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-14T03:40:09.296 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.merge_operator: 2024-09-14T03:40:09.296 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.compaction_filter: None 2024-09-14T03:40:09.296 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.compaction_filter_factory: None 2024-09-14T03:40:09.296 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.sst_partitioner_factory: None 2024-09-14T03:40:09.296 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-14T03:40:09.296 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-14T03:40:09.296 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x558687f87a38) 2024-09-14T03:40:09.296 INFO:journalctl@ceph.mon.c.smithi070.stdout: cache_index_and_filter_blocks: 1 2024-09-14T03:40:09.296 INFO:journalctl@ceph.mon.c.smithi070.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-14T03:40:09.297 INFO:journalctl@ceph.mon.c.smithi070.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-14T03:40:09.297 INFO:journalctl@ceph.mon.c.smithi070.stdout: pin_top_level_index_and_filter: 1 2024-09-14T03:40:09.297 INFO:journalctl@ceph.mon.c.smithi070.stdout: index_type: 0 2024-09-14T03:40:09.297 INFO:journalctl@ceph.mon.c.smithi070.stdout: data_block_index_type: 0 2024-09-14T03:40:09.297 INFO:journalctl@ceph.mon.c.smithi070.stdout: index_shortening: 1 2024-09-14T03:40:09.297 INFO:journalctl@ceph.mon.c.smithi070.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-14T03:40:09.297 INFO:journalctl@ceph.mon.c.smithi070.stdout: hash_index_allow_collision: 1 2024-09-14T03:40:09.297 INFO:journalctl@ceph.mon.c.smithi070.stdout: checksum: 1 2024-09-14T03:40:09.297 INFO:journalctl@ceph.mon.c.smithi070.stdout: no_block_cache: 0 2024-09-14T03:40:09.297 INFO:journalctl@ceph.mon.c.smithi070.stdout: block_cache: 0x558687fbd1f0 2024-09-14T03:40:09.297 INFO:journalctl@ceph.mon.c.smithi070.stdout: block_cache_name: BinnedLRUCache 2024-09-14T03:40:09.297 INFO:journalctl@ceph.mon.c.smithi070.stdout: block_cache_options: 2024-09-14T03:40:09.297 INFO:journalctl@ceph.mon.c.smithi070.stdout: capacity : 536870912 2024-09-14T03:40:09.298 INFO:journalctl@ceph.mon.c.smithi070.stdout: num_shard_bits : 4 2024-09-14T03:40:09.298 INFO:journalctl@ceph.mon.c.smithi070.stdout: strict_capacity_limit : 0 2024-09-14T03:40:09.298 INFO:journalctl@ceph.mon.c.smithi070.stdout: high_pri_pool_ratio: 0.000 2024-09-14T03:40:09.298 INFO:journalctl@ceph.mon.c.smithi070.stdout: block_cache_compressed: (nil) 2024-09-14T03:40:09.298 INFO:journalctl@ceph.mon.c.smithi070.stdout: persistent_cache: (nil) 2024-09-14T03:40:09.298 INFO:journalctl@ceph.mon.c.smithi070.stdout: block_size: 4096 2024-09-14T03:40:09.298 INFO:journalctl@ceph.mon.c.smithi070.stdout: block_size_deviation: 10 2024-09-14T03:40:09.298 INFO:journalctl@ceph.mon.c.smithi070.stdout: block_restart_interval: 16 2024-09-14T03:40:09.298 INFO:journalctl@ceph.mon.c.smithi070.stdout: index_block_restart_interval: 1 2024-09-14T03:40:09.298 INFO:journalctl@ceph.mon.c.smithi070.stdout: metadata_block_size: 4096 2024-09-14T03:40:09.298 INFO:journalctl@ceph.mon.c.smithi070.stdout: partition_filters: 0 2024-09-14T03:40:09.298 INFO:journalctl@ceph.mon.c.smithi070.stdout: use_delta_encoding: 1 2024-09-14T03:40:09.298 INFO:journalctl@ceph.mon.c.smithi070.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-14T03:40:09.298 INFO:journalctl@ceph.mon.c.smithi070.stdout: whole_key_filtering: 1 2024-09-14T03:40:09.299 INFO:journalctl@ceph.mon.c.smithi070.stdout: verify_compression: 0 2024-09-14T03:40:09.299 INFO:journalctl@ceph.mon.c.smithi070.stdout: read_amp_bytes_per_bit: 0 2024-09-14T03:40:09.299 INFO:journalctl@ceph.mon.c.smithi070.stdout: format_version: 4 2024-09-14T03:40:09.299 INFO:journalctl@ceph.mon.c.smithi070.stdout: enable_index_compression: 1 2024-09-14T03:40:09.299 INFO:journalctl@ceph.mon.c.smithi070.stdout: block_align: 0 2024-09-14T03:40:09.299 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.write_buffer_size: 33554432 2024-09-14T03:40:09.299 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.max_write_buffer_number: 2 2024-09-14T03:40:09.299 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.compression: NoCompression 2024-09-14T03:40:09.299 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.bottommost_compression: Disabled 2024-09-14T03:40:09.299 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.prefix_extractor: nullptr 2024-09-14T03:40:09.299 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-14T03:40:09.299 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.num_levels: 7 2024-09-14T03:40:09.299 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-14T03:40:09.299 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-14T03:40:09.300 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-14T03:40:09.300 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-14T03:40:09.300 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-14T03:40:09.300 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-14T03:40:09.300 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-14T03:40:09.300 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:40:09.300 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-14T03:40:09.300 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-14T03:40:09.300 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-14T03:40:09.300 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.compression_opts.level: 32767 2024-09-14T03:40:09.300 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.compression_opts.strategy: 0 2024-09-14T03:40:09.300 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-14T03:40:09.300 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:40:09.301 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-14T03:40:09.301 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.compression_opts.enabled: false 2024-09-14T03:40:09.301 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-14T03:40:09.301 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-14T03:40:09.301 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-14T03:40:09.301 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.target_file_size_base: 67108864 2024-09-14T03:40:09.301 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-14T03:40:09.301 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-14T03:40:09.301 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-14T03:40:09.301 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-14T03:40:09.301 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-14T03:40:09.301 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-14T03:40:09.301 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-14T03:40:09.301 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-14T03:40:09.302 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-14T03:40:09.302 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-14T03:40:09.302 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-14T03:40:09.302 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-14T03:40:09.302 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-14T03:40:09.302 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.arena_block_size: 4194304 2024-09-14T03:40:09.302 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-14T03:40:09.302 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-14T03:40:09.302 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-14T03:40:09.302 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.disable_auto_compactions: 0 2024-09-14T03:40:09.302 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-14T03:40:09.302 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-14T03:40:09.302 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-14T03:40:09.303 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-14T03:40:09.303 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-14T03:40:09.303 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-14T03:40:09.303 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-14T03:40:09.303 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-14T03:40:09.303 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-14T03:40:09.303 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-14T03:40:09.303 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-14T03:40:09.303 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.inplace_update_support: 0 2024-09-14T03:40:09.303 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-14T03:40:09.303 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-14T03:40:09.303 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-14T03:40:09.303 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-14T03:40:09.303 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.bloom_locality: 0 2024-09-14T03:40:09.304 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.max_successive_merges: 0 2024-09-14T03:40:09.304 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-14T03:40:09.304 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.paranoid_file_checks: 0 2024-09-14T03:40:09.304 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.force_consistency_checks: 1 2024-09-14T03:40:09.304 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.report_bg_io_stats: 0 2024-09-14T03:40:09.304 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.ttl: 2592000 2024-09-14T03:40:09.304 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-14T03:40:09.304 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.enable_blob_files: false 2024-09-14T03:40:09.304 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.min_blob_size: 0 2024-09-14T03:40:09.304 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.blob_file_size: 268435456 2024-09-14T03:40:09.304 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-14T03:40:09.304 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-14T03:40:09.304 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-14T03:40:09.305 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-14T03:40:09.305 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-09-14T03:40:09.305 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-09-14T03:40:09.305 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285208829832, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-14T03:40:09.305 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-09-14T03:40:09.305 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-14T03:40:09.305 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285208830252, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1696, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726285208, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "cea3bb85-e0bf-4013-b011-cf1145605315", "db_session_id": "ZJDM5N1LZG7916N1MXUV"}} 2024-09-14T03:40:09.305 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-09-14T03:40:09.305 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285208830680, "job": 1, "event": "recovery_finished"} 2024-09-14T03:40:09.306 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-14T03:40:09.306 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x558687ff0700 2024-09-14T03:40:09.306 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: DB pointer 0x558687fc0000 2024-09-14T03:40:09.306 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-09-14T03:40:09.306 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-14T03:40:09.306 INFO:journalctl@ceph.mon.c.smithi070.stdout: ** DB Stats ** 2024-09-14T03:40:09.306 INFO:journalctl@ceph.mon.c.smithi070.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:40:09.306 INFO:journalctl@ceph.mon.c.smithi070.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-14T03:40:09.306 INFO:journalctl@ceph.mon.c.smithi070.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:40:09.306 INFO:journalctl@ceph.mon.c.smithi070.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:40:09.306 INFO:journalctl@ceph.mon.c.smithi070.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-14T03:40:09.306 INFO:journalctl@ceph.mon.c.smithi070.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-14T03:40:09.306 INFO:journalctl@ceph.mon.c.smithi070.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:40:09.307 INFO:journalctl@ceph.mon.c.smithi070.stdout: 2024-09-14T03:40:09.307 INFO:journalctl@ceph.mon.c.smithi070.stdout: ** Compaction Stats [default] ** 2024-09-14T03:40:09.307 INFO:journalctl@ceph.mon.c.smithi070.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-14T03:40:09.307 INFO:journalctl@ceph.mon.c.smithi070.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-14T03:40:09.307 INFO:journalctl@ceph.mon.c.smithi070.stdout: L0 1/0 1.66 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.1 0.00 0.00 1 0.000 0 0 2024-09-14T03:40:09.307 INFO:journalctl@ceph.mon.c.smithi070.stdout: Sum 1/0 1.66 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.1 0.00 0.00 1 0.000 0 0 2024-09-14T03:40:09.307 INFO:journalctl@ceph.mon.c.smithi070.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.1 0.00 0.00 1 0.000 0 0 2024-09-14T03:40:09.307 INFO:journalctl@ceph.mon.c.smithi070.stdout: 2024-09-14T03:40:09.307 INFO:journalctl@ceph.mon.c.smithi070.stdout: ** Compaction Stats [default] ** 2024-09-14T03:40:09.307 INFO:journalctl@ceph.mon.c.smithi070.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-14T03:40:09.307 INFO:journalctl@ceph.mon.c.smithi070.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-14T03:40:09.307 INFO:journalctl@ceph.mon.c.smithi070.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.1 0.00 0.00 1 0.000 0 0 2024-09-14T03:40:09.307 INFO:journalctl@ceph.mon.c.smithi070.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:40:09.307 INFO:journalctl@ceph.mon.c.smithi070.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-14T03:40:09.308 INFO:journalctl@ceph.mon.c.smithi070.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-14T03:40:09.308 INFO:journalctl@ceph.mon.c.smithi070.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-14T03:40:09.308 INFO:journalctl@ceph.mon.c.smithi070.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-14T03:40:09.308 INFO:journalctl@ceph.mon.c.smithi070.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-14T03:40:09.308 INFO:journalctl@ceph.mon.c.smithi070.stdout: Cumulative compaction: 0.00 GB write, 0.45 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:40:09.309 INFO:journalctl@ceph.mon.c.smithi070.stdout: Interval compaction: 0.00 GB write, 0.45 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:40:09.309 INFO:journalctl@ceph.mon.c.smithi070.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-14T03:40:09.309 INFO:journalctl@ceph.mon.c.smithi070.stdout: 2024-09-14T03:40:09.309 INFO:journalctl@ceph.mon.c.smithi070.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-14T03:40:09.309 INFO:journalctl@ceph.mon.c.smithi070.stdout: 2024-09-14T03:40:09.309 INFO:journalctl@ceph.mon.c.smithi070.stdout: ** Compaction Stats [default] ** 2024-09-14T03:40:09.309 INFO:journalctl@ceph.mon.c.smithi070.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-14T03:40:09.309 INFO:journalctl@ceph.mon.c.smithi070.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-14T03:40:09.310 INFO:journalctl@ceph.mon.c.smithi070.stdout: L0 1/0 1.66 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.1 0.00 0.00 1 0.000 0 0 2024-09-14T03:40:09.310 INFO:journalctl@ceph.mon.c.smithi070.stdout: Sum 1/0 1.66 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.1 0.00 0.00 1 0.000 0 0 2024-09-14T03:40:09.310 INFO:journalctl@ceph.mon.c.smithi070.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-14T03:40:09.310 INFO:journalctl@ceph.mon.c.smithi070.stdout: 2024-09-14T03:40:09.310 INFO:journalctl@ceph.mon.c.smithi070.stdout: ** Compaction Stats [default] ** 2024-09-14T03:40:09.310 INFO:journalctl@ceph.mon.c.smithi070.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-14T03:40:09.311 INFO:journalctl@ceph.mon.c.smithi070.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-14T03:40:09.311 INFO:journalctl@ceph.mon.c.smithi070.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.1 0.00 0.00 1 0.000 0 0 2024-09-14T03:40:09.311 INFO:journalctl@ceph.mon.c.smithi070.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:40:09.311 INFO:journalctl@ceph.mon.c.smithi070.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-14T03:40:09.311 INFO:journalctl@ceph.mon.c.smithi070.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-14T03:40:09.311 INFO:journalctl@ceph.mon.c.smithi070.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-14T03:40:09.311 INFO:journalctl@ceph.mon.c.smithi070.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-14T03:40:09.311 INFO:journalctl@ceph.mon.c.smithi070.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-14T03:40:09.311 INFO:journalctl@ceph.mon.c.smithi070.stdout: Cumulative compaction: 0.00 GB write, 0.45 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:40:09.311 INFO:journalctl@ceph.mon.c.smithi070.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:40:09.312 INFO:journalctl@ceph.mon.c.smithi070.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-14T03:40:09.312 INFO:journalctl@ceph.mon.c.smithi070.stdout: 2024-09-14T03:40:09.312 INFO:journalctl@ceph.mon.c.smithi070.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-14T03:40:09.312 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-09-14T03:40:09.312 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: using public_addrv [v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0] 2024-09-14T03:40:09.312 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: starting mon.c rank -1 at public addrs [v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0] at bind addrs [v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:40:09.312 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: mon.c@-1(???) e0 preinit fsid b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:40:09.312 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: mon.c@-1(synchronizing).mds e1 new map 2024-09-14T03:40:09.312 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: mon.c@-1(synchronizing).mds e1 print_map 2024-09-14T03:40:09.312 INFO:journalctl@ceph.mon.c.smithi070.stdout: e1 2024-09-14T03:40:09.312 INFO:journalctl@ceph.mon.c.smithi070.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-14T03:40:09.312 INFO:journalctl@ceph.mon.c.smithi070.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-14T03:40:09.312 INFO:journalctl@ceph.mon.c.smithi070.stdout: legacy client fscid: -1 2024-09-14T03:40:09.313 INFO:journalctl@ceph.mon.c.smithi070.stdout: 2024-09-14T03:40:09.313 INFO:journalctl@ceph.mon.c.smithi070.stdout: No filesystems configured 2024-09-14T03:40:09.313 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-14T03:40:09.313 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-14T03:40:09.313 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-14T03:40:09.313 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-14T03:40:09.313 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-14T03:40:09.313 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-14T03:40:09.313 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-14T03:40:09.313 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-14T03:40:09.313 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-14T03:40:09.313 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-14T03:40:09.313 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:40:09.314 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:09.314 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:09.314 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:09.314 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:09.314 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:09.314 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi094", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:40:09.314 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:09.314 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:09.314 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: Updating smithi094:/etc/ceph/ceph.conf 2024-09-14T03:40:09.314 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: Updating smithi094:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:40:09.314 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: Updating smithi094:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:40:09.314 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:09.314 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:09.314 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:09.315 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/434194745' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-14T03:40:09.315 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: Updating smithi094:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:40:09.315 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:09.315 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/434194745' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-14T03:40:09.315 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: osdmap e4: 0 total, 0 up, 0 in 2024-09-14T03:40:09.315 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:09.315 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi070:172.21.15.70=a;smithi070:[v2:172.21.15.70:3301,v1:172.21.15.70:6790]=c;smithi094:172.21.15.94=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:09.315 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: Saving service mon spec with placement smithi070:172.21.15.70=a;smithi070:[v2:172.21.15.70:3301,v1:172.21.15.70:6790]=c;smithi094:172.21.15.94=b;count:3 2024-09-14T03:40:09.315 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:09.315 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:09.315 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:09.315 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:09.315 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:09.315 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:40:09.316 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:09.316 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: Deploying daemon mon.b on smithi094 2024-09-14T03:40:09.316 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:09.316 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:09.316 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='client.? 172.21.15.94:0/238013610' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-14T03:40:09.316 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:09.316 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:09.316 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:09.316 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: mon.a calling monitor election 2024-09-14T03:40:09.316 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:09.316 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:09.316 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:09.316 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: mon.b calling monitor election 2024-09-14T03:40:09.316 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:09.317 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:09.317 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:09.317 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:09.317 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:09.317 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-14T03:40:09.317 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: monmap e2: 2 mons at {a=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],b=[v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-14T03:40:09.317 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: fsmap 2024-09-14T03:40:09.317 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: osdmap e4: 0 total, 0 up, 0 in 2024-09-14T03:40:09.317 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: mgrmap e13: y(active, since 40s) 2024-09-14T03:40:09.317 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: overall HEALTH_OK 2024-09-14T03:40:09.317 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:09.317 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:09.317 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:09.317 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:40:09.318 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:09.318 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: Deploying daemon mon.c on smithi070 2024-09-14T03:40:09.318 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='client.? 172.21.15.94:0/3495285269' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-14T03:40:09.318 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:09.318 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:09.318 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 ceph-mon[39288]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-09-14T03:40:09.318 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 bash[39250]: 44ee80c29ee0962628f2ccebe8975350c32d044147afb5468ed661736e2ec58f 2024-09-14T03:40:09.318 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:08 smithi070 systemd[1]: Started Ceph mon.c for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:40:14.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:13 smithi094 ceph-mon[31053]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:14.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:13 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:14.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:13 smithi094 ceph-mon[31053]: mon.a calling monitor election 2024-09-14T03:40:14.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:13 smithi094 ceph-mon[31053]: mon.b calling monitor election 2024-09-14T03:40:14.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:13 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:14.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:13 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:14.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:13 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:14.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:13 smithi094 ceph-mon[31053]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:14.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:13 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:14.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:13 smithi094 ceph-mon[31053]: mon.c calling monitor election 2024-09-14T03:40:14.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:13 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:14.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:13 smithi094 ceph-mon[31053]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:14.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:13 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:14.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:13 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:14.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:13 smithi094 ceph-mon[31053]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:40:14.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:13 smithi094 ceph-mon[31053]: monmap e3: 3 mons at {a=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],b=[v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0],c=[v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-14T03:40:14.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:13 smithi094 ceph-mon[31053]: fsmap 2024-09-14T03:40:14.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:13 smithi094 ceph-mon[31053]: osdmap e4: 0 total, 0 up, 0 in 2024-09-14T03:40:14.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:13 smithi094 ceph-mon[31053]: mgrmap e13: y(active, since 49s) 2024-09-14T03:40:14.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:13 smithi094 ceph-mon[31053]: overall HEALTH_OK 2024-09-14T03:40:14.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:13 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:14.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:13 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:14.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:13 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:14.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:13 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:14.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[27495]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:14.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:14.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[27495]: mon.a calling monitor election 2024-09-14T03:40:14.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[27495]: mon.b calling monitor election 2024-09-14T03:40:14.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:14.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:14.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:14.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[27495]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:14.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:14.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[27495]: mon.c calling monitor election 2024-09-14T03:40:14.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:14.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[27495]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:14.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:14.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:14.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[27495]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:40:14.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[27495]: monmap e3: 3 mons at {a=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],b=[v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0],c=[v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-14T03:40:14.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[27495]: fsmap 2024-09-14T03:40:14.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[27495]: osdmap e4: 0 total, 0 up, 0 in 2024-09-14T03:40:14.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[27495]: mgrmap e13: y(active, since 49s) 2024-09-14T03:40:14.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[27495]: overall HEALTH_OK 2024-09-14T03:40:14.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:14.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:14.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:14.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:14.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[39288]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:14.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:40:14.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[39288]: mon.a calling monitor election 2024-09-14T03:40:14.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[39288]: mon.b calling monitor election 2024-09-14T03:40:14.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:40:14.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:14.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:14.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[39288]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:14.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:14.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[39288]: mon.c calling monitor election 2024-09-14T03:40:14.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:14.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[39288]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:14.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:14.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:14.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[39288]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:40:14.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[39288]: monmap e3: 3 mons at {a=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],b=[v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0],c=[v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-14T03:40:14.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[39288]: fsmap 2024-09-14T03:40:14.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[39288]: osdmap e4: 0 total, 0 up, 0 in 2024-09-14T03:40:14.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[39288]: mgrmap e13: y(active, since 49s) 2024-09-14T03:40:14.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[39288]: overall HEALTH_OK 2024-09-14T03:40:14.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:14.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:14.290 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:14.290 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:13 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:15.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:14 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:15.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:14 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:15.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:14 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:15.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:14 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:15.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:14 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:15.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:14 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:15.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:14 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:15.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:14 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:15.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:14 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:15.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:14 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:15.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:14 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:15.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:14 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:15.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:14 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:40:15.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:14 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:15.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:14 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:15.668 INFO:teuthology.orchestra.run.smithi094.stdout: 2024-09-14T03:40:15.668 INFO:teuthology.orchestra.run.smithi094.stdout:{"epoch":3,"fsid":"b6380d42-724a-11ef-bceb-c7b262605968","modified":"2024-09-14T03:40:08.852192Z","created":"2024-09-14T03:38:26.272229Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:3300","nonce":0},{"type":"v1","addr":"172.21.15.70:6789","nonce":0}]},"addr":"172.21.15.70:6789/0","public_addr":"172.21.15.70:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:3300","nonce":0},{"type":"v1","addr":"172.21.15.94:6789","nonce":0}]},"addr":"172.21.15.94:6789/0","public_addr":"172.21.15.94:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:3301","nonce":0},{"type":"v1","addr":"172.21.15.70:6790","nonce":0}]},"addr":"172.21.15.70:6790/0","public_addr":"172.21.15.70:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-09-14T03:40:15.669 INFO:teuthology.orchestra.run.smithi094.stderr:dumped monmap epoch 3 2024-09-14T03:40:16.258 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:15 smithi070 ceph-mon[39288]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:16.258 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:15 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:16.258 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:15 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:16.258 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:15 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:16.259 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:15 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:16.259 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:15 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:16.259 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:15 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:16.259 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:15 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:16.259 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:15 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:16.259 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:15 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:16.259 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:15 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:40:16.259 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:15 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:40:16.259 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:15 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:16.259 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:15 smithi070 ceph-mon[39288]: from='client.? 172.21.15.94:0/1837901056' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-14T03:40:16.259 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:15 smithi070 ceph-mon[27495]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:16.259 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:15 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:16.260 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:15 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:16.260 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:15 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:16.260 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:15 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:16.260 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:15 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:16.260 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:15 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:16.260 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:15 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:16.260 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:15 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:16.260 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:15 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:16.260 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:15 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:40:16.260 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:15 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:40:16.260 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:15 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:16.260 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:15 smithi070 ceph-mon[27495]: from='client.? 172.21.15.94:0/1837901056' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-14T03:40:16.264 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:15 smithi094 ceph-mon[31053]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:16.264 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:15 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:16.264 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:15 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:16.264 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:15 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:16.264 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:15 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:16.264 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:15 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:16.264 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:15 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:16.264 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:15 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:16.264 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:15 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:16.264 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:15 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:16.264 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:15 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:40:16.264 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:15 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:40:16.265 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:15 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:16.265 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:15 smithi094 ceph-mon[31053]: from='client.? 172.21.15.94:0/1837901056' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-14T03:40:16.288 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-14T03:40:16.288 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph config generate-minimal-conf 2024-09-14T03:40:16.608 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:40:17.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:16 smithi070 ceph-mon[27495]: Updating smithi070:/etc/ceph/ceph.conf 2024-09-14T03:40:17.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:16 smithi070 ceph-mon[27495]: Updating smithi094:/etc/ceph/ceph.conf 2024-09-14T03:40:17.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:16 smithi070 ceph-mon[27495]: Updating smithi094:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:40:17.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:16 smithi070 ceph-mon[27495]: Updating smithi070:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:40:17.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:16 smithi070 ceph-mon[27495]: Reconfiguring mon.a (unknown last config time)... 2024-09-14T03:40:17.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:16 smithi070 ceph-mon[27495]: Reconfiguring daemon mon.a on smithi070 2024-09-14T03:40:17.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:16 smithi070 ceph-mon[39288]: Updating smithi070:/etc/ceph/ceph.conf 2024-09-14T03:40:17.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:16 smithi070 ceph-mon[39288]: Updating smithi094:/etc/ceph/ceph.conf 2024-09-14T03:40:17.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:16 smithi070 ceph-mon[39288]: Updating smithi094:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:40:17.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:16 smithi070 ceph-mon[39288]: Updating smithi070:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:40:17.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:16 smithi070 ceph-mon[39288]: Reconfiguring mon.a (unknown last config time)... 2024-09-14T03:40:17.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:16 smithi070 ceph-mon[39288]: Reconfiguring daemon mon.a on smithi070 2024-09-14T03:40:17.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:16 smithi094 ceph-mon[31053]: Updating smithi070:/etc/ceph/ceph.conf 2024-09-14T03:40:17.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:16 smithi094 ceph-mon[31053]: Updating smithi094:/etc/ceph/ceph.conf 2024-09-14T03:40:17.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:16 smithi094 ceph-mon[31053]: Updating smithi094:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:40:17.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:16 smithi094 ceph-mon[31053]: Updating smithi070:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:40:17.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:16 smithi094 ceph-mon[31053]: Reconfiguring mon.a (unknown last config time)... 2024-09-14T03:40:17.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:16 smithi094 ceph-mon[31053]: Reconfiguring daemon mon.a on smithi070 2024-09-14T03:40:18.001 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:17 smithi070 ceph-mon[27495]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:18.001 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:17 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:18.001 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:17 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:18.001 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:17 smithi070 ceph-mon[27495]: Reconfiguring mon.c (monmap changed)... 2024-09-14T03:40:18.001 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:17 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:40:18.002 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:17 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:40:18.002 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:17 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:18.002 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:17 smithi070 ceph-mon[27495]: Reconfiguring daemon mon.c on smithi070 2024-09-14T03:40:18.002 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:17 smithi070 ceph-mon[39288]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:18.002 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:17 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:18.002 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:17 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:18.002 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:17 smithi070 ceph-mon[39288]: Reconfiguring mon.c (monmap changed)... 2024-09-14T03:40:18.002 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:17 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:40:18.002 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:17 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:40:18.002 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:17 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:18.002 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:17 smithi070 ceph-mon[39288]: Reconfiguring daemon mon.c on smithi070 2024-09-14T03:40:18.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:17 smithi094 ceph-mon[31053]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:18.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:17 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:18.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:17 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:18.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:17 smithi094 ceph-mon[31053]: Reconfiguring mon.c (monmap changed)... 2024-09-14T03:40:18.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:17 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:40:18.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:17 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:40:18.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:17 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:18.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:17 smithi094 ceph-mon[31053]: Reconfiguring daemon mon.c on smithi070 2024-09-14T03:40:19.902 INFO:teuthology.orchestra.run.smithi070.stdout:# minimal ceph.conf for b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:40:19.902 INFO:teuthology.orchestra.run.smithi070.stdout:[global] 2024-09-14T03:40:19.902 INFO:teuthology.orchestra.run.smithi070.stdout: fsid = b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:40:19.902 INFO:teuthology.orchestra.run.smithi070.stdout: mon_host = [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] [v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0] [v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0] 2024-09-14T03:40:20.255 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:19 smithi070 ceph-mon[27495]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:20.255 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:19 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:20.255 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:19 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:20.255 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:19 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:40:20.255 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:19 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:40:20.255 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:19 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:20.255 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:19 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/1245919249' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:20.255 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:19 smithi070 ceph-mon[39288]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:20.256 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:19 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:20.256 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:19 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:20.256 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:19 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:40:20.256 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:19 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:40:20.256 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:19 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:20.256 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:19 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/1245919249' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:20.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:19 smithi094 ceph-mon[31053]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:20.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:19 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:20.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:19 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:20.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:19 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:40:20.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:19 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:40:20.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:19 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:20.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:19 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/1245919249' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:20.570 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-14T03:40:20.570 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2024-09-14T03:40:20.570 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd of=/etc/ceph/ceph.conf 2024-09-14T03:40:20.601 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2024-09-14T03:40:20.601 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:40:20.667 DEBUG:teuthology.orchestra.run.smithi094:> set -ex 2024-09-14T03:40:20.667 DEBUG:teuthology.orchestra.run.smithi094:> sudo dd of=/etc/ceph/ceph.conf 2024-09-14T03:40:20.694 DEBUG:teuthology.orchestra.run.smithi094:> set -ex 2024-09-14T03:40:20.694 DEBUG:teuthology.orchestra.run.smithi094:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:40:20.759 INFO:tasks.cephadm:Adding mgr.y on smithi070 2024-09-14T03:40:20.760 INFO:tasks.cephadm:Adding mgr.x on smithi094 2024-09-14T03:40:20.760 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph orch apply mgr '2;smithi070=y;smithi094=x' 2024-09-14T03:40:21.035 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:40:21.155 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:20 smithi094 ceph-mon[31053]: Reconfiguring mon.b (monmap changed)... 2024-09-14T03:40:21.155 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:20 smithi094 ceph-mon[31053]: Reconfiguring daemon mon.b on smithi094 2024-09-14T03:40:21.155 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:20 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:21.155 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:20 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:21.155 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:20 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:21.155 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:20 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:21.155 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:20 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:21.155 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:20 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:21.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:20 smithi070 ceph-mon[27495]: Reconfiguring mon.b (monmap changed)... 2024-09-14T03:40:21.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:20 smithi070 ceph-mon[27495]: Reconfiguring daemon mon.b on smithi094 2024-09-14T03:40:21.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:20 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:21.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:20 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:21.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:20 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:21.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:20 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:21.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:20 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:21.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:20 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:21.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:20 smithi070 ceph-mon[39288]: Reconfiguring mon.b (monmap changed)... 2024-09-14T03:40:21.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:20 smithi070 ceph-mon[39288]: Reconfiguring daemon mon.b on smithi094 2024-09-14T03:40:21.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:20 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:21.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:20 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:21.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:20 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:21.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:20 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:21.290 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:20 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:21.290 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:20 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:22.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:21 smithi070 ceph-mon[27495]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:22.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:21 smithi070 ceph-mon[39288]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:22.202 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:21 smithi094 ceph-mon[31053]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:22.583 INFO:teuthology.orchestra.run.smithi094.stdout:Scheduled mgr update... 2024-09-14T03:40:23.112 DEBUG:teuthology.orchestra.run.smithi094:mgr.x> sudo journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@mgr.x.service 2024-09-14T03:40:23.115 DEBUG:tasks.cephadm:set 0 configs 2024-09-14T03:40:23.115 INFO:tasks.cephadm:Deploying OSDs... 2024-09-14T03:40:23.115 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2024-09-14T03:40:23.115 DEBUG:teuthology.orchestra.run.smithi070:> dd if=/scratch_devs of=/dev/stdout 2024-09-14T03:40:23.135 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-14T03:40:23.135 DEBUG:teuthology.orchestra.run.smithi070:> stat /dev/vg_nvme/lv_1 2024-09-14T03:40:23.194 INFO:teuthology.orchestra.run.smithi070.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-14T03:40:23.194 INFO:teuthology.orchestra.run.smithi070.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:40:23.195 INFO:teuthology.orchestra.run.smithi070.stdout:Device: 5h/5d Inode: 980 Links: 1 2024-09-14T03:40:23.195 INFO:teuthology.orchestra.run.smithi070.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:40:23.195 INFO:teuthology.orchestra.run.smithi070.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:40:23.195 INFO:teuthology.orchestra.run.smithi070.stdout:Access: 2024-09-14 03:40:08.341770429 +0000 2024-09-14T03:40:23.195 INFO:teuthology.orchestra.run.smithi070.stdout:Modify: 2024-09-14 03:36:24.244463018 +0000 2024-09-14T03:40:23.195 INFO:teuthology.orchestra.run.smithi070.stdout:Change: 2024-09-14 03:36:24.244463018 +0000 2024-09-14T03:40:23.195 INFO:teuthology.orchestra.run.smithi070.stdout: Birth: 2024-09-14 03:36:24.244463018 +0000 2024-09-14T03:40:23.195 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-14T03:40:23.263 INFO:teuthology.orchestra.run.smithi070.stderr:1+0 records in 2024-09-14T03:40:23.263 INFO:teuthology.orchestra.run.smithi070.stderr:1+0 records out 2024-09-14T03:40:23.263 INFO:teuthology.orchestra.run.smithi070.stderr:512 bytes copied, 0.000131783 s, 3.9 MB/s 2024-09-14T03:40:23.264 DEBUG:teuthology.orchestra.run.smithi070:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-14T03:40:23.324 DEBUG:teuthology.orchestra.run.smithi070:> stat /dev/vg_nvme/lv_2 2024-09-14T03:40:23.382 INFO:teuthology.orchestra.run.smithi070.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-14T03:40:23.382 INFO:teuthology.orchestra.run.smithi070.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:40:23.382 INFO:teuthology.orchestra.run.smithi070.stdout:Device: 5h/5d Inode: 972 Links: 1 2024-09-14T03:40:23.382 INFO:teuthology.orchestra.run.smithi070.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:40:23.382 INFO:teuthology.orchestra.run.smithi070.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:40:23.382 INFO:teuthology.orchestra.run.smithi070.stdout:Access: 2024-09-14 03:40:08.341770429 +0000 2024-09-14T03:40:23.382 INFO:teuthology.orchestra.run.smithi070.stdout:Modify: 2024-09-14 03:36:24.244463018 +0000 2024-09-14T03:40:23.382 INFO:teuthology.orchestra.run.smithi070.stdout:Change: 2024-09-14 03:36:24.244463018 +0000 2024-09-14T03:40:23.382 INFO:teuthology.orchestra.run.smithi070.stdout: Birth: 2024-09-14 03:36:24.244463018 +0000 2024-09-14T03:40:23.383 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-14T03:40:23.449 INFO:teuthology.orchestra.run.smithi070.stderr:1+0 records in 2024-09-14T03:40:23.450 INFO:teuthology.orchestra.run.smithi070.stderr:1+0 records out 2024-09-14T03:40:23.450 INFO:teuthology.orchestra.run.smithi070.stderr:512 bytes copied, 0.00011614 s, 4.4 MB/s 2024-09-14T03:40:23.451 DEBUG:teuthology.orchestra.run.smithi070:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-14T03:40:23.510 DEBUG:teuthology.orchestra.run.smithi070:> stat /dev/vg_nvme/lv_3 2024-09-14T03:40:23.569 INFO:teuthology.orchestra.run.smithi070.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-14T03:40:23.569 INFO:teuthology.orchestra.run.smithi070.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:40:23.569 INFO:teuthology.orchestra.run.smithi070.stdout:Device: 5h/5d Inode: 965 Links: 1 2024-09-14T03:40:23.569 INFO:teuthology.orchestra.run.smithi070.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:40:23.569 INFO:teuthology.orchestra.run.smithi070.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:40:23.569 INFO:teuthology.orchestra.run.smithi070.stdout:Access: 2024-09-14 03:40:08.342770399 +0000 2024-09-14T03:40:23.569 INFO:teuthology.orchestra.run.smithi070.stdout:Modify: 2024-09-14 03:36:24.244463018 +0000 2024-09-14T03:40:23.569 INFO:teuthology.orchestra.run.smithi070.stdout:Change: 2024-09-14 03:36:24.244463018 +0000 2024-09-14T03:40:23.569 INFO:teuthology.orchestra.run.smithi070.stdout: Birth: 2024-09-14 03:36:24.244463018 +0000 2024-09-14T03:40:23.570 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-14T03:40:23.636 INFO:teuthology.orchestra.run.smithi070.stderr:1+0 records in 2024-09-14T03:40:23.637 INFO:teuthology.orchestra.run.smithi070.stderr:1+0 records out 2024-09-14T03:40:23.637 INFO:teuthology.orchestra.run.smithi070.stderr:512 bytes copied, 0.000127238 s, 4.0 MB/s 2024-09-14T03:40:23.638 DEBUG:teuthology.orchestra.run.smithi070:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-14T03:40:23.697 DEBUG:teuthology.orchestra.run.smithi070:> stat /dev/vg_nvme/lv_4 2024-09-14T03:40:23.755 INFO:teuthology.orchestra.run.smithi070.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-14T03:40:23.755 INFO:teuthology.orchestra.run.smithi070.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:40:23.755 INFO:teuthology.orchestra.run.smithi070.stdout:Device: 5h/5d Inode: 986 Links: 1 2024-09-14T03:40:23.755 INFO:teuthology.orchestra.run.smithi070.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:40:23.755 INFO:teuthology.orchestra.run.smithi070.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:40:23.755 INFO:teuthology.orchestra.run.smithi070.stdout:Access: 2024-09-14 03:40:08.342770399 +0000 2024-09-14T03:40:23.755 INFO:teuthology.orchestra.run.smithi070.stdout:Modify: 2024-09-14 03:36:24.245462879 +0000 2024-09-14T03:40:23.755 INFO:teuthology.orchestra.run.smithi070.stdout:Change: 2024-09-14 03:36:24.245462879 +0000 2024-09-14T03:40:23.756 INFO:teuthology.orchestra.run.smithi070.stdout: Birth: 2024-09-14 03:36:24.245462879 +0000 2024-09-14T03:40:23.756 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-14T03:40:23.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:23 smithi094 ceph-mon[31053]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:23.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:23 smithi094 ceph-mon[31053]: from='client.24112 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi070=y;smithi094=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:23.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:23 smithi094 ceph-mon[31053]: Saving service mgr spec with placement smithi070=y;smithi094=x;count:2 2024-09-14T03:40:23.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:23 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:23.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:23 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:23.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:23 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:23.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:23 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:23.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:23 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:23.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:23 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:40:23.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:23 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-14T03:40:23.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:23 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:40:23.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:23 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:23.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:23 smithi094 ceph-mon[31053]: Deploying daemon mgr.x on smithi094 2024-09-14T03:40:23.822 INFO:teuthology.orchestra.run.smithi070.stderr:1+0 records in 2024-09-14T03:40:23.822 INFO:teuthology.orchestra.run.smithi070.stderr:1+0 records out 2024-09-14T03:40:23.823 INFO:teuthology.orchestra.run.smithi070.stderr:512 bytes copied, 0.000148002 s, 3.5 MB/s 2024-09-14T03:40:23.824 DEBUG:teuthology.orchestra.run.smithi070:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-14T03:40:23.883 DEBUG:teuthology.orchestra.run.smithi094:> set -ex 2024-09-14T03:40:23.883 DEBUG:teuthology.orchestra.run.smithi094:> dd if=/scratch_devs of=/dev/stdout 2024-09-14T03:40:23.900 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-14T03:40:23.900 DEBUG:teuthology.orchestra.run.smithi094:> stat /dev/vg_nvme/lv_1 2024-09-14T03:40:23.956 INFO:teuthology.orchestra.run.smithi094.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-14T03:40:23.956 INFO:teuthology.orchestra.run.smithi094.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:40:23.956 INFO:teuthology.orchestra.run.smithi094.stdout:Device: 5h/5d Inode: 978 Links: 1 2024-09-14T03:40:23.956 INFO:teuthology.orchestra.run.smithi094.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:40:23.956 INFO:teuthology.orchestra.run.smithi094.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:40:23.956 INFO:teuthology.orchestra.run.smithi094.stdout:Access: 2024-09-14 03:39:59.289956642 +0000 2024-09-14T03:40:23.956 INFO:teuthology.orchestra.run.smithi094.stdout:Modify: 2024-09-14 03:36:24.758404435 +0000 2024-09-14T03:40:23.956 INFO:teuthology.orchestra.run.smithi094.stdout:Change: 2024-09-14 03:36:24.758404435 +0000 2024-09-14T03:40:23.956 INFO:teuthology.orchestra.run.smithi094.stdout: Birth: 2024-09-14 03:36:24.758404435 +0000 2024-09-14T03:40:23.957 DEBUG:teuthology.orchestra.run.smithi094:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-14T03:40:24.020 INFO:teuthology.orchestra.run.smithi094.stderr:1+0 records in 2024-09-14T03:40:24.021 INFO:teuthology.orchestra.run.smithi094.stderr:1+0 records out 2024-09-14T03:40:24.021 INFO:teuthology.orchestra.run.smithi094.stderr:512 bytes copied, 0.00011101 s, 4.6 MB/s 2024-09-14T03:40:24.022 DEBUG:teuthology.orchestra.run.smithi094:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-14T03:40:24.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:23 smithi070 ceph-mon[27495]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:24.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:23 smithi070 ceph-mon[27495]: from='client.24112 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi070=y;smithi094=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:24.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:23 smithi070 ceph-mon[27495]: Saving service mgr spec with placement smithi070=y;smithi094=x;count:2 2024-09-14T03:40:24.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:23 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:24.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:23 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:24.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:23 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:24.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:23 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:24.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:23 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:24.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:23 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:40:24.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:23 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-14T03:40:24.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:23 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:40:24.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:23 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:24.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:23 smithi070 ceph-mon[27495]: Deploying daemon mgr.x on smithi094 2024-09-14T03:40:24.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:23 smithi070 ceph-mon[39288]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:24.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:23 smithi070 ceph-mon[39288]: from='client.24112 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi070=y;smithi094=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:24.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:23 smithi070 ceph-mon[39288]: Saving service mgr spec with placement smithi070=y;smithi094=x;count:2 2024-09-14T03:40:24.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:23 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:24.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:23 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:24.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:23 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:24.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:23 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:24.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:23 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:24.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:23 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:40:24.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:23 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-14T03:40:24.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:23 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:40:24.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:23 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:24.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:23 smithi070 ceph-mon[39288]: Deploying daemon mgr.x on smithi094 2024-09-14T03:40:24.078 DEBUG:teuthology.orchestra.run.smithi094:> stat /dev/vg_nvme/lv_2 2024-09-14T03:40:24.133 INFO:teuthology.orchestra.run.smithi094.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-14T03:40:24.133 INFO:teuthology.orchestra.run.smithi094.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:40:24.134 INFO:teuthology.orchestra.run.smithi094.stdout:Device: 5h/5d Inode: 968 Links: 1 2024-09-14T03:40:24.134 INFO:teuthology.orchestra.run.smithi094.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:40:24.134 INFO:teuthology.orchestra.run.smithi094.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:40:24.134 INFO:teuthology.orchestra.run.smithi094.stdout:Access: 2024-09-14 03:39:59.289956642 +0000 2024-09-14T03:40:24.134 INFO:teuthology.orchestra.run.smithi094.stdout:Modify: 2024-09-14 03:36:24.757404307 +0000 2024-09-14T03:40:24.134 INFO:teuthology.orchestra.run.smithi094.stdout:Change: 2024-09-14 03:36:24.757404307 +0000 2024-09-14T03:40:24.134 INFO:teuthology.orchestra.run.smithi094.stdout: Birth: 2024-09-14 03:36:24.757404307 +0000 2024-09-14T03:40:24.134 DEBUG:teuthology.orchestra.run.smithi094:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-14T03:40:24.197 INFO:teuthology.orchestra.run.smithi094.stderr:1+0 records in 2024-09-14T03:40:24.197 INFO:teuthology.orchestra.run.smithi094.stderr:1+0 records out 2024-09-14T03:40:24.197 INFO:teuthology.orchestra.run.smithi094.stderr:512 bytes copied, 8.9999e-05 s, 5.7 MB/s 2024-09-14T03:40:24.198 DEBUG:teuthology.orchestra.run.smithi094:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-14T03:40:24.254 DEBUG:teuthology.orchestra.run.smithi094:> stat /dev/vg_nvme/lv_3 2024-09-14T03:40:24.309 INFO:teuthology.orchestra.run.smithi094.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-14T03:40:24.309 INFO:teuthology.orchestra.run.smithi094.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:40:24.309 INFO:teuthology.orchestra.run.smithi094.stdout:Device: 5h/5d Inode: 964 Links: 1 2024-09-14T03:40:24.310 INFO:teuthology.orchestra.run.smithi094.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:40:24.310 INFO:teuthology.orchestra.run.smithi094.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:40:24.310 INFO:teuthology.orchestra.run.smithi094.stdout:Access: 2024-09-14 03:39:59.290956626 +0000 2024-09-14T03:40:24.310 INFO:teuthology.orchestra.run.smithi094.stdout:Modify: 2024-09-14 03:36:24.757404307 +0000 2024-09-14T03:40:24.310 INFO:teuthology.orchestra.run.smithi094.stdout:Change: 2024-09-14 03:36:24.757404307 +0000 2024-09-14T03:40:24.310 INFO:teuthology.orchestra.run.smithi094.stdout: Birth: 2024-09-14 03:36:24.757404307 +0000 2024-09-14T03:40:24.310 DEBUG:teuthology.orchestra.run.smithi094:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-14T03:40:24.465 INFO:teuthology.orchestra.run.smithi094.stderr:1+0 records in 2024-09-14T03:40:24.465 INFO:teuthology.orchestra.run.smithi094.stderr:1+0 records out 2024-09-14T03:40:24.465 INFO:teuthology.orchestra.run.smithi094.stderr:512 bytes copied, 9.6417e-05 s, 5.3 MB/s 2024-09-14T03:40:24.466 DEBUG:teuthology.orchestra.run.smithi094:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-14T03:40:24.482 DEBUG:teuthology.orchestra.run.smithi094:> stat /dev/vg_nvme/lv_4 2024-09-14T03:40:24.542 INFO:teuthology.orchestra.run.smithi094.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-14T03:40:24.542 INFO:teuthology.orchestra.run.smithi094.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:40:24.542 INFO:teuthology.orchestra.run.smithi094.stdout:Device: 5h/5d Inode: 970 Links: 1 2024-09-14T03:40:24.542 INFO:teuthology.orchestra.run.smithi094.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:40:24.542 INFO:teuthology.orchestra.run.smithi094.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:40:24.542 INFO:teuthology.orchestra.run.smithi094.stdout:Access: 2024-09-14 03:40:24.421549275 +0000 2024-09-14T03:40:24.542 INFO:teuthology.orchestra.run.smithi094.stdout:Modify: 2024-09-14 03:36:24.757404307 +0000 2024-09-14T03:40:24.542 INFO:teuthology.orchestra.run.smithi094.stdout:Change: 2024-09-14 03:36:24.757404307 +0000 2024-09-14T03:40:24.542 INFO:teuthology.orchestra.run.smithi094.stdout: Birth: 2024-09-14 03:36:24.757404307 +0000 2024-09-14T03:40:24.543 DEBUG:teuthology.orchestra.run.smithi094:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-14T03:40:24.648 INFO:teuthology.orchestra.run.smithi094.stderr:1+0 records in 2024-09-14T03:40:24.648 INFO:teuthology.orchestra.run.smithi094.stderr:1+0 records out 2024-09-14T03:40:24.648 INFO:teuthology.orchestra.run.smithi094.stderr:512 bytes copied, 9.3885e-05 s, 5.5 MB/s 2024-09-14T03:40:24.649 DEBUG:teuthology.orchestra.run.smithi094:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-14T03:40:24.665 INFO:tasks.cephadm:Deploying osd.0 on smithi070 with /dev/vg_nvme/lv_4... 2024-09-14T03:40:24.665 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-14T03:40:24.873 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:40:24 smithi094 systemd[1]: Starting Ceph mgr.x for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:40:24.913 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:40:25.135 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:40:24 smithi094 podman[33016]: 2024-09-14 03:40:24.774769279 +0000 UTC m=+0.015636858 image pull ab51d1bebd1200e205d11a014c20a7f2cf49d932afcbeabdc1c9fc887c00f446 quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6 2024-09-14T03:40:25.135 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:40:24 smithi094 podman[33016]: 2024-09-14 03:40:24.879844901 +0000 UTC m=+0.120712513 container create 8a5c729a2096fe7c37e852dfe69eceec7e330a20db989aed67cf6cd618161755 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=quincy-eb16b97, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-14T03:40:25.135 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:40:25 smithi094 podman[33016]: 2024-09-14 03:40:25.106992602 +0000 UTC m=+0.347860170 container init 8a5c729a2096fe7c37e852dfe69eceec7e330a20db989aed67cf6cd618161755 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-eb16b97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-14T03:40:25.135 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:40:25 smithi094 podman[33016]: 2024-09-14 03:40:25.111921378 +0000 UTC m=+0.352788945 container start 8a5c729a2096fe7c37e852dfe69eceec7e330a20db989aed67cf6cd618161755 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-eb16b97, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-14T03:40:25.135 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:40:25 smithi094 ceph-mgr[33054]: -- 172.21.15.94:0/3926904022 <== mon.1 v2:172.21.15.94:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55b068df0340 con 0x55b068dfd000 2024-09-14T03:40:25.485 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:40:25 smithi094 bash[33016]: 8a5c729a2096fe7c37e852dfe69eceec7e330a20db989aed67cf6cd618161755 2024-09-14T03:40:25.485 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:40:25 smithi094 systemd[1]: Started Ceph mgr.x for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:40:25.485 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:40:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:40:25.227+0000 7f5a67e08ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-14T03:40:25.485 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:40:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:40:25.287+0000 7f5a67e08ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-14T03:40:25.485 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:40:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:40:25.354+0000 7f5a67e08ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-14T03:40:25.485 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:40:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:40:25.483+0000 7f5a67e08ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:40:25.777 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:40:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:40:25.635+0000 7f5a67e08ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-14T03:40:25.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:25 smithi094 ceph-mon[31053]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:25.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:25 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:25.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:25 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:25.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:25 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:25.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:25 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:25.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:25 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:25.994 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:25 smithi070 ceph-mon[27495]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:25.994 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:25 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:25.994 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:25 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:25.994 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:25 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:25.995 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:25 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:25.995 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:25 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:25.995 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:25 smithi070 ceph-mon[39288]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:25.995 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:25 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:25.995 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:25 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:25.995 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:25 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:25.995 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:25 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:25.995 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:25 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:26.220 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:40:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:40:25.902+0000 7f5a67e08ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-14T03:40:26.220 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:40:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:40:25.966+0000 7f5a67e08ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-14T03:40:26.220 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:40:26 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:40:26.133+0000 7f5a67e08ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-14T03:40:27.276 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:40:26 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:40:26.986+0000 7f5a67e08ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-14T03:40:27.277 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:40:27 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:40:27.049+0000 7f5a67e08ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-14T03:40:27.348 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:40:27.368 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi070:vg_nvme/lv_4 2024-09-14T03:40:27.621 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:27 smithi070 ceph-mon[27495]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:27.622 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:27 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:27.622 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:27 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:27.622 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:27 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:27.622 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:27 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:27.622 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:27 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:27.622 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:27 smithi070 ceph-mon[27495]: Reconfiguring mgr.y (unknown last config time)... 2024-09-14T03:40:27.622 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:27 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:40:27.622 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:27 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:40:27.622 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:27 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:27.622 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:27 smithi070 ceph-mon[27495]: Reconfiguring daemon mgr.y on smithi070 2024-09-14T03:40:27.623 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:27 smithi070 ceph-mon[39288]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:27.623 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:27 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:27.623 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:27 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:27.623 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:27 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:27.623 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:27 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:27.623 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:27 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:27.623 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:27 smithi070 ceph-mon[39288]: Reconfiguring mgr.y (unknown last config time)... 2024-09-14T03:40:27.623 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:27 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:40:27.623 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:27 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:40:27.623 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:27 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:27.623 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:27 smithi070 ceph-mon[39288]: Reconfiguring daemon mgr.y on smithi070 2024-09-14T03:40:27.776 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:40:27 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:40:27.749+0000 7f5a67e08ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-14T03:40:27.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:27 smithi094 ceph-mon[31053]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:27.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:27 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:27.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:27 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:27.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:27 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:27.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:27 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:27.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:27 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:27.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:27 smithi094 ceph-mon[31053]: Reconfiguring mgr.y (unknown last config time)... 2024-09-14T03:40:27.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:27 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:40:27.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:27 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:40:27.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:27 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:27.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:27 smithi094 ceph-mon[31053]: Reconfiguring daemon mgr.y on smithi070 2024-09-14T03:40:28.059 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:40:28.161 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:40:27 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:40:27.859+0000 7f5a67e08ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-14T03:40:28.161 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:40:27 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:40:27.978+0000 7f5a67e08ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:40:28.161 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:40:28 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:40:28.035+0000 7f5a67e08ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-14T03:40:28.161 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:40:28 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:40:28.159+0000 7f5a67e08ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-14T03:40:28.416 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:40:28 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:40:28.225+0000 7f5a67e08ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-14T03:40:28.416 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:40:28 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:40:28.282+0000 7f5a67e08ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-14T03:40:28.417 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:40:28 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:40:28.415+0000 7f5a67e08ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-14T03:40:28.679 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:40:28 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:40:28.478+0000 7f5a67e08ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-14T03:40:28.680 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:40:28 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:40:28.542+0000 7f5a67e08ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-14T03:40:28.680 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:40:28 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:40:28.621+0000 7f5a67e08ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-14T03:40:28.680 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:40:28 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:40:28.678+0000 7f5a67e08ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-14T03:40:29.026 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:40:28 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:40:28.734+0000 7f5a67e08ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-14T03:40:29.776 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:40:29 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:40:29.475+0000 7f5a67e08ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-14T03:40:29.957 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:29 smithi070 ceph-mon[39288]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:29.957 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:29 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:29.957 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:29 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:29.957 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:29 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:29.957 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:29 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:29.957 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:29 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:29.957 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:29 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:29.957 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:29 smithi070 ceph-mon[39288]: Standby manager daemon x started 2024-09-14T03:40:29.957 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:29 smithi070 ceph-mon[39288]: from='mgr.? 172.21.15.94:0/1405110424' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:40:29.958 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:29 smithi070 ceph-mon[39288]: from='mgr.? 172.21.15.94:0/1405110424' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:40:29.958 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:29 smithi070 ceph-mon[39288]: from='mgr.? 172.21.15.94:0/1405110424' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:40:29.958 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:29 smithi070 ceph-mon[39288]: from='mgr.? 172.21.15.94:0/1405110424' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:40:29.958 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:29 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:29.958 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:29 smithi070 ceph-mon[27495]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:29.958 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:29 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:29.958 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:29 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:29.958 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:29 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:29.958 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:29 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:29.958 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:29 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:29.959 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:29 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:29.959 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:29 smithi070 ceph-mon[27495]: Standby manager daemon x started 2024-09-14T03:40:29.959 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:29 smithi070 ceph-mon[27495]: from='mgr.? 172.21.15.94:0/1405110424' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:40:29.959 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:29 smithi070 ceph-mon[27495]: from='mgr.? 172.21.15.94:0/1405110424' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:40:29.959 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:29 smithi070 ceph-mon[27495]: from='mgr.? 172.21.15.94:0/1405110424' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:40:29.959 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:29 smithi070 ceph-mon[27495]: from='mgr.? 172.21.15.94:0/1405110424' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:40:29.959 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:29 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:30.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:29 smithi094 ceph-mon[31053]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:30.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:29 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:30.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:29 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:30.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:29 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:30.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:29 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:30.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:29 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:30.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:29 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:30.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:29 smithi094 ceph-mon[31053]: Standby manager daemon x started 2024-09-14T03:40:30.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:29 smithi094 ceph-mon[31053]: from='mgr.? 172.21.15.94:0/1405110424' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:40:30.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:29 smithi094 ceph-mon[31053]: from='mgr.? 172.21.15.94:0/1405110424' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:40:30.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:29 smithi094 ceph-mon[31053]: from='mgr.? 172.21.15.94:0/1405110424' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:40:30.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:29 smithi094 ceph-mon[31053]: from='mgr.? 172.21.15.94:0/1405110424' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:40:30.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:29 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:30.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:30 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:40:30.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:30 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:40:30.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:30 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:30.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:30 smithi070 ceph-mon[27495]: mgrmap e14: y(active, since 65s), standbys: x 2024-09-14T03:40:30.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:30 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:40:30.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:30 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:40:30.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:30 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:40:30.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:30 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:30.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:30 smithi070 ceph-mon[39288]: mgrmap e14: y(active, since 65s), standbys: x 2024-09-14T03:40:30.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:30 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:40:31.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:30 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:40:31.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:30 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:40:31.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:30 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:31.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:30 smithi094 ceph-mon[31053]: mgrmap e14: y(active, since 65s), standbys: x 2024-09-14T03:40:31.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:30 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:40:31.964 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:31 smithi070 ceph-mon[27495]: from='client.24119 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:31.964 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:31 smithi070 ceph-mon[27495]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:31.965 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:31 smithi070 ceph-mon[39288]: from='client.24119 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:31.965 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:31 smithi070 ceph-mon[39288]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:32.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:31 smithi094 ceph-mon[31053]: from='client.24119 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:32.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:31 smithi094 ceph-mon[31053]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:33.151 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:32 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/981729510' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a7fabda1-9e60-42f2-af60-eec626181688"}]: dispatch 2024-09-14T03:40:33.152 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:32 smithi070 ceph-mon[27495]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a7fabda1-9e60-42f2-af60-eec626181688"}]: dispatch 2024-09-14T03:40:33.152 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:32 smithi070 ceph-mon[27495]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a7fabda1-9e60-42f2-af60-eec626181688"}]': finished 2024-09-14T03:40:33.152 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:32 smithi070 ceph-mon[27495]: osdmap e5: 1 total, 0 up, 1 in 2024-09-14T03:40:33.152 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:32 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:40:33.152 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:32 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/981729510' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a7fabda1-9e60-42f2-af60-eec626181688"}]: dispatch 2024-09-14T03:40:33.152 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:32 smithi070 ceph-mon[39288]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a7fabda1-9e60-42f2-af60-eec626181688"}]: dispatch 2024-09-14T03:40:33.152 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:32 smithi070 ceph-mon[39288]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a7fabda1-9e60-42f2-af60-eec626181688"}]': finished 2024-09-14T03:40:33.152 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:32 smithi070 ceph-mon[39288]: osdmap e5: 1 total, 0 up, 1 in 2024-09-14T03:40:33.153 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:32 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:40:33.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:32 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/981729510' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a7fabda1-9e60-42f2-af60-eec626181688"}]: dispatch 2024-09-14T03:40:33.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:32 smithi094 ceph-mon[31053]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a7fabda1-9e60-42f2-af60-eec626181688"}]: dispatch 2024-09-14T03:40:33.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:32 smithi094 ceph-mon[31053]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a7fabda1-9e60-42f2-af60-eec626181688"}]': finished 2024-09-14T03:40:33.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:32 smithi094 ceph-mon[31053]: osdmap e5: 1 total, 0 up, 1 in 2024-09-14T03:40:33.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:32 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:40:34.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:33 smithi094 ceph-mon[31053]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:34.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:33 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/277930011' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:40:34.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:33 smithi070 ceph-mon[27495]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:34.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:33 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/277930011' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:40:34.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:33 smithi070 ceph-mon[39288]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:34.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:33 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/277930011' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:40:35.939 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:35 smithi070 ceph-mon[27495]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:35.939 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:35 smithi070 ceph-mon[39288]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:36.079 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:35 smithi094 ceph-mon[31053]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:38.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:37 smithi094 ceph-mon[31053]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:38.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:37 smithi070 ceph-mon[27495]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:38.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:37 smithi070 ceph-mon[39288]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:39.059 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:38 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-14T03:40:39.059 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:38 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:39.060 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:38 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-14T03:40:39.060 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:38 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:39.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:38 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-14T03:40:39.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:38 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:39.907 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:39 smithi070 ceph-mon[27495]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:39.907 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:39 smithi070 ceph-mon[27495]: Deploying daemon osd.0 on smithi070 2024-09-14T03:40:39.907 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:39 smithi070 ceph-mon[39288]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:39.907 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:39 smithi070 ceph-mon[39288]: Deploying daemon osd.0 on smithi070 2024-09-14T03:40:40.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:39 smithi094 ceph-mon[31053]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:40.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:39 smithi094 ceph-mon[31053]: Deploying daemon osd.0 on smithi070 2024-09-14T03:40:42.108 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:41 smithi070 ceph-mon[27495]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:42.109 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:41 smithi070 ceph-mon[39288]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:42.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:41 smithi094 ceph-mon[31053]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:44.116 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:43 smithi070 ceph-mon[39288]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:44.116 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:43 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:44.117 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:43 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:44.117 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:43 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:44.117 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:43 smithi070 ceph-mon[27495]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:44.117 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:43 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:44.117 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:43 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:44.117 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:43 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:44.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:43 smithi094 ceph-mon[31053]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:44.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:43 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:44.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:43 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:44.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:43 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:46.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:45 smithi070 ceph-mon[27495]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:46.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:45 smithi070 ceph-mon[39288]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:46.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:45 smithi094 ceph-mon[31053]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:46.892 INFO:teuthology.orchestra.run.smithi070.stdout:Created osd(s) 0 on host 'smithi070' 2024-09-14T03:40:47.468 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:47 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:47.468 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:47 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:47.468 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:47 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:47.468 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:47 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:47.468 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:47 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:47.468 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:47 smithi070 ceph-mon[39288]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:47.468 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:47 smithi070 ceph-mon[39288]: from='osd.0 [v2:172.21.15.70:6802/3822384296,v1:172.21.15.70:6803/3822384296]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-14T03:40:47.468 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:47 smithi070 ceph-mon[39288]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-14T03:40:47.469 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:47 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:47.469 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:47 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:47.469 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:47 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:47.469 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:47 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:47.469 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:47 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:47.469 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:47 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:47.469 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:47 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:47.469 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:47 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:47.469 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:47 smithi070 ceph-mon[27495]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:47.469 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:47 smithi070 ceph-mon[27495]: from='osd.0 [v2:172.21.15.70:6802/3822384296,v1:172.21.15.70:6803/3822384296]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-14T03:40:47.469 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:47 smithi070 ceph-mon[27495]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-14T03:40:47.469 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:47 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:47.470 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:47 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:47.470 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:47 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:47.492 DEBUG:teuthology.orchestra.run.smithi070:osd.0> sudo journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.0.service 2024-09-14T03:40:47.495 INFO:tasks.cephadm:Deploying osd.1 on smithi070 with /dev/vg_nvme/lv_3... 2024-09-14T03:40:47.495 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-14T03:40:47.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:47 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:47.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:47 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:47.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:47 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:47.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:47 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:47.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:47 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:47.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:47 smithi094 ceph-mon[31053]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:47.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:47 smithi094 ceph-mon[31053]: from='osd.0 [v2:172.21.15.70:6802/3822384296,v1:172.21.15.70:6803/3822384296]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-14T03:40:47.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:47 smithi094 ceph-mon[31053]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-14T03:40:47.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:47 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:40:47.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:47 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:47.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:47 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:48.295 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:40:48.415 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:48 smithi070 ceph-mon[39288]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-14T03:40:48.415 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:48 smithi070 ceph-mon[39288]: osdmap e6: 1 total, 0 up, 1 in 2024-09-14T03:40:48.416 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:48 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:40:48.416 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:48 smithi070 ceph-mon[39288]: from='osd.0 [v2:172.21.15.70:6802/3822384296,v1:172.21.15.70:6803/3822384296]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-09-14T03:40:48.416 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:48 smithi070 ceph-mon[39288]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-09-14T03:40:48.416 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:48 smithi070 ceph-mon[27495]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-14T03:40:48.416 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:48 smithi070 ceph-mon[27495]: osdmap e6: 1 total, 0 up, 1 in 2024-09-14T03:40:48.416 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:48 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:40:48.416 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:48 smithi070 ceph-mon[27495]: from='osd.0 [v2:172.21.15.70:6802/3822384296,v1:172.21.15.70:6803/3822384296]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-09-14T03:40:48.416 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:48 smithi070 ceph-mon[27495]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-09-14T03:40:48.417 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:40:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0[43672]: 2024-09-14T03:40:48.162+0000 7f98ac2ad640 -1 osd.0 0 waiting for initial osdmap 2024-09-14T03:40:48.417 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:40:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0[43672]: 2024-09-14T03:40:48.164+0000 7f98a90e8640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:40:48.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:48 smithi094 ceph-mon[31053]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-14T03:40:48.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:48 smithi094 ceph-mon[31053]: osdmap e6: 1 total, 0 up, 1 in 2024-09-14T03:40:48.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:48 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:40:48.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:48 smithi094 ceph-mon[31053]: from='osd.0 [v2:172.21.15.70:6802/3822384296,v1:172.21.15.70:6803/3822384296]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-09-14T03:40:48.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:48 smithi094 ceph-mon[31053]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-09-14T03:40:49.321 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:49 smithi070 ceph-mon[27495]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2024-09-14T03:40:49.321 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:49 smithi070 ceph-mon[27495]: osdmap e7: 1 total, 0 up, 1 in 2024-09-14T03:40:49.321 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:49 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:40:49.322 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:49 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:40:49.322 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:49 smithi070 ceph-mon[27495]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:49.322 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:49 smithi070 ceph-mon[39288]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2024-09-14T03:40:49.322 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:49 smithi070 ceph-mon[39288]: osdmap e7: 1 total, 0 up, 1 in 2024-09-14T03:40:49.322 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:49 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:40:49.322 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:49 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:40:49.322 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:49 smithi070 ceph-mon[39288]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:49.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:49 smithi094 ceph-mon[31053]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2024-09-14T03:40:49.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:49 smithi094 ceph-mon[31053]: osdmap e7: 1 total, 0 up, 1 in 2024-09-14T03:40:49.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:49 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:40:49.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:49 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:40:49.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:49 smithi094 ceph-mon[31053]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:40:50.495 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:50 smithi070 ceph-mon[27495]: purged_snaps scrub starts 2024-09-14T03:40:50.495 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:50 smithi070 ceph-mon[27495]: purged_snaps scrub ok 2024-09-14T03:40:50.495 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:50 smithi070 ceph-mon[27495]: osd.0 [v2:172.21.15.70:6802/3822384296,v1:172.21.15.70:6803/3822384296] boot 2024-09-14T03:40:50.495 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:50 smithi070 ceph-mon[27495]: osdmap e8: 1 total, 1 up, 1 in 2024-09-14T03:40:50.495 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:50 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:40:50.495 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:50 smithi070 ceph-mon[39288]: purged_snaps scrub starts 2024-09-14T03:40:50.495 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:50 smithi070 ceph-mon[39288]: purged_snaps scrub ok 2024-09-14T03:40:50.495 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:50 smithi070 ceph-mon[39288]: osd.0 [v2:172.21.15.70:6802/3822384296,v1:172.21.15.70:6803/3822384296] boot 2024-09-14T03:40:50.496 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:50 smithi070 ceph-mon[39288]: osdmap e8: 1 total, 1 up, 1 in 2024-09-14T03:40:50.496 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:50 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:40:50.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:50 smithi094 ceph-mon[31053]: purged_snaps scrub starts 2024-09-14T03:40:50.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:50 smithi094 ceph-mon[31053]: purged_snaps scrub ok 2024-09-14T03:40:50.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:50 smithi094 ceph-mon[31053]: osd.0 [v2:172.21.15.70:6802/3822384296,v1:172.21.15.70:6803/3822384296] boot 2024-09-14T03:40:50.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:50 smithi094 ceph-mon[31053]: osdmap e8: 1 total, 1 up, 1 in 2024-09-14T03:40:50.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:50 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:40:51.494 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:51 smithi070 ceph-mon[27495]: osdmap e9: 1 total, 1 up, 1 in 2024-09-14T03:40:51.494 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:51 smithi070 ceph-mon[27495]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:40:51.494 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:51 smithi070 ceph-mon[27495]: Detected new or changed devices on smithi070 2024-09-14T03:40:51.494 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:51 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:51.494 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:51 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:51.494 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:51 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:40:51.494 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:51 smithi070 ceph-mon[27495]: Adjusting osd_memory_target on smithi070 to 16064M 2024-09-14T03:40:51.494 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:51 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:51.494 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:51 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:51.494 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:51 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:51.495 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:51 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:51.495 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:51 smithi070 ceph-mon[39288]: osdmap e9: 1 total, 1 up, 1 in 2024-09-14T03:40:51.495 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:51 smithi070 ceph-mon[39288]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:40:51.495 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:51 smithi070 ceph-mon[39288]: Detected new or changed devices on smithi070 2024-09-14T03:40:51.495 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:51 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:51.495 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:51 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:51.495 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:51 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:40:51.495 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:51 smithi070 ceph-mon[39288]: Adjusting osd_memory_target on smithi070 to 16064M 2024-09-14T03:40:51.495 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:51 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:51.495 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:51 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:51.496 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:51 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:51.496 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:51 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:51.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:51 smithi094 ceph-mon[31053]: osdmap e9: 1 total, 1 up, 1 in 2024-09-14T03:40:51.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:51 smithi094 ceph-mon[31053]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:40:51.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:51 smithi094 ceph-mon[31053]: Detected new or changed devices on smithi070 2024-09-14T03:40:51.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:51 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:51.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:51 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:51.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:51 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:40:51.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:51 smithi094 ceph-mon[31053]: Adjusting osd_memory_target on smithi070 to 16064M 2024-09-14T03:40:51.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:51 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:51.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:51 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:51.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:51 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:40:51.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:51 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:40:52.186 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:40:52.206 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi070:vg_nvme/lv_3 2024-09-14T03:40:52.448 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:40:53.577 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:53 smithi070 ceph-mon[27495]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:40:53.578 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:53 smithi070 ceph-mon[39288]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:40:53.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:53 smithi094 ceph-mon[31053]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:40:54.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:54 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:40:54.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:54 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:40:54.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:54 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:54.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:54 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:40:54.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:54 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:40:54.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:54 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:54.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:54 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:40:54.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:54 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:40:54.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:54 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:40:55.637 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:55 smithi070 ceph-mon[27495]: from='client.14238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:55.637 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:55 smithi070 ceph-mon[27495]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:40:55.637 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:55 smithi070 ceph-mon[39288]: from='client.14238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:55.637 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:55 smithi070 ceph-mon[39288]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:40:55.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:55 smithi094 ceph-mon[31053]: from='client.14238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:40:55.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:55 smithi094 ceph-mon[31053]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:40:57.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:57 smithi094 ceph-mon[31053]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:40:57.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:57 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/3882935967' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f5b9ef91-59af-4ef0-a101-1c00ba6007ad"}]: dispatch 2024-09-14T03:40:57.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:57 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/3882935967' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f5b9ef91-59af-4ef0-a101-1c00ba6007ad"}]': finished 2024-09-14T03:40:57.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:57 smithi094 ceph-mon[31053]: osdmap e10: 2 total, 1 up, 2 in 2024-09-14T03:40:57.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:57 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:40:57.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:57 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/3643022415' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:40:57.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:57 smithi070 ceph-mon[39288]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:40:57.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:57 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/3882935967' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f5b9ef91-59af-4ef0-a101-1c00ba6007ad"}]: dispatch 2024-09-14T03:40:57.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:57 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/3882935967' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f5b9ef91-59af-4ef0-a101-1c00ba6007ad"}]': finished 2024-09-14T03:40:57.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:57 smithi070 ceph-mon[39288]: osdmap e10: 2 total, 1 up, 2 in 2024-09-14T03:40:57.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:57 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:40:57.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:57 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/3643022415' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:40:57.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:57 smithi070 ceph-mon[27495]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:40:57.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:57 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/3882935967' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f5b9ef91-59af-4ef0-a101-1c00ba6007ad"}]: dispatch 2024-09-14T03:40:57.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:57 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/3882935967' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f5b9ef91-59af-4ef0-a101-1c00ba6007ad"}]': finished 2024-09-14T03:40:57.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:57 smithi070 ceph-mon[27495]: osdmap e10: 2 total, 1 up, 2 in 2024-09-14T03:40:57.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:57 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:40:57.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:57 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/3643022415' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:40:59.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:40:59 smithi094 ceph-mon[31053]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:40:59.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:40:59 smithi070 ceph-mon[39288]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:40:59.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:40:59 smithi070 ceph-mon[27495]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:41:01.743 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:01 smithi070 ceph-mon[27495]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:41:01.743 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:01 smithi070 ceph-mon[39288]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:41:02.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:01 smithi094 ceph-mon[31053]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:41:03.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:03 smithi070 ceph-mon[27495]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:41:03.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:03 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-14T03:41:03.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:03 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:03.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:03 smithi070 ceph-mon[27495]: Deploying daemon osd.1 on smithi070 2024-09-14T03:41:03.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:03 smithi070 ceph-mon[39288]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:41:03.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:03 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-14T03:41:03.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:03 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:03.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:03 smithi070 ceph-mon[39288]: Deploying daemon osd.1 on smithi070 2024-09-14T03:41:04.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:03 smithi094 ceph-mon[31053]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:41:04.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:03 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-14T03:41:04.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:03 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:04.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:03 smithi094 ceph-mon[31053]: Deploying daemon osd.1 on smithi070 2024-09-14T03:41:05.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:05 smithi070 ceph-mon[27495]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:41:05.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:05 smithi070 ceph-mon[39288]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:41:06.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:05 smithi094 ceph-mon[31053]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:41:07.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:07 smithi070 ceph-mon[27495]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:41:07.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:07 smithi070 ceph-mon[39288]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:41:07.829 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:07 smithi094 ceph-mon[31053]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:41:08.769 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:41:08.769 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:08.770 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:08.770 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:08 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:41:08.770 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:08 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:08.770 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:08 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:09.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:08 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:41:09.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:08 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:09.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:08 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:09.992 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:09 smithi070 ceph-mon[27495]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:41:09.993 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:09 smithi070 ceph-mon[39288]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:41:10.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:09 smithi094 ceph-mon[31053]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:41:10.828 INFO:teuthology.orchestra.run.smithi070.stdout:Created osd(s) 1 on host 'smithi070' 2024-09-14T03:41:11.510 DEBUG:teuthology.orchestra.run.smithi070:osd.1> sudo journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.1.service 2024-09-14T03:41:11.512 INFO:tasks.cephadm:Deploying osd.2 on smithi070 with /dev/vg_nvme/lv_2... 2024-09-14T03:41:11.512 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-14T03:41:11.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:11 smithi070 ceph-mon[27495]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:41:11.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:11 smithi070 ceph-mon[27495]: from='osd.1 [v2:172.21.15.70:6810/3986345052,v1:172.21.15.70:6811/3986345052]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-14T03:41:11.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:11 smithi070 ceph-mon[27495]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-14T03:41:11.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:11 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:11.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:11 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:11.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:11 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:11.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:11 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:11.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:11 smithi070 ceph-mon[39288]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:41:11.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:11 smithi070 ceph-mon[39288]: from='osd.1 [v2:172.21.15.70:6810/3986345052,v1:172.21.15.70:6811/3986345052]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-14T03:41:11.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:11 smithi070 ceph-mon[39288]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-14T03:41:11.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:11 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:11.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:11 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:11.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:11 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:11.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:11 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:11.811 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:41:12.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:11 smithi094 ceph-mon[31053]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:41:12.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:11 smithi094 ceph-mon[31053]: from='osd.1 [v2:172.21.15.70:6810/3986345052,v1:172.21.15.70:6811/3986345052]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-14T03:41:12.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:11 smithi094 ceph-mon[31053]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-14T03:41:12.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:11 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:12.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:11 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:12.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:11 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:12.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:11 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:13.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:12 smithi094 ceph-mon[31053]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-14T03:41:13.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:12 smithi094 ceph-mon[31053]: osdmap e11: 2 total, 1 up, 2 in 2024-09-14T03:41:13.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:12 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:41:13.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:12 smithi094 ceph-mon[31053]: from='osd.1 [v2:172.21.15.70:6810/3986345052,v1:172.21.15.70:6811/3986345052]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-09-14T03:41:13.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:12 smithi094 ceph-mon[31053]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-09-14T03:41:13.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:12 smithi070 ceph-mon[39288]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-14T03:41:13.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:12 smithi070 ceph-mon[39288]: osdmap e11: 2 total, 1 up, 2 in 2024-09-14T03:41:13.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:12 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:41:13.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:12 smithi070 ceph-mon[39288]: from='osd.1 [v2:172.21.15.70:6810/3986345052,v1:172.21.15.70:6811/3986345052]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-09-14T03:41:13.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:12 smithi070 ceph-mon[39288]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-09-14T03:41:13.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:12 smithi070 ceph-mon[27495]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-14T03:41:13.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:12 smithi070 ceph-mon[27495]: osdmap e11: 2 total, 1 up, 2 in 2024-09-14T03:41:13.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:12 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:41:13.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:12 smithi070 ceph-mon[27495]: from='osd.1 [v2:172.21.15.70:6810/3986345052,v1:172.21.15.70:6811/3986345052]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-09-14T03:41:13.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:12 smithi070 ceph-mon[27495]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-09-14T03:41:13.037 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:41:12 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1[49601]: 2024-09-14T03:41:12.683+0000 7f961cc6e640 -1 osd.1 0 waiting for initial osdmap 2024-09-14T03:41:13.037 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:41:12 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1[49601]: 2024-09-14T03:41:12.685+0000 7f9617aa5640 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:41:13.971 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:13 smithi070 ceph-mon[27495]: pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:41:13.971 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:13 smithi070 ceph-mon[27495]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2024-09-14T03:41:13.971 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:13 smithi070 ceph-mon[27495]: osdmap e12: 2 total, 1 up, 2 in 2024-09-14T03:41:13.971 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:13 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:41:13.971 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:13 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:41:13.972 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:13 smithi070 ceph-mon[39288]: pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:41:13.972 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:13 smithi070 ceph-mon[39288]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2024-09-14T03:41:13.972 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:13 smithi070 ceph-mon[39288]: osdmap e12: 2 total, 1 up, 2 in 2024-09-14T03:41:13.972 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:13 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:41:13.972 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:13 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:41:14.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:13 smithi094 ceph-mon[31053]: pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:41:14.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:13 smithi094 ceph-mon[31053]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2024-09-14T03:41:14.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:13 smithi094 ceph-mon[31053]: osdmap e12: 2 total, 1 up, 2 in 2024-09-14T03:41:14.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:13 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:41:14.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:13 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:41:15.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:14 smithi094 ceph-mon[31053]: purged_snaps scrub starts 2024-09-14T03:41:15.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:14 smithi094 ceph-mon[31053]: purged_snaps scrub ok 2024-09-14T03:41:15.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:14 smithi094 ceph-mon[31053]: osd.1 [v2:172.21.15.70:6810/3986345052,v1:172.21.15.70:6811/3986345052] boot 2024-09-14T03:41:15.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:14 smithi094 ceph-mon[31053]: osdmap e13: 2 total, 2 up, 2 in 2024-09-14T03:41:15.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:14 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:41:15.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:14 smithi070 ceph-mon[27495]: purged_snaps scrub starts 2024-09-14T03:41:15.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:14 smithi070 ceph-mon[27495]: purged_snaps scrub ok 2024-09-14T03:41:15.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:14 smithi070 ceph-mon[27495]: osd.1 [v2:172.21.15.70:6810/3986345052,v1:172.21.15.70:6811/3986345052] boot 2024-09-14T03:41:15.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:14 smithi070 ceph-mon[27495]: osdmap e13: 2 total, 2 up, 2 in 2024-09-14T03:41:15.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:14 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:41:15.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:14 smithi070 ceph-mon[39288]: purged_snaps scrub starts 2024-09-14T03:41:15.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:14 smithi070 ceph-mon[39288]: purged_snaps scrub ok 2024-09-14T03:41:15.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:14 smithi070 ceph-mon[39288]: osd.1 [v2:172.21.15.70:6810/3986345052,v1:172.21.15.70:6811/3986345052] boot 2024-09-14T03:41:15.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:14 smithi070 ceph-mon[39288]: osdmap e13: 2 total, 2 up, 2 in 2024-09-14T03:41:15.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:14 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:41:15.115 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:41:15.135 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi070:vg_nvme/lv_2 2024-09-14T03:41:15.657 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:41:15.943 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:15 smithi070 ceph-mon[27495]: pgmap v58: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:41:15.944 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:15 smithi070 ceph-mon[27495]: osdmap e14: 2 total, 2 up, 2 in 2024-09-14T03:41:15.944 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:15 smithi070 ceph-mon[39288]: pgmap v58: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:41:15.944 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:15 smithi070 ceph-mon[39288]: osdmap e14: 2 total, 2 up, 2 in 2024-09-14T03:41:16.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:15 smithi094 ceph-mon[31053]: pgmap v58: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:41:16.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:15 smithi094 ceph-mon[31053]: osdmap e14: 2 total, 2 up, 2 in 2024-09-14T03:41:17.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:17 smithi070 ceph-mon[27495]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:41:17.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:17 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:17.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:17 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:17.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:17 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:41:17.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:17 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:41:17.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:17 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:17.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:17 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:17.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:17 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:41:17.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:17 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:17.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:17 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:41:17.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:17 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:17.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:17 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:41:17.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:17 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:17.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:17 smithi070 ceph-mon[39288]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:41:17.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:17 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:17.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:17 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:17.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:17 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:41:17.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:17 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:41:17.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:17 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:17.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:17 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:17.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:17 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:41:17.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:17 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:17.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:17 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:41:17.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:17 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:17.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:17 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:41:17.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:17 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:18.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:17 smithi094 ceph-mon[31053]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:41:18.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:17 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:18.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:17 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:18.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:17 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:41:18.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:17 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:41:18.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:17 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:18.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:17 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:18.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:17 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:41:18.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:17 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:18.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:17 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:41:18.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:17 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:18.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:17 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:41:18.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:17 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:18.854 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:18 smithi070 ceph-mon[27495]: Detected new or changed devices on smithi070 2024-09-14T03:41:18.854 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:18 smithi070 ceph-mon[27495]: Adjusting osd_memory_target on smithi070 to 8032M 2024-09-14T03:41:18.854 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:18 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:41:18.854 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:18 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:41:18.854 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:18 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:18.854 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:18 smithi070 ceph-mon[39288]: Detected new or changed devices on smithi070 2024-09-14T03:41:18.855 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:18 smithi070 ceph-mon[39288]: Adjusting osd_memory_target on smithi070 to 8032M 2024-09-14T03:41:18.855 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:18 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:41:18.855 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:18 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:41:18.855 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:18 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:19.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:18 smithi094 ceph-mon[31053]: Detected new or changed devices on smithi070 2024-09-14T03:41:19.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:18 smithi094 ceph-mon[31053]: Adjusting osd_memory_target on smithi070 to 8032M 2024-09-14T03:41:19.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:18 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:41:19.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:18 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:41:19.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:18 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:19.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:19 smithi070 ceph-mon[27495]: from='client.14265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:19.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:19 smithi070 ceph-mon[27495]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:41:19.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:19 smithi070 ceph-mon[39288]: from='client.14265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:19.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:19 smithi070 ceph-mon[39288]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:41:20.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:19 smithi094 ceph-mon[31053]: from='client.14265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:20.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:19 smithi094 ceph-mon[31053]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:41:20.988 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:20 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/2894834391' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0dc07495-c6a5-4762-8f63-575eb0fe1671"}]: dispatch 2024-09-14T03:41:20.988 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:20 smithi070 ceph-mon[27495]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0dc07495-c6a5-4762-8f63-575eb0fe1671"}]: dispatch 2024-09-14T03:41:20.988 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:20 smithi070 ceph-mon[27495]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0dc07495-c6a5-4762-8f63-575eb0fe1671"}]': finished 2024-09-14T03:41:20.989 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:20 smithi070 ceph-mon[27495]: osdmap e15: 3 total, 2 up, 3 in 2024-09-14T03:41:20.989 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:20 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:41:20.989 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:20 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/2894834391' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0dc07495-c6a5-4762-8f63-575eb0fe1671"}]: dispatch 2024-09-14T03:41:20.989 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:20 smithi070 ceph-mon[39288]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0dc07495-c6a5-4762-8f63-575eb0fe1671"}]: dispatch 2024-09-14T03:41:20.989 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:20 smithi070 ceph-mon[39288]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0dc07495-c6a5-4762-8f63-575eb0fe1671"}]': finished 2024-09-14T03:41:20.989 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:20 smithi070 ceph-mon[39288]: osdmap e15: 3 total, 2 up, 3 in 2024-09-14T03:41:20.989 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:20 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:41:21.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:20 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/2894834391' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0dc07495-c6a5-4762-8f63-575eb0fe1671"}]: dispatch 2024-09-14T03:41:21.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:20 smithi094 ceph-mon[31053]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0dc07495-c6a5-4762-8f63-575eb0fe1671"}]: dispatch 2024-09-14T03:41:21.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:20 smithi094 ceph-mon[31053]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0dc07495-c6a5-4762-8f63-575eb0fe1671"}]': finished 2024-09-14T03:41:21.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:20 smithi094 ceph-mon[31053]: osdmap e15: 3 total, 2 up, 3 in 2024-09-14T03:41:21.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:20 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:41:22.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:21 smithi094 ceph-mon[31053]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:41:22.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:21 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/3641933885' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:41:22.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:21 smithi070 ceph-mon[27495]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:41:22.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:21 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/3641933885' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:41:22.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:21 smithi070 ceph-mon[39288]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:41:22.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:21 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/3641933885' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:41:23.882 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:23 smithi070 ceph-mon[27495]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:41:23.882 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:23 smithi070 ceph-mon[39288]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:41:24.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:23 smithi094 ceph-mon[31053]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:41:25.808 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:25 smithi070 ceph-mon[27495]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:41:25.808 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:25 smithi070 ceph-mon[39288]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:41:26.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:25 smithi094 ceph-mon[31053]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:41:28.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:27 smithi094 ceph-mon[31053]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:41:28.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:27 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-14T03:41:28.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:27 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:28.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:27 smithi094 ceph-mon[31053]: Deploying daemon osd.2 on smithi070 2024-09-14T03:41:28.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:27 smithi070 ceph-mon[39288]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:41:28.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:27 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-14T03:41:28.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:27 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:28.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:27 smithi070 ceph-mon[39288]: Deploying daemon osd.2 on smithi070 2024-09-14T03:41:28.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:27 smithi070 ceph-mon[27495]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:41:28.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:27 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-14T03:41:28.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:27 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:28.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:27 smithi070 ceph-mon[27495]: Deploying daemon osd.2 on smithi070 2024-09-14T03:41:29.933 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:29 smithi070 ceph-mon[27495]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:41:29.933 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:29 smithi070 ceph-mon[39288]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:41:30.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:29 smithi094 ceph-mon[31053]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:41:31.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:31 smithi070 ceph-mon[39288]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:41:31.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:31 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:41:31.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:31 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:31.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:31 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:31.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:31 smithi070 ceph-mon[27495]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:41:31.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:31 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:41:31.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:31 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:31.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:31 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:32.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:31 smithi094 ceph-mon[31053]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:41:32.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:31 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:41:32.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:31 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:32.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:31 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:34.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:33 smithi094 ceph-mon[31053]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:41:34.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:33 smithi070 ceph-mon[27495]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:41:34.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:33 smithi070 ceph-mon[39288]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:41:35.007 INFO:teuthology.orchestra.run.smithi070.stdout:Created osd(s) 2 on host 'smithi070' 2024-09-14T03:41:35.653 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:35 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:35.653 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:35 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:35.653 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:35 smithi070 ceph-mon[27495]: pgmap v70: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:41:35.653 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:35 smithi070 ceph-mon[27495]: from='osd.2 [v2:172.21.15.70:6818/2374764370,v1:172.21.15.70:6819/2374764370]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-14T03:41:35.653 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:35 smithi070 ceph-mon[27495]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-14T03:41:35.653 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:35 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:35.654 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:35 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:41:35.654 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:35 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:35.654 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:35 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:41:35.654 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:35 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:35.654 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:35 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:35.654 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:35 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:35.654 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:35 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:35.654 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:35 smithi070 ceph-mon[39288]: pgmap v70: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:41:35.654 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:35 smithi070 ceph-mon[39288]: from='osd.2 [v2:172.21.15.70:6818/2374764370,v1:172.21.15.70:6819/2374764370]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-14T03:41:35.654 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:35 smithi070 ceph-mon[39288]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-14T03:41:35.655 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:35 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:35.655 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:35 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:41:35.655 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:35 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:35.655 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:35 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:41:35.655 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:35 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:35.655 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:35 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:35.677 DEBUG:teuthology.orchestra.run.smithi070:osd.2> sudo journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.2.service 2024-09-14T03:41:35.679 INFO:tasks.cephadm:Deploying osd.3 on smithi070 with /dev/vg_nvme/lv_1... 2024-09-14T03:41:35.679 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-14T03:41:35.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:35 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:35.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:35 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:35.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:35 smithi094 ceph-mon[31053]: pgmap v70: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:41:35.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:35 smithi094 ceph-mon[31053]: from='osd.2 [v2:172.21.15.70:6818/2374764370,v1:172.21.15.70:6819/2374764370]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-14T03:41:35.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:35 smithi094 ceph-mon[31053]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-14T03:41:35.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:35 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:35.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:35 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:41:35.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:35 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:35.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:35 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:41:35.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:35 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:35.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:35 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:36.477 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:41:36.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:36 smithi070 ceph-mon[27495]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-14T03:41:36.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:36 smithi070 ceph-mon[27495]: osdmap e16: 3 total, 2 up, 3 in 2024-09-14T03:41:36.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:36 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:41:36.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:36 smithi070 ceph-mon[27495]: from='osd.2 [v2:172.21.15.70:6818/2374764370,v1:172.21.15.70:6819/2374764370]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-09-14T03:41:36.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:36 smithi070 ceph-mon[27495]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-09-14T03:41:36.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:36 smithi070 ceph-mon[39288]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-14T03:41:36.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:36 smithi070 ceph-mon[39288]: osdmap e16: 3 total, 2 up, 3 in 2024-09-14T03:41:36.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:36 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:41:36.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:36 smithi070 ceph-mon[39288]: from='osd.2 [v2:172.21.15.70:6818/2374764370,v1:172.21.15.70:6819/2374764370]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-09-14T03:41:36.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:36 smithi070 ceph-mon[39288]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-09-14T03:41:36.537 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:41:36 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2[55574]: 2024-09-14T03:41:36.452+0000 7fbf7f162640 -1 osd.2 0 waiting for initial osdmap 2024-09-14T03:41:36.537 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:41:36 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2[55574]: 2024-09-14T03:41:36.455+0000 7fbf79f99640 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:41:36.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:36 smithi094 ceph-mon[31053]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-14T03:41:36.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:36 smithi094 ceph-mon[31053]: osdmap e16: 3 total, 2 up, 3 in 2024-09-14T03:41:36.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:36 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:41:36.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:36 smithi094 ceph-mon[31053]: from='osd.2 [v2:172.21.15.70:6818/2374764370,v1:172.21.15.70:6819/2374764370]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-09-14T03:41:36.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:36 smithi094 ceph-mon[31053]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-09-14T03:41:37.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:37 smithi070 ceph-mon[27495]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2024-09-14T03:41:37.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:37 smithi070 ceph-mon[27495]: osdmap e17: 3 total, 2 up, 3 in 2024-09-14T03:41:37.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:37 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:41:37.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:37 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:41:37.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:37 smithi070 ceph-mon[27495]: pgmap v73: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:41:37.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:37 smithi070 ceph-mon[39288]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2024-09-14T03:41:37.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:37 smithi070 ceph-mon[39288]: osdmap e17: 3 total, 2 up, 3 in 2024-09-14T03:41:37.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:37 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:41:37.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:37 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:41:37.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:37 smithi070 ceph-mon[39288]: pgmap v73: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:41:37.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:37 smithi094 ceph-mon[31053]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2024-09-14T03:41:37.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:37 smithi094 ceph-mon[31053]: osdmap e17: 3 total, 2 up, 3 in 2024-09-14T03:41:37.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:37 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:41:37.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:37 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:41:37.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:37 smithi094 ceph-mon[31053]: pgmap v73: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-14T03:41:38.772 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:38 smithi070 ceph-mon[27495]: purged_snaps scrub starts 2024-09-14T03:41:38.772 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:38 smithi070 ceph-mon[27495]: purged_snaps scrub ok 2024-09-14T03:41:38.772 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:38 smithi070 ceph-mon[27495]: osd.2 [v2:172.21.15.70:6818/2374764370,v1:172.21.15.70:6819/2374764370] boot 2024-09-14T03:41:38.772 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:38 smithi070 ceph-mon[27495]: osdmap e18: 3 total, 3 up, 3 in 2024-09-14T03:41:38.772 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:38 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:41:38.772 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:38 smithi070 ceph-mon[39288]: purged_snaps scrub starts 2024-09-14T03:41:38.773 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:38 smithi070 ceph-mon[39288]: purged_snaps scrub ok 2024-09-14T03:41:38.773 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:38 smithi070 ceph-mon[39288]: osd.2 [v2:172.21.15.70:6818/2374764370,v1:172.21.15.70:6819/2374764370] boot 2024-09-14T03:41:38.773 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:38 smithi070 ceph-mon[39288]: osdmap e18: 3 total, 3 up, 3 in 2024-09-14T03:41:38.773 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:38 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:41:38.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:38 smithi094 ceph-mon[31053]: purged_snaps scrub starts 2024-09-14T03:41:38.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:38 smithi094 ceph-mon[31053]: purged_snaps scrub ok 2024-09-14T03:41:38.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:38 smithi094 ceph-mon[31053]: osd.2 [v2:172.21.15.70:6818/2374764370,v1:172.21.15.70:6819/2374764370] boot 2024-09-14T03:41:38.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:38 smithi094 ceph-mon[31053]: osdmap e18: 3 total, 3 up, 3 in 2024-09-14T03:41:38.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:38 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:41:39.751 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:39 smithi070 ceph-mon[27495]: osdmap e19: 3 total, 3 up, 3 in 2024-09-14T03:41:39.751 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:39 smithi070 ceph-mon[27495]: pgmap v76: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:41:39.752 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:39 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-14T03:41:39.752 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:39 smithi070 ceph-mon[27495]: Detected new or changed devices on smithi070 2024-09-14T03:41:39.752 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:39 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:39.752 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:39 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:39.752 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:39 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:41:39.752 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:39 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:41:39.752 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:39 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:41:39.752 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:39 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:39.752 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:39 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:39.752 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:39 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:41:39.752 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:39 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:39.753 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:39 smithi070 ceph-mon[39288]: osdmap e19: 3 total, 3 up, 3 in 2024-09-14T03:41:39.753 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:39 smithi070 ceph-mon[39288]: pgmap v76: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:41:39.753 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:39 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-14T03:41:39.753 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:39 smithi070 ceph-mon[39288]: Detected new or changed devices on smithi070 2024-09-14T03:41:39.753 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:39 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:39.753 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:39 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:39.753 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:39 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:41:39.753 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:39 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:41:39.753 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:39 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:41:39.753 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:39 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:39.753 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:39 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:39.753 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:39 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:41:39.753 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:39 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:39.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:39 smithi094 ceph-mon[31053]: osdmap e19: 3 total, 3 up, 3 in 2024-09-14T03:41:39.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:39 smithi094 ceph-mon[31053]: pgmap v76: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:41:39.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:39 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-14T03:41:39.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:39 smithi094 ceph-mon[31053]: Detected new or changed devices on smithi070 2024-09-14T03:41:39.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:39 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:39.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:39 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:39.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:39 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:41:39.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:39 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:41:39.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:39 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:41:39.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:39 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:39.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:39 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:39.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:39 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:41:39.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:39 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:40.452 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:41:40.476 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi070:vg_nvme/lv_1 2024-09-14T03:41:40.726 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:41:40.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:40 smithi094 ceph-mon[31053]: Adjusting osd_memory_target on smithi070 to 5354M 2024-09-14T03:41:40.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:40 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-14T03:41:40.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:40 smithi094 ceph-mon[31053]: osdmap e20: 3 total, 3 up, 3 in 2024-09-14T03:41:40.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:40 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:41:40.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:40 smithi070 ceph-mon[27495]: Adjusting osd_memory_target on smithi070 to 5354M 2024-09-14T03:41:40.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:40 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-14T03:41:40.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:40 smithi070 ceph-mon[27495]: osdmap e20: 3 total, 3 up, 3 in 2024-09-14T03:41:40.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:40 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:41:40.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:40 smithi070 ceph-mon[39288]: Adjusting osd_memory_target on smithi070 to 5354M 2024-09-14T03:41:40.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:40 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-14T03:41:40.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:40 smithi070 ceph-mon[39288]: osdmap e20: 3 total, 3 up, 3 in 2024-09-14T03:41:40.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:40 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:41:41.115 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:41:40 smithi070 sudo[59708]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-14T03:41:41.115 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:41:40 smithi070 sudo[59708]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:41:41.115 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:41:40 smithi070 sudo[59708]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:41:41.115 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:41:40 smithi070 sudo[59708]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:41:41.472 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:41:41 smithi070 sudo[59747]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-14T03:41:41.472 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:41:41 smithi070 sudo[59747]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:41:41.472 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:41:41 smithi070 sudo[59747]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:41:41.472 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:41:41 smithi070 sudo[59747]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:41:41.737 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:41 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-14T03:41:41.737 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:41 smithi070 ceph-mon[27495]: osdmap e21: 3 total, 3 up, 3 in 2024-09-14T03:41:41.737 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:41 smithi070 ceph-mon[27495]: pgmap v79: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:41:41.737 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:41 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:41.738 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:41 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-14T03:41:41.738 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:41 smithi070 ceph-mon[39288]: osdmap e21: 3 total, 3 up, 3 in 2024-09-14T03:41:41.738 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:41 smithi070 ceph-mon[39288]: pgmap v79: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:41:41.738 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:41 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:41.738 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:41:41 smithi070 sudo[59750]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-14T03:41:41.738 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:41:41 smithi070 sudo[59750]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:41:41.738 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:41:41 smithi070 sudo[59750]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:41:41.738 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:41:41 smithi070 sudo[59750]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:41:41.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:41 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-14T03:41:41.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:41 smithi094 ceph-mon[31053]: osdmap e21: 3 total, 3 up, 3 in 2024-09-14T03:41:41.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:41 smithi094 ceph-mon[31053]: pgmap v79: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:41:41.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:41 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:41.988 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:41:41 smithi070 sudo[59761]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-14T03:41:41.988 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:41:41 smithi070 sudo[59761]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:41:41.988 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:41:41 smithi070 sudo[59761]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:41:41.988 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:41:41 smithi070 sudo[59761]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:41:42.250 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:41:41 smithi070 sudo[59784]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-14T03:41:42.250 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:41:41 smithi070 sudo[59784]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:41:42.251 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:41:41 smithi070 sudo[59784]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:41:42.251 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:41:42 smithi070 sudo[59784]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:41:42.504 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:41:42 smithi070 sudo[59808]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-14T03:41:42.504 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:41:42 smithi070 sudo[59808]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:41:42.504 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:41:42 smithi070 sudo[59808]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:41:42.504 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:41:42 smithi070 sudo[59808]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:41:42.759 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:42 smithi070 sudo[59837]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-14T03:41:42.759 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:42 smithi070 sudo[59837]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:41:42.759 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:42 smithi070 sudo[59837]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:41:42.759 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:42 smithi070 sudo[59837]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:41:43.024 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:42 smithi094 ceph-mon[31053]: osdmap e22: 3 total, 3 up, 3 in 2024-09-14T03:41:43.024 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:42 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:41:43.024 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:43 smithi094 sudo[33706]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-14T03:41:43.025 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:43 smithi094 sudo[33706]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:41:43.025 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:43 smithi094 sudo[33706]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:41:43.025 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:43 smithi094 sudo[33706]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:41:43.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:42 smithi070 ceph-mon[39288]: osdmap e22: 3 total, 3 up, 3 in 2024-09-14T03:41:43.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:42 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:41:43.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:42 smithi070 sudo[59841]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-14T03:41:43.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:42 smithi070 sudo[59841]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:41:43.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:42 smithi070 sudo[59841]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:41:43.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:42 smithi070 sudo[59841]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:41:43.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:42 smithi070 ceph-mon[27495]: osdmap e22: 3 total, 3 up, 3 in 2024-09-14T03:41:43.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:42 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:41:43.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:43 smithi094 sudo[33709]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-14T03:41:43.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:43 smithi094 sudo[33709]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:41:43.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:43 smithi094 sudo[33709]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:41:43.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:43 smithi094 sudo[33709]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:41:43.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:43 smithi070 sudo[60000]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-14T03:41:43.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:43 smithi070 sudo[60000]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:41:43.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:43 smithi070 sudo[60000]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:41:43.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:43 smithi070 sudo[60000]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:41:44.145 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[27495]: pgmap v81: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:41:44.145 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[27495]: from='client.24206 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:44.145 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:41:44.146 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:41:44.146 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:41:44.146 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:41:44.146 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:41:44.146 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:41:44.146 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:41:44.146 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:41:44.146 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:41:44.146 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:41:44.146 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:44.146 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:41:44.146 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:44.146 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:41:44.147 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:41:44.147 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:41:44.147 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:43 smithi070 sudo[60010]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-14T03:41:44.147 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:43 smithi070 sudo[60010]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:41:44.147 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:43 smithi070 sudo[60010]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:41:44.147 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:43 smithi070 sudo[60010]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:41:44.147 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[39288]: pgmap v81: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:41:44.147 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[39288]: from='client.24206 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:44.147 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:41:44.147 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:41:44.147 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:41:44.148 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:41:44.148 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:41:44.148 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:41:44.148 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:41:44.148 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:41:44.148 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:41:44.148 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:41:44.148 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:44.148 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:41:44.148 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:44.148 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:41:44.148 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:41:44.148 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:43 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:41:44.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:43 smithi094 ceph-mon[31053]: pgmap v81: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:41:44.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:43 smithi094 ceph-mon[31053]: from='client.24206 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:41:44.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:43 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:41:44.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:43 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:41:44.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:43 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:41:44.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:43 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:41:44.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:43 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:41:44.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:43 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:41:44.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:43 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:41:44.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:43 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:41:44.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:43 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:41:44.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:43 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:41:44.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:43 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:44.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:43 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:41:44.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:43 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:44.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:43 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:41:44.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:43 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:41:44.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:43 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:41:45.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:44 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:41:45.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:44 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:41:45.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:44 smithi070 ceph-mon[39288]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-14T03:41:45.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:44 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:41:45.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:44 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:41:45.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:44 smithi070 ceph-mon[27495]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-14T03:41:45.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:44 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:41:45.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:44 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:41:45.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:44 smithi094 ceph-mon[31053]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-14T03:41:46.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:45 smithi070 ceph-mon[39288]: pgmap v82: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:41:46.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:45 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/3527286688' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "583e5165-ea10-4d03-b731-36cfb1108362"}]: dispatch 2024-09-14T03:41:46.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:45 smithi070 ceph-mon[39288]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "583e5165-ea10-4d03-b731-36cfb1108362"}]: dispatch 2024-09-14T03:41:46.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:45 smithi070 ceph-mon[39288]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "583e5165-ea10-4d03-b731-36cfb1108362"}]': finished 2024-09-14T03:41:46.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:45 smithi070 ceph-mon[39288]: osdmap e23: 4 total, 3 up, 4 in 2024-09-14T03:41:46.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:45 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:41:46.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:45 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/2763794214' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:41:46.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:45 smithi070 ceph-mon[27495]: pgmap v82: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:41:46.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:45 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/3527286688' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "583e5165-ea10-4d03-b731-36cfb1108362"}]: dispatch 2024-09-14T03:41:46.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:45 smithi070 ceph-mon[27495]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "583e5165-ea10-4d03-b731-36cfb1108362"}]: dispatch 2024-09-14T03:41:46.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:45 smithi070 ceph-mon[27495]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "583e5165-ea10-4d03-b731-36cfb1108362"}]': finished 2024-09-14T03:41:46.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:45 smithi070 ceph-mon[27495]: osdmap e23: 4 total, 3 up, 4 in 2024-09-14T03:41:46.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:45 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:41:46.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:45 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/2763794214' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:41:46.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:45 smithi094 ceph-mon[31053]: pgmap v82: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:41:46.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:45 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/3527286688' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "583e5165-ea10-4d03-b731-36cfb1108362"}]: dispatch 2024-09-14T03:41:46.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:45 smithi094 ceph-mon[31053]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "583e5165-ea10-4d03-b731-36cfb1108362"}]: dispatch 2024-09-14T03:41:46.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:45 smithi094 ceph-mon[31053]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "583e5165-ea10-4d03-b731-36cfb1108362"}]': finished 2024-09-14T03:41:46.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:45 smithi094 ceph-mon[31053]: osdmap e23: 4 total, 3 up, 4 in 2024-09-14T03:41:46.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:45 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:41:46.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:45 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/2763794214' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:41:48.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:47 smithi070 ceph-mon[27495]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:41:48.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:47 smithi070 ceph-mon[39288]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:41:48.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:47 smithi094 ceph-mon[31053]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:41:49.860 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:49 smithi070 ceph-mon[27495]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:41:49.860 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:49 smithi070 ceph-mon[39288]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:41:50.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:49 smithi094 ceph-mon[31053]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:41:51.911 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:51 smithi070 ceph-mon[27495]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:41:51.911 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:51 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-14T03:41:51.911 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:51 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:51.911 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:51 smithi070 ceph-mon[39288]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:41:51.911 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:51 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-14T03:41:51.912 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:51 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:52.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:51 smithi094 ceph-mon[31053]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:41:52.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:51 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-14T03:41:52.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:51 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:41:53.066 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:52 smithi070 ceph-mon[27495]: Deploying daemon osd.3 on smithi070 2024-09-14T03:41:53.067 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:52 smithi070 ceph-mon[39288]: Deploying daemon osd.3 on smithi070 2024-09-14T03:41:53.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:52 smithi094 ceph-mon[31053]: Deploying daemon osd.3 on smithi070 2024-09-14T03:41:54.126 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:53 smithi070 ceph-mon[27495]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:41:54.126 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:53 smithi070 ceph-mon[39288]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:41:54.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:53 smithi094 ceph-mon[31053]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:41:56.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:55 smithi070 ceph-mon[27495]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:41:56.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:55 smithi070 ceph-mon[39288]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:41:56.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:55 smithi094 ceph-mon[31053]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:41:57.024 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:56 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:41:57.024 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:56 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:57.025 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:56 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:57.025 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:56 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:41:57.025 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:56 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:57.025 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:56 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:57.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:56 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:41:57.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:56 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:57.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:56 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:58.132 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:57 smithi070 ceph-mon[27495]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:41:58.132 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:57 smithi070 ceph-mon[39288]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:41:58.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:57 smithi094 ceph-mon[31053]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:41:59.634 INFO:teuthology.orchestra.run.smithi070.stdout:Created osd(s) 3 on host 'smithi070' 2024-09-14T03:41:59.956 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:59 smithi070 ceph-mon[27495]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:41:59.957 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:59 smithi070 ceph-mon[27495]: from='osd.3 [v2:172.21.15.70:6826/2294516176,v1:172.21.15.70:6827/2294516176]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-14T03:41:59.957 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:59 smithi070 ceph-mon[27495]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-14T03:41:59.957 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:59 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:59.957 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:41:59 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:59.957 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:59 smithi070 ceph-mon[39288]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:41:59.957 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:59 smithi070 ceph-mon[39288]: from='osd.3 [v2:172.21.15.70:6826/2294516176,v1:172.21.15.70:6827/2294516176]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-14T03:41:59.957 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:59 smithi070 ceph-mon[39288]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-14T03:41:59.957 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:59 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:41:59.957 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:41:59 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:00.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:59 smithi094 ceph-mon[31053]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:42:00.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:59 smithi094 ceph-mon[31053]: from='osd.3 [v2:172.21.15.70:6826/2294516176,v1:172.21.15.70:6827/2294516176]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-14T03:42:00.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:59 smithi094 ceph-mon[31053]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-14T03:42:00.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:59 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:00.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:41:59 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:00.299 DEBUG:teuthology.orchestra.run.smithi070:osd.3> sudo journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.3.service 2024-09-14T03:42:00.302 INFO:tasks.cephadm:Deploying osd.4 on smithi094 with /dev/vg_nvme/lv_4... 2024-09-14T03:42:00.302 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-14T03:42:00.550 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:42:00.945 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:00 smithi094 ceph-mon[31053]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-14T03:42:00.945 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:00 smithi094 ceph-mon[31053]: osdmap e24: 4 total, 3 up, 4 in 2024-09-14T03:42:00.945 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:00 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:42:00.945 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:00 smithi094 ceph-mon[31053]: from='osd.3 [v2:172.21.15.70:6826/2294516176,v1:172.21.15.70:6827/2294516176]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-09-14T03:42:00.945 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:00 smithi094 ceph-mon[31053]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-09-14T03:42:00.945 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:00 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:00.945 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:00 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:01.135 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:00 smithi070 ceph-mon[27495]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-14T03:42:01.135 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:00 smithi070 ceph-mon[27495]: osdmap e24: 4 total, 3 up, 4 in 2024-09-14T03:42:01.135 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:00 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:42:01.135 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:00 smithi070 ceph-mon[27495]: from='osd.3 [v2:172.21.15.70:6826/2294516176,v1:172.21.15.70:6827/2294516176]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-09-14T03:42:01.136 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:00 smithi070 ceph-mon[27495]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-09-14T03:42:01.136 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:00 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:01.136 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:00 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:01.136 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:00 smithi070 ceph-mon[39288]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-14T03:42:01.136 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:00 smithi070 ceph-mon[39288]: osdmap e24: 4 total, 3 up, 4 in 2024-09-14T03:42:01.136 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:00 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:42:01.136 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:00 smithi070 ceph-mon[39288]: from='osd.3 [v2:172.21.15.70:6826/2294516176,v1:172.21.15.70:6827/2294516176]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-09-14T03:42:01.136 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:00 smithi070 ceph-mon[39288]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-09-14T03:42:01.136 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:00 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:01.137 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:00 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:01.137 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:42:00 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3[61924]: 2024-09-14T03:42:00.834+0000 7fd7cb0ac640 -1 osd.3 0 waiting for initial osdmap 2024-09-14T03:42:01.137 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:42:00 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3[61924]: 2024-09-14T03:42:00.837+0000 7fd7c76e6640 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:42:01.967 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:01 smithi094 ceph-mon[31053]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:42:01.967 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:01 smithi094 ceph-mon[31053]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2024-09-14T03:42:01.967 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:01 smithi094 ceph-mon[31053]: osdmap e25: 4 total, 3 up, 4 in 2024-09-14T03:42:01.967 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:01 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:42:01.967 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:01 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:42:02.097 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:01 smithi070 ceph-mon[39288]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:42:02.097 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:01 smithi070 ceph-mon[39288]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2024-09-14T03:42:02.097 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:01 smithi070 ceph-mon[39288]: osdmap e25: 4 total, 3 up, 4 in 2024-09-14T03:42:02.097 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:01 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:42:02.098 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:01 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:42:02.098 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:01 smithi070 ceph-mon[27495]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:42:02.098 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:01 smithi070 ceph-mon[27495]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2024-09-14T03:42:02.098 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:01 smithi070 ceph-mon[27495]: osdmap e25: 4 total, 3 up, 4 in 2024-09-14T03:42:02.098 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:01 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:42:02.098 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:01 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:42:02.767 INFO:teuthology.orchestra.run.smithi094.stdout: 2024-09-14T03:42:02.786 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi094:vg_nvme/lv_4 2024-09-14T03:42:03.024 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:42:03.112 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:02 smithi094 ceph-mon[31053]: purged_snaps scrub starts 2024-09-14T03:42:03.112 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:02 smithi094 ceph-mon[31053]: purged_snaps scrub ok 2024-09-14T03:42:03.112 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:02 smithi094 ceph-mon[31053]: osd.3 [v2:172.21.15.70:6826/2294516176,v1:172.21.15.70:6827/2294516176] boot 2024-09-14T03:42:03.113 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:02 smithi094 ceph-mon[31053]: osdmap e26: 4 total, 4 up, 4 in 2024-09-14T03:42:03.113 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:02 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:42:03.252 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:02 smithi070 ceph-mon[27495]: purged_snaps scrub starts 2024-09-14T03:42:03.252 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:02 smithi070 ceph-mon[27495]: purged_snaps scrub ok 2024-09-14T03:42:03.252 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:02 smithi070 ceph-mon[27495]: osd.3 [v2:172.21.15.70:6826/2294516176,v1:172.21.15.70:6827/2294516176] boot 2024-09-14T03:42:03.253 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:02 smithi070 ceph-mon[27495]: osdmap e26: 4 total, 4 up, 4 in 2024-09-14T03:42:03.253 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:02 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:42:03.253 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:02 smithi070 ceph-mon[39288]: purged_snaps scrub starts 2024-09-14T03:42:03.253 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:02 smithi070 ceph-mon[39288]: purged_snaps scrub ok 2024-09-14T03:42:03.253 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:02 smithi070 ceph-mon[39288]: osd.3 [v2:172.21.15.70:6826/2294516176,v1:172.21.15.70:6827/2294516176] boot 2024-09-14T03:42:03.253 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:02 smithi070 ceph-mon[39288]: osdmap e26: 4 total, 4 up, 4 in 2024-09-14T03:42:03.253 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:02 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:42:04.209 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:03 smithi094 ceph-mon[31053]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:42:04.209 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:03 smithi094 ceph-mon[31053]: osdmap e27: 4 total, 4 up, 4 in 2024-09-14T03:42:04.210 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:03 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:04.210 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:03 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:04.210 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:03 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:42:04.210 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:03 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:42:04.210 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:03 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:42:04.210 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:03 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:42:04.210 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:03 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:04.210 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:03 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:04.210 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:03 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:42:04.210 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:03 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:04.210 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:03 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:42:04.210 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:03 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:04.210 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:03 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:42:04.211 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:03 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:04.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:03 smithi070 ceph-mon[27495]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:42:04.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:03 smithi070 ceph-mon[27495]: osdmap e27: 4 total, 4 up, 4 in 2024-09-14T03:42:04.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:03 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:04.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:03 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:04.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:03 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:42:04.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:03 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:42:04.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:03 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:42:04.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:03 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:42:04.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:03 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:04.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:03 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:04.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:03 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:42:04.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:03 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:04.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:03 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:42:04.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:03 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:04.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:03 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:42:04.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:03 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:04.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:03 smithi070 ceph-mon[39288]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-14T03:42:04.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:03 smithi070 ceph-mon[39288]: osdmap e27: 4 total, 4 up, 4 in 2024-09-14T03:42:04.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:03 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:04.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:03 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:04.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:03 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:42:04.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:03 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:42:04.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:03 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:42:04.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:03 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:42:04.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:03 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:04.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:03 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:04.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:03 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:42:04.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:03 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:04.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:03 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:42:04.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:03 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:04.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:03 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:42:04.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:03 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:05.168 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:04 smithi094 ceph-mon[31053]: Detected new or changed devices on smithi070 2024-09-14T03:42:05.168 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:04 smithi094 ceph-mon[31053]: Adjusting osd_memory_target on smithi070 to 4016M 2024-09-14T03:42:05.169 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:04 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:42:05.169 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:04 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:42:05.169 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:04 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:05.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:04 smithi070 ceph-mon[27495]: Detected new or changed devices on smithi070 2024-09-14T03:42:05.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:04 smithi070 ceph-mon[27495]: Adjusting osd_memory_target on smithi070 to 4016M 2024-09-14T03:42:05.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:04 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:42:05.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:04 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:42:05.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:04 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:05.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:04 smithi070 ceph-mon[39288]: Detected new or changed devices on smithi070 2024-09-14T03:42:05.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:04 smithi070 ceph-mon[39288]: Adjusting osd_memory_target on smithi070 to 4016M 2024-09-14T03:42:05.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:04 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:42:05.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:04 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:42:05.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:04 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:06.109 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:05 smithi094 ceph-mon[31053]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:42:06.110 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:05 smithi094 ceph-mon[31053]: from='client.24184 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi094:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:06.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:05 smithi070 ceph-mon[27495]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:42:06.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:05 smithi070 ceph-mon[27495]: from='client.24184 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi094:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:06.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:05 smithi070 ceph-mon[39288]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:42:06.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:05 smithi070 ceph-mon[39288]: from='client.24184 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi094:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:08.265 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:07 smithi094 ceph-mon[31053]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:42:08.266 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:07 smithi094 ceph-mon[31053]: from='client.? 172.21.15.94:0/3623958642' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "33177119-9632-431c-9180-9d27e7bc0c42"}]: dispatch 2024-09-14T03:42:08.266 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:07 smithi094 ceph-mon[31053]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "33177119-9632-431c-9180-9d27e7bc0c42"}]: dispatch 2024-09-14T03:42:08.266 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:07 smithi094 ceph-mon[31053]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "33177119-9632-431c-9180-9d27e7bc0c42"}]': finished 2024-09-14T03:42:08.266 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:07 smithi094 ceph-mon[31053]: osdmap e28: 5 total, 4 up, 5 in 2024-09-14T03:42:08.266 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:07 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:42:08.266 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:07 smithi094 ceph-mon[31053]: from='client.? 172.21.15.94:0/1829687991' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:42:08.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:07 smithi070 ceph-mon[27495]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:42:08.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:07 smithi070 ceph-mon[27495]: from='client.? 172.21.15.94:0/3623958642' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "33177119-9632-431c-9180-9d27e7bc0c42"}]: dispatch 2024-09-14T03:42:08.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:07 smithi070 ceph-mon[27495]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "33177119-9632-431c-9180-9d27e7bc0c42"}]: dispatch 2024-09-14T03:42:08.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:07 smithi070 ceph-mon[27495]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "33177119-9632-431c-9180-9d27e7bc0c42"}]': finished 2024-09-14T03:42:08.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:07 smithi070 ceph-mon[27495]: osdmap e28: 5 total, 4 up, 5 in 2024-09-14T03:42:08.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:07 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:42:08.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:07 smithi070 ceph-mon[27495]: from='client.? 172.21.15.94:0/1829687991' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:42:08.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:07 smithi070 ceph-mon[39288]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:42:08.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:07 smithi070 ceph-mon[39288]: from='client.? 172.21.15.94:0/3623958642' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "33177119-9632-431c-9180-9d27e7bc0c42"}]: dispatch 2024-09-14T03:42:08.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:07 smithi070 ceph-mon[39288]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "33177119-9632-431c-9180-9d27e7bc0c42"}]: dispatch 2024-09-14T03:42:08.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:07 smithi070 ceph-mon[39288]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "33177119-9632-431c-9180-9d27e7bc0c42"}]': finished 2024-09-14T03:42:08.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:07 smithi070 ceph-mon[39288]: osdmap e28: 5 total, 4 up, 5 in 2024-09-14T03:42:08.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:07 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:42:08.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:07 smithi070 ceph-mon[39288]: from='client.? 172.21.15.94:0/1829687991' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:42:10.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:09 smithi094 ceph-mon[31053]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:42:10.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:09 smithi070 ceph-mon[27495]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:42:10.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:09 smithi070 ceph-mon[39288]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:42:12.067 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:11 smithi094 ceph-mon[31053]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:42:12.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:11 smithi070 ceph-mon[27495]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:42:12.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:11 smithi070 ceph-mon[39288]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:42:13.130 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:12 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-14T03:42:13.130 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:12 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:13.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:12 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-14T03:42:13.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:12 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:13.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:12 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-14T03:42:13.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:12 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:14.127 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:13 smithi094 ceph-mon[31053]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:42:14.127 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:13 smithi094 ceph-mon[31053]: Deploying daemon osd.4 on smithi094 2024-09-14T03:42:14.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:13 smithi070 ceph-mon[27495]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:42:14.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:13 smithi070 ceph-mon[27495]: Deploying daemon osd.4 on smithi094 2024-09-14T03:42:14.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:13 smithi070 ceph-mon[39288]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:42:14.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:13 smithi070 ceph-mon[39288]: Deploying daemon osd.4 on smithi094 2024-09-14T03:42:16.179 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:15 smithi094 ceph-mon[31053]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:42:16.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:15 smithi070 ceph-mon[27495]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:42:16.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:15 smithi070 ceph-mon[39288]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:42:18.057 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:17 smithi094 ceph-mon[31053]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:42:18.058 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:17 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:42:18.058 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:17 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:18.058 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:17 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:18.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:17 smithi070 ceph-mon[27495]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:42:18.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:17 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:42:18.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:17 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:18.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:17 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:18.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:17 smithi070 ceph-mon[39288]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:42:18.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:17 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:42:18.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:17 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:18.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:17 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:19.946 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:19 smithi094 ceph-mon[31053]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:42:19.946 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:19 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:19.946 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:19 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:19.946 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:19 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:19.946 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:19 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:42:19.947 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:19 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:20.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:19 smithi070 ceph-mon[27495]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:42:20.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:19 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:20.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:19 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:20.290 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:19 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:20.290 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:19 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:42:20.290 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:19 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:20.290 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:19 smithi070 ceph-mon[39288]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:42:20.291 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:19 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:20.291 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:19 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:20.291 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:19 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:20.291 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:19 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:42:20.291 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:19 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:20.727 INFO:teuthology.orchestra.run.smithi094.stdout:Created osd(s) 4 on host 'smithi094' 2024-09-14T03:42:21.022 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:20 smithi094 ceph-mon[31053]: from='osd.4 [v2:172.21.15.94:6800/887215534,v1:172.21.15.94:6801/887215534]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-14T03:42:21.022 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:20 smithi094 ceph-mon[31053]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-14T03:42:21.022 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:20 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:42:21.022 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:20 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:21.022 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:20 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:21.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:20 smithi070 ceph-mon[27495]: from='osd.4 [v2:172.21.15.94:6800/887215534,v1:172.21.15.94:6801/887215534]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-14T03:42:21.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:20 smithi070 ceph-mon[27495]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-14T03:42:21.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:20 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:42:21.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:20 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:21.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:20 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:21.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:20 smithi070 ceph-mon[39288]: from='osd.4 [v2:172.21.15.94:6800/887215534,v1:172.21.15.94:6801/887215534]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-14T03:42:21.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:20 smithi070 ceph-mon[39288]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-14T03:42:21.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:20 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:42:21.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:20 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:21.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:20 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:21.312 DEBUG:teuthology.orchestra.run.smithi094:osd.4> sudo journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.4.service 2024-09-14T03:42:21.314 INFO:tasks.cephadm:Deploying osd.5 on smithi094 with /dev/vg_nvme/lv_3... 2024-09-14T03:42:21.315 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-14T03:42:22.083 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:42:22.186 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:21 smithi094 ceph-mon[31053]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:42:22.186 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:21 smithi094 ceph-mon[31053]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-14T03:42:22.187 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:21 smithi094 ceph-mon[31053]: osdmap e29: 5 total, 4 up, 5 in 2024-09-14T03:42:22.187 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:21 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:42:22.187 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:21 smithi094 ceph-mon[31053]: from='osd.4 [v2:172.21.15.94:6800/887215534,v1:172.21.15.94:6801/887215534]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2024-09-14T03:42:22.187 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:21 smithi094 ceph-mon[31053]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2024-09-14T03:42:22.187 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:42:21 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4[36388]: 2024-09-14T03:42:21.879+0000 7f986ba52640 -1 osd.4 0 waiting for initial osdmap 2024-09-14T03:42:22.187 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:42:21 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4[36388]: 2024-09-14T03:42:21.883+0000 7f986708a640 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:42:22.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:21 smithi070 ceph-mon[27495]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:42:22.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:21 smithi070 ceph-mon[27495]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-14T03:42:22.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:21 smithi070 ceph-mon[27495]: osdmap e29: 5 total, 4 up, 5 in 2024-09-14T03:42:22.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:21 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:42:22.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:21 smithi070 ceph-mon[27495]: from='osd.4 [v2:172.21.15.94:6800/887215534,v1:172.21.15.94:6801/887215534]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2024-09-14T03:42:22.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:21 smithi070 ceph-mon[27495]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2024-09-14T03:42:22.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:21 smithi070 ceph-mon[39288]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:42:22.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:21 smithi070 ceph-mon[39288]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-14T03:42:22.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:21 smithi070 ceph-mon[39288]: osdmap e29: 5 total, 4 up, 5 in 2024-09-14T03:42:22.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:21 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:42:22.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:21 smithi070 ceph-mon[39288]: from='osd.4 [v2:172.21.15.94:6800/887215534,v1:172.21.15.94:6801/887215534]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2024-09-14T03:42:22.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:21 smithi070 ceph-mon[39288]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2024-09-14T03:42:23.110 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:22 smithi094 ceph-mon[31053]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]': finished 2024-09-14T03:42:23.110 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:22 smithi094 ceph-mon[31053]: osdmap e30: 5 total, 4 up, 5 in 2024-09-14T03:42:23.110 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:22 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:42:23.110 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:22 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:42:23.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:22 smithi070 ceph-mon[27495]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]': finished 2024-09-14T03:42:23.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:22 smithi070 ceph-mon[27495]: osdmap e30: 5 total, 4 up, 5 in 2024-09-14T03:42:23.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:22 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:42:23.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:22 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:42:23.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:22 smithi070 ceph-mon[39288]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]': finished 2024-09-14T03:42:23.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:22 smithi070 ceph-mon[39288]: osdmap e30: 5 total, 4 up, 5 in 2024-09-14T03:42:23.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:22 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:42:23.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:22 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:42:24.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:23 smithi094 ceph-mon[31053]: purged_snaps scrub starts 2024-09-14T03:42:24.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:23 smithi094 ceph-mon[31053]: purged_snaps scrub ok 2024-09-14T03:42:24.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:23 smithi094 ceph-mon[31053]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:42:24.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:23 smithi094 ceph-mon[31053]: osd.4 [v2:172.21.15.94:6800/887215534,v1:172.21.15.94:6801/887215534] boot 2024-09-14T03:42:24.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:23 smithi094 ceph-mon[31053]: osdmap e31: 5 total, 5 up, 5 in 2024-09-14T03:42:24.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:23 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:42:24.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:23 smithi070 ceph-mon[27495]: purged_snaps scrub starts 2024-09-14T03:42:24.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:23 smithi070 ceph-mon[27495]: purged_snaps scrub ok 2024-09-14T03:42:24.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:23 smithi070 ceph-mon[27495]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:42:24.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:23 smithi070 ceph-mon[27495]: osd.4 [v2:172.21.15.94:6800/887215534,v1:172.21.15.94:6801/887215534] boot 2024-09-14T03:42:24.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:23 smithi070 ceph-mon[27495]: osdmap e31: 5 total, 5 up, 5 in 2024-09-14T03:42:24.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:23 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:42:24.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:23 smithi070 ceph-mon[39288]: purged_snaps scrub starts 2024-09-14T03:42:24.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:23 smithi070 ceph-mon[39288]: purged_snaps scrub ok 2024-09-14T03:42:24.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:23 smithi070 ceph-mon[39288]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-14T03:42:24.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:23 smithi070 ceph-mon[39288]: osd.4 [v2:172.21.15.94:6800/887215534,v1:172.21.15.94:6801/887215534] boot 2024-09-14T03:42:24.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:23 smithi070 ceph-mon[39288]: osdmap e31: 5 total, 5 up, 5 in 2024-09-14T03:42:24.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:23 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:42:25.138 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:24 smithi094 ceph-mon[31053]: osdmap e32: 5 total, 5 up, 5 in 2024-09-14T03:42:25.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:24 smithi070 ceph-mon[39288]: osdmap e32: 5 total, 5 up, 5 in 2024-09-14T03:42:25.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:24 smithi070 ceph-mon[27495]: osdmap e32: 5 total, 5 up, 5 in 2024-09-14T03:42:26.195 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:25 smithi094 ceph-mon[31053]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:42:26.196 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:25 smithi094 ceph-mon[31053]: osdmap e33: 5 total, 5 up, 5 in 2024-09-14T03:42:26.196 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:25 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:26.196 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:25 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:26.196 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:25 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:42:26.196 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:25 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:26.196 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:25 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:26.196 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:25 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:42:26.196 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:25 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:26.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:25 smithi070 ceph-mon[27495]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:42:26.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:25 smithi070 ceph-mon[27495]: osdmap e33: 5 total, 5 up, 5 in 2024-09-14T03:42:26.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:25 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:26.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:25 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:26.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:25 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:42:26.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:25 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:26.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:25 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:26.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:25 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:42:26.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:25 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:26.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:25 smithi070 ceph-mon[39288]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:42:26.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:25 smithi070 ceph-mon[39288]: osdmap e33: 5 total, 5 up, 5 in 2024-09-14T03:42:26.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:25 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:26.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:25 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:26.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:25 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:42:26.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:25 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:26.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:25 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:26.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:25 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:42:26.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:25 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:26.460 INFO:teuthology.orchestra.run.smithi094.stdout: 2024-09-14T03:42:26.480 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi094:vg_nvme/lv_3 2024-09-14T03:42:26.719 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:42:27.125 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:26 smithi094 ceph-mon[31053]: Detected new or changed devices on smithi094 2024-09-14T03:42:27.125 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:26 smithi094 ceph-mon[31053]: Adjusting osd_memory_target on smithi094 to 17088M 2024-09-14T03:42:27.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:26 smithi070 ceph-mon[27495]: Detected new or changed devices on smithi094 2024-09-14T03:42:27.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:26 smithi070 ceph-mon[27495]: Adjusting osd_memory_target on smithi094 to 17088M 2024-09-14T03:42:27.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:26 smithi070 ceph-mon[39288]: Detected new or changed devices on smithi094 2024-09-14T03:42:27.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:26 smithi070 ceph-mon[39288]: Adjusting osd_memory_target on smithi094 to 17088M 2024-09-14T03:42:28.033 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:27 smithi094 ceph-mon[31053]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:42:28.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:27 smithi070 ceph-mon[27495]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:42:28.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:27 smithi070 ceph-mon[39288]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:42:29.019 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:28 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:42:29.020 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:28 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:42:29.020 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:28 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:29.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:28 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:42:29.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:28 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:42:29.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:28 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:29.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:28 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:42:29.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:28 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:42:29.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:28 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:30.126 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:29 smithi094 ceph-mon[31053]: from='client.24211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi094:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:30.127 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:29 smithi094 ceph-mon[31053]: pgmap v115: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:42:30.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:29 smithi070 ceph-mon[27495]: from='client.24211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi094:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:30.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:29 smithi070 ceph-mon[27495]: pgmap v115: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:42:30.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:29 smithi070 ceph-mon[39288]: from='client.24211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi094:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:30.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:29 smithi070 ceph-mon[39288]: pgmap v115: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:42:31.230 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:30 smithi094 ceph-mon[31053]: from='client.? 172.21.15.94:0/2480634020' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eb037130-5a05-42e0-947e-09bfba8fac22"}]: dispatch 2024-09-14T03:42:31.230 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:30 smithi094 ceph-mon[31053]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eb037130-5a05-42e0-947e-09bfba8fac22"}]: dispatch 2024-09-14T03:42:31.231 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:30 smithi094 ceph-mon[31053]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "eb037130-5a05-42e0-947e-09bfba8fac22"}]': finished 2024-09-14T03:42:31.231 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:30 smithi094 ceph-mon[31053]: osdmap e34: 6 total, 5 up, 6 in 2024-09-14T03:42:31.231 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:30 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:42:31.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:30 smithi070 ceph-mon[27495]: from='client.? 172.21.15.94:0/2480634020' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eb037130-5a05-42e0-947e-09bfba8fac22"}]: dispatch 2024-09-14T03:42:31.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:30 smithi070 ceph-mon[27495]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eb037130-5a05-42e0-947e-09bfba8fac22"}]: dispatch 2024-09-14T03:42:31.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:30 smithi070 ceph-mon[27495]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "eb037130-5a05-42e0-947e-09bfba8fac22"}]': finished 2024-09-14T03:42:31.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:30 smithi070 ceph-mon[27495]: osdmap e34: 6 total, 5 up, 6 in 2024-09-14T03:42:31.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:30 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:42:31.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:30 smithi070 ceph-mon[39288]: from='client.? 172.21.15.94:0/2480634020' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eb037130-5a05-42e0-947e-09bfba8fac22"}]: dispatch 2024-09-14T03:42:31.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:30 smithi070 ceph-mon[39288]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eb037130-5a05-42e0-947e-09bfba8fac22"}]: dispatch 2024-09-14T03:42:31.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:30 smithi070 ceph-mon[39288]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "eb037130-5a05-42e0-947e-09bfba8fac22"}]': finished 2024-09-14T03:42:31.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:30 smithi070 ceph-mon[39288]: osdmap e34: 6 total, 5 up, 6 in 2024-09-14T03:42:31.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:30 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:42:32.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:31 smithi094 ceph-mon[31053]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-09-14T03:42:32.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:31 smithi094 ceph-mon[31053]: from='client.? 172.21.15.94:0/1169967928' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:42:32.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:31 smithi070 ceph-mon[27495]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-09-14T03:42:32.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:31 smithi070 ceph-mon[27495]: from='client.? 172.21.15.94:0/1169967928' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:42:32.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:31 smithi070 ceph-mon[39288]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-09-14T03:42:32.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:31 smithi070 ceph-mon[39288]: from='client.? 172.21.15.94:0/1169967928' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:42:33.999 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:33 smithi094 ceph-mon[31053]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-14T03:42:34.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:33 smithi070 ceph-mon[27495]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-14T03:42:34.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:33 smithi070 ceph-mon[39288]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-14T03:42:36.181 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:35 smithi094 ceph-mon[31053]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-09-14T03:42:36.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:35 smithi070 ceph-mon[39288]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-09-14T03:42:36.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:35 smithi070 ceph-mon[27495]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-09-14T03:42:37.143 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:36 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-14T03:42:37.144 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:36 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:37.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:36 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-14T03:42:37.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:36 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:37.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:36 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-14T03:42:37.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:36 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:38.116 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:37 smithi094 ceph-mon[31053]: Deploying daemon osd.5 on smithi094 2024-09-14T03:42:38.116 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:37 smithi094 ceph-mon[31053]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:42:38.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:37 smithi070 ceph-mon[27495]: Deploying daemon osd.5 on smithi094 2024-09-14T03:42:38.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:37 smithi070 ceph-mon[27495]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:42:38.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:37 smithi070 ceph-mon[39288]: Deploying daemon osd.5 on smithi094 2024-09-14T03:42:38.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:37 smithi070 ceph-mon[39288]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:42:40.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:39 smithi094 ceph-mon[31053]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:42:40.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:39 smithi070 ceph-mon[39288]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:42:40.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:39 smithi070 ceph-mon[27495]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:42:42.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:41 smithi094 ceph-mon[31053]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:42:42.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:41 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:42:42.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:41 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:42.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:41 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:42.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:41 smithi070 ceph-mon[27495]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:42:42.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:41 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:42:42.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:41 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:42.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:41 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:42.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:41 smithi070 ceph-mon[39288]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:42:42.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:41 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:42:42.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:41 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:42.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:41 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:44.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:43 smithi094 ceph-mon[31053]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:42:44.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:43 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:44.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:43 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:44.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:43 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:44.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:43 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:42:44.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:43 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:44.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:43 smithi070 ceph-mon[27495]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:42:44.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:43 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:44.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:43 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:44.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:43 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:44.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:43 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:42:44.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:43 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:44.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:43 smithi070 ceph-mon[39288]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:42:44.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:43 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:44.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:43 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:44.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:43 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:44.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:43 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:42:44.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:43 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:44.513 INFO:teuthology.orchestra.run.smithi094.stdout:Created osd(s) 5 on host 'smithi094' 2024-09-14T03:42:45.116 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:44 smithi094 ceph-mon[31053]: from='osd.5 [v2:172.21.15.94:6808/1692579524,v1:172.21.15.94:6809/1692579524]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-14T03:42:45.116 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:44 smithi094 ceph-mon[31053]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-14T03:42:45.116 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:44 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:42:45.117 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:44 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:45.117 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:44 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:45.140 DEBUG:teuthology.orchestra.run.smithi094:osd.5> sudo journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.5.service 2024-09-14T03:42:45.142 INFO:tasks.cephadm:Deploying osd.6 on smithi094 with /dev/vg_nvme/lv_2... 2024-09-14T03:42:45.142 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-14T03:42:45.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:44 smithi070 ceph-mon[27495]: from='osd.5 [v2:172.21.15.94:6808/1692579524,v1:172.21.15.94:6809/1692579524]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-14T03:42:45.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:44 smithi070 ceph-mon[27495]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-14T03:42:45.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:44 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:42:45.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:44 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:45.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:44 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:45.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:44 smithi070 ceph-mon[39288]: from='osd.5 [v2:172.21.15.94:6808/1692579524,v1:172.21.15.94:6809/1692579524]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-14T03:42:45.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:44 smithi070 ceph-mon[39288]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-14T03:42:45.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:44 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:42:45.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:44 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:45.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:44 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:45.921 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:42:46.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:45 smithi094 ceph-mon[31053]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:42:46.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:45 smithi094 ceph-mon[31053]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-14T03:42:46.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:45 smithi094 ceph-mon[31053]: osdmap e35: 6 total, 5 up, 6 in 2024-09-14T03:42:46.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:45 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:42:46.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:45 smithi094 ceph-mon[31053]: from='osd.5 [v2:172.21.15.94:6808/1692579524,v1:172.21.15.94:6809/1692579524]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2024-09-14T03:42:46.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:45 smithi094 ceph-mon[31053]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2024-09-14T03:42:46.027 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:42:45 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[42647]: 2024-09-14T03:42:45.940+0000 7f29e63e7640 -1 osd.5 0 waiting for initial osdmap 2024-09-14T03:42:46.028 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:42:45 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[42647]: 2024-09-14T03:42:45.945+0000 7f29e1a1f640 -1 osd.5 36 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:42:46.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:45 smithi070 ceph-mon[27495]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:42:46.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:45 smithi070 ceph-mon[27495]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-14T03:42:46.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:45 smithi070 ceph-mon[27495]: osdmap e35: 6 total, 5 up, 6 in 2024-09-14T03:42:46.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:45 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:42:46.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:45 smithi070 ceph-mon[27495]: from='osd.5 [v2:172.21.15.94:6808/1692579524,v1:172.21.15.94:6809/1692579524]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2024-09-14T03:42:46.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:45 smithi070 ceph-mon[27495]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2024-09-14T03:42:46.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:45 smithi070 ceph-mon[39288]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:42:46.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:45 smithi070 ceph-mon[39288]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-14T03:42:46.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:45 smithi070 ceph-mon[39288]: osdmap e35: 6 total, 5 up, 6 in 2024-09-14T03:42:46.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:45 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:42:46.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:45 smithi070 ceph-mon[39288]: from='osd.5 [v2:172.21.15.94:6808/1692579524,v1:172.21.15.94:6809/1692579524]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2024-09-14T03:42:46.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:45 smithi070 ceph-mon[39288]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2024-09-14T03:42:47.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:46 smithi094 ceph-mon[31053]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]': finished 2024-09-14T03:42:47.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:46 smithi094 ceph-mon[31053]: osdmap e36: 6 total, 5 up, 6 in 2024-09-14T03:42:47.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:46 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:42:47.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:46 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:42:47.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:46 smithi070 ceph-mon[27495]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]': finished 2024-09-14T03:42:47.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:46 smithi070 ceph-mon[27495]: osdmap e36: 6 total, 5 up, 6 in 2024-09-14T03:42:47.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:46 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:42:47.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:46 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:42:47.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:46 smithi070 ceph-mon[39288]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]': finished 2024-09-14T03:42:47.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:46 smithi070 ceph-mon[39288]: osdmap e36: 6 total, 5 up, 6 in 2024-09-14T03:42:47.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:46 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:42:47.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:46 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:42:48.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:47 smithi094 ceph-mon[31053]: purged_snaps scrub starts 2024-09-14T03:42:48.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:47 smithi094 ceph-mon[31053]: purged_snaps scrub ok 2024-09-14T03:42:48.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:47 smithi094 ceph-mon[31053]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:42:48.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:47 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:42:48.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:47 smithi094 ceph-mon[31053]: osd.5 [v2:172.21.15.94:6808/1692579524,v1:172.21.15.94:6809/1692579524] boot 2024-09-14T03:42:48.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:47 smithi094 ceph-mon[31053]: osdmap e37: 6 total, 6 up, 6 in 2024-09-14T03:42:48.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:47 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:42:48.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:47 smithi070 ceph-mon[27495]: purged_snaps scrub starts 2024-09-14T03:42:48.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:47 smithi070 ceph-mon[27495]: purged_snaps scrub ok 2024-09-14T03:42:48.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:47 smithi070 ceph-mon[27495]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:42:48.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:47 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:42:48.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:47 smithi070 ceph-mon[27495]: osd.5 [v2:172.21.15.94:6808/1692579524,v1:172.21.15.94:6809/1692579524] boot 2024-09-14T03:42:48.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:47 smithi070 ceph-mon[27495]: osdmap e37: 6 total, 6 up, 6 in 2024-09-14T03:42:48.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:47 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:42:48.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:47 smithi070 ceph-mon[39288]: purged_snaps scrub starts 2024-09-14T03:42:48.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:47 smithi070 ceph-mon[39288]: purged_snaps scrub ok 2024-09-14T03:42:48.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:47 smithi070 ceph-mon[39288]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-14T03:42:48.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:47 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:42:48.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:47 smithi070 ceph-mon[39288]: osd.5 [v2:172.21.15.94:6808/1692579524,v1:172.21.15.94:6809/1692579524] boot 2024-09-14T03:42:48.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:47 smithi070 ceph-mon[39288]: osdmap e37: 6 total, 6 up, 6 in 2024-09-14T03:42:48.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:47 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:42:49.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:48 smithi094 ceph-mon[31053]: osdmap e38: 6 total, 6 up, 6 in 2024-09-14T03:42:49.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:48 smithi070 ceph-mon[27495]: osdmap e38: 6 total, 6 up, 6 in 2024-09-14T03:42:49.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:48 smithi070 ceph-mon[39288]: osdmap e38: 6 total, 6 up, 6 in 2024-09-14T03:42:50.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:49 smithi094 ceph-mon[31053]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:42:50.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:49 smithi094 ceph-mon[31053]: osdmap e39: 6 total, 6 up, 6 in 2024-09-14T03:42:50.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:49 smithi094 ceph-mon[31053]: Detected new or changed devices on smithi094 2024-09-14T03:42:50.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:49 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:50.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:49 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:50.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:49 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:42:50.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:49 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:42:50.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:49 smithi094 ceph-mon[31053]: Adjusting osd_memory_target on smithi094 to 8544M 2024-09-14T03:42:50.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:49 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:50.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:49 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:50.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:49 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:42:50.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:49 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:50.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:49 smithi070 ceph-mon[27495]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:42:50.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:49 smithi070 ceph-mon[27495]: osdmap e39: 6 total, 6 up, 6 in 2024-09-14T03:42:50.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:49 smithi070 ceph-mon[27495]: Detected new or changed devices on smithi094 2024-09-14T03:42:50.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:49 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:50.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:49 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:50.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:49 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:42:50.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:49 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:42:50.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:49 smithi070 ceph-mon[27495]: Adjusting osd_memory_target on smithi094 to 8544M 2024-09-14T03:42:50.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:49 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:50.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:49 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:50.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:49 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:42:50.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:49 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:50.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:49 smithi070 ceph-mon[39288]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:42:50.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:49 smithi070 ceph-mon[39288]: osdmap e39: 6 total, 6 up, 6 in 2024-09-14T03:42:50.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:49 smithi070 ceph-mon[39288]: Detected new or changed devices on smithi094 2024-09-14T03:42:50.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:49 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:50.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:49 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:50.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:49 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:42:50.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:49 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:42:50.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:49 smithi070 ceph-mon[39288]: Adjusting osd_memory_target on smithi094 to 8544M 2024-09-14T03:42:50.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:49 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:50.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:49 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:50.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:49 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:42:50.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:49 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:42:50.321 INFO:teuthology.orchestra.run.smithi094.stdout: 2024-09-14T03:42:50.341 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi094:vg_nvme/lv_2 2024-09-14T03:42:50.588 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:42:52.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:51 smithi094 ceph-mon[31053]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:42:52.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:51 smithi070 ceph-mon[27495]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:42:52.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:51 smithi070 ceph-mon[39288]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:42:53.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:52 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:42:53.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:52 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:42:53.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:52 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:53.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:52 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:42:53.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:52 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:42:53.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:52 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:53.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:52 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:42:53.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:52 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:42:53.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:52 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:42:54.135 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:53 smithi094 ceph-mon[31053]: from='client.24238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi094:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:54.135 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:53 smithi094 ceph-mon[31053]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:42:54.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:53 smithi070 ceph-mon[27495]: from='client.24238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi094:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:54.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:53 smithi070 ceph-mon[27495]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:42:54.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:53 smithi070 ceph-mon[39288]: from='client.24238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi094:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:42:54.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:53 smithi070 ceph-mon[39288]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:42:55.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:54 smithi094 ceph-mon[31053]: from='client.? 172.21.15.94:0/1909461997' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f5dca4b5-8906-4eac-84e4-fcb7d1a29674"}]: dispatch 2024-09-14T03:42:55.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:54 smithi094 ceph-mon[31053]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f5dca4b5-8906-4eac-84e4-fcb7d1a29674"}]: dispatch 2024-09-14T03:42:55.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:54 smithi094 ceph-mon[31053]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f5dca4b5-8906-4eac-84e4-fcb7d1a29674"}]': finished 2024-09-14T03:42:55.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:54 smithi094 ceph-mon[31053]: osdmap e40: 7 total, 6 up, 7 in 2024-09-14T03:42:55.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:54 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:42:55.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:54 smithi070 ceph-mon[39288]: from='client.? 172.21.15.94:0/1909461997' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f5dca4b5-8906-4eac-84e4-fcb7d1a29674"}]: dispatch 2024-09-14T03:42:55.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:54 smithi070 ceph-mon[39288]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f5dca4b5-8906-4eac-84e4-fcb7d1a29674"}]: dispatch 2024-09-14T03:42:55.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:54 smithi070 ceph-mon[39288]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f5dca4b5-8906-4eac-84e4-fcb7d1a29674"}]': finished 2024-09-14T03:42:55.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:54 smithi070 ceph-mon[39288]: osdmap e40: 7 total, 6 up, 7 in 2024-09-14T03:42:55.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:54 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:42:55.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:54 smithi070 ceph-mon[27495]: from='client.? 172.21.15.94:0/1909461997' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f5dca4b5-8906-4eac-84e4-fcb7d1a29674"}]: dispatch 2024-09-14T03:42:55.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:54 smithi070 ceph-mon[27495]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f5dca4b5-8906-4eac-84e4-fcb7d1a29674"}]: dispatch 2024-09-14T03:42:55.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:54 smithi070 ceph-mon[27495]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f5dca4b5-8906-4eac-84e4-fcb7d1a29674"}]': finished 2024-09-14T03:42:55.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:54 smithi070 ceph-mon[27495]: osdmap e40: 7 total, 6 up, 7 in 2024-09-14T03:42:55.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:54 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:42:56.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:55 smithi094 ceph-mon[31053]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:42:56.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:55 smithi094 ceph-mon[31053]: from='client.? 172.21.15.94:0/2557913479' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:42:56.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:55 smithi070 ceph-mon[27495]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:42:56.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:55 smithi070 ceph-mon[27495]: from='client.? 172.21.15.94:0/2557913479' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:42:56.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:55 smithi070 ceph-mon[39288]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:42:56.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:55 smithi070 ceph-mon[39288]: from='client.? 172.21.15.94:0/2557913479' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:42:58.228 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:57 smithi094 ceph-mon[31053]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:42:58.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:57 smithi070 ceph-mon[27495]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:42:58.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:57 smithi070 ceph-mon[39288]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:43:00.084 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:42:59 smithi094 ceph-mon[31053]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:43:00.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:42:59 smithi070 ceph-mon[27495]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:43:00.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:42:59 smithi070 ceph-mon[39288]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:43:01.188 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:00 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-14T03:43:01.189 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:00 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:01.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:00 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-14T03:43:01.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:00 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:01.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:00 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-14T03:43:01.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:00 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:02.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:01 smithi094 ceph-mon[31053]: Deploying daemon osd.6 on smithi094 2024-09-14T03:43:02.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:01 smithi094 ceph-mon[31053]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:43:02.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:01 smithi070 ceph-mon[27495]: Deploying daemon osd.6 on smithi094 2024-09-14T03:43:02.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:01 smithi070 ceph-mon[27495]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:43:02.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:01 smithi070 ceph-mon[39288]: Deploying daemon osd.6 on smithi094 2024-09-14T03:43:02.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:01 smithi070 ceph-mon[39288]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:43:04.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:03 smithi094 ceph-mon[31053]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:43:04.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:03 smithi070 ceph-mon[27495]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:43:04.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:03 smithi070 ceph-mon[39288]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:43:06.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:05 smithi094 ceph-mon[31053]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:43:06.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:05 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:43:06.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:05 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:06.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:05 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:06.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:05 smithi070 ceph-mon[27495]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:43:06.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:05 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:43:06.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:05 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:06.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:05 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:06.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:05 smithi070 ceph-mon[39288]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:43:06.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:05 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:43:06.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:05 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:06.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:05 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:08.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:07 smithi094 ceph-mon[31053]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:43:08.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:07 smithi070 ceph-mon[27495]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:43:08.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:07 smithi070 ceph-mon[39288]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:43:08.401 INFO:teuthology.orchestra.run.smithi094.stdout:Created osd(s) 6 on host 'smithi094' 2024-09-14T03:43:09.024 DEBUG:teuthology.orchestra.run.smithi094:osd.6> sudo journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.6.service 2024-09-14T03:43:09.026 INFO:tasks.cephadm:Deploying osd.7 on smithi094 with /dev/vg_nvme/lv_1... 2024-09-14T03:43:09.026 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-14T03:43:09.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:08 smithi094 ceph-mon[31053]: from='osd.6 [v2:172.21.15.94:6816/1107979948,v1:172.21.15.94:6817/1107979948]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-14T03:43:09.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:08 smithi094 ceph-mon[31053]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-14T03:43:09.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:08 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:09.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:08 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:09.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:08 smithi070 ceph-mon[27495]: from='osd.6 [v2:172.21.15.94:6816/1107979948,v1:172.21.15.94:6817/1107979948]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-14T03:43:09.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:08 smithi070 ceph-mon[27495]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-14T03:43:09.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:08 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:09.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:08 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:09.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:08 smithi070 ceph-mon[39288]: from='osd.6 [v2:172.21.15.94:6816/1107979948,v1:172.21.15.94:6817/1107979948]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-14T03:43:09.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:08 smithi070 ceph-mon[39288]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-14T03:43:09.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:09.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:08 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:09.316 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:43:10.257 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:43:09 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[48695]: 2024-09-14T03:43:09.994+0000 7f5f93b69640 -1 osd.6 0 waiting for initial osdmap 2024-09-14T03:43:10.257 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:43:10 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[48695]: 2024-09-14T03:43:10.000+0000 7f5f8d19d640 -1 osd.6 42 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:43:10.258 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:09 smithi094 ceph-mon[31053]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:43:10.258 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:09 smithi094 ceph-mon[31053]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-14T03:43:10.258 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:09 smithi094 ceph-mon[31053]: osdmap e41: 7 total, 6 up, 7 in 2024-09-14T03:43:10.258 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:09 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:43:10.258 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:09 smithi094 ceph-mon[31053]: from='osd.6 [v2:172.21.15.94:6816/1107979948,v1:172.21.15.94:6817/1107979948]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2024-09-14T03:43:10.258 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:09 smithi094 ceph-mon[31053]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2024-09-14T03:43:10.258 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:09 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:10.258 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:09 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:10.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:09 smithi070 ceph-mon[27495]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:43:10.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:09 smithi070 ceph-mon[27495]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-14T03:43:10.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:09 smithi070 ceph-mon[27495]: osdmap e41: 7 total, 6 up, 7 in 2024-09-14T03:43:10.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:09 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:43:10.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:09 smithi070 ceph-mon[27495]: from='osd.6 [v2:172.21.15.94:6816/1107979948,v1:172.21.15.94:6817/1107979948]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2024-09-14T03:43:10.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:09 smithi070 ceph-mon[27495]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2024-09-14T03:43:10.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:09 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:10.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:09 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:10.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:09 smithi070 ceph-mon[39288]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:43:10.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:09 smithi070 ceph-mon[39288]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-14T03:43:10.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:09 smithi070 ceph-mon[39288]: osdmap e41: 7 total, 6 up, 7 in 2024-09-14T03:43:10.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:09 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:43:10.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:09 smithi070 ceph-mon[39288]: from='osd.6 [v2:172.21.15.94:6816/1107979948,v1:172.21.15.94:6817/1107979948]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2024-09-14T03:43:10.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:09 smithi070 ceph-mon[39288]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2024-09-14T03:43:10.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:09 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:10.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:09 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:11.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:10 smithi094 ceph-mon[31053]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]': finished 2024-09-14T03:43:11.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:10 smithi094 ceph-mon[31053]: osdmap e42: 7 total, 6 up, 7 in 2024-09-14T03:43:11.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:10 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:43:11.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:10 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:43:11.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:10 smithi094 ceph-mon[31053]: osd.6 [v2:172.21.15.94:6816/1107979948,v1:172.21.15.94:6817/1107979948] boot 2024-09-14T03:43:11.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:10 smithi094 ceph-mon[31053]: osdmap e43: 7 total, 7 up, 7 in 2024-09-14T03:43:11.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:10 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:43:11.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:10 smithi070 ceph-mon[27495]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]': finished 2024-09-14T03:43:11.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:10 smithi070 ceph-mon[27495]: osdmap e42: 7 total, 6 up, 7 in 2024-09-14T03:43:11.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:10 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:43:11.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:10 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:43:11.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:10 smithi070 ceph-mon[27495]: osd.6 [v2:172.21.15.94:6816/1107979948,v1:172.21.15.94:6817/1107979948] boot 2024-09-14T03:43:11.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:10 smithi070 ceph-mon[27495]: osdmap e43: 7 total, 7 up, 7 in 2024-09-14T03:43:11.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:10 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:43:11.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:10 smithi070 ceph-mon[39288]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]': finished 2024-09-14T03:43:11.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:10 smithi070 ceph-mon[39288]: osdmap e42: 7 total, 6 up, 7 in 2024-09-14T03:43:11.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:10 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:43:11.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:10 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:43:11.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:10 smithi070 ceph-mon[39288]: osd.6 [v2:172.21.15.94:6816/1107979948,v1:172.21.15.94:6817/1107979948] boot 2024-09-14T03:43:11.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:10 smithi070 ceph-mon[39288]: osdmap e43: 7 total, 7 up, 7 in 2024-09-14T03:43:11.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:10 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:43:12.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:11 smithi094 ceph-mon[31053]: purged_snaps scrub starts 2024-09-14T03:43:12.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:11 smithi094 ceph-mon[31053]: purged_snaps scrub ok 2024-09-14T03:43:12.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:11 smithi094 ceph-mon[31053]: pgmap v146: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:43:12.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:11 smithi094 ceph-mon[31053]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-14T03:43:12.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:11 smithi094 ceph-mon[31053]: osdmap e44: 7 total, 7 up, 7 in 2024-09-14T03:43:12.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:11 smithi070 ceph-mon[27495]: purged_snaps scrub starts 2024-09-14T03:43:12.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:11 smithi070 ceph-mon[27495]: purged_snaps scrub ok 2024-09-14T03:43:12.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:11 smithi070 ceph-mon[27495]: pgmap v146: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:43:12.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:11 smithi070 ceph-mon[27495]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-14T03:43:12.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:11 smithi070 ceph-mon[27495]: osdmap e44: 7 total, 7 up, 7 in 2024-09-14T03:43:12.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:11 smithi070 ceph-mon[39288]: purged_snaps scrub starts 2024-09-14T03:43:12.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:11 smithi070 ceph-mon[39288]: purged_snaps scrub ok 2024-09-14T03:43:12.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:11 smithi070 ceph-mon[39288]: pgmap v146: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-14T03:43:12.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:11 smithi070 ceph-mon[39288]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-14T03:43:12.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:11 smithi070 ceph-mon[39288]: osdmap e44: 7 total, 7 up, 7 in 2024-09-14T03:43:12.395 INFO:teuthology.orchestra.run.smithi094.stdout: 2024-09-14T03:43:12.415 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi094:vg_nvme/lv_1 2024-09-14T03:43:12.942 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:43:13.686 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:13 smithi094 ceph-mon[31053]: osdmap e45: 7 total, 7 up, 7 in 2024-09-14T03:43:13.686 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:13 smithi094 ceph-mon[31053]: pgmap v149: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:43:13.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:13 smithi070 ceph-mon[27495]: osdmap e45: 7 total, 7 up, 7 in 2024-09-14T03:43:13.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:13 smithi070 ceph-mon[27495]: pgmap v149: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:43:13.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:13 smithi070 ceph-mon[39288]: osdmap e45: 7 total, 7 up, 7 in 2024-09-14T03:43:13.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:13 smithi070 ceph-mon[39288]: pgmap v149: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:43:15.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:15 smithi070 ceph-mon[27495]: Detected new or changed devices on smithi094 2024-09-14T03:43:15.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:15 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:15.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:15 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:15.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:15 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:43:15.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:15 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:43:15.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:15 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:43:15.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:15 smithi070 ceph-mon[27495]: Adjusting osd_memory_target on smithi094 to 5696M 2024-09-14T03:43:15.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:15 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:15.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:15 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:15.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:15 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:43:15.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:15 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:15.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:15 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:43:15.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:15 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:15.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:15 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:43:15.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:15 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:15.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:15 smithi070 ceph-mon[27495]: pgmap v150: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:43:15.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:15 smithi070 ceph-mon[39288]: Detected new or changed devices on smithi094 2024-09-14T03:43:15.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:15 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:15.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:15 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:15.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:15 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:43:15.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:15 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:43:15.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:15 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:43:15.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:15 smithi070 ceph-mon[39288]: Adjusting osd_memory_target on smithi094 to 5696M 2024-09-14T03:43:15.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:15 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:15.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:15 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:15.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:15 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:43:15.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:15 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:15.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:15 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:43:15.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:15 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:15.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:15 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:43:15.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:15 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:15.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:15 smithi070 ceph-mon[39288]: pgmap v150: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:43:15.758 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:15 smithi094 ceph-mon[31053]: Detected new or changed devices on smithi094 2024-09-14T03:43:15.759 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:15 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:15.759 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:15 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:15.759 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:15 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:43:15.759 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:15 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:43:15.759 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:15 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:43:15.759 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:15 smithi094 ceph-mon[31053]: Adjusting osd_memory_target on smithi094 to 5696M 2024-09-14T03:43:15.759 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:15 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:15.759 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:15 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:15.759 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:15 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:43:15.759 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:15 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:15.760 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:15 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:43:15.760 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:15 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:15.760 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:15 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:43:15.760 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:15 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:15.760 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:15 smithi094 ceph-mon[31053]: pgmap v150: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:43:16.358 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:16 smithi094 ceph-mon[31053]: from='client.24265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi094:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:43:16.359 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:16 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:43:16.359 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:16 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:43:16.359 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:16 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:16.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:16 smithi070 ceph-mon[39288]: from='client.24265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi094:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:43:16.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:16 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:43:16.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:16 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:43:16.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:16 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:16.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:16 smithi070 ceph-mon[27495]: from='client.24265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi094:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:43:16.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:16 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:43:16.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:16 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:43:16.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:16 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:17.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:17 smithi094 ceph-mon[31053]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 95 KiB/s, 0 objects/s recovering 2024-09-14T03:43:17.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:17 smithi070 ceph-mon[27495]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 95 KiB/s, 0 objects/s recovering 2024-09-14T03:43:17.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:17 smithi070 ceph-mon[39288]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 95 KiB/s, 0 objects/s recovering 2024-09-14T03:43:18.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:18 smithi094 ceph-mon[31053]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-14T03:43:18.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:18 smithi094 ceph-mon[31053]: Cluster is now healthy 2024-09-14T03:43:18.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:18 smithi094 ceph-mon[31053]: from='client.? 172.21.15.94:0/3842348611' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "86e3ef7d-6ab8-47c3-a4bb-db923af2b49b"}]: dispatch 2024-09-14T03:43:18.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:18 smithi094 ceph-mon[31053]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "86e3ef7d-6ab8-47c3-a4bb-db923af2b49b"}]: dispatch 2024-09-14T03:43:18.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:18 smithi094 ceph-mon[31053]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "86e3ef7d-6ab8-47c3-a4bb-db923af2b49b"}]': finished 2024-09-14T03:43:18.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:18 smithi094 ceph-mon[31053]: osdmap e46: 8 total, 7 up, 8 in 2024-09-14T03:43:18.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:18 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:43:18.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:18 smithi094 ceph-mon[31053]: from='client.? 172.21.15.94:0/1336575652' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:43:18.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:18 smithi070 ceph-mon[27495]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-14T03:43:18.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:18 smithi070 ceph-mon[27495]: Cluster is now healthy 2024-09-14T03:43:18.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:18 smithi070 ceph-mon[27495]: from='client.? 172.21.15.94:0/3842348611' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "86e3ef7d-6ab8-47c3-a4bb-db923af2b49b"}]: dispatch 2024-09-14T03:43:18.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:18 smithi070 ceph-mon[27495]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "86e3ef7d-6ab8-47c3-a4bb-db923af2b49b"}]: dispatch 2024-09-14T03:43:18.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:18 smithi070 ceph-mon[27495]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "86e3ef7d-6ab8-47c3-a4bb-db923af2b49b"}]': finished 2024-09-14T03:43:18.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:18 smithi070 ceph-mon[27495]: osdmap e46: 8 total, 7 up, 8 in 2024-09-14T03:43:18.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:18 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:43:18.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:18 smithi070 ceph-mon[27495]: from='client.? 172.21.15.94:0/1336575652' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:43:18.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:18 smithi070 ceph-mon[39288]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-14T03:43:18.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:18 smithi070 ceph-mon[39288]: Cluster is now healthy 2024-09-14T03:43:18.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:18 smithi070 ceph-mon[39288]: from='client.? 172.21.15.94:0/3842348611' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "86e3ef7d-6ab8-47c3-a4bb-db923af2b49b"}]: dispatch 2024-09-14T03:43:18.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:18 smithi070 ceph-mon[39288]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "86e3ef7d-6ab8-47c3-a4bb-db923af2b49b"}]: dispatch 2024-09-14T03:43:18.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:18 smithi070 ceph-mon[39288]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "86e3ef7d-6ab8-47c3-a4bb-db923af2b49b"}]': finished 2024-09-14T03:43:18.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:18 smithi070 ceph-mon[39288]: osdmap e46: 8 total, 7 up, 8 in 2024-09-14T03:43:18.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:18 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:43:18.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:18 smithi070 ceph-mon[39288]: from='client.? 172.21.15.94:0/1336575652' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:43:19.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:19 smithi094 ceph-mon[31053]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-09-14T03:43:19.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:19 smithi070 ceph-mon[27495]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-09-14T03:43:19.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:19 smithi070 ceph-mon[39288]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-09-14T03:43:21.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:21 smithi094 ceph-mon[31053]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-14T03:43:22.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:21 smithi070 ceph-mon[27495]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-14T03:43:22.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:21 smithi070 ceph-mon[39288]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-14T03:43:23.553 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:23 smithi094 ceph-mon[31053]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:43:24.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:23 smithi070 ceph-mon[27495]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:43:24.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:23 smithi070 ceph-mon[39288]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:43:24.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:24 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-14T03:43:24.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:24 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:24.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:24 smithi094 ceph-mon[31053]: Deploying daemon osd.7 on smithi094 2024-09-14T03:43:25.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:24 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-14T03:43:25.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:24 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:25.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:24 smithi070 ceph-mon[27495]: Deploying daemon osd.7 on smithi094 2024-09-14T03:43:25.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:24 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-14T03:43:25.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:24 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:25.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:24 smithi070 ceph-mon[39288]: Deploying daemon osd.7 on smithi094 2024-09-14T03:43:25.752 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:25 smithi094 ceph-mon[31053]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:43:26.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:25 smithi070 ceph-mon[27495]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:43:26.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:25 smithi070 ceph-mon[39288]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:43:27.822 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:27 smithi094 ceph-mon[31053]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:43:28.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:27 smithi070 ceph-mon[27495]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:43:28.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:27 smithi070 ceph-mon[39288]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:43:28.805 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:28 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:43:28.805 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:28 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:28.806 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:28 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:29.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:28 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:43:29.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:28 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:29.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:28 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:29.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:28 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:43:29.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:28 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:29.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:28 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:29.766 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:29 smithi094 ceph-mon[31053]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:43:30.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:29 smithi070 ceph-mon[27495]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:43:30.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:29 smithi070 ceph-mon[39288]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:43:31.128 INFO:teuthology.orchestra.run.smithi094.stdout:Created osd(s) 7 on host 'smithi094' 2024-09-14T03:43:31.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:31 smithi094 ceph-mon[31053]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:43:31.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:31 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:31.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:31 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:31.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:31 smithi094 ceph-mon[31053]: from='osd.7 [v2:172.21.15.94:6824/2545649733,v1:172.21.15.94:6825/2545649733]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-14T03:43:31.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:31 smithi094 ceph-mon[31053]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-14T03:43:32.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:31 smithi070 ceph-mon[27495]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:43:32.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:31 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:32.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:31 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:32.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:31 smithi070 ceph-mon[27495]: from='osd.7 [v2:172.21.15.94:6824/2545649733,v1:172.21.15.94:6825/2545649733]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-14T03:43:32.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:31 smithi070 ceph-mon[27495]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-14T03:43:32.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:31 smithi070 ceph-mon[39288]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:43:32.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:31 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:32.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:31 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:32.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:31 smithi070 ceph-mon[39288]: from='osd.7 [v2:172.21.15.94:6824/2545649733,v1:172.21.15.94:6825/2545649733]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-14T03:43:32.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:31 smithi070 ceph-mon[39288]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-14T03:43:32.145 DEBUG:teuthology.orchestra.run.smithi094:osd.7> sudo journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.7.service 2024-09-14T03:43:32.147 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-14T03:43:32.148 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-14T03:43:32.399 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:43:33.407 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:33 smithi094 ceph-mon[31053]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-14T03:43:33.407 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:33 smithi094 ceph-mon[31053]: osdmap e47: 8 total, 7 up, 8 in 2024-09-14T03:43:33.408 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:33 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:43:33.408 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:33 smithi094 ceph-mon[31053]: from='osd.7 [v2:172.21.15.94:6824/2545649733,v1:172.21.15.94:6825/2545649733]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2024-09-14T03:43:33.408 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:33 smithi094 ceph-mon[31053]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2024-09-14T03:43:33.408 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:33 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:33.408 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:33 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:33.408 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:33 smithi094 ceph-mon[31053]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:43:33.408 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:43:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[55202]: 2024-09-14T03:43:33.132+0000 7f30e9123640 -1 osd.7 0 waiting for initial osdmap 2024-09-14T03:43:33.408 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:43:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[55202]: 2024-09-14T03:43:33.138+0000 7f30e3f5a640 -1 osd.7 48 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:43:33.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:33 smithi070 ceph-mon[39288]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-14T03:43:33.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:33 smithi070 ceph-mon[39288]: osdmap e47: 8 total, 7 up, 8 in 2024-09-14T03:43:33.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:33 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:43:33.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:33 smithi070 ceph-mon[39288]: from='osd.7 [v2:172.21.15.94:6824/2545649733,v1:172.21.15.94:6825/2545649733]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2024-09-14T03:43:33.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:33 smithi070 ceph-mon[39288]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2024-09-14T03:43:33.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:33 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:33.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:33 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:33.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:33 smithi070 ceph-mon[39288]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:43:33.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:33 smithi070 ceph-mon[27495]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-14T03:43:33.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:33 smithi070 ceph-mon[27495]: osdmap e47: 8 total, 7 up, 8 in 2024-09-14T03:43:33.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:33 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:43:33.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:33 smithi070 ceph-mon[27495]: from='osd.7 [v2:172.21.15.94:6824/2545649733,v1:172.21.15.94:6825/2545649733]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2024-09-14T03:43:33.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:33 smithi070 ceph-mon[27495]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]: dispatch 2024-09-14T03:43:33.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:33 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:33.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:33 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:33.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:33 smithi070 ceph-mon[27495]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:43:34.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:34 smithi094 ceph-mon[31053]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]': finished 2024-09-14T03:43:34.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:34 smithi094 ceph-mon[31053]: osdmap e48: 8 total, 7 up, 8 in 2024-09-14T03:43:34.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:34 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:43:34.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:34 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:43:34.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:34 smithi070 ceph-mon[27495]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]': finished 2024-09-14T03:43:34.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:34 smithi070 ceph-mon[27495]: osdmap e48: 8 total, 7 up, 8 in 2024-09-14T03:43:34.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:34 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:43:34.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:34 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:43:34.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:34 smithi070 ceph-mon[39288]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi094", "root=default"]}]': finished 2024-09-14T03:43:34.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:34 smithi070 ceph-mon[39288]: osdmap e48: 8 total, 7 up, 8 in 2024-09-14T03:43:34.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:34 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:43:34.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:34 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:43:34.382 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:43:35.000 INFO:teuthology.orchestra.run.smithi070.stdout:{"epoch":49,"num_osds":8,"num_up_osds":8,"osd_up_since":1726285414,"num_in_osds":8,"osd_in_since":1726285397,"num_remapped_pgs":1} 2024-09-14T03:43:35.001 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T03:43:35.234 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:35 smithi094 ceph-mon[31053]: purged_snaps scrub starts 2024-09-14T03:43:35.234 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:35 smithi094 ceph-mon[31053]: purged_snaps scrub ok 2024-09-14T03:43:35.234 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:35 smithi094 ceph-mon[31053]: osd.7 [v2:172.21.15.94:6824/2545649733,v1:172.21.15.94:6825/2545649733] boot 2024-09-14T03:43:35.234 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:35 smithi094 ceph-mon[31053]: osdmap e49: 8 total, 8 up, 8 in 2024-09-14T03:43:35.234 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:35 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:43:35.234 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:35 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/2656089465' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-14T03:43:35.235 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:35 smithi094 ceph-mon[31053]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:43:35.235 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:35 smithi094 ceph-mon[31053]: osdmap e50: 8 total, 8 up, 8 in 2024-09-14T03:43:35.245 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:43:35.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:35 smithi070 ceph-mon[27495]: purged_snaps scrub starts 2024-09-14T03:43:35.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:35 smithi070 ceph-mon[27495]: purged_snaps scrub ok 2024-09-14T03:43:35.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:35 smithi070 ceph-mon[27495]: osd.7 [v2:172.21.15.94:6824/2545649733,v1:172.21.15.94:6825/2545649733] boot 2024-09-14T03:43:35.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:35 smithi070 ceph-mon[27495]: osdmap e49: 8 total, 8 up, 8 in 2024-09-14T03:43:35.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:35 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:43:35.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:35 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/2656089465' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-14T03:43:35.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:35 smithi070 ceph-mon[27495]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:43:35.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:35 smithi070 ceph-mon[27495]: osdmap e50: 8 total, 8 up, 8 in 2024-09-14T03:43:35.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:35 smithi070 ceph-mon[39288]: purged_snaps scrub starts 2024-09-14T03:43:35.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:35 smithi070 ceph-mon[39288]: purged_snaps scrub ok 2024-09-14T03:43:35.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:35 smithi070 ceph-mon[39288]: osd.7 [v2:172.21.15.94:6824/2545649733,v1:172.21.15.94:6825/2545649733] boot 2024-09-14T03:43:35.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:35 smithi070 ceph-mon[39288]: osdmap e49: 8 total, 8 up, 8 in 2024-09-14T03:43:35.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:35 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:43:35.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:35 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/2656089465' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-14T03:43:35.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:35 smithi070 ceph-mon[39288]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-14T03:43:35.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:35 smithi070 ceph-mon[39288]: osdmap e50: 8 total, 8 up, 8 in 2024-09-14T03:43:36.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:36 smithi070 ceph-mon[27495]: Detected new or changed devices on smithi094 2024-09-14T03:43:36.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:36 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:36.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:36 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:36.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:36 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:43:36.409 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:36 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:43:36.409 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:36 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:43:36.409 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:36 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:43:36.409 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:36 smithi070 ceph-mon[27495]: Adjusting osd_memory_target on smithi094 to 4272M 2024-09-14T03:43:36.409 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:36 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:36.409 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:36 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:36.410 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:36 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:43:36.410 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:36 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:36.410 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:36 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:43:36.410 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:36 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:36.410 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:36 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:43:36.410 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:36 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:36.411 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:36 smithi070 ceph-mon[27495]: osdmap e51: 8 total, 8 up, 8 in 2024-09-14T03:43:36.411 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:36 smithi070 ceph-mon[39288]: Detected new or changed devices on smithi094 2024-09-14T03:43:36.411 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:36 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:36.411 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:36 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:36.412 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:36 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:43:36.412 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:36 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:43:36.412 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:36 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:43:36.412 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:36 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:43:36.412 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:36 smithi070 ceph-mon[39288]: Adjusting osd_memory_target on smithi094 to 4272M 2024-09-14T03:43:36.412 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:36 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:36.413 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:36 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:36.413 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:36 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:43:36.413 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:36 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:36.413 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:36 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:43:36.413 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:36 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:36.413 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:36 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:43:36.414 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:36 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:36.414 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:36 smithi070 ceph-mon[39288]: osdmap e51: 8 total, 8 up, 8 in 2024-09-14T03:43:36.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:36 smithi094 ceph-mon[31053]: Detected new or changed devices on smithi094 2024-09-14T03:43:36.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:36 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:36.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:36 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:36.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:36 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:43:36.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:36 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:43:36.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:36 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:43:36.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:36 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:43:36.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:36 smithi094 ceph-mon[31053]: Adjusting osd_memory_target on smithi094 to 4272M 2024-09-14T03:43:36.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:36 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:36.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:36 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:36.529 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:36 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:43:36.529 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:36 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:36.529 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:36 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:43:36.529 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:36 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:36.529 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:36 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:43:36.529 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:36 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:36.530 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:36 smithi094 ceph-mon[31053]: osdmap e51: 8 total, 8 up, 8 in 2024-09-14T03:43:37.025 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:43:37.026 INFO:teuthology.orchestra.run.smithi070.stdout:{"epoch":51,"fsid":"b6380d42-724a-11ef-bceb-c7b262605968","created":"2024-09-14T03:38:29.406762+0000","modified":"2024-09-14T03:43:36.136303+0000","last_up_change":"2024-09-14T03:43:34.130380+0000","last_in_change":"2024-09-14T03:43:17.652311+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:41:38.745294+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"a7fabda1-9e60-42f2-af60-eec626181688","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6802","nonce":3822384296},{"type":"v1","addr":"172.21.15.70:6803","nonce":3822384296}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6804","nonce":3822384296},{"type":"v1","addr":"172.21.15.70:6805","nonce":3822384296}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6808","nonce":3822384296},{"type":"v1","addr":"172.21.15.70:6809","nonce":3822384296}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6806","nonce":3822384296},{"type":"v1","addr":"172.21.15.70:6807","nonce":3822384296}]},"public_addr":"172.21.15.70:6803/3822384296","cluster_addr":"172.21.15.70:6805/3822384296","heartbeat_back_addr":"172.21.15.70:6809/3822384296","heartbeat_front_addr":"172.21.15.70:6807/3822384296","state":["exists","up"]},{"osd":1,"uuid":"f5b9ef91-59af-4ef0-a101-1c00ba6007ad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":32,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6810","nonce":3986345052},{"type":"v1","addr":"172.21.15.70:6811","nonce":3986345052}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6812","nonce":3986345052},{"type":"v1","addr":"172.21.15.70:6813","nonce":3986345052}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6816","nonce":3986345052},{"type":"v1","addr":"172.21.15.70:6817","nonce":3986345052}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6814","nonce":3986345052},{"type":"v1","addr":"172.21.15.70:6815","nonce":3986345052}]},"public_addr":"172.21.15.70:6811/3986345052","cluster_addr":"172.21.15.70:6813/3986345052","heartbeat_back_addr":"172.21.15.70:6817/3986345052","heartbeat_front_addr":"172.21.15.70:6815/3986345052","state":["exists","up"]},{"osd":2,"uuid":"0dc07495-c6a5-4762-8f63-575eb0fe1671","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6818","nonce":2374764370},{"type":"v1","addr":"172.21.15.70:6819","nonce":2374764370}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6820","nonce":2374764370},{"type":"v1","addr":"172.21.15.70:6821","nonce":2374764370}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6824","nonce":2374764370},{"type":"v1","addr":"172.21.15.70:6825","nonce":2374764370}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6822","nonce":2374764370},{"type":"v1","addr":"172.21.15.70:6823","nonce":2374764370}]},"public_addr":"172.21.15.70:6819/2374764370","cluster_addr":"172.21.15.70:6821/2374764370","heartbeat_back_addr":"172.21.15.70:6825/2374764370","heartbeat_front_addr":"172.21.15.70:6823/2374764370","state":["exists","up"]},{"osd":3,"uuid":"583e5165-ea10-4d03-b731-36cfb1108362","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6826","nonce":2294516176},{"type":"v1","addr":"172.21.15.70:6827","nonce":2294516176}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6828","nonce":2294516176},{"type":"v1","addr":"172.21.15.70:6829","nonce":2294516176}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6832","nonce":2294516176},{"type":"v1","addr":"172.21.15.70:6833","nonce":2294516176}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6830","nonce":2294516176},{"type":"v1","addr":"172.21.15.70:6831","nonce":2294516176}]},"public_addr":"172.21.15.70:6827/2294516176","cluster_addr":"172.21.15.70:6829/2294516176","heartbeat_back_addr":"172.21.15.70:6833/2294516176","heartbeat_front_addr":"172.21.15.70:6831/2294516176","state":["exists","up"]},{"osd":4,"uuid":"33177119-9632-431c-9180-9d27e7bc0c42","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6800","nonce":887215534},{"type":"v1","addr":"172.21.15.94:6801","nonce":887215534}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6802","nonce":887215534},{"type":"v1","addr":"172.21.15.94:6803","nonce":887215534}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6806","nonce":887215534},{"type":"v1","addr":"172.21.15.94:6807","nonce":887215534}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6804","nonce":887215534},{"type":"v1","addr":"172.21.15.94:6805","nonce":887215534}]},"public_addr":"172.21.15.94:6801/887215534","cluster_addr":"172.21.15.94:6803/887215534","heartbeat_back_addr":"172.21.15.94:6807/887215534","heartbeat_front_addr":"172.21.15.94:6805/887215534","state":["exists","up"]},{"osd":5,"uuid":"eb037130-5a05-42e0-947e-09bfba8fac22","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":38,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6808","nonce":1692579524},{"type":"v1","addr":"172.21.15.94:6809","nonce":1692579524}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6810","nonce":1692579524},{"type":"v1","addr":"172.21.15.94:6811","nonce":1692579524}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6814","nonce":1692579524},{"type":"v1","addr":"172.21.15.94:6815","nonce":1692579524}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6812","nonce":1692579524},{"type":"v1","addr":"172.21.15.94:6813","nonce":1692579524}]},"public_addr":"172.21.15.94:6809/1692579524","cluster_addr":"172.21.15.94:6811/1692579524","heartbeat_back_addr":"172.21.15.94:6815/1692579524","heartbeat_front_addr":"172.21.15.94:6813/1692579524","state":["exists","up"]},{"osd":6,"uuid":"f5dca4b5-8906-4eac-84e4-fcb7d1a29674","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":44,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6816","nonce":1107979948},{"type":"v1","addr":"172.21.15.94:6817","nonce":1107979948}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6818","nonce":1107979948},{"type":"v1","addr":"172.21.15.94:6819","nonce":1107979948}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6822","nonce":1107979948},{"type":"v1","addr":"172.21.15.94:6823","nonce":1107979948}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6820","nonce":1107979948},{"type":"v1","addr":"172.21.15.94:6821","nonce":1107979948}]},"public_addr":"172.21.15.94:6817/1107979948","cluster_addr":"172.21.15.94:6819/1107979948","heartbeat_back_addr":"172.21.15.94:6823/1107979948","heartbeat_front_addr":"172.21.15.94:6821/1107979948","state":["exists","up"]},{"osd":7,"uuid":"86e3ef7d-6ab8-47c3-a4bb-db923af2b49b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6824","nonce":2545649733},{"type":"v1","addr":"172.21.15.94:6825","nonce":2545649733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6826","nonce":2545649733},{"type":"v1","addr":"172.21.15.94:6827","nonce":2545649733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6830","nonce":2545649733},{"type":"v1","addr":"172.21.15.94:6831","nonce":2545649733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6828","nonce":2545649733},{"type":"v1","addr":"172.21.15.94:6829","nonce":2545649733}]},"public_addr":"172.21.15.94:6825/2545649733","cluster_addr":"172.21.15.94:6827/2545649733","heartbeat_back_addr":"172.21.15.94:6831/2545649733","heartbeat_front_addr":"172.21.15.94:6829/2545649733","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:40:47.681447+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:41:11.801118+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:41:35.764497+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:42:00.424802+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:42:21.618849+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:42:45.384437+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:43:09.350263+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:43:32.472815+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.70:0/2978576061":"2024-09-15T03:39:24.448616+0000","172.21.15.70:6801/673916264":"2024-09-15T03:39:24.448616+0000","172.21.15.70:6800/673916264":"2024-09-15T03:39:24.448616+0000","172.21.15.70:0/3005166882":"2024-09-15T03:39:24.448616+0000","172.21.15.70:0/3134643770":"2024-09-15T03:39:24.448616+0000","172.21.15.70:0/22543101":"2024-09-15T03:39:01.589983+0000","172.21.15.70:0/2447056811":"2024-09-15T03:39:01.589983+0000","172.21.15.70:6801/2254107694":"2024-09-15T03:39:01.589983+0000","172.21.15.70:6800/2254107694":"2024-09-15T03:39:01.589983+0000","172.21.15.70:0/4203647763":"2024-09-15T03:39:01.589983+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:43:37.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:37 smithi070 ceph-mon[27495]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:37.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:37 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/3640386826' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:43:37.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:37 smithi070 ceph-mon[39288]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:37.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:37 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/3640386826' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:43:37.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:37 smithi094 ceph-mon[31053]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:37.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:37 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/3640386826' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:43:37.885 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-14T03:41:38.745294+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-14T03:43:37.885 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-09-14T03:43:38.132 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:43:39.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:39 smithi070 ceph-mon[39288]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:39.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:39 smithi070 ceph-mon[27495]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:39.844 INFO:teuthology.orchestra.run.smithi070.stdout:pg_num: 1 2024-09-14T03:43:40.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:39 smithi094 ceph-mon[31053]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:40.520 INFO:tasks.cephadm:Adding mds.a on smithi070 2024-09-14T03:43:40.520 INFO:tasks.cephadm:Adding mds.b on smithi094 2024-09-14T03:43:40.520 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph orch apply mds all '2;smithi070=a;smithi094=b' 2024-09-14T03:43:40.776 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:43:40.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:40 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/2561035907' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-14T03:43:40.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:40 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/2561035907' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-14T03:43:40.894 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:40 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/2561035907' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-14T03:43:41.842 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:41 smithi094 ceph-mon[31053]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:42.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:41 smithi070 ceph-mon[27495]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:42.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:41 smithi070 ceph-mon[39288]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:42.502 INFO:teuthology.orchestra.run.smithi094.stdout:Scheduled mds.all update... 2024-09-14T03:43:43.066 DEBUG:teuthology.orchestra.run.smithi070:mds.a> sudo journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@mds.a.service 2024-09-14T03:43:43.068 DEBUG:teuthology.orchestra.run.smithi094:mds.b> sudo journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@mds.b.service 2024-09-14T03:43:43.070 INFO:tasks.cephadm:Setting up CephFS filesystem(s)... 2024-09-14T03:43:43.070 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph fs dump --format=json 2024-09-14T03:43:43.753 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:43 smithi070 ceph-mon[27495]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:43.753 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:43 smithi070 ceph-mon[27495]: from='client.24298 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "2;smithi070=a;smithi094=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:43:43.753 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:43 smithi070 ceph-mon[27495]: Saving service mds.all spec with placement smithi070=a;smithi094=b;count:2 2024-09-14T03:43:43.753 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:43 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:43.753 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:43 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:43:43.754 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:43 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:43.754 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:43 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:43:43.754 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:43 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:43.754 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:43 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-14T03:43:43.754 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:43 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-14T03:43:43.754 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:43 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:43.754 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:43 smithi070 ceph-mon[27495]: Deploying daemon mds.a on smithi070 2024-09-14T03:43:43.754 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:43 smithi070 ceph-mon[39288]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:43.754 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:43 smithi070 ceph-mon[39288]: from='client.24298 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "2;smithi070=a;smithi094=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:43:43.754 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:43 smithi070 ceph-mon[39288]: Saving service mds.all spec with placement smithi070=a;smithi094=b;count:2 2024-09-14T03:43:43.754 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:43 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:43.755 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:43 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:43:43.755 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:43 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:43.755 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:43 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:43:43.755 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:43 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:43.755 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:43 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-14T03:43:43.755 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:43 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-14T03:43:43.755 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:43 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:43.755 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:43 smithi070 ceph-mon[39288]: Deploying daemon mds.a on smithi070 2024-09-14T03:43:43.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:43 smithi094 ceph-mon[31053]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:43.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:43 smithi094 ceph-mon[31053]: from='client.24298 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "2;smithi070=a;smithi094=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:43:43.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:43 smithi094 ceph-mon[31053]: Saving service mds.all spec with placement smithi070=a;smithi094=b;count:2 2024-09-14T03:43:43.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:43 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:43.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:43 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:43:43.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:43 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:43.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:43 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:43:43.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:43 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:43.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:43 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-14T03:43:43.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:43 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-14T03:43:43.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:43 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:43.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:43 smithi094 ceph-mon[31053]: Deploying daemon mds.a on smithi070 2024-09-14T03:43:44.450 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:43:45.638 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:45 smithi070 ceph-mon[27495]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:45.638 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:45 smithi070 ceph-mon[39288]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:45.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:45 smithi094 ceph-mon[31053]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:46.402 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:43:46.402 INFO:teuthology.orchestra.run.smithi070.stdout:{"epoch":1,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[]} 2024-09-14T03:43:46.402 INFO:teuthology.orchestra.run.smithi070.stderr:dumped fsmap epoch 1 2024-09-14T03:43:46.503 INFO:journalctl@ceph.mds.a.smithi070.stdout:Sep 14 03:43:46 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-a[67490]: starting mds.a at 2024-09-14T03:43:46.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:46 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/1768059958' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:43:46.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:46 smithi094 ceph-mon[31053]: mds.? [v2:172.21.15.70:6834/421917097,v1:172.21.15.70:6835/421917097] up:boot 2024-09-14T03:43:46.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:46 smithi094 ceph-mon[31053]: fsmap 1 up:standby 2024-09-14T03:43:46.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:46 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-14T03:43:46.785 INFO:journalctl@ceph.mds.a.smithi070.stdout:Sep 14 03:43:46 smithi070 bash[67429]: 52e0dbb7be5086597de29f732d6b365e9675acc0ccf69c4844dfb3eff8876270 2024-09-14T03:43:46.785 INFO:journalctl@ceph.mds.a.smithi070.stdout:Sep 14 03:43:46 smithi070 systemd[1]: Started Ceph mds.a for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:43:46.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:46 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/1768059958' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:43:46.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:46 smithi070 ceph-mon[27495]: mds.? [v2:172.21.15.70:6834/421917097,v1:172.21.15.70:6835/421917097] up:boot 2024-09-14T03:43:46.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:46 smithi070 ceph-mon[27495]: fsmap 1 up:standby 2024-09-14T03:43:46.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:46 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-14T03:43:46.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:46 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/1768059958' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:43:46.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:46 smithi070 ceph-mon[39288]: mds.? [v2:172.21.15.70:6834/421917097,v1:172.21.15.70:6835/421917097] up:boot 2024-09-14T03:43:46.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:46 smithi070 ceph-mon[39288]: fsmap 1 up:standby 2024-09-14T03:43:46.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:46 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-14T03:43:47.983 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:47 smithi094 ceph-mon[31053]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:47.983 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:47 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:47.983 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:47 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:47.983 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:47 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:47.983 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:47 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-14T03:43:47.983 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:47 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-14T03:43:47.983 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:47 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:47.983 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:47 smithi094 ceph-mon[31053]: Deploying daemon mds.b on smithi094 2024-09-14T03:43:48.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:47 smithi070 ceph-mon[27495]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:48.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:47 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:48.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:47 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:48.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:47 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:48.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:47 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-14T03:43:48.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:47 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-14T03:43:48.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:47 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:48.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:47 smithi070 ceph-mon[27495]: Deploying daemon mds.b on smithi094 2024-09-14T03:43:48.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:47 smithi070 ceph-mon[39288]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:48.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:47 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:48.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:47 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:48.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:47 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:48.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:47 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-14T03:43:48.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:47 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-14T03:43:48.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:47 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:48.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:47 smithi070 ceph-mon[39288]: Deploying daemon mds.b on smithi094 2024-09-14T03:43:48.774 INFO:journalctl@ceph.mds.b.smithi094.stdout:Sep 14 03:43:48 smithi094 podman[59360]: 2024-09-14 03:43:48.617281382 +0000 UTC m=+0.136567740 container create e3b5e3e0018900bd5f9f1623ccdf58cbdaf47eed42f4ad6fb3d54c1226f800ae (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-b, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-eb16b97, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-14T03:43:49.137 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph fs dump --format=json 2024-09-14T03:43:49.154 INFO:journalctl@ceph.mds.b.smithi094.stdout:Sep 14 03:43:48 smithi094 podman[59360]: 2024-09-14 03:43:48.870457445 +0000 UTC m=+0.389743811 container init e3b5e3e0018900bd5f9f1623ccdf58cbdaf47eed42f4ad6fb3d54c1226f800ae (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-b, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=quincy-eb16b97, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-14T03:43:49.154 INFO:journalctl@ceph.mds.b.smithi094.stdout:Sep 14 03:43:48 smithi094 podman[59360]: 2024-09-14 03:43:48.875139377 +0000 UTC m=+0.394425726 container start e3b5e3e0018900bd5f9f1623ccdf58cbdaf47eed42f4ad6fb3d54c1226f800ae (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-b, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-eb16b97, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-14T03:43:49.154 INFO:journalctl@ceph.mds.b.smithi094.stdout:Sep 14 03:43:48 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-b[59374]: starting mds.b at 2024-09-14T03:43:49.154 INFO:journalctl@ceph.mds.b.smithi094.stdout:Sep 14 03:43:48 smithi094 bash[59360]: e3b5e3e0018900bd5f9f1623ccdf58cbdaf47eed42f4ad6fb3d54c1226f800ae 2024-09-14T03:43:49.155 INFO:journalctl@ceph.mds.b.smithi094.stdout:Sep 14 03:43:48 smithi094 systemd[1]: Started Ceph mds.b for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:43:49.399 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:43:50.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:50 smithi094 ceph-mon[31053]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:50.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:50 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:50.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:50 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:50.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:50 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:50.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:50 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:50.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:50 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:50.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:50 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:43:50.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:50 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:50.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:50 smithi094 ceph-mon[31053]: mds.? [v2:172.21.15.94:6832/3620660683,v1:172.21.15.94:6833/3620660683] up:boot 2024-09-14T03:43:50.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:50 smithi094 ceph-mon[31053]: fsmap 2 up:standby 2024-09-14T03:43:50.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:50 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-14T03:43:50.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:50 smithi070 ceph-mon[39288]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:50.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:50 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:50.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:50 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:50.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:50 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:50.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:50 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:50.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:50 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:50.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:50 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:43:50.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:50 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:50.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:50 smithi070 ceph-mon[39288]: mds.? [v2:172.21.15.94:6832/3620660683,v1:172.21.15.94:6833/3620660683] up:boot 2024-09-14T03:43:50.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:50 smithi070 ceph-mon[39288]: fsmap 2 up:standby 2024-09-14T03:43:50.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:50 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-14T03:43:50.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:50 smithi070 ceph-mon[27495]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:50.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:50 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:50.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:50 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:50.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:50 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:50.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:50 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:50.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:50 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:50.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:50 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:43:50.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:50 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:50.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:50 smithi070 ceph-mon[27495]: mds.? [v2:172.21.15.94:6832/3620660683,v1:172.21.15.94:6833/3620660683] up:boot 2024-09-14T03:43:50.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:50 smithi070 ceph-mon[27495]: fsmap 2 up:standby 2024-09-14T03:43:50.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:50 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-14T03:43:51.510 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:51 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:51.510 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:51 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:51.510 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:51 smithi070 ceph-mon[27495]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:51.510 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:51 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:51.511 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:51 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:51.511 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:51 smithi070 ceph-mon[39288]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:51.609 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:43:51.609 INFO:teuthology.orchestra.run.smithi070.stdout:{"epoch":3,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":24304,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.94:6833/3620660683","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6832","nonce":3620660683},{"type":"v1","addr":"172.21.15.94:6833","nonce":3620660683}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24323,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.70:6835/421917097","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6834","nonce":421917097},{"type":"v1","addr":"172.21.15.70:6835","nonce":421917097}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[]} 2024-09-14T03:43:51.609 INFO:teuthology.orchestra.run.smithi070.stderr:dumped fsmap epoch 3 2024-09-14T03:43:51.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:51 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:51.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:51 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:51.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:51 smithi094 ceph-mon[31053]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:52.451 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph --format=json-pretty osd lspools 2024-09-14T03:43:52.718 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:43:52.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:52 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/787951468' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:43:52.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:52 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/787951468' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:43:52.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:52 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/787951468' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:43:53.771 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:53 smithi070 ceph-mon[27495]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:53.771 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:53 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:53.771 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:53 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:53.771 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:53 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:53.771 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:53 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:43:53.771 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:53 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:53.771 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:53 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:43:53.771 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:53 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:53.771 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:53 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:43:53.771 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:53 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:53.772 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:53 smithi070 ceph-mon[39288]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:53.772 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:53 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:53.772 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:53 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:53.772 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:53 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:53.772 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:53 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:43:53.772 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:53 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:53.772 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:53 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:43:53.772 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:53 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:53.772 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:53 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:43:53.772 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:53 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:54.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:53 smithi094 ceph-mon[31053]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:54.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:53 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:54.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:53 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:54.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:53 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:54.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:53 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:43:54.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:53 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:54.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:53 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:43:54.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:53 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:43:54.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:53 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:43:54.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:53 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:43:54.396 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:43:54.397 INFO:teuthology.orchestra.run.smithi070.stdout:[ 2024-09-14T03:43:54.397 INFO:teuthology.orchestra.run.smithi070.stdout: { 2024-09-14T03:43:54.397 INFO:teuthology.orchestra.run.smithi070.stdout: "poolnum": 1, 2024-09-14T03:43:54.397 INFO:teuthology.orchestra.run.smithi070.stdout: "poolname": ".mgr" 2024-09-14T03:43:54.397 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-09-14T03:43:54.397 INFO:teuthology.orchestra.run.smithi070.stdout:] 2024-09-14T03:43:55.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:54 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/3059795134' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2024-09-14T03:43:55.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:54 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/3059795134' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2024-09-14T03:43:55.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:54 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/3059795134' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2024-09-14T03:43:55.100 DEBUG:tasks.cephfs.filesystem:Creating filesystem 'cephfs' 2024-09-14T03:43:55.100 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph osd pool create cephfs_metadata --pg_num_min 64 2024-09-14T03:43:55.344 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:43:55.752 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:55 smithi070 ceph-mon[27495]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:55.753 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:55 smithi070 ceph-mon[39288]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:56.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:55 smithi094 ceph-mon[31053]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:57.721 INFO:teuthology.orchestra.run.smithi070.stderr:pool 'cephfs_metadata' created 2024-09-14T03:43:58.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:57 smithi094 ceph-mon[31053]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:58.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:57 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/1196549905' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-09-14T03:43:58.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:57 smithi094 ceph-mon[31053]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-09-14T03:43:58.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:57 smithi070 ceph-mon[27495]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:58.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:57 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/1196549905' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-09-14T03:43:58.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:57 smithi070 ceph-mon[27495]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-09-14T03:43:58.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:57 smithi070 ceph-mon[39288]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:58.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:57 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/1196549905' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-09-14T03:43:58.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:57 smithi070 ceph-mon[39288]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-09-14T03:43:58.455 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph osd pool create cephfs_data 64 --pg_num_min 64 --target_size_ratio 0.9 2024-09-14T03:43:58.697 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:43:58.737 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:58 smithi070 ceph-mon[27495]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2024-09-14T03:43:59.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:58 smithi094 ceph-mon[31053]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2024-09-14T03:43:59.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:58 smithi094 ceph-mon[31053]: osdmap e52: 8 total, 8 up, 8 in 2024-09-14T03:43:59.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:58 smithi094 ceph-mon[31053]: osdmap e53: 8 total, 8 up, 8 in 2024-09-14T03:43:59.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:58 smithi070 ceph-mon[27495]: osdmap e52: 8 total, 8 up, 8 in 2024-09-14T03:43:59.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:58 smithi070 ceph-mon[27495]: osdmap e53: 8 total, 8 up, 8 in 2024-09-14T03:43:59.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:58 smithi070 ceph-mon[39288]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2024-09-14T03:43:59.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:58 smithi070 ceph-mon[39288]: osdmap e52: 8 total, 8 up, 8 in 2024-09-14T03:43:59.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:58 smithi070 ceph-mon[39288]: osdmap e53: 8 total, 8 up, 8 in 2024-09-14T03:43:59.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:59 smithi070 ceph-mon[39288]: pgmap v179: 33 pgs: 9 creating+peering, 23 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:59.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:59 smithi070 ceph-mon[39288]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:43:59.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:43:59 smithi070 ceph-mon[39288]: osdmap e54: 8 total, 8 up, 8 in 2024-09-14T03:43:59.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:59 smithi070 ceph-mon[27495]: pgmap v179: 33 pgs: 9 creating+peering, 23 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:43:59.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:59 smithi070 ceph-mon[27495]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:43:59.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:43:59 smithi070 ceph-mon[27495]: osdmap e54: 8 total, 8 up, 8 in 2024-09-14T03:44:00.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:59 smithi094 ceph-mon[31053]: pgmap v179: 33 pgs: 9 creating+peering, 23 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:00.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:59 smithi094 ceph-mon[31053]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:44:00.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:43:59 smithi094 ceph-mon[31053]: osdmap e54: 8 total, 8 up, 8 in 2024-09-14T03:44:00.858 INFO:teuthology.orchestra.run.smithi070.stderr:pool 'cephfs_data' created 2024-09-14T03:44:01.165 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:00 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/1458614541' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-09-14T03:44:01.165 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:00 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/1458614541' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-09-14T03:44:01.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:00 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/1458614541' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-09-14T03:44:01.481 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph fs new cephfs cephfs_metadata cephfs_data 2024-09-14T03:44:01.723 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:44:02.101 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:01 smithi070 ceph-mon[27495]: pgmap v182: 33 pgs: 9 creating+peering, 9 unknown, 15 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:02.102 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:01 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/1458614541' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2024-09-14T03:44:02.102 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:01 smithi070 ceph-mon[27495]: osdmap e55: 8 total, 8 up, 8 in 2024-09-14T03:44:02.102 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:01 smithi070 ceph-mon[27495]: osdmap e56: 8 total, 8 up, 8 in 2024-09-14T03:44:02.102 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:01 smithi070 ceph-mon[39288]: pgmap v182: 33 pgs: 9 creating+peering, 9 unknown, 15 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:02.102 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:01 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/1458614541' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2024-09-14T03:44:02.102 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:01 smithi070 ceph-mon[39288]: osdmap e55: 8 total, 8 up, 8 in 2024-09-14T03:44:02.102 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:01 smithi070 ceph-mon[39288]: osdmap e56: 8 total, 8 up, 8 in 2024-09-14T03:44:02.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:01 smithi094 ceph-mon[31053]: pgmap v182: 33 pgs: 9 creating+peering, 9 unknown, 15 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:02.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:01 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/1458614541' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2024-09-14T03:44:02.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:01 smithi094 ceph-mon[31053]: osdmap e55: 8 total, 8 up, 8 in 2024-09-14T03:44:02.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:01 smithi094 ceph-mon[31053]: osdmap e56: 8 total, 8 up, 8 in 2024-09-14T03:44:03.875 INFO:teuthology.orchestra.run.smithi070.stderr:new fs with metadata pool 2 and data pool 3 2024-09-14T03:44:04.245 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:03 smithi070 ceph-mon[39288]: pgmap v185: 97 pgs: 16 creating+peering, 55 unknown, 26 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:04.246 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:03 smithi070 ceph-mon[39288]: osdmap e57: 8 total, 8 up, 8 in 2024-09-14T03:44:04.246 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:03 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/1685281676' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-09-14T03:44:04.246 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:03 smithi070 ceph-mon[39288]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-09-14T03:44:04.246 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:03 smithi070 ceph-mon[39288]: daemon mds.a assigned to filesystem cephfs as rank 0 2024-09-14T03:44:04.246 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:03 smithi070 ceph-mon[27495]: pgmap v185: 97 pgs: 16 creating+peering, 55 unknown, 26 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:04.246 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:03 smithi070 ceph-mon[27495]: osdmap e57: 8 total, 8 up, 8 in 2024-09-14T03:44:04.246 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:03 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/1685281676' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-09-14T03:44:04.246 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:03 smithi070 ceph-mon[27495]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-09-14T03:44:04.246 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:03 smithi070 ceph-mon[27495]: daemon mds.a assigned to filesystem cephfs as rank 0 2024-09-14T03:44:04.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:03 smithi094 ceph-mon[31053]: pgmap v185: 97 pgs: 16 creating+peering, 55 unknown, 26 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:04.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:03 smithi094 ceph-mon[31053]: osdmap e57: 8 total, 8 up, 8 in 2024-09-14T03:44:04.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:03 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/1685281676' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-09-14T03:44:04.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:03 smithi094 ceph-mon[31053]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-09-14T03:44:04.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:03 smithi094 ceph-mon[31053]: daemon mds.a assigned to filesystem cephfs as rank 0 2024-09-14T03:44:04.661 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T03:44:04.904 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:44:05.021 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:04 smithi070 ceph-mon[27495]: osdmap e58: 8 total, 8 up, 8 in 2024-09-14T03:44:05.022 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:04 smithi070 ceph-mon[27495]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2024-09-14T03:44:05.022 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:04 smithi070 ceph-mon[27495]: fsmap cephfs:1 {0=a=up:creating} 1 up:standby 2024-09-14T03:44:05.022 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:04 smithi070 ceph-mon[27495]: daemon mds.a is now active in filesystem cephfs as rank 0 2024-09-14T03:44:05.022 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:04 smithi070 ceph-mon[39288]: osdmap e58: 8 total, 8 up, 8 in 2024-09-14T03:44:05.022 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:04 smithi070 ceph-mon[39288]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2024-09-14T03:44:05.022 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:04 smithi070 ceph-mon[39288]: fsmap cephfs:1 {0=a=up:creating} 1 up:standby 2024-09-14T03:44:05.022 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:04 smithi070 ceph-mon[39288]: daemon mds.a is now active in filesystem cephfs as rank 0 2024-09-14T03:44:05.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:04 smithi094 ceph-mon[31053]: osdmap e58: 8 total, 8 up, 8 in 2024-09-14T03:44:05.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:04 smithi094 ceph-mon[31053]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2024-09-14T03:44:05.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:04 smithi094 ceph-mon[31053]: fsmap cephfs:1 {0=a=up:creating} 1 up:standby 2024-09-14T03:44:05.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:04 smithi094 ceph-mon[31053]: daemon mds.a is now active in filesystem cephfs as rank 0 2024-09-14T03:44:06.001 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:05 smithi070 ceph-mon[27495]: pgmap v188: 97 pgs: 7 creating+peering, 27 unknown, 63 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:06.002 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:05 smithi070 ceph-mon[27495]: Health check cleared: POOL_APP_NOT_ENABLED (was: 2 pool(s) do not have an application enabled) 2024-09-14T03:44:06.002 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:05 smithi070 ceph-mon[27495]: Cluster is now healthy 2024-09-14T03:44:06.002 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:05 smithi070 ceph-mon[27495]: mds.? [v2:172.21.15.70:6834/421917097,v1:172.21.15.70:6835/421917097] up:active 2024-09-14T03:44:06.002 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:05 smithi070 ceph-mon[27495]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:44:06.002 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:05 smithi070 ceph-mon[27495]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:44:06.002 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:05 smithi070 ceph-mon[39288]: pgmap v188: 97 pgs: 7 creating+peering, 27 unknown, 63 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:06.002 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:05 smithi070 ceph-mon[39288]: Health check cleared: POOL_APP_NOT_ENABLED (was: 2 pool(s) do not have an application enabled) 2024-09-14T03:44:06.002 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:05 smithi070 ceph-mon[39288]: Cluster is now healthy 2024-09-14T03:44:06.002 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:05 smithi070 ceph-mon[39288]: mds.? [v2:172.21.15.70:6834/421917097,v1:172.21.15.70:6835/421917097] up:active 2024-09-14T03:44:06.003 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:05 smithi070 ceph-mon[39288]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:44:06.003 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:05 smithi070 ceph-mon[39288]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:44:06.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:05 smithi094 ceph-mon[31053]: pgmap v188: 97 pgs: 7 creating+peering, 27 unknown, 63 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:06.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:05 smithi094 ceph-mon[31053]: Health check cleared: POOL_APP_NOT_ENABLED (was: 2 pool(s) do not have an application enabled) 2024-09-14T03:44:06.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:05 smithi094 ceph-mon[31053]: Cluster is now healthy 2024-09-14T03:44:06.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:05 smithi094 ceph-mon[31053]: mds.? [v2:172.21.15.70:6834/421917097,v1:172.21.15.70:6835/421917097] up:active 2024-09-14T03:44:06.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:05 smithi094 ceph-mon[31053]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:44:06.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:05 smithi094 ceph-mon[31053]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:44:06.535 INFO:journalctl@ceph.mds.a.smithi070.stdout:Sep 14 03:44:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-a[67490]: 2024-09-14T03:44:06.453+0000 7ff8c2013640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-09-14T03:44:06.631 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:44:06.632 INFO:teuthology.orchestra.run.smithi070.stdout:{"epoch":58,"fsid":"b6380d42-724a-11ef-bceb-c7b262605968","created":"2024-09-14T03:38:29.406762+0000","modified":"2024-09-14T03:44:03.568878+0000","last_up_change":"2024-09-14T03:43:34.130380+0000","last_in_change":"2024-09-14T03:43:17.652311+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:41:38.745294+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-14T03:43:57.402183+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-14T03:44:00.803078+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"a7fabda1-9e60-42f2-af60-eec626181688","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6802","nonce":3822384296},{"type":"v1","addr":"172.21.15.70:6803","nonce":3822384296}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6804","nonce":3822384296},{"type":"v1","addr":"172.21.15.70:6805","nonce":3822384296}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6808","nonce":3822384296},{"type":"v1","addr":"172.21.15.70:6809","nonce":3822384296}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6806","nonce":3822384296},{"type":"v1","addr":"172.21.15.70:6807","nonce":3822384296}]},"public_addr":"172.21.15.70:6803/3822384296","cluster_addr":"172.21.15.70:6805/3822384296","heartbeat_back_addr":"172.21.15.70:6809/3822384296","heartbeat_front_addr":"172.21.15.70:6807/3822384296","state":["exists","up"]},{"osd":1,"uuid":"f5b9ef91-59af-4ef0-a101-1c00ba6007ad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6810","nonce":3986345052},{"type":"v1","addr":"172.21.15.70:6811","nonce":3986345052}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6812","nonce":3986345052},{"type":"v1","addr":"172.21.15.70:6813","nonce":3986345052}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6816","nonce":3986345052},{"type":"v1","addr":"172.21.15.70:6817","nonce":3986345052}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6814","nonce":3986345052},{"type":"v1","addr":"172.21.15.70:6815","nonce":3986345052}]},"public_addr":"172.21.15.70:6811/3986345052","cluster_addr":"172.21.15.70:6813/3986345052","heartbeat_back_addr":"172.21.15.70:6817/3986345052","heartbeat_front_addr":"172.21.15.70:6815/3986345052","state":["exists","up"]},{"osd":2,"uuid":"0dc07495-c6a5-4762-8f63-575eb0fe1671","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6818","nonce":2374764370},{"type":"v1","addr":"172.21.15.70:6819","nonce":2374764370}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6820","nonce":2374764370},{"type":"v1","addr":"172.21.15.70:6821","nonce":2374764370}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6824","nonce":2374764370},{"type":"v1","addr":"172.21.15.70:6825","nonce":2374764370}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6822","nonce":2374764370},{"type":"v1","addr":"172.21.15.70:6823","nonce":2374764370}]},"public_addr":"172.21.15.70:6819/2374764370","cluster_addr":"172.21.15.70:6821/2374764370","heartbeat_back_addr":"172.21.15.70:6825/2374764370","heartbeat_front_addr":"172.21.15.70:6823/2374764370","state":["exists","up"]},{"osd":3,"uuid":"583e5165-ea10-4d03-b731-36cfb1108362","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6826","nonce":2294516176},{"type":"v1","addr":"172.21.15.70:6827","nonce":2294516176}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6828","nonce":2294516176},{"type":"v1","addr":"172.21.15.70:6829","nonce":2294516176}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6832","nonce":2294516176},{"type":"v1","addr":"172.21.15.70:6833","nonce":2294516176}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6830","nonce":2294516176},{"type":"v1","addr":"172.21.15.70:6831","nonce":2294516176}]},"public_addr":"172.21.15.70:6827/2294516176","cluster_addr":"172.21.15.70:6829/2294516176","heartbeat_back_addr":"172.21.15.70:6833/2294516176","heartbeat_front_addr":"172.21.15.70:6831/2294516176","state":["exists","up"]},{"osd":4,"uuid":"33177119-9632-431c-9180-9d27e7bc0c42","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6800","nonce":887215534},{"type":"v1","addr":"172.21.15.94:6801","nonce":887215534}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6802","nonce":887215534},{"type":"v1","addr":"172.21.15.94:6803","nonce":887215534}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6806","nonce":887215534},{"type":"v1","addr":"172.21.15.94:6807","nonce":887215534}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6804","nonce":887215534},{"type":"v1","addr":"172.21.15.94:6805","nonce":887215534}]},"public_addr":"172.21.15.94:6801/887215534","cluster_addr":"172.21.15.94:6803/887215534","heartbeat_back_addr":"172.21.15.94:6807/887215534","heartbeat_front_addr":"172.21.15.94:6805/887215534","state":["exists","up"]},{"osd":5,"uuid":"eb037130-5a05-42e0-947e-09bfba8fac22","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6808","nonce":1692579524},{"type":"v1","addr":"172.21.15.94:6809","nonce":1692579524}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6810","nonce":1692579524},{"type":"v1","addr":"172.21.15.94:6811","nonce":1692579524}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6814","nonce":1692579524},{"type":"v1","addr":"172.21.15.94:6815","nonce":1692579524}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6812","nonce":1692579524},{"type":"v1","addr":"172.21.15.94:6813","nonce":1692579524}]},"public_addr":"172.21.15.94:6809/1692579524","cluster_addr":"172.21.15.94:6811/1692579524","heartbeat_back_addr":"172.21.15.94:6815/1692579524","heartbeat_front_addr":"172.21.15.94:6813/1692579524","state":["exists","up"]},{"osd":6,"uuid":"f5dca4b5-8906-4eac-84e4-fcb7d1a29674","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6816","nonce":1107979948},{"type":"v1","addr":"172.21.15.94:6817","nonce":1107979948}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6818","nonce":1107979948},{"type":"v1","addr":"172.21.15.94:6819","nonce":1107979948}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6822","nonce":1107979948},{"type":"v1","addr":"172.21.15.94:6823","nonce":1107979948}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6820","nonce":1107979948},{"type":"v1","addr":"172.21.15.94:6821","nonce":1107979948}]},"public_addr":"172.21.15.94:6817/1107979948","cluster_addr":"172.21.15.94:6819/1107979948","heartbeat_back_addr":"172.21.15.94:6823/1107979948","heartbeat_front_addr":"172.21.15.94:6821/1107979948","state":["exists","up"]},{"osd":7,"uuid":"86e3ef7d-6ab8-47c3-a4bb-db923af2b49b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6824","nonce":2545649733},{"type":"v1","addr":"172.21.15.94:6825","nonce":2545649733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6826","nonce":2545649733},{"type":"v1","addr":"172.21.15.94:6827","nonce":2545649733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6830","nonce":2545649733},{"type":"v1","addr":"172.21.15.94:6831","nonce":2545649733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6828","nonce":2545649733},{"type":"v1","addr":"172.21.15.94:6829","nonce":2545649733}]},"public_addr":"172.21.15.94:6825/2545649733","cluster_addr":"172.21.15.94:6827/2545649733","heartbeat_back_addr":"172.21.15.94:6831/2545649733","heartbeat_front_addr":"172.21.15.94:6829/2545649733","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:40:47.681447+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:41:11.801118+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:41:35.764497+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:42:00.424802+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:42:21.618849+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:42:45.384437+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:43:09.350263+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:43:32.472815+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.70:0/2978576061":"2024-09-15T03:39:24.448616+0000","172.21.15.70:6801/673916264":"2024-09-15T03:39:24.448616+0000","172.21.15.70:6800/673916264":"2024-09-15T03:39:24.448616+0000","172.21.15.70:0/3005166882":"2024-09-15T03:39:24.448616+0000","172.21.15.70:0/3134643770":"2024-09-15T03:39:24.448616+0000","172.21.15.70:0/22543101":"2024-09-15T03:39:01.589983+0000","172.21.15.70:0/2447056811":"2024-09-15T03:39:01.589983+0000","172.21.15.70:6801/2254107694":"2024-09-15T03:39:01.589983+0000","172.21.15.70:6800/2254107694":"2024-09-15T03:39:01.589983+0000","172.21.15.70:0/4203647763":"2024-09-15T03:39:01.589983+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:44:07.178 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:06 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/4233464611' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:44:07.216 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:06 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/4233464611' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:44:07.216 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:06 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/4233464611' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:44:07.241 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T03:44:07.498 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:44:08.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:07 smithi094 ceph-mon[31053]: pgmap v189: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.1 KiB/s wr, 4 op/s 2024-09-14T03:44:08.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:07 smithi070 ceph-mon[27495]: pgmap v189: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.1 KiB/s wr, 4 op/s 2024-09-14T03:44:08.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:07 smithi070 ceph-mon[39288]: pgmap v189: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.1 KiB/s wr, 4 op/s 2024-09-14T03:44:09.429 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:44:09.430 INFO:teuthology.orchestra.run.smithi070.stdout:{"epoch":58,"fsid":"b6380d42-724a-11ef-bceb-c7b262605968","created":"2024-09-14T03:38:29.406762+0000","modified":"2024-09-14T03:44:03.568878+0000","last_up_change":"2024-09-14T03:43:34.130380+0000","last_in_change":"2024-09-14T03:43:17.652311+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:41:38.745294+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-14T03:43:57.402183+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-14T03:44:00.803078+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"a7fabda1-9e60-42f2-af60-eec626181688","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6802","nonce":3822384296},{"type":"v1","addr":"172.21.15.70:6803","nonce":3822384296}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6804","nonce":3822384296},{"type":"v1","addr":"172.21.15.70:6805","nonce":3822384296}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6808","nonce":3822384296},{"type":"v1","addr":"172.21.15.70:6809","nonce":3822384296}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6806","nonce":3822384296},{"type":"v1","addr":"172.21.15.70:6807","nonce":3822384296}]},"public_addr":"172.21.15.70:6803/3822384296","cluster_addr":"172.21.15.70:6805/3822384296","heartbeat_back_addr":"172.21.15.70:6809/3822384296","heartbeat_front_addr":"172.21.15.70:6807/3822384296","state":["exists","up"]},{"osd":1,"uuid":"f5b9ef91-59af-4ef0-a101-1c00ba6007ad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6810","nonce":3986345052},{"type":"v1","addr":"172.21.15.70:6811","nonce":3986345052}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6812","nonce":3986345052},{"type":"v1","addr":"172.21.15.70:6813","nonce":3986345052}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6816","nonce":3986345052},{"type":"v1","addr":"172.21.15.70:6817","nonce":3986345052}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6814","nonce":3986345052},{"type":"v1","addr":"172.21.15.70:6815","nonce":3986345052}]},"public_addr":"172.21.15.70:6811/3986345052","cluster_addr":"172.21.15.70:6813/3986345052","heartbeat_back_addr":"172.21.15.70:6817/3986345052","heartbeat_front_addr":"172.21.15.70:6815/3986345052","state":["exists","up"]},{"osd":2,"uuid":"0dc07495-c6a5-4762-8f63-575eb0fe1671","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6818","nonce":2374764370},{"type":"v1","addr":"172.21.15.70:6819","nonce":2374764370}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6820","nonce":2374764370},{"type":"v1","addr":"172.21.15.70:6821","nonce":2374764370}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6824","nonce":2374764370},{"type":"v1","addr":"172.21.15.70:6825","nonce":2374764370}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6822","nonce":2374764370},{"type":"v1","addr":"172.21.15.70:6823","nonce":2374764370}]},"public_addr":"172.21.15.70:6819/2374764370","cluster_addr":"172.21.15.70:6821/2374764370","heartbeat_back_addr":"172.21.15.70:6825/2374764370","heartbeat_front_addr":"172.21.15.70:6823/2374764370","state":["exists","up"]},{"osd":3,"uuid":"583e5165-ea10-4d03-b731-36cfb1108362","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6826","nonce":2294516176},{"type":"v1","addr":"172.21.15.70:6827","nonce":2294516176}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6828","nonce":2294516176},{"type":"v1","addr":"172.21.15.70:6829","nonce":2294516176}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6832","nonce":2294516176},{"type":"v1","addr":"172.21.15.70:6833","nonce":2294516176}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6830","nonce":2294516176},{"type":"v1","addr":"172.21.15.70:6831","nonce":2294516176}]},"public_addr":"172.21.15.70:6827/2294516176","cluster_addr":"172.21.15.70:6829/2294516176","heartbeat_back_addr":"172.21.15.70:6833/2294516176","heartbeat_front_addr":"172.21.15.70:6831/2294516176","state":["exists","up"]},{"osd":4,"uuid":"33177119-9632-431c-9180-9d27e7bc0c42","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6800","nonce":887215534},{"type":"v1","addr":"172.21.15.94:6801","nonce":887215534}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6802","nonce":887215534},{"type":"v1","addr":"172.21.15.94:6803","nonce":887215534}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6806","nonce":887215534},{"type":"v1","addr":"172.21.15.94:6807","nonce":887215534}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6804","nonce":887215534},{"type":"v1","addr":"172.21.15.94:6805","nonce":887215534}]},"public_addr":"172.21.15.94:6801/887215534","cluster_addr":"172.21.15.94:6803/887215534","heartbeat_back_addr":"172.21.15.94:6807/887215534","heartbeat_front_addr":"172.21.15.94:6805/887215534","state":["exists","up"]},{"osd":5,"uuid":"eb037130-5a05-42e0-947e-09bfba8fac22","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6808","nonce":1692579524},{"type":"v1","addr":"172.21.15.94:6809","nonce":1692579524}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6810","nonce":1692579524},{"type":"v1","addr":"172.21.15.94:6811","nonce":1692579524}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6814","nonce":1692579524},{"type":"v1","addr":"172.21.15.94:6815","nonce":1692579524}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6812","nonce":1692579524},{"type":"v1","addr":"172.21.15.94:6813","nonce":1692579524}]},"public_addr":"172.21.15.94:6809/1692579524","cluster_addr":"172.21.15.94:6811/1692579524","heartbeat_back_addr":"172.21.15.94:6815/1692579524","heartbeat_front_addr":"172.21.15.94:6813/1692579524","state":["exists","up"]},{"osd":6,"uuid":"f5dca4b5-8906-4eac-84e4-fcb7d1a29674","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6816","nonce":1107979948},{"type":"v1","addr":"172.21.15.94:6817","nonce":1107979948}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6818","nonce":1107979948},{"type":"v1","addr":"172.21.15.94:6819","nonce":1107979948}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6822","nonce":1107979948},{"type":"v1","addr":"172.21.15.94:6823","nonce":1107979948}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6820","nonce":1107979948},{"type":"v1","addr":"172.21.15.94:6821","nonce":1107979948}]},"public_addr":"172.21.15.94:6817/1107979948","cluster_addr":"172.21.15.94:6819/1107979948","heartbeat_back_addr":"172.21.15.94:6823/1107979948","heartbeat_front_addr":"172.21.15.94:6821/1107979948","state":["exists","up"]},{"osd":7,"uuid":"86e3ef7d-6ab8-47c3-a4bb-db923af2b49b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6824","nonce":2545649733},{"type":"v1","addr":"172.21.15.94:6825","nonce":2545649733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6826","nonce":2545649733},{"type":"v1","addr":"172.21.15.94:6827","nonce":2545649733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6830","nonce":2545649733},{"type":"v1","addr":"172.21.15.94:6831","nonce":2545649733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6828","nonce":2545649733},{"type":"v1","addr":"172.21.15.94:6829","nonce":2545649733}]},"public_addr":"172.21.15.94:6825/2545649733","cluster_addr":"172.21.15.94:6827/2545649733","heartbeat_back_addr":"172.21.15.94:6831/2545649733","heartbeat_front_addr":"172.21.15.94:6829/2545649733","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:40:47.681447+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:41:11.801118+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:41:35.764497+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:42:00.424802+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:42:21.618849+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:42:45.384437+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:43:09.350263+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:43:32.472815+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.70:0/2978576061":"2024-09-15T03:39:24.448616+0000","172.21.15.70:6801/673916264":"2024-09-15T03:39:24.448616+0000","172.21.15.70:6800/673916264":"2024-09-15T03:39:24.448616+0000","172.21.15.70:0/3005166882":"2024-09-15T03:39:24.448616+0000","172.21.15.70:0/3134643770":"2024-09-15T03:39:24.448616+0000","172.21.15.70:0/22543101":"2024-09-15T03:39:01.589983+0000","172.21.15.70:0/2447056811":"2024-09-15T03:39:01.589983+0000","172.21.15.70:6801/2254107694":"2024-09-15T03:39:01.589983+0000","172.21.15.70:6800/2254107694":"2024-09-15T03:39:01.589983+0000","172.21.15.70:0/4203647763":"2024-09-15T03:39:01.589983+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:44:10.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:09 smithi070 ceph-mon[27495]: pgmap v190: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 KiB/s wr, 6 op/s 2024-09-14T03:44:10.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:09 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/1057546258' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:44:10.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:09 smithi070 ceph-mon[39288]: pgmap v190: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 KiB/s wr, 6 op/s 2024-09-14T03:44:10.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:09 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/1057546258' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:44:10.070 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph fs set cephfs standby_count_wanted 0 2024-09-14T03:44:10.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:09 smithi094 ceph-mon[31053]: pgmap v190: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 KiB/s wr, 6 op/s 2024-09-14T03:44:10.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:09 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/1057546258' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:44:10.311 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:44:12.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:11 smithi094 ceph-mon[31053]: pgmap v191: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-09-14T03:44:12.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:11 smithi070 ceph-mon[27495]: pgmap v191: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-09-14T03:44:12.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:11 smithi070 ceph-mon[39288]: pgmap v191: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-09-14T03:44:13.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:12 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/2695466346' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-09-14T03:44:13.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:12 smithi070 ceph-mon[27495]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-09-14T03:44:13.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:12 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/2695466346' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-09-14T03:44:13.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:12 smithi070 ceph-mon[39288]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-09-14T03:44:13.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:12 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/2695466346' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-09-14T03:44:13.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:12 smithi094 ceph-mon[31053]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-09-14T03:44:13.655 DEBUG:tasks.cephfs.filesystem:fs_config: {} 2024-09-14T03:44:13.655 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph fs set cephfs allow_standby_replay false 2024-09-14T03:44:13.905 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:44:14.151 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:13 smithi070 ceph-mon[27495]: pgmap v192: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-14T03:44:14.151 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:13 smithi070 ceph-mon[27495]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2024-09-14T03:44:14.151 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:13 smithi070 ceph-mon[27495]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:44:14.151 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:13 smithi070 ceph-mon[39288]: pgmap v192: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-14T03:44:14.151 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:13 smithi070 ceph-mon[39288]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2024-09-14T03:44:14.151 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:13 smithi070 ceph-mon[39288]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:44:14.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:13 smithi094 ceph-mon[31053]: pgmap v192: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-14T03:44:14.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:13 smithi094 ceph-mon[31053]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2024-09-14T03:44:14.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:13 smithi094 ceph-mon[31053]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:44:16.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:15 smithi094 ceph-mon[31053]: pgmap v193: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-09-14T03:44:16.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:15 smithi070 ceph-mon[27495]: pgmap v193: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-09-14T03:44:16.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:15 smithi070 ceph-mon[39288]: pgmap v193: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-09-14T03:44:17.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:16 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/2274972823' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]: dispatch 2024-09-14T03:44:17.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:16 smithi094 ceph-mon[31053]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]: dispatch 2024-09-14T03:44:17.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:16 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/2274972823' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]: dispatch 2024-09-14T03:44:17.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:16 smithi070 ceph-mon[39288]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]: dispatch 2024-09-14T03:44:17.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:16 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/2274972823' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]: dispatch 2024-09-14T03:44:17.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:16 smithi070 ceph-mon[27495]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]: dispatch 2024-09-14T03:44:17.696 DEBUG:tasks.cephfs.filesystem:Not Creating any subvols for filesystem 'cephfs' 2024-09-14T03:44:17.696 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph fs dump --format=json 2024-09-14T03:44:17.950 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:44:18.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:17 smithi070 ceph-mon[27495]: pgmap v194: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-14T03:44:18.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:17 smithi070 ceph-mon[27495]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]': finished 2024-09-14T03:44:18.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:17 smithi070 ceph-mon[27495]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:44:18.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:17 smithi070 ceph-mon[39288]: pgmap v194: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-14T03:44:18.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:17 smithi070 ceph-mon[39288]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]': finished 2024-09-14T03:44:18.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:17 smithi070 ceph-mon[39288]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:44:18.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:17 smithi094 ceph-mon[31053]: pgmap v194: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-14T03:44:18.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:17 smithi094 ceph-mon[31053]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]': finished 2024-09-14T03:44:18.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:17 smithi094 ceph-mon[31053]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:44:19.922 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:44:19.922 INFO:teuthology.orchestra.run.smithi070.stdout:{"epoch":8,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":24304,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.94:6833/3620660683","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6832","nonce":3620660683},{"type":"v1","addr":"172.21.15.94:6833","nonce":3620660683}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":8,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-14T03:44:03.568995+0000","modified":"2024-09-14T03:44:16.900718+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":1,"in":[0],"up":{"mds_0":24323},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24323":{"gid":24323,"name":"a","rank":0,"incarnation":4,"state":"up:active","state_seq":6,"addr":"172.21.15.70:6835/421917097","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6834","nonce":421917097},{"type":"v1","addr":"172.21.15.70:6835","nonce":421917097}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","standby_count_wanted":0},"id":1}]} 2024-09-14T03:44:19.922 INFO:teuthology.orchestra.run.smithi070.stderr:dumped fsmap epoch 8 2024-09-14T03:44:20.238 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:19 smithi070 ceph-mon[27495]: pgmap v195: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 85 B/s wr, 1 op/s 2024-09-14T03:44:20.238 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:19 smithi070 ceph-mon[39288]: pgmap v195: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 85 B/s wr, 1 op/s 2024-09-14T03:44:20.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:19 smithi094 ceph-mon[31053]: pgmap v195: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 85 B/s wr, 1 op/s 2024-09-14T03:44:20.658 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T03:44:20.906 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:44:21.164 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:20 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/3822476497' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:44:21.164 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:20 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/3822476497' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:44:21.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:20 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/3822476497' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:44:22.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:21 smithi070 ceph-mon[27495]: pgmap v196: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-14T03:44:22.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:21 smithi070 ceph-mon[39288]: pgmap v196: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-14T03:44:22.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:21 smithi094 ceph-mon[31053]: pgmap v196: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-14T03:44:22.761 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:44:22.761 INFO:teuthology.orchestra.run.smithi070.stdout:{"epoch":58,"fsid":"b6380d42-724a-11ef-bceb-c7b262605968","created":"2024-09-14T03:38:29.406762+0000","modified":"2024-09-14T03:44:03.568878+0000","last_up_change":"2024-09-14T03:43:34.130380+0000","last_in_change":"2024-09-14T03:43:17.652311+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:41:38.745294+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-14T03:43:57.402183+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-14T03:44:00.803078+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"a7fabda1-9e60-42f2-af60-eec626181688","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6802","nonce":3822384296},{"type":"v1","addr":"172.21.15.70:6803","nonce":3822384296}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6804","nonce":3822384296},{"type":"v1","addr":"172.21.15.70:6805","nonce":3822384296}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6808","nonce":3822384296},{"type":"v1","addr":"172.21.15.70:6809","nonce":3822384296}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6806","nonce":3822384296},{"type":"v1","addr":"172.21.15.70:6807","nonce":3822384296}]},"public_addr":"172.21.15.70:6803/3822384296","cluster_addr":"172.21.15.70:6805/3822384296","heartbeat_back_addr":"172.21.15.70:6809/3822384296","heartbeat_front_addr":"172.21.15.70:6807/3822384296","state":["exists","up"]},{"osd":1,"uuid":"f5b9ef91-59af-4ef0-a101-1c00ba6007ad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6810","nonce":3986345052},{"type":"v1","addr":"172.21.15.70:6811","nonce":3986345052}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6812","nonce":3986345052},{"type":"v1","addr":"172.21.15.70:6813","nonce":3986345052}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6816","nonce":3986345052},{"type":"v1","addr":"172.21.15.70:6817","nonce":3986345052}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6814","nonce":3986345052},{"type":"v1","addr":"172.21.15.70:6815","nonce":3986345052}]},"public_addr":"172.21.15.70:6811/3986345052","cluster_addr":"172.21.15.70:6813/3986345052","heartbeat_back_addr":"172.21.15.70:6817/3986345052","heartbeat_front_addr":"172.21.15.70:6815/3986345052","state":["exists","up"]},{"osd":2,"uuid":"0dc07495-c6a5-4762-8f63-575eb0fe1671","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6818","nonce":2374764370},{"type":"v1","addr":"172.21.15.70:6819","nonce":2374764370}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6820","nonce":2374764370},{"type":"v1","addr":"172.21.15.70:6821","nonce":2374764370}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6824","nonce":2374764370},{"type":"v1","addr":"172.21.15.70:6825","nonce":2374764370}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6822","nonce":2374764370},{"type":"v1","addr":"172.21.15.70:6823","nonce":2374764370}]},"public_addr":"172.21.15.70:6819/2374764370","cluster_addr":"172.21.15.70:6821/2374764370","heartbeat_back_addr":"172.21.15.70:6825/2374764370","heartbeat_front_addr":"172.21.15.70:6823/2374764370","state":["exists","up"]},{"osd":3,"uuid":"583e5165-ea10-4d03-b731-36cfb1108362","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6826","nonce":2294516176},{"type":"v1","addr":"172.21.15.70:6827","nonce":2294516176}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6828","nonce":2294516176},{"type":"v1","addr":"172.21.15.70:6829","nonce":2294516176}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6832","nonce":2294516176},{"type":"v1","addr":"172.21.15.70:6833","nonce":2294516176}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6830","nonce":2294516176},{"type":"v1","addr":"172.21.15.70:6831","nonce":2294516176}]},"public_addr":"172.21.15.70:6827/2294516176","cluster_addr":"172.21.15.70:6829/2294516176","heartbeat_back_addr":"172.21.15.70:6833/2294516176","heartbeat_front_addr":"172.21.15.70:6831/2294516176","state":["exists","up"]},{"osd":4,"uuid":"33177119-9632-431c-9180-9d27e7bc0c42","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6800","nonce":887215534},{"type":"v1","addr":"172.21.15.94:6801","nonce":887215534}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6802","nonce":887215534},{"type":"v1","addr":"172.21.15.94:6803","nonce":887215534}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6806","nonce":887215534},{"type":"v1","addr":"172.21.15.94:6807","nonce":887215534}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6804","nonce":887215534},{"type":"v1","addr":"172.21.15.94:6805","nonce":887215534}]},"public_addr":"172.21.15.94:6801/887215534","cluster_addr":"172.21.15.94:6803/887215534","heartbeat_back_addr":"172.21.15.94:6807/887215534","heartbeat_front_addr":"172.21.15.94:6805/887215534","state":["exists","up"]},{"osd":5,"uuid":"eb037130-5a05-42e0-947e-09bfba8fac22","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6808","nonce":1692579524},{"type":"v1","addr":"172.21.15.94:6809","nonce":1692579524}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6810","nonce":1692579524},{"type":"v1","addr":"172.21.15.94:6811","nonce":1692579524}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6814","nonce":1692579524},{"type":"v1","addr":"172.21.15.94:6815","nonce":1692579524}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6812","nonce":1692579524},{"type":"v1","addr":"172.21.15.94:6813","nonce":1692579524}]},"public_addr":"172.21.15.94:6809/1692579524","cluster_addr":"172.21.15.94:6811/1692579524","heartbeat_back_addr":"172.21.15.94:6815/1692579524","heartbeat_front_addr":"172.21.15.94:6813/1692579524","state":["exists","up"]},{"osd":6,"uuid":"f5dca4b5-8906-4eac-84e4-fcb7d1a29674","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6816","nonce":1107979948},{"type":"v1","addr":"172.21.15.94:6817","nonce":1107979948}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6818","nonce":1107979948},{"type":"v1","addr":"172.21.15.94:6819","nonce":1107979948}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6822","nonce":1107979948},{"type":"v1","addr":"172.21.15.94:6823","nonce":1107979948}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6820","nonce":1107979948},{"type":"v1","addr":"172.21.15.94:6821","nonce":1107979948}]},"public_addr":"172.21.15.94:6817/1107979948","cluster_addr":"172.21.15.94:6819/1107979948","heartbeat_back_addr":"172.21.15.94:6823/1107979948","heartbeat_front_addr":"172.21.15.94:6821/1107979948","state":["exists","up"]},{"osd":7,"uuid":"86e3ef7d-6ab8-47c3-a4bb-db923af2b49b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6824","nonce":2545649733},{"type":"v1","addr":"172.21.15.94:6825","nonce":2545649733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6826","nonce":2545649733},{"type":"v1","addr":"172.21.15.94:6827","nonce":2545649733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6830","nonce":2545649733},{"type":"v1","addr":"172.21.15.94:6831","nonce":2545649733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6828","nonce":2545649733},{"type":"v1","addr":"172.21.15.94:6829","nonce":2545649733}]},"public_addr":"172.21.15.94:6825/2545649733","cluster_addr":"172.21.15.94:6827/2545649733","heartbeat_back_addr":"172.21.15.94:6831/2545649733","heartbeat_front_addr":"172.21.15.94:6829/2545649733","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:40:47.681447+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:41:11.801118+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:41:35.764497+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:42:00.424802+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:42:21.618849+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:42:45.384437+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:43:09.350263+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:43:32.472815+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.70:0/2978576061":"2024-09-15T03:39:24.448616+0000","172.21.15.70:6801/673916264":"2024-09-15T03:39:24.448616+0000","172.21.15.70:6800/673916264":"2024-09-15T03:39:24.448616+0000","172.21.15.70:0/3005166882":"2024-09-15T03:39:24.448616+0000","172.21.15.70:0/3134643770":"2024-09-15T03:39:24.448616+0000","172.21.15.70:0/22543101":"2024-09-15T03:39:01.589983+0000","172.21.15.70:0/2447056811":"2024-09-15T03:39:01.589983+0000","172.21.15.70:6801/2254107694":"2024-09-15T03:39:01.589983+0000","172.21.15.70:6800/2254107694":"2024-09-15T03:39:01.589983+0000","172.21.15.70:0/4203647763":"2024-09-15T03:39:01.589983+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:44:23.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:22 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/2878162914' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:44:23.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:22 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/2878162914' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:44:23.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:22 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/2878162914' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:44:23.428 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-14T03:44:23.428 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T03:44:23.676 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:44:24.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:23 smithi070 ceph-mon[27495]: pgmap v197: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:24.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:23 smithi070 ceph-mon[39288]: pgmap v197: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:24.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:23 smithi094 ceph-mon[31053]: pgmap v197: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:25.119 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:24 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.18", "id": [7, 2]}]: dispatch 2024-09-14T03:44:25.120 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:24 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3", "id": [0, 1]}]: dispatch 2024-09-14T03:44:25.120 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:24 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [6, 2]}]: dispatch 2024-09-14T03:44:25.120 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:24 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [0, 2, 4, 1]}]: dispatch 2024-09-14T03:44:25.120 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:24 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.26", "id": [4, 3]}]: dispatch 2024-09-14T03:44:25.120 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:24 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.36", "id": [5, 2]}]: dispatch 2024-09-14T03:44:25.120 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:24 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.18", "id": [7, 2]}]: dispatch 2024-09-14T03:44:25.120 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:24 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3", "id": [0, 1]}]: dispatch 2024-09-14T03:44:25.120 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:24 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [6, 2]}]: dispatch 2024-09-14T03:44:25.120 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:24 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [0, 2, 4, 1]}]: dispatch 2024-09-14T03:44:25.120 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:24 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.26", "id": [4, 3]}]: dispatch 2024-09-14T03:44:25.121 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:24 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.36", "id": [5, 2]}]: dispatch 2024-09-14T03:44:25.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:24 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.18", "id": [7, 2]}]: dispatch 2024-09-14T03:44:25.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:24 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3", "id": [0, 1]}]: dispatch 2024-09-14T03:44:25.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:24 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [6, 2]}]: dispatch 2024-09-14T03:44:25.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:24 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [0, 2, 4, 1]}]: dispatch 2024-09-14T03:44:25.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:24 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.26", "id": [4, 3]}]: dispatch 2024-09-14T03:44:25.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:24 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.36", "id": [5, 2]}]: dispatch 2024-09-14T03:44:25.507 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:44:25.508 INFO:teuthology.orchestra.run.smithi070.stderr:dumped all 2024-09-14T03:44:26.217 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:25 smithi070 ceph-mon[27495]: pgmap v198: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:26.217 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:25 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.18", "id": [7, 2]}]': finished 2024-09-14T03:44:26.217 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:25 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3", "id": [0, 1]}]': finished 2024-09-14T03:44:26.217 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:25 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [6, 2]}]': finished 2024-09-14T03:44:26.217 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:25 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [0, 2, 4, 1]}]': finished 2024-09-14T03:44:26.217 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:25 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.26", "id": [4, 3]}]': finished 2024-09-14T03:44:26.217 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:25 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.36", "id": [5, 2]}]': finished 2024-09-14T03:44:26.217 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:25 smithi070 ceph-mon[27495]: osdmap e59: 8 total, 8 up, 8 in 2024-09-14T03:44:26.218 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:25 smithi070 ceph-mon[39288]: pgmap v198: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:26.218 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:25 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.18", "id": [7, 2]}]': finished 2024-09-14T03:44:26.218 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:25 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3", "id": [0, 1]}]': finished 2024-09-14T03:44:26.218 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:25 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [6, 2]}]': finished 2024-09-14T03:44:26.218 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:25 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [0, 2, 4, 1]}]': finished 2024-09-14T03:44:26.218 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:25 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.26", "id": [4, 3]}]': finished 2024-09-14T03:44:26.218 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:25 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.36", "id": [5, 2]}]': finished 2024-09-14T03:44:26.218 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:25 smithi070 ceph-mon[39288]: osdmap e59: 8 total, 8 up, 8 in 2024-09-14T03:44:26.241 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":199,"stamp":"2024-09-14T03:44:24.924700+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":146,"num_write_kb":1641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111,"ondisk_log_size":111,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":291,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385148,"kb_used_data":6284,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747576068,"statfs":{"total":767960285184,"available":765517893632,"internally_reserved":0,"allocated":6434816,"data_stored":3700198,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.424723"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:02.871359+0000","last_change":"2024-09-14T03:44:01.882774+0000","last_active":"2024-09-14T03:44:02.871359+0000","last_peered":"2024-09-14T03:44:02.871359+0000","last_clean":"2024-09-14T03:44:02.871359+0000","last_became_active":"2024-09-14T03:44:01.882481+0000","last_became_peered":"2024-09-14T03:44:01.882481+0000","last_unstale":"2024-09-14T03:44:02.871359+0000","last_undegraded":"2024-09-14T03:44:02.871359+0000","last_fullsized":"2024-09-14T03:44:02.871359+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:49:25.209573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:02.870886+0000","last_change":"2024-09-14T03:44:01.868415+0000","last_active":"2024-09-14T03:44:02.870886+0000","last_peered":"2024-09-14T03:44:02.870886+0000","last_clean":"2024-09-14T03:44:02.870886+0000","last_became_active":"2024-09-14T03:44:01.868152+0000","last_became_peered":"2024-09-14T03:44:01.868152+0000","last_unstale":"2024-09-14T03:44:02.870886+0000","last_undegraded":"2024-09-14T03:44:02.870886+0000","last_fullsized":"2024-09-14T03:44:02.870886+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:56:00.270070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.873161+0000","last_change":"2024-09-14T03:44:01.871459+0000","last_active":"2024-09-14T03:44:01.873161+0000","last_peered":"2024-09-14T03:44:01.873161+0000","last_clean":"2024-09-14T03:44:01.873161+0000","last_became_active":"2024-09-14T03:44:01.870749+0000","last_became_peered":"2024-09-14T03:44:01.870749+0000","last_unstale":"2024-09-14T03:44:01.873161+0000","last_undegraded":"2024-09-14T03:44:01.873161+0000","last_fullsized":"2024-09-14T03:44:01.873161+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:54.391363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.883540+0000","last_change":"2024-09-14T03:44:01.882503+0000","last_active":"2024-09-14T03:44:01.883540+0000","last_peered":"2024-09-14T03:44:01.883540+0000","last_clean":"2024-09-14T03:44:01.883540+0000","last_became_active":"2024-09-14T03:44:01.882250+0000","last_became_peered":"2024-09-14T03:44:01.882250+0000","last_unstale":"2024-09-14T03:44:01.883540+0000","last_undegraded":"2024-09-14T03:44:01.883540+0000","last_fullsized":"2024-09-14T03:44:01.883540+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:46:29.551285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:02.870869+0000","last_change":"2024-09-14T03:44:01.880333+0000","last_active":"2024-09-14T03:44:02.870869+0000","last_peered":"2024-09-14T03:44:02.870869+0000","last_clean":"2024-09-14T03:44:02.870869+0000","last_became_active":"2024-09-14T03:44:01.880010+0000","last_became_peered":"2024-09-14T03:44:01.880010+0000","last_unstale":"2024-09-14T03:44:02.870869+0000","last_undegraded":"2024-09-14T03:44:02.870869+0000","last_fullsized":"2024-09-14T03:44:02.870869+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:45:02.741425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.881438+0000","last_change":"2024-09-14T03:44:01.880710+0000","last_active":"2024-09-14T03:44:01.881438+0000","last_peered":"2024-09-14T03:44:01.881438+0000","last_clean":"2024-09-14T03:44:01.881438+0000","last_became_active":"2024-09-14T03:44:01.880237+0000","last_became_peered":"2024-09-14T03:44:01.880237+0000","last_unstale":"2024-09-14T03:44:01.881438+0000","last_undegraded":"2024-09-14T03:44:01.881438+0000","last_fullsized":"2024-09-14T03:44:01.881438+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:56:49.808707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:02.870816+0000","last_change":"2024-09-14T03:44:01.878590+0000","last_active":"2024-09-14T03:44:02.870816+0000","last_peered":"2024-09-14T03:44:02.870816+0000","last_clean":"2024-09-14T03:44:02.870816+0000","last_became_active":"2024-09-14T03:44:01.878281+0000","last_became_peered":"2024-09-14T03:44:01.878281+0000","last_unstale":"2024-09-14T03:44:02.870816+0000","last_undegraded":"2024-09-14T03:44:02.870816+0000","last_fullsized":"2024-09-14T03:44:02.870816+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:41:57.164227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.046509+0000","last_change":"2024-09-14T03:44:01.877057+0000","last_active":"2024-09-14T03:44:03.046509+0000","last_peered":"2024-09-14T03:44:03.046509+0000","last_clean":"2024-09-14T03:44:03.046509+0000","last_became_active":"2024-09-14T03:44:01.876727+0000","last_became_peered":"2024-09-14T03:44:01.876727+0000","last_unstale":"2024-09-14T03:44:03.046509+0000","last_undegraded":"2024-09-14T03:44:03.046509+0000","last_fullsized":"2024-09-14T03:44:03.046509+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:31:24.716316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:02.871547+0000","last_change":"2024-09-14T03:44:01.884865+0000","last_active":"2024-09-14T03:44:02.871547+0000","last_peered":"2024-09-14T03:44:02.871547+0000","last_clean":"2024-09-14T03:44:02.871547+0000","last_became_active":"2024-09-14T03:44:01.884136+0000","last_became_peered":"2024-09-14T03:44:01.884136+0000","last_unstale":"2024-09-14T03:44:02.871547+0000","last_undegraded":"2024-09-14T03:44:02.871547+0000","last_fullsized":"2024-09-14T03:44:02.871547+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:59.903382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.877394+0000","last_change":"2024-09-14T03:44:01.875821+0000","last_active":"2024-09-14T03:44:01.877394+0000","last_peered":"2024-09-14T03:44:01.877394+0000","last_clean":"2024-09-14T03:44:01.877394+0000","last_became_active":"2024-09-14T03:44:01.873644+0000","last_became_peered":"2024-09-14T03:44:01.873644+0000","last_unstale":"2024-09-14T03:44:01.877394+0000","last_undegraded":"2024-09-14T03:44:01.877394+0000","last_fullsized":"2024-09-14T03:44:01.877394+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:42:35.043594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.876878+0000","last_change":"2024-09-14T03:44:01.875017+0000","last_active":"2024-09-14T03:44:01.876878+0000","last_peered":"2024-09-14T03:44:01.876878+0000","last_clean":"2024-09-14T03:44:01.876878+0000","last_became_active":"2024-09-14T03:44:01.874687+0000","last_became_peered":"2024-09-14T03:44:01.874687+0000","last_unstale":"2024-09-14T03:44:01.876878+0000","last_undegraded":"2024-09-14T03:44:01.876878+0000","last_fullsized":"2024-09-14T03:44:01.876878+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:29:09.035342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.887082+0000","last_change":"2024-09-14T03:44:01.883313+0000","last_active":"2024-09-14T03:44:01.887082+0000","last_peered":"2024-09-14T03:44:01.887082+0000","last_clean":"2024-09-14T03:44:01.887082+0000","last_became_active":"2024-09-14T03:44:01.882770+0000","last_became_peered":"2024-09-14T03:44:01.882770+0000","last_unstale":"2024-09-14T03:44:01.887082+0000","last_undegraded":"2024-09-14T03:44:01.887082+0000","last_fullsized":"2024-09-14T03:44:01.887082+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:57:19.402319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.046697+0000","last_change":"2024-09-14T03:44:01.878245+0000","last_active":"2024-09-14T03:44:03.046697+0000","last_peered":"2024-09-14T03:44:03.046697+0000","last_clean":"2024-09-14T03:44:03.046697+0000","last_became_active":"2024-09-14T03:44:01.877861+0000","last_became_peered":"2024-09-14T03:44:01.877861+0000","last_unstale":"2024-09-14T03:44:03.046697+0000","last_undegraded":"2024-09-14T03:44:03.046697+0000","last_fullsized":"2024-09-14T03:44:03.046697+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:08:29.902090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.878648+0000","last_change":"2024-09-14T03:44:01.877391+0000","last_active":"2024-09-14T03:44:01.878648+0000","last_peered":"2024-09-14T03:44:01.878648+0000","last_clean":"2024-09-14T03:44:01.878648+0000","last_became_active":"2024-09-14T03:44:01.876797+0000","last_became_peered":"2024-09-14T03:44:01.876797+0000","last_unstale":"2024-09-14T03:44:01.878648+0000","last_undegraded":"2024-09-14T03:44:01.878648+0000","last_fullsized":"2024-09-14T03:44:01.878648+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:06:02.891762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:02.870321+0000","last_change":"2024-09-14T03:44:01.876700+0000","last_active":"2024-09-14T03:44:02.870321+0000","last_peered":"2024-09-14T03:44:02.870321+0000","last_clean":"2024-09-14T03:44:02.870321+0000","last_became_active":"2024-09-14T03:44:01.876091+0000","last_became_peered":"2024-09-14T03:44:01.876091+0000","last_unstale":"2024-09-14T03:44:02.870321+0000","last_undegraded":"2024-09-14T03:44:02.870321+0000","last_fullsized":"2024-09-14T03:44:02.870321+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:25:41.750127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.879498+0000","last_change":"2024-09-14T03:44:01.877120+0000","last_active":"2024-09-14T03:44:01.879498+0000","last_peered":"2024-09-14T03:44:01.879498+0000","last_clean":"2024-09-14T03:44:01.879498+0000","last_became_active":"2024-09-14T03:44:01.876793+0000","last_became_peered":"2024-09-14T03:44:01.876793+0000","last_unstale":"2024-09-14T03:44:01.879498+0000","last_undegraded":"2024-09-14T03:44:01.879498+0000","last_fullsized":"2024-09-14T03:44:01.879498+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:23:56.450728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.886538+0000","last_change":"2024-09-14T03:44:01.885108+0000","last_active":"2024-09-14T03:44:01.886538+0000","last_peered":"2024-09-14T03:44:01.886538+0000","last_clean":"2024-09-14T03:44:01.886538+0000","last_became_active":"2024-09-14T03:44:01.884857+0000","last_became_peered":"2024-09-14T03:44:01.884857+0000","last_unstale":"2024-09-14T03:44:01.886538+0000","last_undegraded":"2024-09-14T03:44:01.886538+0000","last_fullsized":"2024-09-14T03:44:01.886538+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:31.448617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.877369+0000","last_change":"2024-09-14T03:44:01.876503+0000","last_active":"2024-09-14T03:44:01.877369+0000","last_peered":"2024-09-14T03:44:01.877369+0000","last_clean":"2024-09-14T03:44:01.877369+0000","last_became_active":"2024-09-14T03:44:01.876227+0000","last_became_peered":"2024-09-14T03:44:01.876227+0000","last_unstale":"2024-09-14T03:44:01.877369+0000","last_undegraded":"2024-09-14T03:44:01.877369+0000","last_fullsized":"2024-09-14T03:44:01.877369+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:44:50.132152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:02.871173+0000","last_change":"2024-09-14T03:44:01.883708+0000","last_active":"2024-09-14T03:44:02.871173+0000","last_peered":"2024-09-14T03:44:02.871173+0000","last_clean":"2024-09-14T03:44:02.871173+0000","last_became_active":"2024-09-14T03:44:01.883440+0000","last_became_peered":"2024-09-14T03:44:01.883440+0000","last_unstale":"2024-09-14T03:44:02.871173+0000","last_undegraded":"2024-09-14T03:44:02.871173+0000","last_fullsized":"2024-09-14T03:44:02.871173+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:10:34.658062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.889254+0000","last_change":"2024-09-14T03:44:01.888565+0000","last_active":"2024-09-14T03:44:01.889254+0000","last_peered":"2024-09-14T03:44:01.889254+0000","last_clean":"2024-09-14T03:44:01.889254+0000","last_became_active":"2024-09-14T03:44:01.888191+0000","last_became_peered":"2024-09-14T03:44:01.888191+0000","last_unstale":"2024-09-14T03:44:01.889254+0000","last_undegraded":"2024-09-14T03:44:01.889254+0000","last_fullsized":"2024-09-14T03:44:01.889254+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:52:23.782664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.877272+0000","last_change":"2024-09-14T03:44:01.874642+0000","last_active":"2024-09-14T03:44:01.877272+0000","last_peered":"2024-09-14T03:44:01.877272+0000","last_clean":"2024-09-14T03:44:01.877272+0000","last_became_active":"2024-09-14T03:44:01.874384+0000","last_became_peered":"2024-09-14T03:44:01.874384+0000","last_unstale":"2024-09-14T03:44:01.877272+0000","last_undegraded":"2024-09-14T03:44:01.877272+0000","last_fullsized":"2024-09-14T03:44:01.877272+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:35:05.535173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.887238+0000","last_change":"2024-09-14T03:44:01.886893+0000","last_active":"2024-09-14T03:44:01.887238+0000","last_peered":"2024-09-14T03:44:01.887238+0000","last_clean":"2024-09-14T03:44:01.887238+0000","last_became_active":"2024-09-14T03:44:01.886594+0000","last_became_peered":"2024-09-14T03:44:01.886594+0000","last_unstale":"2024-09-14T03:44:01.887238+0000","last_undegraded":"2024-09-14T03:44:01.887238+0000","last_fullsized":"2024-09-14T03:44:01.887238+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:07:37.154358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.046662+0000","last_change":"2024-09-14T03:44:01.868698+0000","last_active":"2024-09-14T03:44:03.046662+0000","last_peered":"2024-09-14T03:44:03.046662+0000","last_clean":"2024-09-14T03:44:03.046662+0000","last_became_active":"2024-09-14T03:44:01.868442+0000","last_became_peered":"2024-09-14T03:44:01.868442+0000","last_unstale":"2024-09-14T03:44:03.046662+0000","last_undegraded":"2024-09-14T03:44:03.046662+0000","last_fullsized":"2024-09-14T03:44:03.046662+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:41:18.296424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.886249+0000","last_change":"2024-09-14T03:44:01.882708+0000","last_active":"2024-09-14T03:44:01.886249+0000","last_peered":"2024-09-14T03:44:01.886249+0000","last_clean":"2024-09-14T03:44:01.886249+0000","last_became_active":"2024-09-14T03:44:01.882443+0000","last_became_peered":"2024-09-14T03:44:01.882443+0000","last_unstale":"2024-09-14T03:44:01.886249+0000","last_undegraded":"2024-09-14T03:44:01.886249+0000","last_fullsized":"2024-09-14T03:44:01.886249+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:46:48.306263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.046997+0000","last_change":"2024-09-14T03:44:01.884940+0000","last_active":"2024-09-14T03:44:03.046997+0000","last_peered":"2024-09-14T03:44:03.046997+0000","last_clean":"2024-09-14T03:44:03.046997+0000","last_became_active":"2024-09-14T03:44:01.884304+0000","last_became_peered":"2024-09-14T03:44:01.884304+0000","last_unstale":"2024-09-14T03:44:03.046997+0000","last_undegraded":"2024-09-14T03:44:03.046997+0000","last_fullsized":"2024-09-14T03:44:03.046997+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:01:49.061637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.879514+0000","last_change":"2024-09-14T03:44:01.878050+0000","last_active":"2024-09-14T03:44:01.879514+0000","last_peered":"2024-09-14T03:44:01.879514+0000","last_clean":"2024-09-14T03:44:01.879514+0000","last_became_active":"2024-09-14T03:44:01.877768+0000","last_became_peered":"2024-09-14T03:44:01.877768+0000","last_unstale":"2024-09-14T03:44:01.879514+0000","last_undegraded":"2024-09-14T03:44:01.879514+0000","last_fullsized":"2024-09-14T03:44:01.879514+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:22:18.836309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:02.870322+0000","last_change":"2024-09-14T03:44:01.874564+0000","last_active":"2024-09-14T03:44:02.870322+0000","last_peered":"2024-09-14T03:44:02.870322+0000","last_clean":"2024-09-14T03:44:02.870322+0000","last_became_active":"2024-09-14T03:44:01.874297+0000","last_became_peered":"2024-09-14T03:44:01.874297+0000","last_unstale":"2024-09-14T03:44:02.870322+0000","last_undegraded":"2024-09-14T03:44:02.870322+0000","last_fullsized":"2024-09-14T03:44:02.870322+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:07.027610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.046813+0000","last_change":"2024-09-14T03:44:01.884486+0000","last_active":"2024-09-14T03:44:03.046813+0000","last_peered":"2024-09-14T03:44:03.046813+0000","last_clean":"2024-09-14T03:44:03.046813+0000","last_became_active":"2024-09-14T03:44:01.884218+0000","last_became_peered":"2024-09-14T03:44:01.884218+0000","last_unstale":"2024-09-14T03:44:03.046813+0000","last_undegraded":"2024-09-14T03:44:03.046813+0000","last_fullsized":"2024-09-14T03:44:03.046813+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:59:18.153239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.879455+0000","last_change":"2024-09-14T03:44:01.878737+0000","last_active":"2024-09-14T03:44:01.879455+0000","last_peered":"2024-09-14T03:44:01.879455+0000","last_clean":"2024-09-14T03:44:01.879455+0000","last_became_active":"2024-09-14T03:44:01.878504+0000","last_became_peered":"2024-09-14T03:44:01.878504+0000","last_unstale":"2024-09-14T03:44:01.879455+0000","last_undegraded":"2024-09-14T03:44:01.879455+0000","last_fullsized":"2024-09-14T03:44:01.879455+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:41:35.420957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.880939+0000","last_change":"2024-09-14T03:44:01.880509+0000","last_active":"2024-09-14T03:44:01.880939+0000","last_peered":"2024-09-14T03:44:01.880939+0000","last_clean":"2024-09-14T03:44:01.880939+0000","last_became_active":"2024-09-14T03:44:01.879855+0000","last_became_peered":"2024-09-14T03:44:01.879855+0000","last_unstale":"2024-09-14T03:44:01.880939+0000","last_undegraded":"2024-09-14T03:44:01.880939+0000","last_fullsized":"2024-09-14T03:44:01.880939+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:05:31.166902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.870149+0000","last_change":"2024-09-14T03:44:01.869523+0000","last_active":"2024-09-14T03:44:01.870149+0000","last_peered":"2024-09-14T03:44:01.870149+0000","last_clean":"2024-09-14T03:44:01.870149+0000","last_became_active":"2024-09-14T03:44:01.869225+0000","last_became_peered":"2024-09-14T03:44:01.869225+0000","last_unstale":"2024-09-14T03:44:01.870149+0000","last_undegraded":"2024-09-14T03:44:01.870149+0000","last_fullsized":"2024-09-14T03:44:01.870149+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:43:15.638187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.876583+0000","last_change":"2024-09-14T03:44:01.876014+0000","last_active":"2024-09-14T03:44:01.876583+0000","last_peered":"2024-09-14T03:44:01.876583+0000","last_clean":"2024-09-14T03:44:01.876583+0000","last_became_active":"2024-09-14T03:44:01.875496+0000","last_became_peered":"2024-09-14T03:44:01.875496+0000","last_unstale":"2024-09-14T03:44:01.876583+0000","last_undegraded":"2024-09-14T03:44:01.876583+0000","last_fullsized":"2024-09-14T03:44:01.876583+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:03:45.344199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.869587+0000","last_change":"2024-09-14T03:44:01.865496+0000","last_active":"2024-09-14T03:44:01.869587+0000","last_peered":"2024-09-14T03:44:01.869587+0000","last_clean":"2024-09-14T03:44:01.869587+0000","last_became_active":"2024-09-14T03:44:01.865190+0000","last_became_peered":"2024-09-14T03:44:01.865190+0000","last_unstale":"2024-09-14T03:44:01.869587+0000","last_undegraded":"2024-09-14T03:44:01.869587+0000","last_fullsized":"2024-09-14T03:44:01.869587+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:18:50.037675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"58'1","reported_seq":14,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.878410+0000","last_change":"2024-09-14T03:43:58.729188+0000","last_active":"2024-09-14T03:44:03.878410+0000","last_peered":"2024-09-14T03:44:03.878410+0000","last_clean":"2024-09-14T03:44:03.878410+0000","last_became_active":"2024-09-14T03:43:58.728927+0000","last_became_peered":"2024-09-14T03:43:58.728927+0000","last_unstale":"2024-09-14T03:44:03.878410+0000","last_undegraded":"2024-09-14T03:44:03.878410+0000","last_fullsized":"2024-09-14T03:44:03.878410+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:13:16.419701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.879471+0000","last_change":"2024-09-14T03:44:01.879101+0000","last_active":"2024-09-14T03:44:01.879471+0000","last_peered":"2024-09-14T03:44:01.879471+0000","last_clean":"2024-09-14T03:44:01.879471+0000","last_became_active":"2024-09-14T03:44:01.878801+0000","last_became_peered":"2024-09-14T03:44:01.878801+0000","last_unstale":"2024-09-14T03:44:01.879471+0000","last_undegraded":"2024-09-14T03:44:01.879471+0000","last_fullsized":"2024-09-14T03:44:01.879471+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:13.447251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"58'1","reported_seq":14,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.881113+0000","last_change":"2024-09-14T03:43:58.730444+0000","last_active":"2024-09-14T03:44:03.881113+0000","last_peered":"2024-09-14T03:44:03.881113+0000","last_clean":"2024-09-14T03:44:03.881113+0000","last_became_active":"2024-09-14T03:43:58.730172+0000","last_became_peered":"2024-09-14T03:43:58.730172+0000","last_unstale":"2024-09-14T03:44:03.881113+0000","last_undegraded":"2024-09-14T03:44:03.881113+0000","last_fullsized":"2024-09-14T03:44:03.881113+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:03:44.810562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":13,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.046789+0000","last_change":"2024-09-14T03:43:58.731458+0000","last_active":"2024-09-14T03:44:03.046789+0000","last_peered":"2024-09-14T03:44:03.046789+0000","last_clean":"2024-09-14T03:44:03.046789+0000","last_became_active":"2024-09-14T03:43:58.731206+0000","last_became_peered":"2024-09-14T03:43:58.731206+0000","last_unstale":"2024-09-14T03:44:03.046789+0000","last_undegraded":"2024-09-14T03:44:03.046789+0000","last_fullsized":"2024-09-14T03:44:03.046789+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:37:06.670539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.046550+0000","last_change":"2024-09-14T03:44:01.869781+0000","last_active":"2024-09-14T03:44:03.046550+0000","last_peered":"2024-09-14T03:44:03.046550+0000","last_clean":"2024-09-14T03:44:03.046550+0000","last_became_active":"2024-09-14T03:44:01.869521+0000","last_became_peered":"2024-09-14T03:44:01.869521+0000","last_unstale":"2024-09-14T03:44:03.046550+0000","last_undegraded":"2024-09-14T03:44:03.046550+0000","last_fullsized":"2024-09-14T03:44:03.046550+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:35:18.565206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":13,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.862828+0000","last_change":"2024-09-14T03:43:58.734065+0000","last_active":"2024-09-14T03:44:01.862828+0000","last_peered":"2024-09-14T03:44:01.862828+0000","last_clean":"2024-09-14T03:44:01.862828+0000","last_became_active":"2024-09-14T03:43:58.733752+0000","last_became_peered":"2024-09-14T03:43:58.733752+0000","last_unstale":"2024-09-14T03:44:01.862828+0000","last_undegraded":"2024-09-14T03:44:01.862828+0000","last_fullsized":"2024-09-14T03:44:01.862828+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:54:52.048737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:02.870312+0000","last_change":"2024-09-14T03:44:01.873748+0000","last_active":"2024-09-14T03:44:02.870312+0000","last_peered":"2024-09-14T03:44:02.870312+0000","last_clean":"2024-09-14T03:44:02.870312+0000","last_became_active":"2024-09-14T03:44:01.873473+0000","last_became_peered":"2024-09-14T03:44:01.873473+0000","last_unstale":"2024-09-14T03:44:02.870312+0000","last_undegraded":"2024-09-14T03:44:02.870312+0000","last_fullsized":"2024-09-14T03:44:02.870312+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:04:39.289903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":13,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.862232+0000","last_change":"2024-09-14T03:43:58.735608+0000","last_active":"2024-09-14T03:44:01.862232+0000","last_peered":"2024-09-14T03:44:01.862232+0000","last_clean":"2024-09-14T03:44:01.862232+0000","last_became_active":"2024-09-14T03:43:58.735301+0000","last_became_peered":"2024-09-14T03:43:58.735301+0000","last_unstale":"2024-09-14T03:44:01.862232+0000","last_undegraded":"2024-09-14T03:44:01.862232+0000","last_fullsized":"2024-09-14T03:44:01.862232+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:44:56.523073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.875923+0000","last_change":"2024-09-14T03:44:01.874777+0000","last_active":"2024-09-14T03:44:01.875923+0000","last_peered":"2024-09-14T03:44:01.875923+0000","last_clean":"2024-09-14T03:44:01.875923+0000","last_became_active":"2024-09-14T03:44:01.874473+0000","last_became_peered":"2024-09-14T03:44:01.874473+0000","last_unstale":"2024-09-14T03:44:01.875923+0000","last_undegraded":"2024-09-14T03:44:01.875923+0000","last_fullsized":"2024-09-14T03:44:01.875923+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:49:02.042635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":13,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.862878+0000","last_change":"2024-09-14T03:43:58.732969+0000","last_active":"2024-09-14T03:44:01.862878+0000","last_peered":"2024-09-14T03:44:01.862878+0000","last_clean":"2024-09-14T03:44:01.862878+0000","last_became_active":"2024-09-14T03:43:58.732492+0000","last_became_peered":"2024-09-14T03:43:58.732492+0000","last_unstale":"2024-09-14T03:44:01.862878+0000","last_undegraded":"2024-09-14T03:44:01.862878+0000","last_fullsized":"2024-09-14T03:44:01.862878+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:50:28.719447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.877453+0000","last_change":"2024-09-14T03:44:01.875679+0000","last_active":"2024-09-14T03:44:01.877453+0000","last_peered":"2024-09-14T03:44:01.877453+0000","last_clean":"2024-09-14T03:44:01.877453+0000","last_became_active":"2024-09-14T03:44:01.875422+0000","last_became_peered":"2024-09-14T03:44:01.875422+0000","last_unstale":"2024-09-14T03:44:01.877453+0000","last_undegraded":"2024-09-14T03:44:01.877453+0000","last_fullsized":"2024-09-14T03:44:01.877453+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:13:00.349373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"58'3","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.886706+0000","last_change":"2024-09-14T03:43:58.735066+0000","last_active":"2024-09-14T03:44:03.886706+0000","last_peered":"2024-09-14T03:44:03.886706+0000","last_clean":"2024-09-14T03:44:03.886706+0000","last_became_active":"2024-09-14T03:43:58.734514+0000","last_became_peered":"2024-09-14T03:43:58.734514+0000","last_unstale":"2024-09-14T03:44:03.886706+0000","last_undegraded":"2024-09-14T03:44:03.886706+0000","last_fullsized":"2024-09-14T03:44:03.886706+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:41:55.478295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.883377+0000","last_change":"2024-09-14T03:44:01.881319+0000","last_active":"2024-09-14T03:44:01.883377+0000","last_peered":"2024-09-14T03:44:01.883377+0000","last_clean":"2024-09-14T03:44:01.883377+0000","last_became_active":"2024-09-14T03:44:01.881032+0000","last_became_peered":"2024-09-14T03:44:01.881032+0000","last_unstale":"2024-09-14T03:44:01.883377+0000","last_undegraded":"2024-09-14T03:44:01.883377+0000","last_fullsized":"2024-09-14T03:44:01.883377+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:35:41.762014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"58'6","reported_seq":19,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.885331+0000","last_change":"2024-09-14T03:43:58.734778+0000","last_active":"2024-09-14T03:44:03.885331+0000","last_peered":"2024-09-14T03:44:03.885331+0000","last_clean":"2024-09-14T03:44:03.885331+0000","last_became_active":"2024-09-14T03:43:58.734380+0000","last_became_peered":"2024-09-14T03:43:58.734380+0000","last_unstale":"2024-09-14T03:44:03.885331+0000","last_undegraded":"2024-09-14T03:44:03.885331+0000","last_fullsized":"2024-09-14T03:44:03.885331+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:55:13.561065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:02.870755+0000","last_change":"2024-09-14T03:44:01.880139+0000","last_active":"2024-09-14T03:44:02.870755+0000","last_peered":"2024-09-14T03:44:02.870755+0000","last_clean":"2024-09-14T03:44:02.870755+0000","last_became_active":"2024-09-14T03:44:01.879840+0000","last_became_peered":"2024-09-14T03:44:01.879840+0000","last_unstale":"2024-09-14T03:44:02.870755+0000","last_undegraded":"2024-09-14T03:44:02.870755+0000","last_fullsized":"2024-09-14T03:44:02.870755+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:31:34.003676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"58'2","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.884938+0000","last_change":"2024-09-14T03:43:58.728768+0000","last_active":"2024-09-14T03:44:03.884938+0000","last_peered":"2024-09-14T03:44:03.884938+0000","last_clean":"2024-09-14T03:44:03.884938+0000","last_became_active":"2024-09-14T03:43:58.727893+0000","last_became_peered":"2024-09-14T03:43:58.727893+0000","last_unstale":"2024-09-14T03:44:03.884938+0000","last_undegraded":"2024-09-14T03:44:03.884938+0000","last_fullsized":"2024-09-14T03:44:03.884938+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:54:48.554486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.878664+0000","last_change":"2024-09-14T03:44:01.877540+0000","last_active":"2024-09-14T03:44:01.878664+0000","last_peered":"2024-09-14T03:44:01.878664+0000","last_clean":"2024-09-14T03:44:01.878664+0000","last_became_active":"2024-09-14T03:44:01.877147+0000","last_became_peered":"2024-09-14T03:44:01.877147+0000","last_unstale":"2024-09-14T03:44:01.878664+0000","last_undegraded":"2024-09-14T03:44:01.878664+0000","last_fullsized":"2024-09-14T03:44:01.878664+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:17:57.724648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":13,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.865997+0000","last_change":"2024-09-14T03:43:58.732812+0000","last_active":"2024-09-14T03:44:01.865997+0000","last_peered":"2024-09-14T03:44:01.865997+0000","last_clean":"2024-09-14T03:44:01.865997+0000","last_became_active":"2024-09-14T03:43:58.732537+0000","last_became_peered":"2024-09-14T03:43:58.732537+0000","last_unstale":"2024-09-14T03:44:01.865997+0000","last_undegraded":"2024-09-14T03:44:01.865997+0000","last_fullsized":"2024-09-14T03:44:01.865997+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:26:11.551182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:02.870910+0000","last_change":"2024-09-14T03:44:01.881140+0000","last_active":"2024-09-14T03:44:02.870910+0000","last_peered":"2024-09-14T03:44:02.870910+0000","last_clean":"2024-09-14T03:44:02.870910+0000","last_became_active":"2024-09-14T03:44:01.880868+0000","last_became_peered":"2024-09-14T03:44:01.880868+0000","last_unstale":"2024-09-14T03:44:02.870910+0000","last_undegraded":"2024-09-14T03:44:02.870910+0000","last_fullsized":"2024-09-14T03:44:02.870910+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:29:05.729804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":13,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.862264+0000","last_change":"2024-09-14T03:43:58.730505+0000","last_active":"2024-09-14T03:44:01.862264+0000","last_peered":"2024-09-14T03:44:01.862264+0000","last_clean":"2024-09-14T03:44:01.862264+0000","last_became_active":"2024-09-14T03:43:58.729821+0000","last_became_peered":"2024-09-14T03:43:58.729821+0000","last_unstale":"2024-09-14T03:44:01.862264+0000","last_undegraded":"2024-09-14T03:44:01.862264+0000","last_fullsized":"2024-09-14T03:44:01.862264+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:17:44.091667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.876565+0000","last_change":"2024-09-14T03:44:01.874477+0000","last_active":"2024-09-14T03:44:01.876565+0000","last_peered":"2024-09-14T03:44:01.876565+0000","last_clean":"2024-09-14T03:44:01.876565+0000","last_became_active":"2024-09-14T03:44:01.873714+0000","last_became_peered":"2024-09-14T03:44:01.873714+0000","last_unstale":"2024-09-14T03:44:01.876565+0000","last_undegraded":"2024-09-14T03:44:01.876565+0000","last_fullsized":"2024-09-14T03:44:01.876565+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:54:21.601768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"58'8","reported_seq":21,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.885624+0000","last_change":"2024-09-14T03:43:58.726387+0000","last_active":"2024-09-14T03:44:03.885624+0000","last_peered":"2024-09-14T03:44:03.885624+0000","last_clean":"2024-09-14T03:44:03.885624+0000","last_became_active":"2024-09-14T03:43:58.726115+0000","last_became_peered":"2024-09-14T03:43:58.726115+0000","last_unstale":"2024-09-14T03:44:03.885624+0000","last_undegraded":"2024-09-14T03:44:03.885624+0000","last_fullsized":"2024-09-14T03:44:03.885624+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:18:04.176970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:02.870755+0000","last_change":"2024-09-14T03:44:01.878624+0000","last_active":"2024-09-14T03:44:02.870755+0000","last_peered":"2024-09-14T03:44:02.870755+0000","last_clean":"2024-09-14T03:44:02.870755+0000","last_became_active":"2024-09-14T03:44:01.878343+0000","last_became_peered":"2024-09-14T03:44:01.878343+0000","last_unstale":"2024-09-14T03:44:02.870755+0000","last_undegraded":"2024-09-14T03:44:02.870755+0000","last_fullsized":"2024-09-14T03:44:02.870755+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:44:05.205671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"58'1","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.873677+0000","last_change":"2024-09-14T03:43:58.728489+0000","last_active":"2024-09-14T03:44:03.873677+0000","last_peered":"2024-09-14T03:44:03.873677+0000","last_clean":"2024-09-14T03:44:03.873677+0000","last_became_active":"2024-09-14T03:43:58.728182+0000","last_became_peered":"2024-09-14T03:43:58.728182+0000","last_unstale":"2024-09-14T03:44:03.873677+0000","last_undegraded":"2024-09-14T03:44:03.873677+0000","last_fullsized":"2024-09-14T03:44:03.873677+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:56.310710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:02.871343+0000","last_change":"2024-09-14T03:44:01.880754+0000","last_active":"2024-09-14T03:44:02.871343+0000","last_peered":"2024-09-14T03:44:02.871343+0000","last_clean":"2024-09-14T03:44:02.871343+0000","last_became_active":"2024-09-14T03:44:01.880483+0000","last_became_peered":"2024-09-14T03:44:01.880483+0000","last_unstale":"2024-09-14T03:44:02.871343+0000","last_undegraded":"2024-09-14T03:44:02.871343+0000","last_fullsized":"2024-09-14T03:44:02.871343+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:32:53.245007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"58'2","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.880815+0000","last_change":"2024-09-14T03:43:58.726366+0000","last_active":"2024-09-14T03:44:03.880815+0000","last_peered":"2024-09-14T03:44:03.880815+0000","last_clean":"2024-09-14T03:44:03.880815+0000","last_became_active":"2024-09-14T03:43:58.726090+0000","last_became_peered":"2024-09-14T03:43:58.726090+0000","last_unstale":"2024-09-14T03:44:03.880815+0000","last_undegraded":"2024-09-14T03:44:03.880815+0000","last_fullsized":"2024-09-14T03:44:03.880815+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:45:33.572820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:02.871390+0000","last_change":"2024-09-14T03:44:01.883704+0000","last_active":"2024-09-14T03:44:02.871390+0000","last_peered":"2024-09-14T03:44:02.871390+0000","last_clean":"2024-09-14T03:44:02.871390+0000","last_became_active":"2024-09-14T03:44:01.883448+0000","last_became_peered":"2024-09-14T03:44:01.883448+0000","last_unstale":"2024-09-14T03:44:02.871390+0000","last_undegraded":"2024-09-14T03:44:02.871390+0000","last_fullsized":"2024-09-14T03:44:02.871390+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:16:27.946770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"58'1","reported_seq":14,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.885569+0000","last_change":"2024-09-14T03:43:58.733205+0000","last_active":"2024-09-14T03:44:03.885569+0000","last_peered":"2024-09-14T03:44:03.885569+0000","last_clean":"2024-09-14T03:44:03.885569+0000","last_became_active":"2024-09-14T03:43:58.732929+0000","last_became_peered":"2024-09-14T03:43:58.732929+0000","last_unstale":"2024-09-14T03:44:03.885569+0000","last_undegraded":"2024-09-14T03:44:03.885569+0000","last_fullsized":"2024-09-14T03:44:03.885569+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:31:13.660965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.886970+0000","last_change":"2024-09-14T03:44:01.886158+0000","last_active":"2024-09-14T03:44:01.886970+0000","last_peered":"2024-09-14T03:44:01.886970+0000","last_clean":"2024-09-14T03:44:01.886970+0000","last_became_active":"2024-09-14T03:44:01.884747+0000","last_became_peered":"2024-09-14T03:44:01.884747+0000","last_unstale":"2024-09-14T03:44:01.886970+0000","last_undegraded":"2024-09-14T03:44:01.886970+0000","last_fullsized":"2024-09-14T03:44:01.886970+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:49:56.265783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":13,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.858323+0000","last_change":"2024-09-14T03:43:58.732401+0000","last_active":"2024-09-14T03:44:01.858323+0000","last_peered":"2024-09-14T03:44:01.858323+0000","last_clean":"2024-09-14T03:44:01.858323+0000","last_became_active":"2024-09-14T03:43:58.732067+0000","last_became_peered":"2024-09-14T03:43:58.732067+0000","last_unstale":"2024-09-14T03:44:01.858323+0000","last_undegraded":"2024-09-14T03:44:01.858323+0000","last_fullsized":"2024-09-14T03:44:01.858323+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:34:00.566245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:02.870791+0000","last_change":"2024-09-14T03:44:01.881736+0000","last_active":"2024-09-14T03:44:02.870791+0000","last_peered":"2024-09-14T03:44:02.870791+0000","last_clean":"2024-09-14T03:44:02.870791+0000","last_became_active":"2024-09-14T03:44:01.881219+0000","last_became_peered":"2024-09-14T03:44:01.881219+0000","last_unstale":"2024-09-14T03:44:02.870791+0000","last_undegraded":"2024-09-14T03:44:02.870791+0000","last_fullsized":"2024-09-14T03:44:02.870791+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:24:23.342066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"22'72","reported_seq":23,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:02.870745+0000","last_change":"2024-09-14T03:43:36.147227+0000","last_active":"2024-09-14T03:44:02.870745+0000","last_peered":"2024-09-14T03:44:02.870745+0000","last_clean":"2024-09-14T03:44:02.870745+0000","last_became_active":"2024-09-14T03:43:36.141953+0000","last_became_peered":"2024-09-14T03:43:36.141953+0000","last_unstale":"2024-09-14T03:44:02.870745+0000","last_undegraded":"2024-09-14T03:44:02.870745+0000","last_fullsized":"2024-09-14T03:44:02.870745+0000","mapping_epoch":50,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":51,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:39.458815+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:39.458815+0000","last_clean_scrub_stamp":"2024-09-14T03:41:39.458815+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:38:43.056328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":13,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.863031+0000","last_change":"2024-09-14T03:43:58.733557+0000","last_active":"2024-09-14T03:44:01.863031+0000","last_peered":"2024-09-14T03:44:01.863031+0000","last_clean":"2024-09-14T03:44:01.863031+0000","last_became_active":"2024-09-14T03:43:58.733256+0000","last_became_peered":"2024-09-14T03:43:58.733256+0000","last_unstale":"2024-09-14T03:44:01.863031+0000","last_undegraded":"2024-09-14T03:44:01.863031+0000","last_fullsized":"2024-09-14T03:44:01.863031+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:09:12.987156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.046665+0000","last_change":"2024-09-14T03:44:01.875525+0000","last_active":"2024-09-14T03:44:03.046665+0000","last_peered":"2024-09-14T03:44:03.046665+0000","last_clean":"2024-09-14T03:44:03.046665+0000","last_became_active":"2024-09-14T03:44:01.875241+0000","last_became_peered":"2024-09-14T03:44:01.875241+0000","last_unstale":"2024-09-14T03:44:03.046665+0000","last_undegraded":"2024-09-14T03:44:03.046665+0000","last_fullsized":"2024-09-14T03:44:03.046665+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:34:47.346963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:02.870635+0000","last_change":"2024-09-14T03:43:58.732376+0000","last_active":"2024-09-14T03:44:02.870635+0000","last_peered":"2024-09-14T03:44:02.870635+0000","last_clean":"2024-09-14T03:44:02.870635+0000","last_became_active":"2024-09-14T03:43:58.731812+0000","last_became_peered":"2024-09-14T03:43:58.731812+0000","last_unstale":"2024-09-14T03:44:02.870635+0000","last_undegraded":"2024-09-14T03:44:02.870635+0000","last_fullsized":"2024-09-14T03:44:02.870635+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:35:26.262383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.877445+0000","last_change":"2024-09-14T03:44:01.876511+0000","last_active":"2024-09-14T03:44:01.877445+0000","last_peered":"2024-09-14T03:44:01.877445+0000","last_clean":"2024-09-14T03:44:01.877445+0000","last_became_active":"2024-09-14T03:44:01.876225+0000","last_became_peered":"2024-09-14T03:44:01.876225+0000","last_unstale":"2024-09-14T03:44:01.877445+0000","last_undegraded":"2024-09-14T03:44:01.877445+0000","last_fullsized":"2024-09-14T03:44:01.877445+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:36:40.283622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":13,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.858583+0000","last_change":"2024-09-14T03:43:58.727129+0000","last_active":"2024-09-14T03:44:01.858583+0000","last_peered":"2024-09-14T03:44:01.858583+0000","last_clean":"2024-09-14T03:44:01.858583+0000","last_became_active":"2024-09-14T03:43:58.726879+0000","last_became_peered":"2024-09-14T03:43:58.726879+0000","last_unstale":"2024-09-14T03:44:01.858583+0000","last_undegraded":"2024-09-14T03:44:01.858583+0000","last_fullsized":"2024-09-14T03:44:01.858583+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:54:41.748651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:02.870239+0000","last_change":"2024-09-14T03:44:01.871005+0000","last_active":"2024-09-14T03:44:02.870239+0000","last_peered":"2024-09-14T03:44:02.870239+0000","last_clean":"2024-09-14T03:44:02.870239+0000","last_became_active":"2024-09-14T03:44:01.870707+0000","last_became_peered":"2024-09-14T03:44:01.870707+0000","last_unstale":"2024-09-14T03:44:02.870239+0000","last_undegraded":"2024-09-14T03:44:02.870239+0000","last_fullsized":"2024-09-14T03:44:02.870239+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:30:22.758581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":13,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.866074+0000","last_change":"2024-09-14T03:43:58.733038+0000","last_active":"2024-09-14T03:44:01.866074+0000","last_peered":"2024-09-14T03:44:01.866074+0000","last_clean":"2024-09-14T03:44:01.866074+0000","last_became_active":"2024-09-14T03:43:58.732753+0000","last_became_peered":"2024-09-14T03:43:58.732753+0000","last_unstale":"2024-09-14T03:44:01.866074+0000","last_undegraded":"2024-09-14T03:44:01.866074+0000","last_fullsized":"2024-09-14T03:44:01.866074+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:40:49.754211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.881906+0000","last_change":"2024-09-14T03:44:01.881600+0000","last_active":"2024-09-14T03:44:01.881906+0000","last_peered":"2024-09-14T03:44:01.881906+0000","last_clean":"2024-09-14T03:44:01.881906+0000","last_became_active":"2024-09-14T03:44:01.881001+0000","last_became_peered":"2024-09-14T03:44:01.881001+0000","last_unstale":"2024-09-14T03:44:01.881906+0000","last_undegraded":"2024-09-14T03:44:01.881906+0000","last_fullsized":"2024-09-14T03:44:01.881906+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:26:45.052191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":13,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:02.870489+0000","last_change":"2024-09-14T03:43:58.733621+0000","last_active":"2024-09-14T03:44:02.870489+0000","last_peered":"2024-09-14T03:44:02.870489+0000","last_clean":"2024-09-14T03:44:02.870489+0000","last_became_active":"2024-09-14T03:43:58.733357+0000","last_became_peered":"2024-09-14T03:43:58.733357+0000","last_unstale":"2024-09-14T03:44:02.870489+0000","last_undegraded":"2024-09-14T03:44:02.870489+0000","last_fullsized":"2024-09-14T03:44:02.870489+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:25:53.791926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.880813+0000","last_change":"2024-09-14T03:44:01.880200+0000","last_active":"2024-09-14T03:44:01.880813+0000","last_peered":"2024-09-14T03:44:01.880813+0000","last_clean":"2024-09-14T03:44:01.880813+0000","last_became_active":"2024-09-14T03:44:01.879925+0000","last_became_peered":"2024-09-14T03:44:01.879925+0000","last_unstale":"2024-09-14T03:44:01.880813+0000","last_undegraded":"2024-09-14T03:44:01.880813+0000","last_fullsized":"2024-09-14T03:44:01.880813+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:55:24.738215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:02.871083+0000","last_change":"2024-09-14T03:43:58.733230+0000","last_active":"2024-09-14T03:44:02.871083+0000","last_peered":"2024-09-14T03:44:02.871083+0000","last_clean":"2024-09-14T03:44:02.871083+0000","last_became_active":"2024-09-14T03:43:58.732868+0000","last_became_peered":"2024-09-14T03:43:58.732868+0000","last_unstale":"2024-09-14T03:44:02.871083+0000","last_undegraded":"2024-09-14T03:44:02.871083+0000","last_fullsized":"2024-09-14T03:44:02.871083+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:15:12.184851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:02.870449+0000","last_change":"2024-09-14T03:44:01.878736+0000","last_active":"2024-09-14T03:44:02.870449+0000","last_peered":"2024-09-14T03:44:02.870449+0000","last_clean":"2024-09-14T03:44:02.870449+0000","last_became_active":"2024-09-14T03:44:01.877390+0000","last_became_peered":"2024-09-14T03:44:01.877390+0000","last_unstale":"2024-09-14T03:44:02.870449+0000","last_undegraded":"2024-09-14T03:44:02.870449+0000","last_fullsized":"2024-09-14T03:44:02.870449+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:17:42.812575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"58'1","reported_seq":14,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.884532+0000","last_change":"2024-09-14T03:43:58.733688+0000","last_active":"2024-09-14T03:44:03.884532+0000","last_peered":"2024-09-14T03:44:03.884532+0000","last_clean":"2024-09-14T03:44:03.884532+0000","last_became_active":"2024-09-14T03:43:58.733457+0000","last_became_peered":"2024-09-14T03:43:58.733457+0000","last_unstale":"2024-09-14T03:44:03.884532+0000","last_undegraded":"2024-09-14T03:44:03.884532+0000","last_fullsized":"2024-09-14T03:44:03.884532+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:09:01.069988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.047037+0000","last_change":"2024-09-14T03:44:01.870045+0000","last_active":"2024-09-14T03:44:03.047037+0000","last_peered":"2024-09-14T03:44:03.047037+0000","last_clean":"2024-09-14T03:44:03.047037+0000","last_became_active":"2024-09-14T03:44:01.869780+0000","last_became_peered":"2024-09-14T03:44:01.869780+0000","last_unstale":"2024-09-14T03:44:03.047037+0000","last_undegraded":"2024-09-14T03:44:03.047037+0000","last_fullsized":"2024-09-14T03:44:03.047037+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:04:58.634704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"58'2","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.883496+0000","last_change":"2024-09-14T03:43:58.735364+0000","last_active":"2024-09-14T03:44:03.883496+0000","last_peered":"2024-09-14T03:44:03.883496+0000","last_clean":"2024-09-14T03:44:03.883496+0000","last_became_active":"2024-09-14T03:43:58.734833+0000","last_became_peered":"2024-09-14T03:43:58.734833+0000","last_unstale":"2024-09-14T03:44:03.883496+0000","last_undegraded":"2024-09-14T03:44:03.883496+0000","last_fullsized":"2024-09-14T03:44:03.883496+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:58:37.428713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.879209+0000","last_change":"2024-09-14T03:44:01.878573+0000","last_active":"2024-09-14T03:44:01.879209+0000","last_peered":"2024-09-14T03:44:01.879209+0000","last_clean":"2024-09-14T03:44:01.879209+0000","last_became_active":"2024-09-14T03:44:01.878316+0000","last_became_peered":"2024-09-14T03:44:01.878316+0000","last_unstale":"2024-09-14T03:44:01.879209+0000","last_undegraded":"2024-09-14T03:44:01.879209+0000","last_fullsized":"2024-09-14T03:44:01.879209+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:45:21.129163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":14,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:02.870626+0000","last_change":"2024-09-14T03:43:58.730843+0000","last_active":"2024-09-14T03:44:02.870626+0000","last_peered":"2024-09-14T03:44:02.870626+0000","last_clean":"2024-09-14T03:44:02.870626+0000","last_became_active":"2024-09-14T03:43:58.730538+0000","last_became_peered":"2024-09-14T03:43:58.730538+0000","last_unstale":"2024-09-14T03:44:02.870626+0000","last_undegraded":"2024-09-14T03:44:02.870626+0000","last_fullsized":"2024-09-14T03:44:02.870626+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:04:57.059345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.880062+0000","last_change":"2024-09-14T03:44:01.874355+0000","last_active":"2024-09-14T03:44:01.880062+0000","last_peered":"2024-09-14T03:44:01.880062+0000","last_clean":"2024-09-14T03:44:01.880062+0000","last_became_active":"2024-09-14T03:44:01.874049+0000","last_became_peered":"2024-09-14T03:44:01.874049+0000","last_unstale":"2024-09-14T03:44:01.880062+0000","last_undegraded":"2024-09-14T03:44:01.880062+0000","last_fullsized":"2024-09-14T03:44:01.880062+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:53:30.306175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":13,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:02.870452+0000","last_change":"2024-09-14T03:43:58.728080+0000","last_active":"2024-09-14T03:44:02.870452+0000","last_peered":"2024-09-14T03:44:02.870452+0000","last_clean":"2024-09-14T03:44:02.870452+0000","last_became_active":"2024-09-14T03:43:58.727820+0000","last_became_peered":"2024-09-14T03:43:58.727820+0000","last_unstale":"2024-09-14T03:44:02.870452+0000","last_undegraded":"2024-09-14T03:44:02.870452+0000","last_fullsized":"2024-09-14T03:44:02.870452+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:37:58.166430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.046619+0000","last_change":"2024-09-14T03:44:01.869311+0000","last_active":"2024-09-14T03:44:03.046619+0000","last_peered":"2024-09-14T03:44:03.046619+0000","last_clean":"2024-09-14T03:44:03.046619+0000","last_became_active":"2024-09-14T03:44:01.869065+0000","last_became_peered":"2024-09-14T03:44:01.869065+0000","last_unstale":"2024-09-14T03:44:03.046619+0000","last_undegraded":"2024-09-14T03:44:03.046619+0000","last_fullsized":"2024-09-14T03:44:03.046619+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:45:09.508019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":13,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:02.870616+0000","last_change":"2024-09-14T03:43:58.728931+0000","last_active":"2024-09-14T03:44:02.870616+0000","last_peered":"2024-09-14T03:44:02.870616+0000","last_clean":"2024-09-14T03:44:02.870616+0000","last_became_active":"2024-09-14T03:43:58.728647+0000","last_became_peered":"2024-09-14T03:43:58.728647+0000","last_unstale":"2024-09-14T03:44:02.870616+0000","last_undegraded":"2024-09-14T03:44:02.870616+0000","last_fullsized":"2024-09-14T03:44:02.870616+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:09:38.769523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.046725+0000","last_change":"2024-09-14T03:44:01.869027+0000","last_active":"2024-09-14T03:44:03.046725+0000","last_peered":"2024-09-14T03:44:03.046725+0000","last_clean":"2024-09-14T03:44:03.046725+0000","last_became_active":"2024-09-14T03:44:01.868757+0000","last_became_peered":"2024-09-14T03:44:01.868757+0000","last_unstale":"2024-09-14T03:44:03.046725+0000","last_undegraded":"2024-09-14T03:44:03.046725+0000","last_fullsized":"2024-09-14T03:44:03.046725+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:47:57.070651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"58'2","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.882342+0000","last_change":"2024-09-14T03:43:58.737705+0000","last_active":"2024-09-14T03:44:03.882342+0000","last_peered":"2024-09-14T03:44:03.882342+0000","last_clean":"2024-09-14T03:44:03.882342+0000","last_became_active":"2024-09-14T03:43:58.736546+0000","last_became_peered":"2024-09-14T03:43:58.736546+0000","last_unstale":"2024-09-14T03:44:03.882342+0000","last_undegraded":"2024-09-14T03:44:03.882342+0000","last_fullsized":"2024-09-14T03:44:03.882342+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:27:12.542706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.879558+0000","last_change":"2024-09-14T03:44:01.878897+0000","last_active":"2024-09-14T03:44:01.879558+0000","last_peered":"2024-09-14T03:44:01.879558+0000","last_clean":"2024-09-14T03:44:01.879558+0000","last_became_active":"2024-09-14T03:44:01.878393+0000","last_became_peered":"2024-09-14T03:44:01.878393+0000","last_unstale":"2024-09-14T03:44:01.879558+0000","last_undegraded":"2024-09-14T03:44:01.879558+0000","last_fullsized":"2024-09-14T03:44:01.879558+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:38:38.897169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.889144+0000","last_change":"2024-09-14T03:44:01.888591+0000","last_active":"2024-09-14T03:44:01.889144+0000","last_peered":"2024-09-14T03:44:01.889144+0000","last_clean":"2024-09-14T03:44:01.889144+0000","last_became_active":"2024-09-14T03:44:01.888257+0000","last_became_peered":"2024-09-14T03:44:01.888257+0000","last_unstale":"2024-09-14T03:44:01.889144+0000","last_undegraded":"2024-09-14T03:44:01.889144+0000","last_fullsized":"2024-09-14T03:44:01.889144+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:24:40.992358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"58'2","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.883883+0000","last_change":"2024-09-14T03:43:58.734680+0000","last_active":"2024-09-14T03:44:03.883883+0000","last_peered":"2024-09-14T03:44:03.883883+0000","last_clean":"2024-09-14T03:44:03.883883+0000","last_became_active":"2024-09-14T03:43:58.734408+0000","last_became_peered":"2024-09-14T03:43:58.734408+0000","last_unstale":"2024-09-14T03:44:03.883883+0000","last_undegraded":"2024-09-14T03:44:03.883883+0000","last_fullsized":"2024-09-14T03:44:03.883883+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:49:02.536404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.886954+0000","last_change":"2024-09-14T03:44:01.885572+0000","last_active":"2024-09-14T03:44:01.886954+0000","last_peered":"2024-09-14T03:44:01.886954+0000","last_clean":"2024-09-14T03:44:01.886954+0000","last_became_active":"2024-09-14T03:44:01.885110+0000","last_became_peered":"2024-09-14T03:44:01.885110+0000","last_unstale":"2024-09-14T03:44:01.886954+0000","last_undegraded":"2024-09-14T03:44:01.886954+0000","last_fullsized":"2024-09-14T03:44:01.886954+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:10.439239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1d","version":"58'2","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.882627+0000","last_change":"2024-09-14T03:43:58.731296+0000","last_active":"2024-09-14T03:44:03.882627+0000","last_peered":"2024-09-14T03:44:03.882627+0000","last_clean":"2024-09-14T03:44:03.882627+0000","last_became_active":"2024-09-14T03:43:58.730951+0000","last_became_peered":"2024-09-14T03:43:58.730951+0000","last_unstale":"2024-09-14T03:44:03.882627+0000","last_undegraded":"2024-09-14T03:44:03.882627+0000","last_fullsized":"2024-09-14T03:44:03.882627+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:26:14.801414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.877436+0000","last_change":"2024-09-14T03:44:01.874935+0000","last_active":"2024-09-14T03:44:01.877436+0000","last_peered":"2024-09-14T03:44:01.877436+0000","last_clean":"2024-09-14T03:44:01.877436+0000","last_became_active":"2024-09-14T03:44:01.874668+0000","last_became_peered":"2024-09-14T03:44:01.874668+0000","last_unstale":"2024-09-14T03:44:01.877436+0000","last_undegraded":"2024-09-14T03:44:01.877436+0000","last_fullsized":"2024-09-14T03:44:01.877436+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:24:31.404958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"58'1","reported_seq":14,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.878803+0000","last_change":"2024-09-14T03:43:58.734206+0000","last_active":"2024-09-14T03:44:03.878803+0000","last_peered":"2024-09-14T03:44:03.878803+0000","last_clean":"2024-09-14T03:44:03.878803+0000","last_became_active":"2024-09-14T03:43:58.733960+0000","last_became_peered":"2024-09-14T03:43:58.733960+0000","last_unstale":"2024-09-14T03:44:03.878803+0000","last_undegraded":"2024-09-14T03:44:03.878803+0000","last_fullsized":"2024-09-14T03:44:03.878803+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:44.073503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.046939+0000","last_change":"2024-09-14T03:44:01.883550+0000","last_active":"2024-09-14T03:44:03.046939+0000","last_peered":"2024-09-14T03:44:03.046939+0000","last_clean":"2024-09-14T03:44:03.046939+0000","last_became_active":"2024-09-14T03:44:01.883265+0000","last_became_peered":"2024-09-14T03:44:01.883265+0000","last_unstale":"2024-09-14T03:44:03.046939+0000","last_undegraded":"2024-09-14T03:44:03.046939+0000","last_fullsized":"2024-09-14T03:44:03.046939+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:52:04.512990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"58'4","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.884598+0000","last_change":"2024-09-14T03:43:58.731954+0000","last_active":"2024-09-14T03:44:03.884598+0000","last_peered":"2024-09-14T03:44:03.884598+0000","last_clean":"2024-09-14T03:44:03.884598+0000","last_became_active":"2024-09-14T03:43:58.731688+0000","last_became_peered":"2024-09-14T03:43:58.731688+0000","last_unstale":"2024-09-14T03:44:03.884598+0000","last_undegraded":"2024-09-14T03:44:03.884598+0000","last_fullsized":"2024-09-14T03:44:03.884598+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:30:10.811609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3563520,"data_stored":3542208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397516,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298512,"kb_used_data":1148,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446640,"statfs":{"total":95995035648,"available":95689359360,"internally_reserved":0,"allocated":1175552,"data_stored":831572,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52300000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63200000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61499999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69199999999999995}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64700000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70399999999999996}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67700000000000005}]}]},{"osd":6,"up_from":43,"seq":184683593744,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298504,"kb_used_data":1148,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446648,"statfs":{"total":95995035648,"available":95689367552,"internally_reserved":0,"allocated":1175552,"data_stored":832217,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.54700000000000004}]},{"osd":1,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.58299999999999996}]},{"osd":2,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.52400000000000002}]},{"osd":3,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.50800000000000001}]},{"osd":4,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.375},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.33600000000000002}]},{"osd":5,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.46300000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48499999999999999}]}]},{"osd":5,"up_from":37,"seq":158913789974,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297928,"kb_used_data":572,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447224,"statfs":{"total":95995035648,"available":95689957376,"internally_reserved":0,"allocated":585728,"data_stored":241939,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:43:47 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.442},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.51400000000000001}]},{"osd":1,"last update":"Sat Sep 14 03:43:47 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.40300000000000002}]},{"osd":2,"last update":"Sat Sep 14 03:43:47 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.41799999999999998}]},{"osd":3,"last update":"Sat Sep 14 03:43:47 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.497},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.45900000000000002}]},{"osd":4,"last update":"Sat Sep 14 03:43:47 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.379,"15min":0.379},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.38200000000000001}]},{"osd":6,"last update":"Sat Sep 14 03:44:11 2024","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.432},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.48199999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47099999999999997}]}]},{"osd":4,"up_from":31,"seq":133143986202,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297932,"kb_used_data":576,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447220,"statfs":{"total":95995035648,"available":95689953280,"internally_reserved":0,"allocated":589824,"data_stored":241308,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:43:24 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.54000000000000004}]},{"osd":1,"last update":"Sat Sep 14 03:43:24 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.61199999999999999}]},{"osd":2,"last update":"Sat Sep 14 03:43:24 2024","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.66900000000000004}]},{"osd":3,"last update":"Sat Sep 14 03:43:24 2024","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.628},{"interface":"front","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.58699999999999997}]},{"osd":5,"last update":"Sat Sep 14 03:43:49 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.42299999999999999}]},{"osd":6,"last update":"Sat Sep 14 03:44:11 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.627,"5min":0.627,"15min":0.627},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.47999999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55300000000000005}]}]},{"osd":3,"up_from":26,"seq":111669149726,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297936,"kb_used_data":576,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447216,"statfs":{"total":95995035648,"available":95689949184,"internally_reserved":0,"allocated":589824,"data_stored":241953,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:44:03 2024","interfaces":[{"interface":"back","average":{"1min":0.34599999999999997,"5min":0.38400000000000001,"15min":0.39000000000000001},"min":{"1min":0.245,"5min":0.185,"15min":0.185},"max":{"1min":0.43099999999999999,"5min":0.48199999999999998,"15min":0.48199999999999998},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.40200000000000002,"15min":0.40799999999999997},"min":{"1min":0.254,"5min":0.218,"15min":0.218},"max":{"1min":0.48099999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.36399999999999999}]},{"osd":1,"last update":"Sat Sep 14 03:44:03 2024","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.41599999999999998,"15min":0.41999999999999998},"min":{"1min":0.28799999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.5,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.36599999999999999,"5min":0.39300000000000002,"15min":0.39700000000000002},"min":{"1min":0.30499999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.47199999999999998,"5min":0.53700000000000003,"15min":0.53700000000000003},"last":0.318}]},{"osd":2,"last update":"Sat Sep 14 03:44:03 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.434,"15min":0.441},"min":{"1min":0.28999999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.52300000000000002,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.42299999999999999,"15min":0.42799999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.54400000000000004,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.378}]},{"osd":4,"last update":"Sat Sep 14 03:43:29 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.47199999999999998}]},{"osd":5,"last update":"Sat Sep 14 03:43:52 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.48299999999999998}]},{"osd":6,"last update":"Sat Sep 14 03:44:11 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.501},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.53700000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45200000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411363,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297912,"kb_used_data":556,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447240,"statfs":{"total":95995035648,"available":95689973760,"internally_reserved":0,"allocated":569344,"data_stored":240001,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:43:45 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.36299999999999999,"15min":0.35599999999999998},"min":{"1min":0.26200000000000001,"5min":0.22,"15min":0.22},"max":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.371,"15min":0.36599999999999999},"min":{"1min":0.27800000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.56200000000000006,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.46100000000000002}]},{"osd":1,"last update":"Sat Sep 14 03:43:45 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.40000000000000002,"15min":0.39800000000000002},"min":{"1min":0.32800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.441,"5min":0.39200000000000002,"15min":0.38400000000000001},"min":{"1min":0.29199999999999998,"5min":0.245,"15min":0.245},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.40200000000000002}]},{"osd":3,"last update":"Sat Sep 14 03:44:08 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.44800000000000001,"15min":0.45100000000000001},"min":{"1min":0.34999999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.53300000000000003,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.496},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.44800000000000001,"15min":0.45800000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.56000000000000005,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.50600000000000001}]},{"osd":4,"last update":"Sat Sep 14 03:43:24 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.57399999999999995}]},{"osd":5,"last update":"Sat Sep 14 03:43:48 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.438}]},{"osd":6,"last update":"Sat Sep 14 03:44:12 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.47699999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55100000000000005}]}]},{"osd":1,"up_from":13,"seq":55834574889,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297916,"kb_used_data":556,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447236,"statfs":{"total":95995035648,"available":95689969664,"internally_reserved":0,"allocated":569344,"data_stored":239827,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:44:20 2024","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.32000000000000001,"15min":0.29599999999999999},"min":{"1min":0.249,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.51900000000000002,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.374,"5min":0.34399999999999997,"15min":0.31900000000000001},"min":{"1min":0.27000000000000002,"5min":0.189,"15min":0.189},"max":{"1min":0.56799999999999995,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.49399999999999999}]},{"osd":2,"last update":"Sat Sep 14 03:43:39 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.35499999999999998,"15min":0.34300000000000003},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.36199999999999999,"15min":0.34699999999999998},"min":{"1min":0.29499999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.58199999999999996}]},{"osd":3,"last update":"Sat Sep 14 03:44:07 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.42299999999999999,"15min":0.42199999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.435,"15min":0.437},"min":{"1min":0.27500000000000002,"5min":0.247,"15min":0.247},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.63800000000000001}]},{"osd":4,"last update":"Sat Sep 14 03:43:25 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.65400000000000003}]},{"osd":5,"last update":"Sat Sep 14 03:43:47 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.78300000000000003}]},{"osd":6,"last update":"Sat Sep 14 03:44:10 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.747},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.81999999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.84899999999999998}]}]},{"osd":0,"up_from":8,"seq":34359738413,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298508,"kb_used_data":1152,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446644,"statfs":{"total":95995035648,"available":95689363456,"internally_reserved":0,"allocated":1179648,"data_stored":831381,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 14 03:44:17 2024","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":0.35699999999999998,"15min":0.34899999999999998},"min":{"1min":0.25700000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.46500000000000002,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.36299999999999999,"15min":0.34799999999999998},"min":{"1min":0.246,"5min":0.216,"15min":0.216},"max":{"1min":0.48999999999999999,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.38}]},{"osd":2,"last update":"Sat Sep 14 03:43:43 2024","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.41299999999999998,"15min":0.40999999999999998},"min":{"1min":0.29099999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.54900000000000004,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.442},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.39700000000000002,"15min":0.39500000000000002},"min":{"1min":0.28799999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.505,"5min":0.505,"15min":0.505},"last":0.5}]},{"osd":3,"last update":"Sat Sep 14 03:44:04 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.44,"15min":0.44700000000000001},"min":{"1min":0.32500000000000001,"5min":0.307,"15min":0.307},"max":{"1min":0.54300000000000004,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.41499999999999998,"15min":0.42099999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.35099999999999998}]},{"osd":4,"last update":"Sat Sep 14 03:43:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.56299999999999994}]},{"osd":5,"last update":"Sat Sep 14 03:43:48 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.59499999999999997}]},{"osd":6,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.54800000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53200000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1642,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T03:44:26.246 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T03:44:26.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:25 smithi094 ceph-mon[31053]: pgmap v198: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:26.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:25 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.18", "id": [7, 2]}]': finished 2024-09-14T03:44:26.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:25 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3", "id": [0, 1]}]': finished 2024-09-14T03:44:26.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:25 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [6, 2]}]': finished 2024-09-14T03:44:26.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:25 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [0, 2, 4, 1]}]': finished 2024-09-14T03:44:26.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:25 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.26", "id": [4, 3]}]': finished 2024-09-14T03:44:26.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:25 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.36", "id": [5, 2]}]': finished 2024-09-14T03:44:26.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:25 smithi094 ceph-mon[31053]: osdmap e59: 8 total, 8 up, 8 in 2024-09-14T03:44:26.489 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:44:27.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:26 smithi094 ceph-mon[31053]: from='client.24398 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:44:27.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:26 smithi094 ceph-mon[31053]: osdmap e60: 8 total, 8 up, 8 in 2024-09-14T03:44:27.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:26 smithi070 ceph-mon[27495]: from='client.24398 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:44:27.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:26 smithi070 ceph-mon[27495]: osdmap e60: 8 total, 8 up, 8 in 2024-09-14T03:44:27.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:26 smithi070 ceph-mon[39288]: from='client.24398 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:44:27.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:26 smithi070 ceph-mon[39288]: osdmap e60: 8 total, 8 up, 8 in 2024-09-14T03:44:28.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:27 smithi070 ceph-mon[27495]: pgmap v201: 97 pgs: 2 activating, 95 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:28.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:27 smithi070 ceph-mon[39288]: pgmap v201: 97 pgs: 2 activating, 95 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:28.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:27 smithi094 ceph-mon[31053]: pgmap v201: 97 pgs: 2 activating, 95 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:28.413 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:44:28.415 INFO:teuthology.orchestra.run.smithi070.stderr:dumped all 2024-09-14T03:44:29.231 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":201,"stamp":"2024-09-14T03:44:26.502995+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":146,"num_write_kb":1641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111,"ondisk_log_size":111,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":291,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385172,"kb_used_data":6308,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747576044,"statfs":{"total":767960285184,"available":765517869056,"internally_reserved":0,"allocated":6459392,"data_stored":3713580,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001879"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.877339+0000","last_change":"2024-09-14T03:44:01.882774+0000","last_active":"2024-09-14T03:44:03.877339+0000","last_peered":"2024-09-14T03:44:03.877339+0000","last_clean":"2024-09-14T03:44:03.877339+0000","last_became_active":"2024-09-14T03:44:01.882481+0000","last_became_peered":"2024-09-14T03:44:01.882481+0000","last_unstale":"2024-09-14T03:44:03.877339+0000","last_undegraded":"2024-09-14T03:44:03.877339+0000","last_fullsized":"2024-09-14T03:44:03.877339+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:49:25.209573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:02.870886+0000","last_change":"2024-09-14T03:44:01.868415+0000","last_active":"2024-09-14T03:44:02.870886+0000","last_peered":"2024-09-14T03:44:02.870886+0000","last_clean":"2024-09-14T03:44:02.870886+0000","last_became_active":"2024-09-14T03:44:01.868152+0000","last_became_peered":"2024-09-14T03:44:01.868152+0000","last_unstale":"2024-09-14T03:44:02.870886+0000","last_undegraded":"2024-09-14T03:44:02.870886+0000","last_fullsized":"2024-09-14T03:44:02.870886+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:56:00.270070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.880085+0000","last_change":"2024-09-14T03:44:01.871459+0000","last_active":"2024-09-14T03:44:03.880085+0000","last_peered":"2024-09-14T03:44:03.880085+0000","last_clean":"2024-09-14T03:44:03.880085+0000","last_became_active":"2024-09-14T03:44:01.870749+0000","last_became_peered":"2024-09-14T03:44:01.870749+0000","last_unstale":"2024-09-14T03:44:03.880085+0000","last_undegraded":"2024-09-14T03:44:03.880085+0000","last_fullsized":"2024-09-14T03:44:03.880085+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:54.391363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.883540+0000","last_change":"2024-09-14T03:44:01.882503+0000","last_active":"2024-09-14T03:44:01.883540+0000","last_peered":"2024-09-14T03:44:01.883540+0000","last_clean":"2024-09-14T03:44:01.883540+0000","last_became_active":"2024-09-14T03:44:01.882250+0000","last_became_peered":"2024-09-14T03:44:01.882250+0000","last_unstale":"2024-09-14T03:44:01.883540+0000","last_undegraded":"2024-09-14T03:44:01.883540+0000","last_fullsized":"2024-09-14T03:44:01.883540+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:46:29.551285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.877043+0000","last_change":"2024-09-14T03:44:01.880333+0000","last_active":"2024-09-14T03:44:03.877043+0000","last_peered":"2024-09-14T03:44:03.877043+0000","last_clean":"2024-09-14T03:44:03.877043+0000","last_became_active":"2024-09-14T03:44:01.880010+0000","last_became_peered":"2024-09-14T03:44:01.880010+0000","last_unstale":"2024-09-14T03:44:03.877043+0000","last_undegraded":"2024-09-14T03:44:03.877043+0000","last_fullsized":"2024-09-14T03:44:03.877043+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:45:02.741425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.881438+0000","last_change":"2024-09-14T03:44:01.880710+0000","last_active":"2024-09-14T03:44:01.881438+0000","last_peered":"2024-09-14T03:44:01.881438+0000","last_clean":"2024-09-14T03:44:01.881438+0000","last_became_active":"2024-09-14T03:44:01.880237+0000","last_became_peered":"2024-09-14T03:44:01.880237+0000","last_unstale":"2024-09-14T03:44:01.881438+0000","last_undegraded":"2024-09-14T03:44:01.881438+0000","last_fullsized":"2024-09-14T03:44:01.881438+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:56:49.808707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.876722+0000","last_change":"2024-09-14T03:44:01.878590+0000","last_active":"2024-09-14T03:44:03.876722+0000","last_peered":"2024-09-14T03:44:03.876722+0000","last_clean":"2024-09-14T03:44:03.876722+0000","last_became_active":"2024-09-14T03:44:01.878281+0000","last_became_peered":"2024-09-14T03:44:01.878281+0000","last_unstale":"2024-09-14T03:44:03.876722+0000","last_undegraded":"2024-09-14T03:44:03.876722+0000","last_fullsized":"2024-09-14T03:44:03.876722+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:41:57.164227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.046509+0000","last_change":"2024-09-14T03:44:01.877057+0000","last_active":"2024-09-14T03:44:03.046509+0000","last_peered":"2024-09-14T03:44:03.046509+0000","last_clean":"2024-09-14T03:44:03.046509+0000","last_became_active":"2024-09-14T03:44:01.876727+0000","last_became_peered":"2024-09-14T03:44:01.876727+0000","last_unstale":"2024-09-14T03:44:03.046509+0000","last_undegraded":"2024-09-14T03:44:03.046509+0000","last_fullsized":"2024-09-14T03:44:03.046509+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:31:24.716316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.877597+0000","last_change":"2024-09-14T03:44:01.884865+0000","last_active":"2024-09-14T03:44:03.877597+0000","last_peered":"2024-09-14T03:44:03.877597+0000","last_clean":"2024-09-14T03:44:03.877597+0000","last_became_active":"2024-09-14T03:44:01.884136+0000","last_became_peered":"2024-09-14T03:44:01.884136+0000","last_unstale":"2024-09-14T03:44:03.877597+0000","last_undegraded":"2024-09-14T03:44:03.877597+0000","last_fullsized":"2024-09-14T03:44:03.877597+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:59.903382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":16,"reported_epoch":60,"state":"activating","last_fresh":"2024-09-14T03:44:25.928176+0000","last_change":"2024-09-14T03:44:25.928176+0000","last_active":"2024-09-14T03:44:24.930301+0000","last_peered":"2024-09-14T03:44:03.881437+0000","last_clean":"2024-09-14T03:44:03.881437+0000","last_became_active":"2024-09-14T03:44:01.873644+0000","last_became_peered":"2024-09-14T03:44:01.873644+0000","last_unstale":"2024-09-14T03:44:25.928176+0000","last_undegraded":"2024-09-14T03:44:25.928176+0000","last_fullsized":"2024-09-14T03:44:25.928176+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:34.312039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":["6","1","2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929430+0000","last_change":"2024-09-14T03:44:01.875017+0000","last_active":"2024-09-14T03:44:24.929430+0000","last_peered":"2024-09-14T03:44:24.929430+0000","last_clean":"2024-09-14T03:44:24.929430+0000","last_became_active":"2024-09-14T03:44:01.874687+0000","last_became_peered":"2024-09-14T03:44:01.874687+0000","last_unstale":"2024-09-14T03:44:24.929430+0000","last_undegraded":"2024-09-14T03:44:24.929430+0000","last_fullsized":"2024-09-14T03:44:24.929430+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:29:09.035342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.887082+0000","last_change":"2024-09-14T03:44:01.883313+0000","last_active":"2024-09-14T03:44:01.887082+0000","last_peered":"2024-09-14T03:44:01.887082+0000","last_clean":"2024-09-14T03:44:01.887082+0000","last_became_active":"2024-09-14T03:44:01.882770+0000","last_became_peered":"2024-09-14T03:44:01.882770+0000","last_unstale":"2024-09-14T03:44:01.887082+0000","last_undegraded":"2024-09-14T03:44:01.887082+0000","last_fullsized":"2024-09-14T03:44:01.887082+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:57:19.402319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.046697+0000","last_change":"2024-09-14T03:44:01.878245+0000","last_active":"2024-09-14T03:44:03.046697+0000","last_peered":"2024-09-14T03:44:03.046697+0000","last_clean":"2024-09-14T03:44:03.046697+0000","last_became_active":"2024-09-14T03:44:01.877861+0000","last_became_peered":"2024-09-14T03:44:01.877861+0000","last_unstale":"2024-09-14T03:44:03.046697+0000","last_undegraded":"2024-09-14T03:44:03.046697+0000","last_fullsized":"2024-09-14T03:44:03.046697+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:08:29.902090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.879249+0000","last_change":"2024-09-14T03:44:01.877391+0000","last_active":"2024-09-14T03:44:03.879249+0000","last_peered":"2024-09-14T03:44:03.879249+0000","last_clean":"2024-09-14T03:44:03.879249+0000","last_became_active":"2024-09-14T03:44:01.876797+0000","last_became_peered":"2024-09-14T03:44:01.876797+0000","last_unstale":"2024-09-14T03:44:03.879249+0000","last_undegraded":"2024-09-14T03:44:03.879249+0000","last_fullsized":"2024-09-14T03:44:03.879249+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:06:02.891762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:02.870321+0000","last_change":"2024-09-14T03:44:01.876700+0000","last_active":"2024-09-14T03:44:02.870321+0000","last_peered":"2024-09-14T03:44:02.870321+0000","last_clean":"2024-09-14T03:44:02.870321+0000","last_became_active":"2024-09-14T03:44:01.876091+0000","last_became_peered":"2024-09-14T03:44:01.876091+0000","last_unstale":"2024-09-14T03:44:02.870321+0000","last_undegraded":"2024-09-14T03:44:02.870321+0000","last_fullsized":"2024-09-14T03:44:02.870321+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:25:41.750127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.879498+0000","last_change":"2024-09-14T03:44:01.877120+0000","last_active":"2024-09-14T03:44:01.879498+0000","last_peered":"2024-09-14T03:44:01.879498+0000","last_clean":"2024-09-14T03:44:01.879498+0000","last_became_active":"2024-09-14T03:44:01.876793+0000","last_became_peered":"2024-09-14T03:44:01.876793+0000","last_unstale":"2024-09-14T03:44:01.879498+0000","last_undegraded":"2024-09-14T03:44:01.879498+0000","last_fullsized":"2024-09-14T03:44:01.879498+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:23:56.450728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.886538+0000","last_change":"2024-09-14T03:44:01.885108+0000","last_active":"2024-09-14T03:44:01.886538+0000","last_peered":"2024-09-14T03:44:01.886538+0000","last_clean":"2024-09-14T03:44:01.886538+0000","last_became_active":"2024-09-14T03:44:01.884857+0000","last_became_peered":"2024-09-14T03:44:01.884857+0000","last_unstale":"2024-09-14T03:44:01.886538+0000","last_undegraded":"2024-09-14T03:44:01.886538+0000","last_fullsized":"2024-09-14T03:44:01.886538+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:31.448617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.877369+0000","last_change":"2024-09-14T03:44:01.876503+0000","last_active":"2024-09-14T03:44:01.877369+0000","last_peered":"2024-09-14T03:44:01.877369+0000","last_clean":"2024-09-14T03:44:01.877369+0000","last_became_active":"2024-09-14T03:44:01.876227+0000","last_became_peered":"2024-09-14T03:44:01.876227+0000","last_unstale":"2024-09-14T03:44:01.877369+0000","last_undegraded":"2024-09-14T03:44:01.877369+0000","last_fullsized":"2024-09-14T03:44:01.877369+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:44:50.132152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.877249+0000","last_change":"2024-09-14T03:44:01.883708+0000","last_active":"2024-09-14T03:44:03.877249+0000","last_peered":"2024-09-14T03:44:03.877249+0000","last_clean":"2024-09-14T03:44:03.877249+0000","last_became_active":"2024-09-14T03:44:01.883440+0000","last_became_peered":"2024-09-14T03:44:01.883440+0000","last_unstale":"2024-09-14T03:44:03.877249+0000","last_undegraded":"2024-09-14T03:44:03.877249+0000","last_fullsized":"2024-09-14T03:44:03.877249+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:10:34.658062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929540+0000","last_change":"2024-09-14T03:44:01.888565+0000","last_active":"2024-09-14T03:44:24.929540+0000","last_peered":"2024-09-14T03:44:24.929540+0000","last_clean":"2024-09-14T03:44:24.929540+0000","last_became_active":"2024-09-14T03:44:01.888191+0000","last_became_peered":"2024-09-14T03:44:01.888191+0000","last_unstale":"2024-09-14T03:44:24.929540+0000","last_undegraded":"2024-09-14T03:44:24.929540+0000","last_fullsized":"2024-09-14T03:44:24.929540+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:52:23.782664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.879582+0000","last_change":"2024-09-14T03:44:01.874642+0000","last_active":"2024-09-14T03:44:03.879582+0000","last_peered":"2024-09-14T03:44:03.879582+0000","last_clean":"2024-09-14T03:44:03.879582+0000","last_became_active":"2024-09-14T03:44:01.874384+0000","last_became_peered":"2024-09-14T03:44:01.874384+0000","last_unstale":"2024-09-14T03:44:03.879582+0000","last_undegraded":"2024-09-14T03:44:03.879582+0000","last_fullsized":"2024-09-14T03:44:03.879582+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:35:05.535173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.887238+0000","last_change":"2024-09-14T03:44:01.886893+0000","last_active":"2024-09-14T03:44:01.887238+0000","last_peered":"2024-09-14T03:44:01.887238+0000","last_clean":"2024-09-14T03:44:01.887238+0000","last_became_active":"2024-09-14T03:44:01.886594+0000","last_became_peered":"2024-09-14T03:44:01.886594+0000","last_unstale":"2024-09-14T03:44:01.887238+0000","last_undegraded":"2024-09-14T03:44:01.887238+0000","last_fullsized":"2024-09-14T03:44:01.887238+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:07:37.154358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.046662+0000","last_change":"2024-09-14T03:44:01.868698+0000","last_active":"2024-09-14T03:44:03.046662+0000","last_peered":"2024-09-14T03:44:03.046662+0000","last_clean":"2024-09-14T03:44:03.046662+0000","last_became_active":"2024-09-14T03:44:01.868442+0000","last_became_peered":"2024-09-14T03:44:01.868442+0000","last_unstale":"2024-09-14T03:44:03.046662+0000","last_undegraded":"2024-09-14T03:44:03.046662+0000","last_fullsized":"2024-09-14T03:44:03.046662+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:41:18.296424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.886249+0000","last_change":"2024-09-14T03:44:01.882708+0000","last_active":"2024-09-14T03:44:01.886249+0000","last_peered":"2024-09-14T03:44:01.886249+0000","last_clean":"2024-09-14T03:44:01.886249+0000","last_became_active":"2024-09-14T03:44:01.882443+0000","last_became_peered":"2024-09-14T03:44:01.882443+0000","last_unstale":"2024-09-14T03:44:01.886249+0000","last_undegraded":"2024-09-14T03:44:01.886249+0000","last_fullsized":"2024-09-14T03:44:01.886249+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:46:48.306263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.046997+0000","last_change":"2024-09-14T03:44:01.884940+0000","last_active":"2024-09-14T03:44:03.046997+0000","last_peered":"2024-09-14T03:44:03.046997+0000","last_clean":"2024-09-14T03:44:03.046997+0000","last_became_active":"2024-09-14T03:44:01.884304+0000","last_became_peered":"2024-09-14T03:44:01.884304+0000","last_unstale":"2024-09-14T03:44:03.046997+0000","last_undegraded":"2024-09-14T03:44:03.046997+0000","last_fullsized":"2024-09-14T03:44:03.046997+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:01:49.061637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":16,"reported_epoch":60,"state":"activating","last_fresh":"2024-09-14T03:44:25.928238+0000","last_change":"2024-09-14T03:44:25.928238+0000","last_active":"2024-09-14T03:44:24.929140+0000","last_peered":"2024-09-14T03:44:03.881581+0000","last_clean":"2024-09-14T03:44:03.881581+0000","last_became_active":"2024-09-14T03:44:01.877768+0000","last_became_peered":"2024-09-14T03:44:01.877768+0000","last_unstale":"2024-09-14T03:44:25.928238+0000","last_undegraded":"2024-09-14T03:44:25.928238+0000","last_fullsized":"2024-09-14T03:44:25.928238+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:00:53.758328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":["6","3","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:02.870322+0000","last_change":"2024-09-14T03:44:01.874564+0000","last_active":"2024-09-14T03:44:02.870322+0000","last_peered":"2024-09-14T03:44:02.870322+0000","last_clean":"2024-09-14T03:44:02.870322+0000","last_became_active":"2024-09-14T03:44:01.874297+0000","last_became_peered":"2024-09-14T03:44:01.874297+0000","last_unstale":"2024-09-14T03:44:02.870322+0000","last_undegraded":"2024-09-14T03:44:02.870322+0000","last_fullsized":"2024-09-14T03:44:02.870322+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:07.027610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.046813+0000","last_change":"2024-09-14T03:44:01.884486+0000","last_active":"2024-09-14T03:44:03.046813+0000","last_peered":"2024-09-14T03:44:03.046813+0000","last_clean":"2024-09-14T03:44:03.046813+0000","last_became_active":"2024-09-14T03:44:01.884218+0000","last_became_peered":"2024-09-14T03:44:01.884218+0000","last_unstale":"2024-09-14T03:44:03.046813+0000","last_undegraded":"2024-09-14T03:44:03.046813+0000","last_fullsized":"2024-09-14T03:44:03.046813+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:59:18.153239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.881307+0000","last_change":"2024-09-14T03:44:01.878737+0000","last_active":"2024-09-14T03:44:03.881307+0000","last_peered":"2024-09-14T03:44:03.881307+0000","last_clean":"2024-09-14T03:44:03.881307+0000","last_became_active":"2024-09-14T03:44:01.878504+0000","last_became_peered":"2024-09-14T03:44:01.878504+0000","last_unstale":"2024-09-14T03:44:03.881307+0000","last_undegraded":"2024-09-14T03:44:03.881307+0000","last_fullsized":"2024-09-14T03:44:03.881307+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:41:35.420957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.880939+0000","last_change":"2024-09-14T03:44:01.880509+0000","last_active":"2024-09-14T03:44:01.880939+0000","last_peered":"2024-09-14T03:44:01.880939+0000","last_clean":"2024-09-14T03:44:01.880939+0000","last_became_active":"2024-09-14T03:44:01.879855+0000","last_became_peered":"2024-09-14T03:44:01.879855+0000","last_unstale":"2024-09-14T03:44:01.880939+0000","last_undegraded":"2024-09-14T03:44:01.880939+0000","last_fullsized":"2024-09-14T03:44:01.880939+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:05:31.166902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.870149+0000","last_change":"2024-09-14T03:44:01.869523+0000","last_active":"2024-09-14T03:44:01.870149+0000","last_peered":"2024-09-14T03:44:01.870149+0000","last_clean":"2024-09-14T03:44:01.870149+0000","last_became_active":"2024-09-14T03:44:01.869225+0000","last_became_peered":"2024-09-14T03:44:01.869225+0000","last_unstale":"2024-09-14T03:44:01.870149+0000","last_undegraded":"2024-09-14T03:44:01.870149+0000","last_fullsized":"2024-09-14T03:44:01.870149+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:43:15.638187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929040+0000","last_change":"2024-09-14T03:44:01.876014+0000","last_active":"2024-09-14T03:44:24.929040+0000","last_peered":"2024-09-14T03:44:24.929040+0000","last_clean":"2024-09-14T03:44:24.929040+0000","last_became_active":"2024-09-14T03:44:01.875496+0000","last_became_peered":"2024-09-14T03:44:01.875496+0000","last_unstale":"2024-09-14T03:44:24.929040+0000","last_undegraded":"2024-09-14T03:44:24.929040+0000","last_fullsized":"2024-09-14T03:44:24.929040+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:03:45.344199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.869587+0000","last_change":"2024-09-14T03:44:01.865496+0000","last_active":"2024-09-14T03:44:01.869587+0000","last_peered":"2024-09-14T03:44:01.869587+0000","last_clean":"2024-09-14T03:44:01.869587+0000","last_became_active":"2024-09-14T03:44:01.865190+0000","last_became_peered":"2024-09-14T03:44:01.865190+0000","last_unstale":"2024-09-14T03:44:01.869587+0000","last_undegraded":"2024-09-14T03:44:01.869587+0000","last_fullsized":"2024-09-14T03:44:01.869587+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:18:50.037675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"58'1","reported_seq":14,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.878410+0000","last_change":"2024-09-14T03:43:58.729188+0000","last_active":"2024-09-14T03:44:03.878410+0000","last_peered":"2024-09-14T03:44:03.878410+0000","last_clean":"2024-09-14T03:44:03.878410+0000","last_became_active":"2024-09-14T03:43:58.728927+0000","last_became_peered":"2024-09-14T03:43:58.728927+0000","last_unstale":"2024-09-14T03:44:03.878410+0000","last_undegraded":"2024-09-14T03:44:03.878410+0000","last_fullsized":"2024-09-14T03:44:03.878410+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:13:16.419701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.879704+0000","last_change":"2024-09-14T03:44:01.879101+0000","last_active":"2024-09-14T03:44:03.879704+0000","last_peered":"2024-09-14T03:44:03.879704+0000","last_clean":"2024-09-14T03:44:03.879704+0000","last_became_active":"2024-09-14T03:44:01.878801+0000","last_became_peered":"2024-09-14T03:44:01.878801+0000","last_unstale":"2024-09-14T03:44:03.879704+0000","last_undegraded":"2024-09-14T03:44:03.879704+0000","last_fullsized":"2024-09-14T03:44:03.879704+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:13.447251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"58'1","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930120+0000","last_change":"2024-09-14T03:43:58.730444+0000","last_active":"2024-09-14T03:44:24.930120+0000","last_peered":"2024-09-14T03:44:24.930120+0000","last_clean":"2024-09-14T03:44:24.930120+0000","last_became_active":"2024-09-14T03:43:58.730172+0000","last_became_peered":"2024-09-14T03:43:58.730172+0000","last_unstale":"2024-09-14T03:44:24.930120+0000","last_undegraded":"2024-09-14T03:44:24.930120+0000","last_fullsized":"2024-09-14T03:44:24.930120+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:03:44.810562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":13,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.046789+0000","last_change":"2024-09-14T03:43:58.731458+0000","last_active":"2024-09-14T03:44:03.046789+0000","last_peered":"2024-09-14T03:44:03.046789+0000","last_clean":"2024-09-14T03:44:03.046789+0000","last_became_active":"2024-09-14T03:43:58.731206+0000","last_became_peered":"2024-09-14T03:43:58.731206+0000","last_unstale":"2024-09-14T03:44:03.046789+0000","last_undegraded":"2024-09-14T03:44:03.046789+0000","last_fullsized":"2024-09-14T03:44:03.046789+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:37:06.670539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.046550+0000","last_change":"2024-09-14T03:44:01.869781+0000","last_active":"2024-09-14T03:44:03.046550+0000","last_peered":"2024-09-14T03:44:03.046550+0000","last_clean":"2024-09-14T03:44:03.046550+0000","last_became_active":"2024-09-14T03:44:01.869521+0000","last_became_peered":"2024-09-14T03:44:01.869521+0000","last_unstale":"2024-09-14T03:44:03.046550+0000","last_undegraded":"2024-09-14T03:44:03.046550+0000","last_fullsized":"2024-09-14T03:44:03.046550+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:35:18.565206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":13,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.862828+0000","last_change":"2024-09-14T03:43:58.734065+0000","last_active":"2024-09-14T03:44:01.862828+0000","last_peered":"2024-09-14T03:44:01.862828+0000","last_clean":"2024-09-14T03:44:01.862828+0000","last_became_active":"2024-09-14T03:43:58.733752+0000","last_became_peered":"2024-09-14T03:43:58.733752+0000","last_unstale":"2024-09-14T03:44:01.862828+0000","last_undegraded":"2024-09-14T03:44:01.862828+0000","last_fullsized":"2024-09-14T03:44:01.862828+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:54:52.048737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:02.870312+0000","last_change":"2024-09-14T03:44:01.873748+0000","last_active":"2024-09-14T03:44:02.870312+0000","last_peered":"2024-09-14T03:44:02.870312+0000","last_clean":"2024-09-14T03:44:02.870312+0000","last_became_active":"2024-09-14T03:44:01.873473+0000","last_became_peered":"2024-09-14T03:44:01.873473+0000","last_unstale":"2024-09-14T03:44:02.870312+0000","last_undegraded":"2024-09-14T03:44:02.870312+0000","last_fullsized":"2024-09-14T03:44:02.870312+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:04:39.289903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929850+0000","last_change":"2024-09-14T03:43:58.735608+0000","last_active":"2024-09-14T03:44:24.929850+0000","last_peered":"2024-09-14T03:44:24.929850+0000","last_clean":"2024-09-14T03:44:24.929850+0000","last_became_active":"2024-09-14T03:43:58.735301+0000","last_became_peered":"2024-09-14T03:43:58.735301+0000","last_unstale":"2024-09-14T03:44:24.929850+0000","last_undegraded":"2024-09-14T03:44:24.929850+0000","last_fullsized":"2024-09-14T03:44:24.929850+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:44:56.523073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.875923+0000","last_change":"2024-09-14T03:44:01.874777+0000","last_active":"2024-09-14T03:44:01.875923+0000","last_peered":"2024-09-14T03:44:01.875923+0000","last_clean":"2024-09-14T03:44:01.875923+0000","last_became_active":"2024-09-14T03:44:01.874473+0000","last_became_peered":"2024-09-14T03:44:01.874473+0000","last_unstale":"2024-09-14T03:44:01.875923+0000","last_undegraded":"2024-09-14T03:44:01.875923+0000","last_fullsized":"2024-09-14T03:44:01.875923+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:49:02.042635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":13,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.862878+0000","last_change":"2024-09-14T03:43:58.732969+0000","last_active":"2024-09-14T03:44:01.862878+0000","last_peered":"2024-09-14T03:44:01.862878+0000","last_clean":"2024-09-14T03:44:01.862878+0000","last_became_active":"2024-09-14T03:43:58.732492+0000","last_became_peered":"2024-09-14T03:43:58.732492+0000","last_unstale":"2024-09-14T03:44:01.862878+0000","last_undegraded":"2024-09-14T03:44:01.862878+0000","last_fullsized":"2024-09-14T03:44:01.862878+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:50:28.719447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.880721+0000","last_change":"2024-09-14T03:44:01.875679+0000","last_active":"2024-09-14T03:44:03.880721+0000","last_peered":"2024-09-14T03:44:03.880721+0000","last_clean":"2024-09-14T03:44:03.880721+0000","last_became_active":"2024-09-14T03:44:01.875422+0000","last_became_peered":"2024-09-14T03:44:01.875422+0000","last_unstale":"2024-09-14T03:44:03.880721+0000","last_undegraded":"2024-09-14T03:44:03.880721+0000","last_fullsized":"2024-09-14T03:44:03.880721+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:13:00.349373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"58'3","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.886706+0000","last_change":"2024-09-14T03:43:58.735066+0000","last_active":"2024-09-14T03:44:03.886706+0000","last_peered":"2024-09-14T03:44:03.886706+0000","last_clean":"2024-09-14T03:44:03.886706+0000","last_became_active":"2024-09-14T03:43:58.734514+0000","last_became_peered":"2024-09-14T03:43:58.734514+0000","last_unstale":"2024-09-14T03:44:03.886706+0000","last_undegraded":"2024-09-14T03:44:03.886706+0000","last_fullsized":"2024-09-14T03:44:03.886706+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:41:55.478295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.883377+0000","last_change":"2024-09-14T03:44:01.881319+0000","last_active":"2024-09-14T03:44:01.883377+0000","last_peered":"2024-09-14T03:44:01.883377+0000","last_clean":"2024-09-14T03:44:01.883377+0000","last_became_active":"2024-09-14T03:44:01.881032+0000","last_became_peered":"2024-09-14T03:44:01.881032+0000","last_unstale":"2024-09-14T03:44:01.883377+0000","last_undegraded":"2024-09-14T03:44:01.883377+0000","last_fullsized":"2024-09-14T03:44:01.883377+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:35:41.762014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"58'6","reported_seq":21,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929594+0000","last_change":"2024-09-14T03:43:58.734778+0000","last_active":"2024-09-14T03:44:24.929594+0000","last_peered":"2024-09-14T03:44:24.929594+0000","last_clean":"2024-09-14T03:44:24.929594+0000","last_became_active":"2024-09-14T03:43:58.734380+0000","last_became_peered":"2024-09-14T03:43:58.734380+0000","last_unstale":"2024-09-14T03:44:24.929594+0000","last_undegraded":"2024-09-14T03:44:24.929594+0000","last_fullsized":"2024-09-14T03:44:24.929594+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:55:13.561065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.876979+0000","last_change":"2024-09-14T03:44:01.880139+0000","last_active":"2024-09-14T03:44:03.876979+0000","last_peered":"2024-09-14T03:44:03.876979+0000","last_clean":"2024-09-14T03:44:03.876979+0000","last_became_active":"2024-09-14T03:44:01.879840+0000","last_became_peered":"2024-09-14T03:44:01.879840+0000","last_unstale":"2024-09-14T03:44:03.876979+0000","last_undegraded":"2024-09-14T03:44:03.876979+0000","last_fullsized":"2024-09-14T03:44:03.876979+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:31:34.003676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"58'2","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.885139+0000","last_change":"2024-09-14T03:43:58.728768+0000","last_active":"2024-09-14T03:44:03.885139+0000","last_peered":"2024-09-14T03:44:03.885139+0000","last_clean":"2024-09-14T03:44:03.885139+0000","last_became_active":"2024-09-14T03:43:58.727893+0000","last_became_peered":"2024-09-14T03:43:58.727893+0000","last_unstale":"2024-09-14T03:44:03.885139+0000","last_undegraded":"2024-09-14T03:44:03.885139+0000","last_fullsized":"2024-09-14T03:44:03.885139+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:54:48.554486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.879380+0000","last_change":"2024-09-14T03:44:01.877540+0000","last_active":"2024-09-14T03:44:03.879380+0000","last_peered":"2024-09-14T03:44:03.879380+0000","last_clean":"2024-09-14T03:44:03.879380+0000","last_became_active":"2024-09-14T03:44:01.877147+0000","last_became_peered":"2024-09-14T03:44:01.877147+0000","last_unstale":"2024-09-14T03:44:03.879380+0000","last_undegraded":"2024-09-14T03:44:03.879380+0000","last_fullsized":"2024-09-14T03:44:03.879380+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:17:57.724648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":13,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.865997+0000","last_change":"2024-09-14T03:43:58.732812+0000","last_active":"2024-09-14T03:44:01.865997+0000","last_peered":"2024-09-14T03:44:01.865997+0000","last_clean":"2024-09-14T03:44:01.865997+0000","last_became_active":"2024-09-14T03:43:58.732537+0000","last_became_peered":"2024-09-14T03:43:58.732537+0000","last_unstale":"2024-09-14T03:44:01.865997+0000","last_undegraded":"2024-09-14T03:44:01.865997+0000","last_fullsized":"2024-09-14T03:44:01.865997+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:26:11.551182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.877040+0000","last_change":"2024-09-14T03:44:01.881140+0000","last_active":"2024-09-14T03:44:03.877040+0000","last_peered":"2024-09-14T03:44:03.877040+0000","last_clean":"2024-09-14T03:44:03.877040+0000","last_became_active":"2024-09-14T03:44:01.880868+0000","last_became_peered":"2024-09-14T03:44:01.880868+0000","last_unstale":"2024-09-14T03:44:03.877040+0000","last_undegraded":"2024-09-14T03:44:03.877040+0000","last_fullsized":"2024-09-14T03:44:03.877040+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:29:05.729804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929350+0000","last_change":"2024-09-14T03:43:58.730505+0000","last_active":"2024-09-14T03:44:24.929350+0000","last_peered":"2024-09-14T03:44:24.929350+0000","last_clean":"2024-09-14T03:44:24.929350+0000","last_became_active":"2024-09-14T03:43:58.729821+0000","last_became_peered":"2024-09-14T03:43:58.729821+0000","last_unstale":"2024-09-14T03:44:24.929350+0000","last_undegraded":"2024-09-14T03:44:24.929350+0000","last_fullsized":"2024-09-14T03:44:24.929350+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:17:44.091667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929155+0000","last_change":"2024-09-14T03:44:01.874477+0000","last_active":"2024-09-14T03:44:24.929155+0000","last_peered":"2024-09-14T03:44:24.929155+0000","last_clean":"2024-09-14T03:44:24.929155+0000","last_became_active":"2024-09-14T03:44:01.873714+0000","last_became_peered":"2024-09-14T03:44:01.873714+0000","last_unstale":"2024-09-14T03:44:24.929155+0000","last_undegraded":"2024-09-14T03:44:24.929155+0000","last_fullsized":"2024-09-14T03:44:24.929155+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:54:21.601768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"58'8","reported_seq":21,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.885624+0000","last_change":"2024-09-14T03:43:58.726387+0000","last_active":"2024-09-14T03:44:03.885624+0000","last_peered":"2024-09-14T03:44:03.885624+0000","last_clean":"2024-09-14T03:44:03.885624+0000","last_became_active":"2024-09-14T03:43:58.726115+0000","last_became_peered":"2024-09-14T03:43:58.726115+0000","last_unstale":"2024-09-14T03:44:03.885624+0000","last_undegraded":"2024-09-14T03:44:03.885624+0000","last_fullsized":"2024-09-14T03:44:03.885624+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:18:04.176970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.876861+0000","last_change":"2024-09-14T03:44:01.878624+0000","last_active":"2024-09-14T03:44:03.876861+0000","last_peered":"2024-09-14T03:44:03.876861+0000","last_clean":"2024-09-14T03:44:03.876861+0000","last_became_active":"2024-09-14T03:44:01.878343+0000","last_became_peered":"2024-09-14T03:44:01.878343+0000","last_unstale":"2024-09-14T03:44:03.876861+0000","last_undegraded":"2024-09-14T03:44:03.876861+0000","last_fullsized":"2024-09-14T03:44:03.876861+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:44:05.205671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"58'1","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.873677+0000","last_change":"2024-09-14T03:43:58.728489+0000","last_active":"2024-09-14T03:44:03.873677+0000","last_peered":"2024-09-14T03:44:03.873677+0000","last_clean":"2024-09-14T03:44:03.873677+0000","last_became_active":"2024-09-14T03:43:58.728182+0000","last_became_peered":"2024-09-14T03:43:58.728182+0000","last_unstale":"2024-09-14T03:44:03.873677+0000","last_undegraded":"2024-09-14T03:44:03.873677+0000","last_fullsized":"2024-09-14T03:44:03.873677+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:56.310710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.877775+0000","last_change":"2024-09-14T03:44:01.880754+0000","last_active":"2024-09-14T03:44:03.877775+0000","last_peered":"2024-09-14T03:44:03.877775+0000","last_clean":"2024-09-14T03:44:03.877775+0000","last_became_active":"2024-09-14T03:44:01.880483+0000","last_became_peered":"2024-09-14T03:44:01.880483+0000","last_unstale":"2024-09-14T03:44:03.877775+0000","last_undegraded":"2024-09-14T03:44:03.877775+0000","last_fullsized":"2024-09-14T03:44:03.877775+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:32:53.245007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"58'2","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.880815+0000","last_change":"2024-09-14T03:43:58.726366+0000","last_active":"2024-09-14T03:44:03.880815+0000","last_peered":"2024-09-14T03:44:03.880815+0000","last_clean":"2024-09-14T03:44:03.880815+0000","last_became_active":"2024-09-14T03:43:58.726090+0000","last_became_peered":"2024-09-14T03:43:58.726090+0000","last_unstale":"2024-09-14T03:44:03.880815+0000","last_undegraded":"2024-09-14T03:44:03.880815+0000","last_fullsized":"2024-09-14T03:44:03.880815+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:45:33.572820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.877463+0000","last_change":"2024-09-14T03:44:01.883704+0000","last_active":"2024-09-14T03:44:03.877463+0000","last_peered":"2024-09-14T03:44:03.877463+0000","last_clean":"2024-09-14T03:44:03.877463+0000","last_became_active":"2024-09-14T03:44:01.883448+0000","last_became_peered":"2024-09-14T03:44:01.883448+0000","last_unstale":"2024-09-14T03:44:03.877463+0000","last_undegraded":"2024-09-14T03:44:03.877463+0000","last_fullsized":"2024-09-14T03:44:03.877463+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:16:27.946770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"58'1","reported_seq":14,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.885569+0000","last_change":"2024-09-14T03:43:58.733205+0000","last_active":"2024-09-14T03:44:03.885569+0000","last_peered":"2024-09-14T03:44:03.885569+0000","last_clean":"2024-09-14T03:44:03.885569+0000","last_became_active":"2024-09-14T03:43:58.732929+0000","last_became_peered":"2024-09-14T03:43:58.732929+0000","last_unstale":"2024-09-14T03:44:03.885569+0000","last_undegraded":"2024-09-14T03:44:03.885569+0000","last_fullsized":"2024-09-14T03:44:03.885569+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:31:13.660965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.886970+0000","last_change":"2024-09-14T03:44:01.886158+0000","last_active":"2024-09-14T03:44:01.886970+0000","last_peered":"2024-09-14T03:44:01.886970+0000","last_clean":"2024-09-14T03:44:01.886970+0000","last_became_active":"2024-09-14T03:44:01.884747+0000","last_became_peered":"2024-09-14T03:44:01.884747+0000","last_unstale":"2024-09-14T03:44:01.886970+0000","last_undegraded":"2024-09-14T03:44:01.886970+0000","last_fullsized":"2024-09-14T03:44:01.886970+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:49:56.265783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":13,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.858323+0000","last_change":"2024-09-14T03:43:58.732401+0000","last_active":"2024-09-14T03:44:01.858323+0000","last_peered":"2024-09-14T03:44:01.858323+0000","last_clean":"2024-09-14T03:44:01.858323+0000","last_became_active":"2024-09-14T03:43:58.732067+0000","last_became_peered":"2024-09-14T03:43:58.732067+0000","last_unstale":"2024-09-14T03:44:01.858323+0000","last_undegraded":"2024-09-14T03:44:01.858323+0000","last_fullsized":"2024-09-14T03:44:01.858323+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:34:00.566245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.877102+0000","last_change":"2024-09-14T03:44:01.881736+0000","last_active":"2024-09-14T03:44:03.877102+0000","last_peered":"2024-09-14T03:44:03.877102+0000","last_clean":"2024-09-14T03:44:03.877102+0000","last_became_active":"2024-09-14T03:44:01.881219+0000","last_became_peered":"2024-09-14T03:44:01.881219+0000","last_unstale":"2024-09-14T03:44:03.877102+0000","last_undegraded":"2024-09-14T03:44:03.877102+0000","last_fullsized":"2024-09-14T03:44:03.877102+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:24:23.342066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"22'72","reported_seq":24,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.876821+0000","last_change":"2024-09-14T03:43:36.147227+0000","last_active":"2024-09-14T03:44:03.876821+0000","last_peered":"2024-09-14T03:44:03.876821+0000","last_clean":"2024-09-14T03:44:03.876821+0000","last_became_active":"2024-09-14T03:43:36.141953+0000","last_became_peered":"2024-09-14T03:43:36.141953+0000","last_unstale":"2024-09-14T03:44:03.876821+0000","last_undegraded":"2024-09-14T03:44:03.876821+0000","last_fullsized":"2024-09-14T03:44:03.876821+0000","mapping_epoch":50,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":51,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:39.458815+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:39.458815+0000","last_clean_scrub_stamp":"2024-09-14T03:41:39.458815+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:38:43.056328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":13,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.863031+0000","last_change":"2024-09-14T03:43:58.733557+0000","last_active":"2024-09-14T03:44:01.863031+0000","last_peered":"2024-09-14T03:44:01.863031+0000","last_clean":"2024-09-14T03:44:01.863031+0000","last_became_active":"2024-09-14T03:43:58.733256+0000","last_became_peered":"2024-09-14T03:43:58.733256+0000","last_unstale":"2024-09-14T03:44:01.863031+0000","last_undegraded":"2024-09-14T03:44:01.863031+0000","last_fullsized":"2024-09-14T03:44:01.863031+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:09:12.987156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.046665+0000","last_change":"2024-09-14T03:44:01.875525+0000","last_active":"2024-09-14T03:44:03.046665+0000","last_peered":"2024-09-14T03:44:03.046665+0000","last_clean":"2024-09-14T03:44:03.046665+0000","last_became_active":"2024-09-14T03:44:01.875241+0000","last_became_peered":"2024-09-14T03:44:01.875241+0000","last_unstale":"2024-09-14T03:44:03.046665+0000","last_undegraded":"2024-09-14T03:44:03.046665+0000","last_fullsized":"2024-09-14T03:44:03.046665+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:34:47.346963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.876707+0000","last_change":"2024-09-14T03:43:58.732376+0000","last_active":"2024-09-14T03:44:03.876707+0000","last_peered":"2024-09-14T03:44:03.876707+0000","last_clean":"2024-09-14T03:44:03.876707+0000","last_became_active":"2024-09-14T03:43:58.731812+0000","last_became_peered":"2024-09-14T03:43:58.731812+0000","last_unstale":"2024-09-14T03:44:03.876707+0000","last_undegraded":"2024-09-14T03:44:03.876707+0000","last_fullsized":"2024-09-14T03:44:03.876707+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:35:26.262383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.878782+0000","last_change":"2024-09-14T03:44:01.876511+0000","last_active":"2024-09-14T03:44:03.878782+0000","last_peered":"2024-09-14T03:44:03.878782+0000","last_clean":"2024-09-14T03:44:03.878782+0000","last_became_active":"2024-09-14T03:44:01.876225+0000","last_became_peered":"2024-09-14T03:44:01.876225+0000","last_unstale":"2024-09-14T03:44:03.878782+0000","last_undegraded":"2024-09-14T03:44:03.878782+0000","last_fullsized":"2024-09-14T03:44:03.878782+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:36:40.283622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":13,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.858583+0000","last_change":"2024-09-14T03:43:58.727129+0000","last_active":"2024-09-14T03:44:01.858583+0000","last_peered":"2024-09-14T03:44:01.858583+0000","last_clean":"2024-09-14T03:44:01.858583+0000","last_became_active":"2024-09-14T03:43:58.726879+0000","last_became_peered":"2024-09-14T03:43:58.726879+0000","last_unstale":"2024-09-14T03:44:01.858583+0000","last_undegraded":"2024-09-14T03:44:01.858583+0000","last_fullsized":"2024-09-14T03:44:01.858583+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:54:41.748651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:02.870239+0000","last_change":"2024-09-14T03:44:01.871005+0000","last_active":"2024-09-14T03:44:02.870239+0000","last_peered":"2024-09-14T03:44:02.870239+0000","last_clean":"2024-09-14T03:44:02.870239+0000","last_became_active":"2024-09-14T03:44:01.870707+0000","last_became_peered":"2024-09-14T03:44:01.870707+0000","last_unstale":"2024-09-14T03:44:02.870239+0000","last_undegraded":"2024-09-14T03:44:02.870239+0000","last_fullsized":"2024-09-14T03:44:02.870239+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:30:22.758581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":13,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.866074+0000","last_change":"2024-09-14T03:43:58.733038+0000","last_active":"2024-09-14T03:44:01.866074+0000","last_peered":"2024-09-14T03:44:01.866074+0000","last_clean":"2024-09-14T03:44:01.866074+0000","last_became_active":"2024-09-14T03:43:58.732753+0000","last_became_peered":"2024-09-14T03:43:58.732753+0000","last_unstale":"2024-09-14T03:44:01.866074+0000","last_undegraded":"2024-09-14T03:44:01.866074+0000","last_fullsized":"2024-09-14T03:44:01.866074+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:40:49.754211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.881906+0000","last_change":"2024-09-14T03:44:01.881600+0000","last_active":"2024-09-14T03:44:01.881906+0000","last_peered":"2024-09-14T03:44:01.881906+0000","last_clean":"2024-09-14T03:44:01.881906+0000","last_became_active":"2024-09-14T03:44:01.881001+0000","last_became_peered":"2024-09-14T03:44:01.881001+0000","last_unstale":"2024-09-14T03:44:01.881906+0000","last_undegraded":"2024-09-14T03:44:01.881906+0000","last_fullsized":"2024-09-14T03:44:01.881906+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:26:45.052191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":13,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:02.870489+0000","last_change":"2024-09-14T03:43:58.733621+0000","last_active":"2024-09-14T03:44:02.870489+0000","last_peered":"2024-09-14T03:44:02.870489+0000","last_clean":"2024-09-14T03:44:02.870489+0000","last_became_active":"2024-09-14T03:43:58.733357+0000","last_became_peered":"2024-09-14T03:43:58.733357+0000","last_unstale":"2024-09-14T03:44:02.870489+0000","last_undegraded":"2024-09-14T03:44:02.870489+0000","last_fullsized":"2024-09-14T03:44:02.870489+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:25:53.791926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.880813+0000","last_change":"2024-09-14T03:44:01.880200+0000","last_active":"2024-09-14T03:44:01.880813+0000","last_peered":"2024-09-14T03:44:01.880813+0000","last_clean":"2024-09-14T03:44:01.880813+0000","last_became_active":"2024-09-14T03:44:01.879925+0000","last_became_peered":"2024-09-14T03:44:01.879925+0000","last_unstale":"2024-09-14T03:44:01.880813+0000","last_undegraded":"2024-09-14T03:44:01.880813+0000","last_fullsized":"2024-09-14T03:44:01.880813+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:55:24.738215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.878524+0000","last_change":"2024-09-14T03:43:58.733230+0000","last_active":"2024-09-14T03:44:03.878524+0000","last_peered":"2024-09-14T03:44:03.878524+0000","last_clean":"2024-09-14T03:44:03.878524+0000","last_became_active":"2024-09-14T03:43:58.732868+0000","last_became_peered":"2024-09-14T03:43:58.732868+0000","last_unstale":"2024-09-14T03:44:03.878524+0000","last_undegraded":"2024-09-14T03:44:03.878524+0000","last_fullsized":"2024-09-14T03:44:03.878524+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:15:12.184851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:02.870449+0000","last_change":"2024-09-14T03:44:01.878736+0000","last_active":"2024-09-14T03:44:02.870449+0000","last_peered":"2024-09-14T03:44:02.870449+0000","last_clean":"2024-09-14T03:44:02.870449+0000","last_became_active":"2024-09-14T03:44:01.877390+0000","last_became_peered":"2024-09-14T03:44:01.877390+0000","last_unstale":"2024-09-14T03:44:02.870449+0000","last_undegraded":"2024-09-14T03:44:02.870449+0000","last_fullsized":"2024-09-14T03:44:02.870449+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:17:42.812575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"58'1","reported_seq":14,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.884532+0000","last_change":"2024-09-14T03:43:58.733688+0000","last_active":"2024-09-14T03:44:03.884532+0000","last_peered":"2024-09-14T03:44:03.884532+0000","last_clean":"2024-09-14T03:44:03.884532+0000","last_became_active":"2024-09-14T03:43:58.733457+0000","last_became_peered":"2024-09-14T03:43:58.733457+0000","last_unstale":"2024-09-14T03:44:03.884532+0000","last_undegraded":"2024-09-14T03:44:03.884532+0000","last_fullsized":"2024-09-14T03:44:03.884532+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:09:01.069988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.047037+0000","last_change":"2024-09-14T03:44:01.870045+0000","last_active":"2024-09-14T03:44:03.047037+0000","last_peered":"2024-09-14T03:44:03.047037+0000","last_clean":"2024-09-14T03:44:03.047037+0000","last_became_active":"2024-09-14T03:44:01.869780+0000","last_became_peered":"2024-09-14T03:44:01.869780+0000","last_unstale":"2024-09-14T03:44:03.047037+0000","last_undegraded":"2024-09-14T03:44:03.047037+0000","last_fullsized":"2024-09-14T03:44:03.047037+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:04:58.634704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"58'2","reported_seq":17,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929736+0000","last_change":"2024-09-14T03:43:58.735364+0000","last_active":"2024-09-14T03:44:24.929736+0000","last_peered":"2024-09-14T03:44:24.929736+0000","last_clean":"2024-09-14T03:44:24.929736+0000","last_became_active":"2024-09-14T03:43:58.734833+0000","last_became_peered":"2024-09-14T03:43:58.734833+0000","last_unstale":"2024-09-14T03:44:24.929736+0000","last_undegraded":"2024-09-14T03:44:24.929736+0000","last_fullsized":"2024-09-14T03:44:24.929736+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:58:37.428713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.879573+0000","last_change":"2024-09-14T03:44:01.878573+0000","last_active":"2024-09-14T03:44:03.879573+0000","last_peered":"2024-09-14T03:44:03.879573+0000","last_clean":"2024-09-14T03:44:03.879573+0000","last_became_active":"2024-09-14T03:44:01.878316+0000","last_became_peered":"2024-09-14T03:44:01.878316+0000","last_unstale":"2024-09-14T03:44:03.879573+0000","last_undegraded":"2024-09-14T03:44:03.879573+0000","last_fullsized":"2024-09-14T03:44:03.879573+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:45:21.129163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.876680+0000","last_change":"2024-09-14T03:43:58.730843+0000","last_active":"2024-09-14T03:44:03.876680+0000","last_peered":"2024-09-14T03:44:03.876680+0000","last_clean":"2024-09-14T03:44:03.876680+0000","last_became_active":"2024-09-14T03:43:58.730538+0000","last_became_peered":"2024-09-14T03:43:58.730538+0000","last_unstale":"2024-09-14T03:44:03.876680+0000","last_undegraded":"2024-09-14T03:44:03.876680+0000","last_fullsized":"2024-09-14T03:44:03.876680+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:04:57.059345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.880062+0000","last_change":"2024-09-14T03:44:01.874355+0000","last_active":"2024-09-14T03:44:01.880062+0000","last_peered":"2024-09-14T03:44:01.880062+0000","last_clean":"2024-09-14T03:44:01.880062+0000","last_became_active":"2024-09-14T03:44:01.874049+0000","last_became_peered":"2024-09-14T03:44:01.874049+0000","last_unstale":"2024-09-14T03:44:01.880062+0000","last_undegraded":"2024-09-14T03:44:01.880062+0000","last_fullsized":"2024-09-14T03:44:01.880062+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:53:30.306175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":13,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:02.870452+0000","last_change":"2024-09-14T03:43:58.728080+0000","last_active":"2024-09-14T03:44:02.870452+0000","last_peered":"2024-09-14T03:44:02.870452+0000","last_clean":"2024-09-14T03:44:02.870452+0000","last_became_active":"2024-09-14T03:43:58.727820+0000","last_became_peered":"2024-09-14T03:43:58.727820+0000","last_unstale":"2024-09-14T03:44:02.870452+0000","last_undegraded":"2024-09-14T03:44:02.870452+0000","last_fullsized":"2024-09-14T03:44:02.870452+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:37:58.166430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.046619+0000","last_change":"2024-09-14T03:44:01.869311+0000","last_active":"2024-09-14T03:44:03.046619+0000","last_peered":"2024-09-14T03:44:03.046619+0000","last_clean":"2024-09-14T03:44:03.046619+0000","last_became_active":"2024-09-14T03:44:01.869065+0000","last_became_peered":"2024-09-14T03:44:01.869065+0000","last_unstale":"2024-09-14T03:44:03.046619+0000","last_undegraded":"2024-09-14T03:44:03.046619+0000","last_fullsized":"2024-09-14T03:44:03.046619+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:45:09.508019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":13,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:02.870616+0000","last_change":"2024-09-14T03:43:58.728931+0000","last_active":"2024-09-14T03:44:02.870616+0000","last_peered":"2024-09-14T03:44:02.870616+0000","last_clean":"2024-09-14T03:44:02.870616+0000","last_became_active":"2024-09-14T03:43:58.728647+0000","last_became_peered":"2024-09-14T03:43:58.728647+0000","last_unstale":"2024-09-14T03:44:02.870616+0000","last_undegraded":"2024-09-14T03:44:02.870616+0000","last_fullsized":"2024-09-14T03:44:02.870616+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:09:38.769523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.046725+0000","last_change":"2024-09-14T03:44:01.869027+0000","last_active":"2024-09-14T03:44:03.046725+0000","last_peered":"2024-09-14T03:44:03.046725+0000","last_clean":"2024-09-14T03:44:03.046725+0000","last_became_active":"2024-09-14T03:44:01.868757+0000","last_became_peered":"2024-09-14T03:44:01.868757+0000","last_unstale":"2024-09-14T03:44:03.046725+0000","last_undegraded":"2024-09-14T03:44:03.046725+0000","last_fullsized":"2024-09-14T03:44:03.046725+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:47:57.070651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"58'2","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.882434+0000","last_change":"2024-09-14T03:43:58.737705+0000","last_active":"2024-09-14T03:44:03.882434+0000","last_peered":"2024-09-14T03:44:03.882434+0000","last_clean":"2024-09-14T03:44:03.882434+0000","last_became_active":"2024-09-14T03:43:58.736546+0000","last_became_peered":"2024-09-14T03:43:58.736546+0000","last_unstale":"2024-09-14T03:44:03.882434+0000","last_undegraded":"2024-09-14T03:44:03.882434+0000","last_fullsized":"2024-09-14T03:44:03.882434+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:27:12.542706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.879558+0000","last_change":"2024-09-14T03:44:01.878897+0000","last_active":"2024-09-14T03:44:01.879558+0000","last_peered":"2024-09-14T03:44:01.879558+0000","last_clean":"2024-09-14T03:44:01.879558+0000","last_became_active":"2024-09-14T03:44:01.878393+0000","last_became_peered":"2024-09-14T03:44:01.878393+0000","last_unstale":"2024-09-14T03:44:01.879558+0000","last_undegraded":"2024-09-14T03:44:01.879558+0000","last_fullsized":"2024-09-14T03:44:01.879558+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:38:38.897169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.889144+0000","last_change":"2024-09-14T03:44:01.888591+0000","last_active":"2024-09-14T03:44:01.889144+0000","last_peered":"2024-09-14T03:44:01.889144+0000","last_clean":"2024-09-14T03:44:01.889144+0000","last_became_active":"2024-09-14T03:44:01.888257+0000","last_became_peered":"2024-09-14T03:44:01.888257+0000","last_unstale":"2024-09-14T03:44:01.889144+0000","last_undegraded":"2024-09-14T03:44:01.889144+0000","last_fullsized":"2024-09-14T03:44:01.889144+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:24:40.992358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"58'2","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.884060+0000","last_change":"2024-09-14T03:43:58.734680+0000","last_active":"2024-09-14T03:44:03.884060+0000","last_peered":"2024-09-14T03:44:03.884060+0000","last_clean":"2024-09-14T03:44:03.884060+0000","last_became_active":"2024-09-14T03:43:58.734408+0000","last_became_peered":"2024-09-14T03:43:58.734408+0000","last_unstale":"2024-09-14T03:44:03.884060+0000","last_undegraded":"2024-09-14T03:44:03.884060+0000","last_fullsized":"2024-09-14T03:44:03.884060+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:49:02.536404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":10,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-14T03:44:01.886954+0000","last_change":"2024-09-14T03:44:01.885572+0000","last_active":"2024-09-14T03:44:01.886954+0000","last_peered":"2024-09-14T03:44:01.886954+0000","last_clean":"2024-09-14T03:44:01.886954+0000","last_became_active":"2024-09-14T03:44:01.885110+0000","last_became_peered":"2024-09-14T03:44:01.885110+0000","last_unstale":"2024-09-14T03:44:01.886954+0000","last_undegraded":"2024-09-14T03:44:01.886954+0000","last_fullsized":"2024-09-14T03:44:01.886954+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:10.439239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1d","version":"58'2","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.882824+0000","last_change":"2024-09-14T03:43:58.731296+0000","last_active":"2024-09-14T03:44:03.882824+0000","last_peered":"2024-09-14T03:44:03.882824+0000","last_clean":"2024-09-14T03:44:03.882824+0000","last_became_active":"2024-09-14T03:43:58.730951+0000","last_became_peered":"2024-09-14T03:43:58.730951+0000","last_unstale":"2024-09-14T03:44:03.882824+0000","last_undegraded":"2024-09-14T03:44:03.882824+0000","last_fullsized":"2024-09-14T03:44:03.882824+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:26:14.801414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.880575+0000","last_change":"2024-09-14T03:44:01.874935+0000","last_active":"2024-09-14T03:44:03.880575+0000","last_peered":"2024-09-14T03:44:03.880575+0000","last_clean":"2024-09-14T03:44:03.880575+0000","last_became_active":"2024-09-14T03:44:01.874668+0000","last_became_peered":"2024-09-14T03:44:01.874668+0000","last_unstale":"2024-09-14T03:44:03.880575+0000","last_undegraded":"2024-09-14T03:44:03.880575+0000","last_fullsized":"2024-09-14T03:44:03.880575+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:24:31.404958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"58'1","reported_seq":14,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.878803+0000","last_change":"2024-09-14T03:43:58.734206+0000","last_active":"2024-09-14T03:44:03.878803+0000","last_peered":"2024-09-14T03:44:03.878803+0000","last_clean":"2024-09-14T03:44:03.878803+0000","last_became_active":"2024-09-14T03:43:58.733960+0000","last_became_peered":"2024-09-14T03:43:58.733960+0000","last_unstale":"2024-09-14T03:44:03.878803+0000","last_undegraded":"2024-09-14T03:44:03.878803+0000","last_fullsized":"2024-09-14T03:44:03.878803+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:44.073503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":11,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.046939+0000","last_change":"2024-09-14T03:44:01.883550+0000","last_active":"2024-09-14T03:44:03.046939+0000","last_peered":"2024-09-14T03:44:03.046939+0000","last_clean":"2024-09-14T03:44:03.046939+0000","last_became_active":"2024-09-14T03:44:01.883265+0000","last_became_peered":"2024-09-14T03:44:01.883265+0000","last_unstale":"2024-09-14T03:44:03.046939+0000","last_undegraded":"2024-09-14T03:44:03.046939+0000","last_fullsized":"2024-09-14T03:44:03.046939+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:52:04.512990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"58'4","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-14T03:44:03.884689+0000","last_change":"2024-09-14T03:43:58.731954+0000","last_active":"2024-09-14T03:44:03.884689+0000","last_peered":"2024-09-14T03:44:03.884689+0000","last_clean":"2024-09-14T03:44:03.884689+0000","last_became_active":"2024-09-14T03:43:58.731688+0000","last_became_peered":"2024-09-14T03:43:58.731688+0000","last_unstale":"2024-09-14T03:44:03.884689+0000","last_undegraded":"2024-09-14T03:44:03.884689+0000","last_fullsized":"2024-09-14T03:44:03.884689+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:30:10.811609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3563520,"data_stored":3542208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397517,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298512,"kb_used_data":1148,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446640,"statfs":{"total":95995035648,"available":95689359360,"internally_reserved":0,"allocated":1175552,"data_stored":831572,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44700000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54500000000000004}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57699999999999996}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58699999999999997}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56100000000000005}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47399999999999998}]}]},{"osd":6,"up_from":43,"seq":184683593745,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298516,"kb_used_data":1160,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446636,"statfs":{"total":95995035648,"available":95689355264,"internally_reserved":0,"allocated":1187840,"data_stored":838908,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.39800000000000002}]},{"osd":1,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.53400000000000003}]},{"osd":2,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.52300000000000002}]},{"osd":3,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.50600000000000001}]},{"osd":4,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.372}]},{"osd":5,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.45500000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48499999999999999}]}]},{"osd":5,"up_from":37,"seq":158913789974,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297928,"kb_used_data":572,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447224,"statfs":{"total":95995035648,"available":95689957376,"internally_reserved":0,"allocated":585728,"data_stored":241939,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:43:47 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.442},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.51400000000000001}]},{"osd":1,"last update":"Sat Sep 14 03:43:47 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.40300000000000002}]},{"osd":2,"last update":"Sat Sep 14 03:43:47 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.41799999999999998}]},{"osd":3,"last update":"Sat Sep 14 03:43:47 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.497},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.45900000000000002}]},{"osd":4,"last update":"Sat Sep 14 03:43:47 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.379,"15min":0.379},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.38200000000000001}]},{"osd":6,"last update":"Sat Sep 14 03:44:11 2024","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.432},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.48199999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47099999999999997}]}]},{"osd":4,"up_from":31,"seq":133143986202,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297932,"kb_used_data":576,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447220,"statfs":{"total":95995035648,"available":95689953280,"internally_reserved":0,"allocated":589824,"data_stored":241308,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:43:24 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.54000000000000004}]},{"osd":1,"last update":"Sat Sep 14 03:43:24 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.61199999999999999}]},{"osd":2,"last update":"Sat Sep 14 03:43:24 2024","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.66900000000000004}]},{"osd":3,"last update":"Sat Sep 14 03:43:24 2024","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.628},{"interface":"front","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.58699999999999997}]},{"osd":5,"last update":"Sat Sep 14 03:43:49 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.42299999999999999}]},{"osd":6,"last update":"Sat Sep 14 03:44:11 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.627,"5min":0.627,"15min":0.627},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.47999999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55300000000000005}]}]},{"osd":3,"up_from":26,"seq":111669149726,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297936,"kb_used_data":576,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447216,"statfs":{"total":95995035648,"available":95689949184,"internally_reserved":0,"allocated":589824,"data_stored":241953,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:44:03 2024","interfaces":[{"interface":"back","average":{"1min":0.34599999999999997,"5min":0.38400000000000001,"15min":0.39000000000000001},"min":{"1min":0.245,"5min":0.185,"15min":0.185},"max":{"1min":0.43099999999999999,"5min":0.48199999999999998,"15min":0.48199999999999998},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.40200000000000002,"15min":0.40799999999999997},"min":{"1min":0.254,"5min":0.218,"15min":0.218},"max":{"1min":0.48099999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.36399999999999999}]},{"osd":1,"last update":"Sat Sep 14 03:44:03 2024","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.41599999999999998,"15min":0.41999999999999998},"min":{"1min":0.28799999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.5,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.36599999999999999,"5min":0.39300000000000002,"15min":0.39700000000000002},"min":{"1min":0.30499999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.47199999999999998,"5min":0.53700000000000003,"15min":0.53700000000000003},"last":0.318}]},{"osd":2,"last update":"Sat Sep 14 03:44:03 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.434,"15min":0.441},"min":{"1min":0.28999999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.52300000000000002,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.42299999999999999,"15min":0.42799999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.54400000000000004,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.378}]},{"osd":4,"last update":"Sat Sep 14 03:43:29 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.47199999999999998}]},{"osd":5,"last update":"Sat Sep 14 03:43:52 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.48299999999999998}]},{"osd":6,"last update":"Sat Sep 14 03:44:11 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.501},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.53700000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45200000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411363,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297912,"kb_used_data":556,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447240,"statfs":{"total":95995035648,"available":95689973760,"internally_reserved":0,"allocated":569344,"data_stored":240001,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:43:45 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.36299999999999999,"15min":0.35599999999999998},"min":{"1min":0.26200000000000001,"5min":0.22,"15min":0.22},"max":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.371,"15min":0.36599999999999999},"min":{"1min":0.27800000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.56200000000000006,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.46100000000000002}]},{"osd":1,"last update":"Sat Sep 14 03:43:45 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.40000000000000002,"15min":0.39800000000000002},"min":{"1min":0.32800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.441,"5min":0.39200000000000002,"15min":0.38400000000000001},"min":{"1min":0.29199999999999998,"5min":0.245,"15min":0.245},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.40200000000000002}]},{"osd":3,"last update":"Sat Sep 14 03:44:08 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.44800000000000001,"15min":0.45100000000000001},"min":{"1min":0.34999999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.53300000000000003,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.496},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.44800000000000001,"15min":0.45800000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.56000000000000005,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.50600000000000001}]},{"osd":4,"last update":"Sat Sep 14 03:43:24 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.57399999999999995}]},{"osd":5,"last update":"Sat Sep 14 03:43:48 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.438}]},{"osd":6,"last update":"Sat Sep 14 03:44:12 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.47699999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55100000000000005}]}]},{"osd":1,"up_from":13,"seq":55834574889,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297916,"kb_used_data":556,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447236,"statfs":{"total":95995035648,"available":95689969664,"internally_reserved":0,"allocated":569344,"data_stored":239827,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:44:20 2024","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.32000000000000001,"15min":0.29599999999999999},"min":{"1min":0.249,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.51900000000000002,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.374,"5min":0.34399999999999997,"15min":0.31900000000000001},"min":{"1min":0.27000000000000002,"5min":0.189,"15min":0.189},"max":{"1min":0.56799999999999995,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.49399999999999999}]},{"osd":2,"last update":"Sat Sep 14 03:43:39 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.35499999999999998,"15min":0.34300000000000003},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.36199999999999999,"15min":0.34699999999999998},"min":{"1min":0.29499999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.58199999999999996}]},{"osd":3,"last update":"Sat Sep 14 03:44:07 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.42299999999999999,"15min":0.42199999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.435,"15min":0.437},"min":{"1min":0.27500000000000002,"5min":0.247,"15min":0.247},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.63800000000000001}]},{"osd":4,"last update":"Sat Sep 14 03:43:25 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.65400000000000003}]},{"osd":5,"last update":"Sat Sep 14 03:43:47 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.78300000000000003}]},{"osd":6,"last update":"Sat Sep 14 03:44:10 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.747},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.81999999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.84899999999999998}]}]},{"osd":0,"up_from":8,"seq":34359738414,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298520,"kb_used_data":1164,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446632,"statfs":{"total":95995035648,"available":95689351168,"internally_reserved":0,"allocated":1191936,"data_stored":838072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 14 03:44:17 2024","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":0.35699999999999998,"15min":0.34899999999999998},"min":{"1min":0.25700000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.46500000000000002,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.36299999999999999,"15min":0.34799999999999998},"min":{"1min":0.246,"5min":0.216,"15min":0.216},"max":{"1min":0.48999999999999999,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.33200000000000002}]},{"osd":2,"last update":"Sat Sep 14 03:43:43 2024","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.41299999999999998,"15min":0.40999999999999998},"min":{"1min":0.29099999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.54900000000000004,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.39700000000000002,"15min":0.39500000000000002},"min":{"1min":0.28799999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.505,"5min":0.505,"15min":0.505},"last":0.379}]},{"osd":3,"last update":"Sat Sep 14 03:44:04 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.44,"15min":0.44700000000000001},"min":{"1min":0.32500000000000001,"5min":0.307,"15min":0.307},"max":{"1min":0.54300000000000004,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.41499999999999998,"15min":0.42099999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.26100000000000001}]},{"osd":4,"last update":"Sat Sep 14 03:43:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.432},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.38800000000000001}]},{"osd":5,"last update":"Sat Sep 14 03:43:48 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.45300000000000001}]},{"osd":6,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.41799999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46999999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1642,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T03:44:29.237 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T03:44:29.481 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:44:30.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:29 smithi094 ceph-mon[31053]: from='client.24404 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:44:30.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:29 smithi094 ceph-mon[31053]: pgmap v202: 97 pgs: 2 activating, 95 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:30.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:29 smithi070 ceph-mon[27495]: from='client.24404 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:44:30.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:29 smithi070 ceph-mon[27495]: pgmap v202: 97 pgs: 2 activating, 95 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:30.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:29 smithi070 ceph-mon[39288]: from='client.24404 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:44:30.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:29 smithi070 ceph-mon[39288]: pgmap v202: 97 pgs: 2 activating, 95 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:31.299 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:44:31.300 INFO:teuthology.orchestra.run.smithi070.stderr:dumped all 2024-09-14T03:44:32.070 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":203,"stamp":"2024-09-14T03:44:30.503930+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":146,"num_write_kb":1641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111,"ondisk_log_size":111,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":290,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385332,"kb_used_data":6464,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575884,"statfs":{"total":767960285184,"available":765517705216,"internally_reserved":0,"allocated":6619136,"data_stored":3799569,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001656"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933508+0000","last_change":"2024-09-14T03:44:01.882774+0000","last_active":"2024-09-14T03:44:24.933508+0000","last_peered":"2024-09-14T03:44:24.933508+0000","last_clean":"2024-09-14T03:44:24.933508+0000","last_became_active":"2024-09-14T03:44:01.882481+0000","last_became_peered":"2024-09-14T03:44:01.882481+0000","last_unstale":"2024-09-14T03:44:24.933508+0000","last_undegraded":"2024-09-14T03:44:24.933508+0000","last_fullsized":"2024-09-14T03:44:24.933508+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:49:25.209573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929696+0000","last_change":"2024-09-14T03:44:01.868415+0000","last_active":"2024-09-14T03:44:24.929696+0000","last_peered":"2024-09-14T03:44:24.929696+0000","last_clean":"2024-09-14T03:44:24.929696+0000","last_became_active":"2024-09-14T03:44:01.868152+0000","last_became_peered":"2024-09-14T03:44:01.868152+0000","last_unstale":"2024-09-14T03:44:24.929696+0000","last_undegraded":"2024-09-14T03:44:24.929696+0000","last_fullsized":"2024-09-14T03:44:24.929696+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:56:00.270070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929890+0000","last_change":"2024-09-14T03:44:01.871459+0000","last_active":"2024-09-14T03:44:24.929890+0000","last_peered":"2024-09-14T03:44:24.929890+0000","last_clean":"2024-09-14T03:44:24.929890+0000","last_became_active":"2024-09-14T03:44:01.870749+0000","last_became_peered":"2024-09-14T03:44:01.870749+0000","last_unstale":"2024-09-14T03:44:24.929890+0000","last_undegraded":"2024-09-14T03:44:24.929890+0000","last_fullsized":"2024-09-14T03:44:24.929890+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:54.391363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928911+0000","last_change":"2024-09-14T03:44:01.882503+0000","last_active":"2024-09-14T03:44:24.928911+0000","last_peered":"2024-09-14T03:44:24.928911+0000","last_clean":"2024-09-14T03:44:24.928911+0000","last_became_active":"2024-09-14T03:44:01.882250+0000","last_became_peered":"2024-09-14T03:44:01.882250+0000","last_unstale":"2024-09-14T03:44:24.928911+0000","last_undegraded":"2024-09-14T03:44:24.928911+0000","last_fullsized":"2024-09-14T03:44:24.928911+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:46:29.551285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.932873+0000","last_change":"2024-09-14T03:44:01.880333+0000","last_active":"2024-09-14T03:44:24.932873+0000","last_peered":"2024-09-14T03:44:24.932873+0000","last_clean":"2024-09-14T03:44:24.932873+0000","last_became_active":"2024-09-14T03:44:01.880010+0000","last_became_peered":"2024-09-14T03:44:01.880010+0000","last_unstale":"2024-09-14T03:44:24.932873+0000","last_undegraded":"2024-09-14T03:44:24.932873+0000","last_fullsized":"2024-09-14T03:44:24.932873+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:45:02.741425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928684+0000","last_change":"2024-09-14T03:44:01.880710+0000","last_active":"2024-09-14T03:44:24.928684+0000","last_peered":"2024-09-14T03:44:24.928684+0000","last_clean":"2024-09-14T03:44:24.928684+0000","last_became_active":"2024-09-14T03:44:01.880237+0000","last_became_peered":"2024-09-14T03:44:01.880237+0000","last_unstale":"2024-09-14T03:44:24.928684+0000","last_undegraded":"2024-09-14T03:44:24.928684+0000","last_fullsized":"2024-09-14T03:44:24.928684+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:56:49.808707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.932581+0000","last_change":"2024-09-14T03:44:01.878590+0000","last_active":"2024-09-14T03:44:24.932581+0000","last_peered":"2024-09-14T03:44:24.932581+0000","last_clean":"2024-09-14T03:44:24.932581+0000","last_became_active":"2024-09-14T03:44:01.878281+0000","last_became_peered":"2024-09-14T03:44:01.878281+0000","last_unstale":"2024-09-14T03:44:24.932581+0000","last_undegraded":"2024-09-14T03:44:24.932581+0000","last_fullsized":"2024-09-14T03:44:24.932581+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:41:57.164227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.925754+0000","last_change":"2024-09-14T03:44:01.877057+0000","last_active":"2024-09-14T03:44:24.925754+0000","last_peered":"2024-09-14T03:44:24.925754+0000","last_clean":"2024-09-14T03:44:24.925754+0000","last_became_active":"2024-09-14T03:44:01.876727+0000","last_became_peered":"2024-09-14T03:44:01.876727+0000","last_unstale":"2024-09-14T03:44:24.925754+0000","last_undegraded":"2024-09-14T03:44:24.925754+0000","last_fullsized":"2024-09-14T03:44:24.925754+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:31:24.716316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933832+0000","last_change":"2024-09-14T03:44:01.884865+0000","last_active":"2024-09-14T03:44:24.933832+0000","last_peered":"2024-09-14T03:44:24.933832+0000","last_clean":"2024-09-14T03:44:24.933832+0000","last_became_active":"2024-09-14T03:44:01.884136+0000","last_became_peered":"2024-09-14T03:44:01.884136+0000","last_unstale":"2024-09-14T03:44:24.933832+0000","last_undegraded":"2024-09-14T03:44:24.933832+0000","last_fullsized":"2024-09-14T03:44:24.933832+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:59.903382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":16,"reported_epoch":60,"state":"activating","last_fresh":"2024-09-14T03:44:25.928176+0000","last_change":"2024-09-14T03:44:25.928176+0000","last_active":"2024-09-14T03:44:24.930301+0000","last_peered":"2024-09-14T03:44:03.881437+0000","last_clean":"2024-09-14T03:44:03.881437+0000","last_became_active":"2024-09-14T03:44:01.873644+0000","last_became_peered":"2024-09-14T03:44:01.873644+0000","last_unstale":"2024-09-14T03:44:25.928176+0000","last_undegraded":"2024-09-14T03:44:25.928176+0000","last_fullsized":"2024-09-14T03:44:25.928176+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:34.312039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":["6","1","2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929430+0000","last_change":"2024-09-14T03:44:01.875017+0000","last_active":"2024-09-14T03:44:24.929430+0000","last_peered":"2024-09-14T03:44:24.929430+0000","last_clean":"2024-09-14T03:44:24.929430+0000","last_became_active":"2024-09-14T03:44:01.874687+0000","last_became_peered":"2024-09-14T03:44:01.874687+0000","last_unstale":"2024-09-14T03:44:24.929430+0000","last_undegraded":"2024-09-14T03:44:24.929430+0000","last_fullsized":"2024-09-14T03:44:24.929430+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:29:09.035342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926391+0000","last_change":"2024-09-14T03:44:01.883313+0000","last_active":"2024-09-14T03:44:24.926391+0000","last_peered":"2024-09-14T03:44:24.926391+0000","last_clean":"2024-09-14T03:44:24.926391+0000","last_became_active":"2024-09-14T03:44:01.882770+0000","last_became_peered":"2024-09-14T03:44:01.882770+0000","last_unstale":"2024-09-14T03:44:24.926391+0000","last_undegraded":"2024-09-14T03:44:24.926391+0000","last_fullsized":"2024-09-14T03:44:24.926391+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:57:19.402319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926070+0000","last_change":"2024-09-14T03:44:01.878245+0000","last_active":"2024-09-14T03:44:24.926070+0000","last_peered":"2024-09-14T03:44:24.926070+0000","last_clean":"2024-09-14T03:44:24.926070+0000","last_became_active":"2024-09-14T03:44:01.877861+0000","last_became_peered":"2024-09-14T03:44:01.877861+0000","last_unstale":"2024-09-14T03:44:24.926070+0000","last_undegraded":"2024-09-14T03:44:24.926070+0000","last_fullsized":"2024-09-14T03:44:24.926070+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:08:29.902090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930473+0000","last_change":"2024-09-14T03:44:01.877391+0000","last_active":"2024-09-14T03:44:24.930473+0000","last_peered":"2024-09-14T03:44:24.930473+0000","last_clean":"2024-09-14T03:44:24.930473+0000","last_became_active":"2024-09-14T03:44:01.876797+0000","last_became_peered":"2024-09-14T03:44:01.876797+0000","last_unstale":"2024-09-14T03:44:24.930473+0000","last_undegraded":"2024-09-14T03:44:24.930473+0000","last_fullsized":"2024-09-14T03:44:24.930473+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:06:02.891762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928601+0000","last_change":"2024-09-14T03:44:01.876700+0000","last_active":"2024-09-14T03:44:24.928601+0000","last_peered":"2024-09-14T03:44:24.928601+0000","last_clean":"2024-09-14T03:44:24.928601+0000","last_became_active":"2024-09-14T03:44:01.876091+0000","last_became_peered":"2024-09-14T03:44:01.876091+0000","last_unstale":"2024-09-14T03:44:24.928601+0000","last_undegraded":"2024-09-14T03:44:24.928601+0000","last_fullsized":"2024-09-14T03:44:24.928601+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:25:41.750127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930443+0000","last_change":"2024-09-14T03:44:01.877120+0000","last_active":"2024-09-14T03:44:24.930443+0000","last_peered":"2024-09-14T03:44:24.930443+0000","last_clean":"2024-09-14T03:44:24.930443+0000","last_became_active":"2024-09-14T03:44:01.876793+0000","last_became_peered":"2024-09-14T03:44:01.876793+0000","last_unstale":"2024-09-14T03:44:24.930443+0000","last_undegraded":"2024-09-14T03:44:24.930443+0000","last_fullsized":"2024-09-14T03:44:24.930443+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:23:56.450728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930751+0000","last_change":"2024-09-14T03:44:01.885108+0000","last_active":"2024-09-14T03:44:24.930751+0000","last_peered":"2024-09-14T03:44:24.930751+0000","last_clean":"2024-09-14T03:44:24.930751+0000","last_became_active":"2024-09-14T03:44:01.884857+0000","last_became_peered":"2024-09-14T03:44:01.884857+0000","last_unstale":"2024-09-14T03:44:24.930751+0000","last_undegraded":"2024-09-14T03:44:24.930751+0000","last_fullsized":"2024-09-14T03:44:24.930751+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:31.448617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928709+0000","last_change":"2024-09-14T03:44:01.876503+0000","last_active":"2024-09-14T03:44:24.928709+0000","last_peered":"2024-09-14T03:44:24.928709+0000","last_clean":"2024-09-14T03:44:24.928709+0000","last_became_active":"2024-09-14T03:44:01.876227+0000","last_became_peered":"2024-09-14T03:44:01.876227+0000","last_unstale":"2024-09-14T03:44:24.928709+0000","last_undegraded":"2024-09-14T03:44:24.928709+0000","last_fullsized":"2024-09-14T03:44:24.928709+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:44:50.132152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933287+0000","last_change":"2024-09-14T03:44:01.883708+0000","last_active":"2024-09-14T03:44:24.933287+0000","last_peered":"2024-09-14T03:44:24.933287+0000","last_clean":"2024-09-14T03:44:24.933287+0000","last_became_active":"2024-09-14T03:44:01.883440+0000","last_became_peered":"2024-09-14T03:44:01.883440+0000","last_unstale":"2024-09-14T03:44:24.933287+0000","last_undegraded":"2024-09-14T03:44:24.933287+0000","last_fullsized":"2024-09-14T03:44:24.933287+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:10:34.658062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929540+0000","last_change":"2024-09-14T03:44:01.888565+0000","last_active":"2024-09-14T03:44:24.929540+0000","last_peered":"2024-09-14T03:44:24.929540+0000","last_clean":"2024-09-14T03:44:24.929540+0000","last_became_active":"2024-09-14T03:44:01.888191+0000","last_became_peered":"2024-09-14T03:44:01.888191+0000","last_unstale":"2024-09-14T03:44:24.929540+0000","last_undegraded":"2024-09-14T03:44:24.929540+0000","last_fullsized":"2024-09-14T03:44:24.929540+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:52:23.782664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930632+0000","last_change":"2024-09-14T03:44:01.874642+0000","last_active":"2024-09-14T03:44:24.930632+0000","last_peered":"2024-09-14T03:44:24.930632+0000","last_clean":"2024-09-14T03:44:24.930632+0000","last_became_active":"2024-09-14T03:44:01.874384+0000","last_became_peered":"2024-09-14T03:44:01.874384+0000","last_unstale":"2024-09-14T03:44:24.930632+0000","last_undegraded":"2024-09-14T03:44:24.930632+0000","last_fullsized":"2024-09-14T03:44:24.930632+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:35:05.535173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928767+0000","last_change":"2024-09-14T03:44:01.886893+0000","last_active":"2024-09-14T03:44:24.928767+0000","last_peered":"2024-09-14T03:44:24.928767+0000","last_clean":"2024-09-14T03:44:24.928767+0000","last_became_active":"2024-09-14T03:44:01.886594+0000","last_became_peered":"2024-09-14T03:44:01.886594+0000","last_unstale":"2024-09-14T03:44:24.928767+0000","last_undegraded":"2024-09-14T03:44:24.928767+0000","last_fullsized":"2024-09-14T03:44:24.928767+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:07:37.154358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.925870+0000","last_change":"2024-09-14T03:44:01.868698+0000","last_active":"2024-09-14T03:44:24.925870+0000","last_peered":"2024-09-14T03:44:24.925870+0000","last_clean":"2024-09-14T03:44:24.925870+0000","last_became_active":"2024-09-14T03:44:01.868442+0000","last_became_peered":"2024-09-14T03:44:01.868442+0000","last_unstale":"2024-09-14T03:44:24.925870+0000","last_undegraded":"2024-09-14T03:44:24.925870+0000","last_fullsized":"2024-09-14T03:44:24.925870+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:41:18.296424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930530+0000","last_change":"2024-09-14T03:44:01.882708+0000","last_active":"2024-09-14T03:44:24.930530+0000","last_peered":"2024-09-14T03:44:24.930530+0000","last_clean":"2024-09-14T03:44:24.930530+0000","last_became_active":"2024-09-14T03:44:01.882443+0000","last_became_peered":"2024-09-14T03:44:01.882443+0000","last_unstale":"2024-09-14T03:44:24.930530+0000","last_undegraded":"2024-09-14T03:44:24.930530+0000","last_fullsized":"2024-09-14T03:44:24.930530+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:46:48.306263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926452+0000","last_change":"2024-09-14T03:44:01.884940+0000","last_active":"2024-09-14T03:44:24.926452+0000","last_peered":"2024-09-14T03:44:24.926452+0000","last_clean":"2024-09-14T03:44:24.926452+0000","last_became_active":"2024-09-14T03:44:01.884304+0000","last_became_peered":"2024-09-14T03:44:01.884304+0000","last_unstale":"2024-09-14T03:44:24.926452+0000","last_undegraded":"2024-09-14T03:44:24.926452+0000","last_fullsized":"2024-09-14T03:44:24.926452+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:01:49.061637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":16,"reported_epoch":60,"state":"activating","last_fresh":"2024-09-14T03:44:25.928238+0000","last_change":"2024-09-14T03:44:25.928238+0000","last_active":"2024-09-14T03:44:24.929140+0000","last_peered":"2024-09-14T03:44:03.881581+0000","last_clean":"2024-09-14T03:44:03.881581+0000","last_became_active":"2024-09-14T03:44:01.877768+0000","last_became_peered":"2024-09-14T03:44:01.877768+0000","last_unstale":"2024-09-14T03:44:25.928238+0000","last_undegraded":"2024-09-14T03:44:25.928238+0000","last_fullsized":"2024-09-14T03:44:25.928238+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:00:53.758328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":["6","3","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928051+0000","last_change":"2024-09-14T03:44:01.874564+0000","last_active":"2024-09-14T03:44:24.928051+0000","last_peered":"2024-09-14T03:44:24.928051+0000","last_clean":"2024-09-14T03:44:24.928051+0000","last_became_active":"2024-09-14T03:44:01.874297+0000","last_became_peered":"2024-09-14T03:44:01.874297+0000","last_unstale":"2024-09-14T03:44:24.928051+0000","last_undegraded":"2024-09-14T03:44:24.928051+0000","last_fullsized":"2024-09-14T03:44:24.928051+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:07.027610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926133+0000","last_change":"2024-09-14T03:44:01.884486+0000","last_active":"2024-09-14T03:44:24.926133+0000","last_peered":"2024-09-14T03:44:24.926133+0000","last_clean":"2024-09-14T03:44:24.926133+0000","last_became_active":"2024-09-14T03:44:01.884218+0000","last_became_peered":"2024-09-14T03:44:01.884218+0000","last_unstale":"2024-09-14T03:44:24.926133+0000","last_undegraded":"2024-09-14T03:44:24.926133+0000","last_fullsized":"2024-09-14T03:44:24.926133+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:59:18.153239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930230+0000","last_change":"2024-09-14T03:44:01.878737+0000","last_active":"2024-09-14T03:44:24.930230+0000","last_peered":"2024-09-14T03:44:24.930230+0000","last_clean":"2024-09-14T03:44:24.930230+0000","last_became_active":"2024-09-14T03:44:01.878504+0000","last_became_peered":"2024-09-14T03:44:01.878504+0000","last_unstale":"2024-09-14T03:44:24.930230+0000","last_undegraded":"2024-09-14T03:44:24.930230+0000","last_fullsized":"2024-09-14T03:44:24.930230+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:41:35.420957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928860+0000","last_change":"2024-09-14T03:44:01.880509+0000","last_active":"2024-09-14T03:44:24.928860+0000","last_peered":"2024-09-14T03:44:24.928860+0000","last_clean":"2024-09-14T03:44:24.928860+0000","last_became_active":"2024-09-14T03:44:01.879855+0000","last_became_peered":"2024-09-14T03:44:01.879855+0000","last_unstale":"2024-09-14T03:44:24.928860+0000","last_undegraded":"2024-09-14T03:44:24.928860+0000","last_fullsized":"2024-09-14T03:44:24.928860+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:05:31.166902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928290+0000","last_change":"2024-09-14T03:44:01.869523+0000","last_active":"2024-09-14T03:44:24.928290+0000","last_peered":"2024-09-14T03:44:24.928290+0000","last_clean":"2024-09-14T03:44:24.928290+0000","last_became_active":"2024-09-14T03:44:01.869225+0000","last_became_peered":"2024-09-14T03:44:01.869225+0000","last_unstale":"2024-09-14T03:44:24.928290+0000","last_undegraded":"2024-09-14T03:44:24.928290+0000","last_fullsized":"2024-09-14T03:44:24.928290+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:43:15.638187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929040+0000","last_change":"2024-09-14T03:44:01.876014+0000","last_active":"2024-09-14T03:44:24.929040+0000","last_peered":"2024-09-14T03:44:24.929040+0000","last_clean":"2024-09-14T03:44:24.929040+0000","last_became_active":"2024-09-14T03:44:01.875496+0000","last_became_peered":"2024-09-14T03:44:01.875496+0000","last_unstale":"2024-09-14T03:44:24.929040+0000","last_undegraded":"2024-09-14T03:44:24.929040+0000","last_fullsized":"2024-09-14T03:44:24.929040+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:03:45.344199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926322+0000","last_change":"2024-09-14T03:44:01.865496+0000","last_active":"2024-09-14T03:44:24.926322+0000","last_peered":"2024-09-14T03:44:24.926322+0000","last_clean":"2024-09-14T03:44:24.926322+0000","last_became_active":"2024-09-14T03:44:01.865190+0000","last_became_peered":"2024-09-14T03:44:01.865190+0000","last_unstale":"2024-09-14T03:44:24.926322+0000","last_undegraded":"2024-09-14T03:44:24.926322+0000","last_fullsized":"2024-09-14T03:44:24.926322+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:18:50.037675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"58'1","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926154+0000","last_change":"2024-09-14T03:43:58.729188+0000","last_active":"2024-09-14T03:44:24.926154+0000","last_peered":"2024-09-14T03:44:24.926154+0000","last_clean":"2024-09-14T03:44:24.926154+0000","last_became_active":"2024-09-14T03:43:58.728927+0000","last_became_peered":"2024-09-14T03:43:58.728927+0000","last_unstale":"2024-09-14T03:44:24.926154+0000","last_undegraded":"2024-09-14T03:44:24.926154+0000","last_fullsized":"2024-09-14T03:44:24.926154+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:13:16.419701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930296+0000","last_change":"2024-09-14T03:44:01.879101+0000","last_active":"2024-09-14T03:44:24.930296+0000","last_peered":"2024-09-14T03:44:24.930296+0000","last_clean":"2024-09-14T03:44:24.930296+0000","last_became_active":"2024-09-14T03:44:01.878801+0000","last_became_peered":"2024-09-14T03:44:01.878801+0000","last_unstale":"2024-09-14T03:44:24.930296+0000","last_undegraded":"2024-09-14T03:44:24.930296+0000","last_fullsized":"2024-09-14T03:44:24.930296+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:13.447251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"58'1","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930120+0000","last_change":"2024-09-14T03:43:58.730444+0000","last_active":"2024-09-14T03:44:24.930120+0000","last_peered":"2024-09-14T03:44:24.930120+0000","last_clean":"2024-09-14T03:44:24.930120+0000","last_became_active":"2024-09-14T03:43:58.730172+0000","last_became_peered":"2024-09-14T03:43:58.730172+0000","last_unstale":"2024-09-14T03:44:24.930120+0000","last_undegraded":"2024-09-14T03:44:24.930120+0000","last_fullsized":"2024-09-14T03:44:24.930120+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:03:44.810562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.925963+0000","last_change":"2024-09-14T03:43:58.731458+0000","last_active":"2024-09-14T03:44:24.925963+0000","last_peered":"2024-09-14T03:44:24.925963+0000","last_clean":"2024-09-14T03:44:24.925963+0000","last_became_active":"2024-09-14T03:43:58.731206+0000","last_became_peered":"2024-09-14T03:43:58.731206+0000","last_unstale":"2024-09-14T03:44:24.925963+0000","last_undegraded":"2024-09-14T03:44:24.925963+0000","last_fullsized":"2024-09-14T03:44:24.925963+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:37:06.670539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.925827+0000","last_change":"2024-09-14T03:44:01.869781+0000","last_active":"2024-09-14T03:44:24.925827+0000","last_peered":"2024-09-14T03:44:24.925827+0000","last_clean":"2024-09-14T03:44:24.925827+0000","last_became_active":"2024-09-14T03:44:01.869521+0000","last_became_peered":"2024-09-14T03:44:01.869521+0000","last_unstale":"2024-09-14T03:44:24.925827+0000","last_undegraded":"2024-09-14T03:44:24.925827+0000","last_fullsized":"2024-09-14T03:44:24.925827+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:35:18.565206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":25,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-14T03:44:25.932578+0000","last_change":"2024-09-14T03:44:25.932578+0000","last_active":"2024-09-14T03:44:25.932578+0000","last_peered":"2024-09-14T03:44:25.932578+0000","last_clean":"2024-09-14T03:44:25.932578+0000","last_became_active":"2024-09-14T03:44:25.932288+0000","last_became_peered":"2024-09-14T03:44:25.932288+0000","last_unstale":"2024-09-14T03:44:25.932578+0000","last_undegraded":"2024-09-14T03:44:25.932578+0000","last_fullsized":"2024-09-14T03:44:25.932578+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:24:41.091353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928474+0000","last_change":"2024-09-14T03:44:01.873748+0000","last_active":"2024-09-14T03:44:24.928474+0000","last_peered":"2024-09-14T03:44:24.928474+0000","last_clean":"2024-09-14T03:44:24.928474+0000","last_became_active":"2024-09-14T03:44:01.873473+0000","last_became_peered":"2024-09-14T03:44:01.873473+0000","last_unstale":"2024-09-14T03:44:24.928474+0000","last_undegraded":"2024-09-14T03:44:24.928474+0000","last_fullsized":"2024-09-14T03:44:24.928474+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:04:39.289903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929850+0000","last_change":"2024-09-14T03:43:58.735608+0000","last_active":"2024-09-14T03:44:24.929850+0000","last_peered":"2024-09-14T03:44:24.929850+0000","last_clean":"2024-09-14T03:44:24.929850+0000","last_became_active":"2024-09-14T03:43:58.735301+0000","last_became_peered":"2024-09-14T03:43:58.735301+0000","last_unstale":"2024-09-14T03:44:24.929850+0000","last_undegraded":"2024-09-14T03:44:24.929850+0000","last_fullsized":"2024-09-14T03:44:24.929850+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:44:56.523073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930382+0000","last_change":"2024-09-14T03:44:01.874777+0000","last_active":"2024-09-14T03:44:24.930382+0000","last_peered":"2024-09-14T03:44:24.930382+0000","last_clean":"2024-09-14T03:44:24.930382+0000","last_became_active":"2024-09-14T03:44:01.874473+0000","last_became_peered":"2024-09-14T03:44:01.874473+0000","last_unstale":"2024-09-14T03:44:24.930382+0000","last_undegraded":"2024-09-14T03:44:24.930382+0000","last_fullsized":"2024-09-14T03:44:24.930382+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:49:02.042635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930539+0000","last_change":"2024-09-14T03:43:58.732969+0000","last_active":"2024-09-14T03:44:24.930539+0000","last_peered":"2024-09-14T03:44:24.930539+0000","last_clean":"2024-09-14T03:44:24.930539+0000","last_became_active":"2024-09-14T03:43:58.732492+0000","last_became_peered":"2024-09-14T03:43:58.732492+0000","last_unstale":"2024-09-14T03:44:24.930539+0000","last_undegraded":"2024-09-14T03:44:24.930539+0000","last_fullsized":"2024-09-14T03:44:24.930539+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:50:28.719447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930714+0000","last_change":"2024-09-14T03:44:01.875679+0000","last_active":"2024-09-14T03:44:24.930714+0000","last_peered":"2024-09-14T03:44:24.930714+0000","last_clean":"2024-09-14T03:44:24.930714+0000","last_became_active":"2024-09-14T03:44:01.875422+0000","last_became_peered":"2024-09-14T03:44:01.875422+0000","last_unstale":"2024-09-14T03:44:24.930714+0000","last_undegraded":"2024-09-14T03:44:24.930714+0000","last_fullsized":"2024-09-14T03:44:24.930714+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:13:00.349373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"58'3","reported_seq":18,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929910+0000","last_change":"2024-09-14T03:43:58.735066+0000","last_active":"2024-09-14T03:44:24.929910+0000","last_peered":"2024-09-14T03:44:24.929910+0000","last_clean":"2024-09-14T03:44:24.929910+0000","last_became_active":"2024-09-14T03:43:58.734514+0000","last_became_peered":"2024-09-14T03:43:58.734514+0000","last_unstale":"2024-09-14T03:44:24.929910+0000","last_undegraded":"2024-09-14T03:44:24.929910+0000","last_fullsized":"2024-09-14T03:44:24.929910+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:41:55.478295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926573+0000","last_change":"2024-09-14T03:44:01.881319+0000","last_active":"2024-09-14T03:44:24.926573+0000","last_peered":"2024-09-14T03:44:24.926573+0000","last_clean":"2024-09-14T03:44:24.926573+0000","last_became_active":"2024-09-14T03:44:01.881032+0000","last_became_peered":"2024-09-14T03:44:01.881032+0000","last_unstale":"2024-09-14T03:44:24.926573+0000","last_undegraded":"2024-09-14T03:44:24.926573+0000","last_fullsized":"2024-09-14T03:44:24.926573+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:35:41.762014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"58'6","reported_seq":21,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929594+0000","last_change":"2024-09-14T03:43:58.734778+0000","last_active":"2024-09-14T03:44:24.929594+0000","last_peered":"2024-09-14T03:44:24.929594+0000","last_clean":"2024-09-14T03:44:24.929594+0000","last_became_active":"2024-09-14T03:43:58.734380+0000","last_became_peered":"2024-09-14T03:43:58.734380+0000","last_unstale":"2024-09-14T03:44:24.929594+0000","last_undegraded":"2024-09-14T03:44:24.929594+0000","last_fullsized":"2024-09-14T03:44:24.929594+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:55:13.561065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933222+0000","last_change":"2024-09-14T03:44:01.880139+0000","last_active":"2024-09-14T03:44:24.933222+0000","last_peered":"2024-09-14T03:44:24.933222+0000","last_clean":"2024-09-14T03:44:24.933222+0000","last_became_active":"2024-09-14T03:44:01.879840+0000","last_became_peered":"2024-09-14T03:44:01.879840+0000","last_unstale":"2024-09-14T03:44:24.933222+0000","last_undegraded":"2024-09-14T03:44:24.933222+0000","last_fullsized":"2024-09-14T03:44:24.933222+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:31:34.003676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"58'2","reported_seq":17,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930377+0000","last_change":"2024-09-14T03:43:58.728768+0000","last_active":"2024-09-14T03:44:24.930377+0000","last_peered":"2024-09-14T03:44:24.930377+0000","last_clean":"2024-09-14T03:44:24.930377+0000","last_became_active":"2024-09-14T03:43:58.727893+0000","last_became_peered":"2024-09-14T03:43:58.727893+0000","last_unstale":"2024-09-14T03:44:24.930377+0000","last_undegraded":"2024-09-14T03:44:24.930377+0000","last_fullsized":"2024-09-14T03:44:24.930377+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:54:48.554486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930509+0000","last_change":"2024-09-14T03:44:01.877540+0000","last_active":"2024-09-14T03:44:24.930509+0000","last_peered":"2024-09-14T03:44:24.930509+0000","last_clean":"2024-09-14T03:44:24.930509+0000","last_became_active":"2024-09-14T03:44:01.877147+0000","last_became_peered":"2024-09-14T03:44:01.877147+0000","last_unstale":"2024-09-14T03:44:24.930509+0000","last_undegraded":"2024-09-14T03:44:24.930509+0000","last_fullsized":"2024-09-14T03:44:24.930509+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:17:57.724648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928948+0000","last_change":"2024-09-14T03:43:58.732812+0000","last_active":"2024-09-14T03:44:24.928948+0000","last_peered":"2024-09-14T03:44:24.928948+0000","last_clean":"2024-09-14T03:44:24.928948+0000","last_became_active":"2024-09-14T03:43:58.732537+0000","last_became_peered":"2024-09-14T03:43:58.732537+0000","last_unstale":"2024-09-14T03:44:24.928948+0000","last_undegraded":"2024-09-14T03:44:24.928948+0000","last_fullsized":"2024-09-14T03:44:24.928948+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:26:11.551182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.932936+0000","last_change":"2024-09-14T03:44:01.881140+0000","last_active":"2024-09-14T03:44:24.932936+0000","last_peered":"2024-09-14T03:44:24.932936+0000","last_clean":"2024-09-14T03:44:24.932936+0000","last_became_active":"2024-09-14T03:44:01.880868+0000","last_became_peered":"2024-09-14T03:44:01.880868+0000","last_unstale":"2024-09-14T03:44:24.932936+0000","last_undegraded":"2024-09-14T03:44:24.932936+0000","last_fullsized":"2024-09-14T03:44:24.932936+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:29:05.729804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929350+0000","last_change":"2024-09-14T03:43:58.730505+0000","last_active":"2024-09-14T03:44:24.929350+0000","last_peered":"2024-09-14T03:44:24.929350+0000","last_clean":"2024-09-14T03:44:24.929350+0000","last_became_active":"2024-09-14T03:43:58.729821+0000","last_became_peered":"2024-09-14T03:43:58.729821+0000","last_unstale":"2024-09-14T03:44:24.929350+0000","last_undegraded":"2024-09-14T03:44:24.929350+0000","last_fullsized":"2024-09-14T03:44:24.929350+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:17:44.091667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929155+0000","last_change":"2024-09-14T03:44:01.874477+0000","last_active":"2024-09-14T03:44:24.929155+0000","last_peered":"2024-09-14T03:44:24.929155+0000","last_clean":"2024-09-14T03:44:24.929155+0000","last_became_active":"2024-09-14T03:44:01.873714+0000","last_became_peered":"2024-09-14T03:44:01.873714+0000","last_unstale":"2024-09-14T03:44:24.929155+0000","last_undegraded":"2024-09-14T03:44:24.929155+0000","last_fullsized":"2024-09-14T03:44:24.929155+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:54:21.601768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"58'8","reported_seq":23,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928144+0000","last_change":"2024-09-14T03:43:58.726387+0000","last_active":"2024-09-14T03:44:24.928144+0000","last_peered":"2024-09-14T03:44:24.928144+0000","last_clean":"2024-09-14T03:44:24.928144+0000","last_became_active":"2024-09-14T03:43:58.726115+0000","last_became_peered":"2024-09-14T03:43:58.726115+0000","last_unstale":"2024-09-14T03:44:24.928144+0000","last_undegraded":"2024-09-14T03:44:24.928144+0000","last_fullsized":"2024-09-14T03:44:24.928144+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:18:04.176970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.932701+0000","last_change":"2024-09-14T03:44:01.878624+0000","last_active":"2024-09-14T03:44:24.932701+0000","last_peered":"2024-09-14T03:44:24.932701+0000","last_clean":"2024-09-14T03:44:24.932701+0000","last_became_active":"2024-09-14T03:44:01.878343+0000","last_became_peered":"2024-09-14T03:44:01.878343+0000","last_unstale":"2024-09-14T03:44:24.932701+0000","last_undegraded":"2024-09-14T03:44:24.932701+0000","last_fullsized":"2024-09-14T03:44:24.932701+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:44:05.205671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"58'1","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926865+0000","last_change":"2024-09-14T03:43:58.728489+0000","last_active":"2024-09-14T03:44:24.926865+0000","last_peered":"2024-09-14T03:44:24.926865+0000","last_clean":"2024-09-14T03:44:24.926865+0000","last_became_active":"2024-09-14T03:43:58.728182+0000","last_became_peered":"2024-09-14T03:43:58.728182+0000","last_unstale":"2024-09-14T03:44:24.926865+0000","last_undegraded":"2024-09-14T03:44:24.926865+0000","last_fullsized":"2024-09-14T03:44:24.926865+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:56.310710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933262+0000","last_change":"2024-09-14T03:44:01.880754+0000","last_active":"2024-09-14T03:44:24.933262+0000","last_peered":"2024-09-14T03:44:24.933262+0000","last_clean":"2024-09-14T03:44:24.933262+0000","last_became_active":"2024-09-14T03:44:01.880483+0000","last_became_peered":"2024-09-14T03:44:01.880483+0000","last_unstale":"2024-09-14T03:44:24.933262+0000","last_undegraded":"2024-09-14T03:44:24.933262+0000","last_fullsized":"2024-09-14T03:44:24.933262+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:32:53.245007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"58'2","reported_seq":17,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928841+0000","last_change":"2024-09-14T03:43:58.726366+0000","last_active":"2024-09-14T03:44:24.928841+0000","last_peered":"2024-09-14T03:44:24.928841+0000","last_clean":"2024-09-14T03:44:24.928841+0000","last_became_active":"2024-09-14T03:43:58.726090+0000","last_became_peered":"2024-09-14T03:43:58.726090+0000","last_unstale":"2024-09-14T03:44:24.928841+0000","last_undegraded":"2024-09-14T03:44:24.928841+0000","last_fullsized":"2024-09-14T03:44:24.928841+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:45:33.572820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933584+0000","last_change":"2024-09-14T03:44:01.883704+0000","last_active":"2024-09-14T03:44:24.933584+0000","last_peered":"2024-09-14T03:44:24.933584+0000","last_clean":"2024-09-14T03:44:24.933584+0000","last_became_active":"2024-09-14T03:44:01.883448+0000","last_became_peered":"2024-09-14T03:44:01.883448+0000","last_unstale":"2024-09-14T03:44:24.933584+0000","last_undegraded":"2024-09-14T03:44:24.933584+0000","last_fullsized":"2024-09-14T03:44:24.933584+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:16:27.946770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"58'1","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929063+0000","last_change":"2024-09-14T03:43:58.733205+0000","last_active":"2024-09-14T03:44:24.929063+0000","last_peered":"2024-09-14T03:44:24.929063+0000","last_clean":"2024-09-14T03:44:24.929063+0000","last_became_active":"2024-09-14T03:43:58.732929+0000","last_became_peered":"2024-09-14T03:43:58.732929+0000","last_unstale":"2024-09-14T03:44:24.929063+0000","last_undegraded":"2024-09-14T03:44:24.929063+0000","last_fullsized":"2024-09-14T03:44:24.929063+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:31:13.660965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929096+0000","last_change":"2024-09-14T03:44:01.886158+0000","last_active":"2024-09-14T03:44:24.929096+0000","last_peered":"2024-09-14T03:44:24.929096+0000","last_clean":"2024-09-14T03:44:24.929096+0000","last_became_active":"2024-09-14T03:44:01.884747+0000","last_became_peered":"2024-09-14T03:44:01.884747+0000","last_unstale":"2024-09-14T03:44:24.929096+0000","last_undegraded":"2024-09-14T03:44:24.929096+0000","last_fullsized":"2024-09-14T03:44:24.929096+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:49:56.265783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928891+0000","last_change":"2024-09-14T03:43:58.732401+0000","last_active":"2024-09-14T03:44:24.928891+0000","last_peered":"2024-09-14T03:44:24.928891+0000","last_clean":"2024-09-14T03:44:24.928891+0000","last_became_active":"2024-09-14T03:43:58.732067+0000","last_became_peered":"2024-09-14T03:43:58.732067+0000","last_unstale":"2024-09-14T03:44:24.928891+0000","last_undegraded":"2024-09-14T03:44:24.928891+0000","last_fullsized":"2024-09-14T03:44:24.928891+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:34:00.566245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933105+0000","last_change":"2024-09-14T03:44:01.881736+0000","last_active":"2024-09-14T03:44:24.933105+0000","last_peered":"2024-09-14T03:44:24.933105+0000","last_clean":"2024-09-14T03:44:24.933105+0000","last_became_active":"2024-09-14T03:44:01.881219+0000","last_became_peered":"2024-09-14T03:44:01.881219+0000","last_unstale":"2024-09-14T03:44:24.933105+0000","last_undegraded":"2024-09-14T03:44:24.933105+0000","last_fullsized":"2024-09-14T03:44:24.933105+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:24:23.342066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"22'72","reported_seq":25,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933249+0000","last_change":"2024-09-14T03:43:36.147227+0000","last_active":"2024-09-14T03:44:24.933249+0000","last_peered":"2024-09-14T03:44:24.933249+0000","last_clean":"2024-09-14T03:44:24.933249+0000","last_became_active":"2024-09-14T03:43:36.141953+0000","last_became_peered":"2024-09-14T03:43:36.141953+0000","last_unstale":"2024-09-14T03:44:24.933249+0000","last_undegraded":"2024-09-14T03:44:24.933249+0000","last_fullsized":"2024-09-14T03:44:24.933249+0000","mapping_epoch":50,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":51,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:39.458815+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:39.458815+0000","last_clean_scrub_stamp":"2024-09-14T03:41:39.458815+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:38:43.056328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930279+0000","last_change":"2024-09-14T03:43:58.733557+0000","last_active":"2024-09-14T03:44:24.930279+0000","last_peered":"2024-09-14T03:44:24.930279+0000","last_clean":"2024-09-14T03:44:24.930279+0000","last_became_active":"2024-09-14T03:43:58.733256+0000","last_became_peered":"2024-09-14T03:43:58.733256+0000","last_unstale":"2024-09-14T03:44:24.930279+0000","last_undegraded":"2024-09-14T03:44:24.930279+0000","last_fullsized":"2024-09-14T03:44:24.930279+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:09:12.987156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.925983+0000","last_change":"2024-09-14T03:44:01.875525+0000","last_active":"2024-09-14T03:44:24.925983+0000","last_peered":"2024-09-14T03:44:24.925983+0000","last_clean":"2024-09-14T03:44:24.925983+0000","last_became_active":"2024-09-14T03:44:01.875241+0000","last_became_peered":"2024-09-14T03:44:01.875241+0000","last_unstale":"2024-09-14T03:44:24.925983+0000","last_undegraded":"2024-09-14T03:44:24.925983+0000","last_fullsized":"2024-09-14T03:44:24.925983+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:34:47.346963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933143+0000","last_change":"2024-09-14T03:43:58.732376+0000","last_active":"2024-09-14T03:44:24.933143+0000","last_peered":"2024-09-14T03:44:24.933143+0000","last_clean":"2024-09-14T03:44:24.933143+0000","last_became_active":"2024-09-14T03:43:58.731812+0000","last_became_peered":"2024-09-14T03:43:58.731812+0000","last_unstale":"2024-09-14T03:44:24.933143+0000","last_undegraded":"2024-09-14T03:44:24.933143+0000","last_fullsized":"2024-09-14T03:44:24.933143+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:35:26.262383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930295+0000","last_change":"2024-09-14T03:44:01.876511+0000","last_active":"2024-09-14T03:44:24.930295+0000","last_peered":"2024-09-14T03:44:24.930295+0000","last_clean":"2024-09-14T03:44:24.930295+0000","last_became_active":"2024-09-14T03:44:01.876225+0000","last_became_peered":"2024-09-14T03:44:01.876225+0000","last_unstale":"2024-09-14T03:44:24.930295+0000","last_undegraded":"2024-09-14T03:44:24.930295+0000","last_fullsized":"2024-09-14T03:44:24.930295+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:36:40.283622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928369+0000","last_change":"2024-09-14T03:43:58.727129+0000","last_active":"2024-09-14T03:44:24.928369+0000","last_peered":"2024-09-14T03:44:24.928369+0000","last_clean":"2024-09-14T03:44:24.928369+0000","last_became_active":"2024-09-14T03:43:58.726879+0000","last_became_peered":"2024-09-14T03:43:58.726879+0000","last_unstale":"2024-09-14T03:44:24.928369+0000","last_undegraded":"2024-09-14T03:44:24.928369+0000","last_fullsized":"2024-09-14T03:44:24.928369+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:54:41.748651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928166+0000","last_change":"2024-09-14T03:44:01.871005+0000","last_active":"2024-09-14T03:44:24.928166+0000","last_peered":"2024-09-14T03:44:24.928166+0000","last_clean":"2024-09-14T03:44:24.928166+0000","last_became_active":"2024-09-14T03:44:01.870707+0000","last_became_peered":"2024-09-14T03:44:01.870707+0000","last_unstale":"2024-09-14T03:44:24.928166+0000","last_undegraded":"2024-09-14T03:44:24.928166+0000","last_fullsized":"2024-09-14T03:44:24.928166+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:30:22.758581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929035+0000","last_change":"2024-09-14T03:43:58.733038+0000","last_active":"2024-09-14T03:44:24.929035+0000","last_peered":"2024-09-14T03:44:24.929035+0000","last_clean":"2024-09-14T03:44:24.929035+0000","last_became_active":"2024-09-14T03:43:58.732753+0000","last_became_peered":"2024-09-14T03:43:58.732753+0000","last_unstale":"2024-09-14T03:44:24.929035+0000","last_undegraded":"2024-09-14T03:44:24.929035+0000","last_fullsized":"2024-09-14T03:44:24.929035+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:40:49.754211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":22,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-14T03:44:26.430320+0000","last_change":"2024-09-14T03:44:26.430320+0000","last_active":"2024-09-14T03:44:26.430320+0000","last_peered":"2024-09-14T03:44:26.430320+0000","last_clean":"2024-09-14T03:44:26.430320+0000","last_became_active":"2024-09-14T03:44:26.429935+0000","last_became_peered":"2024-09-14T03:44:26.429935+0000","last_unstale":"2024-09-14T03:44:26.430320+0000","last_undegraded":"2024-09-14T03:44:26.430320+0000","last_fullsized":"2024-09-14T03:44:26.430320+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:12:35.577538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929025+0000","last_change":"2024-09-14T03:43:58.733621+0000","last_active":"2024-09-14T03:44:24.929025+0000","last_peered":"2024-09-14T03:44:24.929025+0000","last_clean":"2024-09-14T03:44:24.929025+0000","last_became_active":"2024-09-14T03:43:58.733357+0000","last_became_peered":"2024-09-14T03:43:58.733357+0000","last_unstale":"2024-09-14T03:44:24.929025+0000","last_undegraded":"2024-09-14T03:44:24.929025+0000","last_fullsized":"2024-09-14T03:44:24.929025+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:25:53.791926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929608+0000","last_change":"2024-09-14T03:44:01.880200+0000","last_active":"2024-09-14T03:44:24.929608+0000","last_peered":"2024-09-14T03:44:24.929608+0000","last_clean":"2024-09-14T03:44:24.929608+0000","last_became_active":"2024-09-14T03:44:01.879925+0000","last_became_peered":"2024-09-14T03:44:01.879925+0000","last_unstale":"2024-09-14T03:44:24.929608+0000","last_undegraded":"2024-09-14T03:44:24.929608+0000","last_fullsized":"2024-09-14T03:44:24.929608+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:55:24.738215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933352+0000","last_change":"2024-09-14T03:43:58.733230+0000","last_active":"2024-09-14T03:44:24.933352+0000","last_peered":"2024-09-14T03:44:24.933352+0000","last_clean":"2024-09-14T03:44:24.933352+0000","last_became_active":"2024-09-14T03:43:58.732868+0000","last_became_peered":"2024-09-14T03:43:58.732868+0000","last_unstale":"2024-09-14T03:44:24.933352+0000","last_undegraded":"2024-09-14T03:44:24.933352+0000","last_fullsized":"2024-09-14T03:44:24.933352+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:15:12.184851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929025+0000","last_change":"2024-09-14T03:44:01.878736+0000","last_active":"2024-09-14T03:44:24.929025+0000","last_peered":"2024-09-14T03:44:24.929025+0000","last_clean":"2024-09-14T03:44:24.929025+0000","last_became_active":"2024-09-14T03:44:01.877390+0000","last_became_peered":"2024-09-14T03:44:01.877390+0000","last_unstale":"2024-09-14T03:44:24.929025+0000","last_undegraded":"2024-09-14T03:44:24.929025+0000","last_fullsized":"2024-09-14T03:44:24.929025+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:17:42.812575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"58'1","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929778+0000","last_change":"2024-09-14T03:43:58.733688+0000","last_active":"2024-09-14T03:44:24.929778+0000","last_peered":"2024-09-14T03:44:24.929778+0000","last_clean":"2024-09-14T03:44:24.929778+0000","last_became_active":"2024-09-14T03:43:58.733457+0000","last_became_peered":"2024-09-14T03:43:58.733457+0000","last_unstale":"2024-09-14T03:44:24.929778+0000","last_undegraded":"2024-09-14T03:44:24.929778+0000","last_fullsized":"2024-09-14T03:44:24.929778+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:09:01.069988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926470+0000","last_change":"2024-09-14T03:44:01.870045+0000","last_active":"2024-09-14T03:44:24.926470+0000","last_peered":"2024-09-14T03:44:24.926470+0000","last_clean":"2024-09-14T03:44:24.926470+0000","last_became_active":"2024-09-14T03:44:01.869780+0000","last_became_peered":"2024-09-14T03:44:01.869780+0000","last_unstale":"2024-09-14T03:44:24.926470+0000","last_undegraded":"2024-09-14T03:44:24.926470+0000","last_fullsized":"2024-09-14T03:44:24.926470+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:04:58.634704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"58'2","reported_seq":17,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929736+0000","last_change":"2024-09-14T03:43:58.735364+0000","last_active":"2024-09-14T03:44:24.929736+0000","last_peered":"2024-09-14T03:44:24.929736+0000","last_clean":"2024-09-14T03:44:24.929736+0000","last_became_active":"2024-09-14T03:43:58.734833+0000","last_became_peered":"2024-09-14T03:43:58.734833+0000","last_unstale":"2024-09-14T03:44:24.929736+0000","last_undegraded":"2024-09-14T03:44:24.929736+0000","last_fullsized":"2024-09-14T03:44:24.929736+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:58:37.428713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930534+0000","last_change":"2024-09-14T03:44:01.878573+0000","last_active":"2024-09-14T03:44:24.930534+0000","last_peered":"2024-09-14T03:44:24.930534+0000","last_clean":"2024-09-14T03:44:24.930534+0000","last_became_active":"2024-09-14T03:44:01.878316+0000","last_became_peered":"2024-09-14T03:44:01.878316+0000","last_unstale":"2024-09-14T03:44:24.930534+0000","last_undegraded":"2024-09-14T03:44:24.930534+0000","last_fullsized":"2024-09-14T03:44:24.930534+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:45:21.129163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933129+0000","last_change":"2024-09-14T03:43:58.730843+0000","last_active":"2024-09-14T03:44:24.933129+0000","last_peered":"2024-09-14T03:44:24.933129+0000","last_clean":"2024-09-14T03:44:24.933129+0000","last_became_active":"2024-09-14T03:43:58.730538+0000","last_became_peered":"2024-09-14T03:43:58.730538+0000","last_unstale":"2024-09-14T03:44:24.933129+0000","last_undegraded":"2024-09-14T03:44:24.933129+0000","last_fullsized":"2024-09-14T03:44:24.933129+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:04:57.059345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926481+0000","last_change":"2024-09-14T03:44:01.874355+0000","last_active":"2024-09-14T03:44:24.926481+0000","last_peered":"2024-09-14T03:44:24.926481+0000","last_clean":"2024-09-14T03:44:24.926481+0000","last_became_active":"2024-09-14T03:44:01.874049+0000","last_became_peered":"2024-09-14T03:44:01.874049+0000","last_unstale":"2024-09-14T03:44:24.926481+0000","last_undegraded":"2024-09-14T03:44:24.926481+0000","last_fullsized":"2024-09-14T03:44:24.926481+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:53:30.306175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928677+0000","last_change":"2024-09-14T03:43:58.728080+0000","last_active":"2024-09-14T03:44:24.928677+0000","last_peered":"2024-09-14T03:44:24.928677+0000","last_clean":"2024-09-14T03:44:24.928677+0000","last_became_active":"2024-09-14T03:43:58.727820+0000","last_became_peered":"2024-09-14T03:43:58.727820+0000","last_unstale":"2024-09-14T03:44:24.928677+0000","last_undegraded":"2024-09-14T03:44:24.928677+0000","last_fullsized":"2024-09-14T03:44:24.928677+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:37:58.166430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.925857+0000","last_change":"2024-09-14T03:44:01.869311+0000","last_active":"2024-09-14T03:44:24.925857+0000","last_peered":"2024-09-14T03:44:24.925857+0000","last_clean":"2024-09-14T03:44:24.925857+0000","last_became_active":"2024-09-14T03:44:01.869065+0000","last_became_peered":"2024-09-14T03:44:01.869065+0000","last_unstale":"2024-09-14T03:44:24.925857+0000","last_undegraded":"2024-09-14T03:44:24.925857+0000","last_fullsized":"2024-09-14T03:44:24.925857+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:45:09.508019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929475+0000","last_change":"2024-09-14T03:43:58.728931+0000","last_active":"2024-09-14T03:44:24.929475+0000","last_peered":"2024-09-14T03:44:24.929475+0000","last_clean":"2024-09-14T03:44:24.929475+0000","last_became_active":"2024-09-14T03:43:58.728647+0000","last_became_peered":"2024-09-14T03:43:58.728647+0000","last_unstale":"2024-09-14T03:44:24.929475+0000","last_undegraded":"2024-09-14T03:44:24.929475+0000","last_fullsized":"2024-09-14T03:44:24.929475+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:09:38.769523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":23,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-14T03:44:26.430230+0000","last_change":"2024-09-14T03:44:26.430230+0000","last_active":"2024-09-14T03:44:26.430230+0000","last_peered":"2024-09-14T03:44:26.430230+0000","last_clean":"2024-09-14T03:44:26.430230+0000","last_became_active":"2024-09-14T03:44:26.429880+0000","last_became_peered":"2024-09-14T03:44:26.429880+0000","last_unstale":"2024-09-14T03:44:26.430230+0000","last_undegraded":"2024-09-14T03:44:26.430230+0000","last_fullsized":"2024-09-14T03:44:26.430230+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:39.132572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"58'2","reported_seq":18,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933131+0000","last_change":"2024-09-14T03:43:58.737705+0000","last_active":"2024-09-14T03:44:24.933131+0000","last_peered":"2024-09-14T03:44:24.933131+0000","last_clean":"2024-09-14T03:44:24.933131+0000","last_became_active":"2024-09-14T03:43:58.736546+0000","last_became_peered":"2024-09-14T03:43:58.736546+0000","last_unstale":"2024-09-14T03:44:24.933131+0000","last_undegraded":"2024-09-14T03:44:24.933131+0000","last_fullsized":"2024-09-14T03:44:24.933131+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:27:12.542706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":10,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-14T03:44:26.429678+0000","last_change":"2024-09-14T03:44:26.429678+0000","last_active":"2024-09-14T03:44:26.429678+0000","last_peered":"2024-09-14T03:44:26.429678+0000","last_clean":"2024-09-14T03:44:26.429678+0000","last_became_active":"2024-09-14T03:44:26.429364+0000","last_became_peered":"2024-09-14T03:44:26.429364+0000","last_unstale":"2024-09-14T03:44:26.429678+0000","last_undegraded":"2024-09-14T03:44:26.429678+0000","last_fullsized":"2024-09-14T03:44:26.429678+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:29:31.230108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929526+0000","last_change":"2024-09-14T03:44:01.888591+0000","last_active":"2024-09-14T03:44:24.929526+0000","last_peered":"2024-09-14T03:44:24.929526+0000","last_clean":"2024-09-14T03:44:24.929526+0000","last_became_active":"2024-09-14T03:44:01.888257+0000","last_became_peered":"2024-09-14T03:44:01.888257+0000","last_unstale":"2024-09-14T03:44:24.929526+0000","last_undegraded":"2024-09-14T03:44:24.929526+0000","last_fullsized":"2024-09-14T03:44:24.929526+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:24:40.992358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"58'2","reported_seq":18,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933222+0000","last_change":"2024-09-14T03:43:58.734680+0000","last_active":"2024-09-14T03:44:24.933222+0000","last_peered":"2024-09-14T03:44:24.933222+0000","last_clean":"2024-09-14T03:44:24.933222+0000","last_became_active":"2024-09-14T03:43:58.734408+0000","last_became_peered":"2024-09-14T03:43:58.734408+0000","last_unstale":"2024-09-14T03:44:24.933222+0000","last_undegraded":"2024-09-14T03:44:24.933222+0000","last_fullsized":"2024-09-14T03:44:24.933222+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:49:02.536404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929006+0000","last_change":"2024-09-14T03:44:01.885572+0000","last_active":"2024-09-14T03:44:24.929006+0000","last_peered":"2024-09-14T03:44:24.929006+0000","last_clean":"2024-09-14T03:44:24.929006+0000","last_became_active":"2024-09-14T03:44:01.885110+0000","last_became_peered":"2024-09-14T03:44:01.885110+0000","last_unstale":"2024-09-14T03:44:24.929006+0000","last_undegraded":"2024-09-14T03:44:24.929006+0000","last_fullsized":"2024-09-14T03:44:24.929006+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:10.439239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1d","version":"58'2","reported_seq":18,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933206+0000","last_change":"2024-09-14T03:43:58.731296+0000","last_active":"2024-09-14T03:44:24.933206+0000","last_peered":"2024-09-14T03:44:24.933206+0000","last_clean":"2024-09-14T03:44:24.933206+0000","last_became_active":"2024-09-14T03:43:58.730951+0000","last_became_peered":"2024-09-14T03:43:58.730951+0000","last_unstale":"2024-09-14T03:44:24.933206+0000","last_undegraded":"2024-09-14T03:44:24.933206+0000","last_fullsized":"2024-09-14T03:44:24.933206+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:26:14.801414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930670+0000","last_change":"2024-09-14T03:44:01.874935+0000","last_active":"2024-09-14T03:44:24.930670+0000","last_peered":"2024-09-14T03:44:24.930670+0000","last_clean":"2024-09-14T03:44:24.930670+0000","last_became_active":"2024-09-14T03:44:01.874668+0000","last_became_peered":"2024-09-14T03:44:01.874668+0000","last_unstale":"2024-09-14T03:44:24.930670+0000","last_undegraded":"2024-09-14T03:44:24.930670+0000","last_fullsized":"2024-09-14T03:44:24.930670+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:24:31.404958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"58'1","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926437+0000","last_change":"2024-09-14T03:43:58.734206+0000","last_active":"2024-09-14T03:44:24.926437+0000","last_peered":"2024-09-14T03:44:24.926437+0000","last_clean":"2024-09-14T03:44:24.926437+0000","last_became_active":"2024-09-14T03:43:58.733960+0000","last_became_peered":"2024-09-14T03:43:58.733960+0000","last_unstale":"2024-09-14T03:44:24.926437+0000","last_undegraded":"2024-09-14T03:44:24.926437+0000","last_fullsized":"2024-09-14T03:44:24.926437+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:44.073503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926323+0000","last_change":"2024-09-14T03:44:01.883550+0000","last_active":"2024-09-14T03:44:24.926323+0000","last_peered":"2024-09-14T03:44:24.926323+0000","last_clean":"2024-09-14T03:44:24.926323+0000","last_became_active":"2024-09-14T03:44:01.883265+0000","last_became_peered":"2024-09-14T03:44:01.883265+0000","last_unstale":"2024-09-14T03:44:24.926323+0000","last_undegraded":"2024-09-14T03:44:24.926323+0000","last_fullsized":"2024-09-14T03:44:24.926323+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:52:04.512990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"58'4","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930891+0000","last_change":"2024-09-14T03:43:58.731954+0000","last_active":"2024-09-14T03:44:24.930891+0000","last_peered":"2024-09-14T03:44:24.930891+0000","last_clean":"2024-09-14T03:44:24.930891+0000","last_became_active":"2024-09-14T03:43:58.731688+0000","last_became_peered":"2024-09-14T03:43:58.731688+0000","last_unstale":"2024-09-14T03:44:24.930891+0000","last_undegraded":"2024-09-14T03:44:24.930891+0000","last_fullsized":"2024-09-14T03:44:24.930891+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:30:10.811609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3563520,"data_stored":3542208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397518,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298528,"kb_used_data":1172,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446624,"statfs":{"total":95995035648,"available":95689342976,"internally_reserved":0,"allocated":1200128,"data_stored":844812,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51800000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.499}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.436}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55700000000000005}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60399999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39900000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57299999999999995}]}]},{"osd":6,"up_from":43,"seq":184683593745,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298516,"kb_used_data":1160,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446636,"statfs":{"total":95995035648,"available":95689355264,"internally_reserved":0,"allocated":1187840,"data_stored":838908,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.39800000000000002}]},{"osd":1,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.53400000000000003}]},{"osd":2,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.52300000000000002}]},{"osd":3,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.50600000000000001}]},{"osd":4,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.372}]},{"osd":5,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.45500000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48499999999999999}]}]},{"osd":5,"up_from":37,"seq":158913789975,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297956,"kb_used_data":596,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447196,"statfs":{"total":95995035648,"available":95689928704,"internally_reserved":0,"allocated":610304,"data_stored":255179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:43:47 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.45900000000000002}]},{"osd":1,"last update":"Sat Sep 14 03:43:47 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.34999999999999998}]},{"osd":2,"last update":"Sat Sep 14 03:43:47 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.378},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.58399999999999996}]},{"osd":3,"last update":"Sat Sep 14 03:43:47 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.50700000000000001}]},{"osd":4,"last update":"Sat Sep 14 03:43:47 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.379,"15min":0.379},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"last":0.442},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.52900000000000003}]},{"osd":6,"last update":"Sat Sep 14 03:44:11 2024","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.61399999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47799999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986203,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297960,"kb_used_data":600,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447192,"statfs":{"total":95995035648,"available":95689924608,"internally_reserved":0,"allocated":614400,"data_stored":254548,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:44:26 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.51000000000000001,"15min":0.51200000000000001},"min":{"1min":0.35499999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.82599999999999996},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.54500000000000004,"15min":0.54700000000000004},"min":{"1min":0.40500000000000003,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.80500000000000005}]},{"osd":1,"last update":"Sat Sep 14 03:44:26 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.48399999999999999,"15min":0.47599999999999998},"min":{"1min":0.34100000000000003,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.73199999999999998},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.39100000000000001,"5min":0.377,"15min":0.377},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.753}]},{"osd":2,"last update":"Sat Sep 14 03:44:26 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.495,"15min":0.48999999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.51100000000000001,"15min":0.505},"min":{"1min":0.38,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.81599999999999995}]},{"osd":3,"last update":"Sat Sep 14 03:44:26 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.505,"15min":0.49299999999999999},"min":{"1min":0.35599999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.78800000000000003},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.50700000000000001,"15min":0.501},"min":{"1min":0.35899999999999999,"5min":0.318,"15min":0.318},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.77400000000000002}]},{"osd":5,"last update":"Sat Sep 14 03:43:49 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.497}]},{"osd":6,"last update":"Sat Sep 14 03:44:11 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.627,"5min":0.627,"15min":0.627},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.55300000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60399999999999998}]}]},{"osd":3,"up_from":26,"seq":111669149727,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297960,"kb_used_data":600,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447192,"statfs":{"total":95995035648,"available":95689924608,"internally_reserved":0,"allocated":614400,"data_stored":255193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:44:03 2024","interfaces":[{"interface":"back","average":{"1min":0.34599999999999997,"5min":0.38400000000000001,"15min":0.39000000000000001},"min":{"1min":0.245,"5min":0.185,"15min":0.185},"max":{"1min":0.43099999999999999,"5min":0.48199999999999998,"15min":0.48199999999999998},"last":0.379},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.40200000000000002,"15min":0.40799999999999997},"min":{"1min":0.254,"5min":0.218,"15min":0.218},"max":{"1min":0.48099999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.35899999999999999}]},{"osd":1,"last update":"Sat Sep 14 03:44:03 2024","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.41599999999999998,"15min":0.41999999999999998},"min":{"1min":0.28799999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.5,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.435},{"interface":"front","average":{"1min":0.36599999999999999,"5min":0.39300000000000002,"15min":0.39700000000000002},"min":{"1min":0.30499999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.47199999999999998,"5min":0.53700000000000003,"15min":0.53700000000000003},"last":0.40799999999999997}]},{"osd":2,"last update":"Sat Sep 14 03:44:03 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.434,"15min":0.441},"min":{"1min":0.28999999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.52300000000000002,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.42299999999999999,"15min":0.42799999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.54400000000000004,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.45700000000000002}]},{"osd":4,"last update":"Sat Sep 14 03:43:29 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.56999999999999995}]},{"osd":5,"last update":"Sat Sep 14 03:43:52 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.55800000000000005}]},{"osd":6,"last update":"Sat Sep 14 03:44:11 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.60599999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.503}]}]},{"osd":2,"up_from":18,"seq":77309411364,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297940,"kb_used_data":580,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447212,"statfs":{"total":95995035648,"available":95689945088,"internally_reserved":0,"allocated":593920,"data_stored":253241,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:43:45 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.36299999999999999,"15min":0.35599999999999998},"min":{"1min":0.26200000000000001,"5min":0.22,"15min":0.22},"max":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.309},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.371,"15min":0.36599999999999999},"min":{"1min":0.27800000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.56200000000000006,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.27700000000000002}]},{"osd":1,"last update":"Sat Sep 14 03:43:45 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.40000000000000002,"15min":0.39800000000000002},"min":{"1min":0.32800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.441,"5min":0.39200000000000002,"15min":0.38400000000000001},"min":{"1min":0.29199999999999998,"5min":0.245,"15min":0.245},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.35599999999999998}]},{"osd":3,"last update":"Sat Sep 14 03:44:08 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.44800000000000001,"15min":0.45100000000000001},"min":{"1min":0.34999999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.53300000000000003,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.372},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.44800000000000001,"15min":0.45800000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.56000000000000005,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.33000000000000002}]},{"osd":4,"last update":"Sat Sep 14 03:44:25 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.55400000000000005,"15min":0.55700000000000005},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.78100000000000003,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.59199999999999997,"15min":0.59299999999999997},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.76000000000000001,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.68799999999999994}]},{"osd":5,"last update":"Sat Sep 14 03:43:48 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.63600000000000001}]},{"osd":6,"last update":"Sat Sep 14 03:44:12 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.67700000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.623}]}]},{"osd":1,"up_from":13,"seq":55834574890,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297936,"kb_used_data":580,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447216,"statfs":{"total":95995035648,"available":95689949184,"internally_reserved":0,"allocated":593920,"data_stored":253067,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:44:20 2024","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.32000000000000001,"15min":0.29599999999999999},"min":{"1min":0.249,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.51900000000000002,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.254},{"interface":"front","average":{"1min":0.374,"5min":0.34399999999999997,"15min":0.31900000000000001},"min":{"1min":0.27000000000000002,"5min":0.189,"15min":0.189},"max":{"1min":0.56799999999999995,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.311}]},{"osd":2,"last update":"Sat Sep 14 03:43:39 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.35499999999999998,"15min":0.34300000000000003},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.36199999999999999,"15min":0.34699999999999998},"min":{"1min":0.29499999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.32400000000000001}]},{"osd":3,"last update":"Sat Sep 14 03:44:07 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.42299999999999999,"15min":0.42199999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.435,"15min":0.437},"min":{"1min":0.27500000000000002,"5min":0.247,"15min":0.247},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.27200000000000002}]},{"osd":4,"last update":"Sat Sep 14 03:44:28 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.77200000000000002,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.58099999999999996}]},{"osd":5,"last update":"Sat Sep 14 03:43:47 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.55900000000000005}]},{"osd":6,"last update":"Sat Sep 14 03:44:10 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.52900000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48599999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738415,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298536,"kb_used_data":1176,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446616,"statfs":{"total":95995035648,"available":95689334784,"internally_reserved":0,"allocated":1204224,"data_stored":844621,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 14 03:44:17 2024","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":0.35699999999999998,"15min":0.34899999999999998},"min":{"1min":0.25700000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.46500000000000002,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.36299999999999999,"15min":0.34799999999999998},"min":{"1min":0.246,"5min":0.216,"15min":0.216},"max":{"1min":0.48999999999999999,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.252}]},{"osd":2,"last update":"Sat Sep 14 03:43:43 2024","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.41299999999999998,"15min":0.40999999999999998},"min":{"1min":0.29099999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.54900000000000004,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.39700000000000002,"15min":0.39500000000000002},"min":{"1min":0.28799999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.505,"5min":0.505,"15min":0.505},"last":0.35199999999999998}]},{"osd":3,"last update":"Sat Sep 14 03:44:04 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.44,"15min":0.44700000000000001},"min":{"1min":0.32500000000000001,"5min":0.307,"15min":0.307},"max":{"1min":0.54300000000000004,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.41499999999999998,"15min":0.42099999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.313}]},{"osd":4,"last update":"Sat Sep 14 03:44:26 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.55700000000000005,"15min":0.56499999999999995},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.70099999999999996,"5min":0.754,"15min":0.754},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.55600000000000005,"15min":0.55800000000000005},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.59199999999999997}]},{"osd":5,"last update":"Sat Sep 14 03:43:48 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.56499999999999995}]},{"osd":6,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.498},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.57699999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53000000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1642,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T03:44:32.077 INFO:tasks.cephadm.ceph_manager.ceph:PG 3.36 is not active+clean 2024-09-14T03:44:32.077 INFO:tasks.cephadm.ceph_manager.ceph:{'pgid': '3.36', 'version': "0'0", 'reported_seq': 16, 'reported_epoch': 60, 'state': 'activating', 'last_fresh': '2024-09-14T03:44:25.928176+0000', 'last_change': '2024-09-14T03:44:25.928176+0000', 'last_active': '2024-09-14T03:44:24.930301+0000', 'last_peered': '2024-09-14T03:44:03.881437+0000', 'last_clean': '2024-09-14T03:44:03.881437+0000', 'last_became_active': '2024-09-14T03:44:01.873644+0000', 'last_became_peered': '2024-09-14T03:44:01.873644+0000', 'last_unstale': '2024-09-14T03:44:25.928176+0000', 'last_undegraded': '2024-09-14T03:44:25.928176+0000', 'last_fullsized': '2024-09-14T03:44:25.928176+0000', 'mapping_epoch': 59, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 55, 'last_epoch_clean': 56, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-14T03:44:00.853212+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-14T03:44:00.853212+0000', 'last_clean_scrub_stamp': '2024-09-14T03:44:00.853212+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-15T03:59:34.312039+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [6, 2, 1], 'acting': [6, 2, 1], 'avail_no_missing': ['6', '1', '2', '5'], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 6, 'acting_primary': 6, 'purged_snaps': []} 2024-09-14T03:44:32.077 INFO:tasks.cephadm.ceph_manager.ceph:PG 3.26 is not active+clean 2024-09-14T03:44:32.077 INFO:tasks.cephadm.ceph_manager.ceph:{'pgid': '3.26', 'version': "0'0", 'reported_seq': 16, 'reported_epoch': 60, 'state': 'activating', 'last_fresh': '2024-09-14T03:44:25.928238+0000', 'last_change': '2024-09-14T03:44:25.928238+0000', 'last_active': '2024-09-14T03:44:24.929140+0000', 'last_peered': '2024-09-14T03:44:03.881581+0000', 'last_clean': '2024-09-14T03:44:03.881581+0000', 'last_became_active': '2024-09-14T03:44:01.877768+0000', 'last_became_peered': '2024-09-14T03:44:01.877768+0000', 'last_unstale': '2024-09-14T03:44:25.928238+0000', 'last_undegraded': '2024-09-14T03:44:25.928238+0000', 'last_fullsized': '2024-09-14T03:44:25.928238+0000', 'mapping_epoch': 59, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 55, 'last_epoch_clean': 56, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-09-14T03:44:00.853212+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-09-14T03:44:00.853212+0000', 'last_clean_scrub_stamp': '2024-09-14T03:44:00.853212+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2024-09-15T05:00:53.758328+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [6, 7, 3], 'acting': [6, 7, 3], 'avail_no_missing': ['6', '3', '4', '7'], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 6, 'acting_primary': 6, 'purged_snaps': []} 2024-09-14T03:44:32.078 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph status --format=json 2024-09-14T03:44:32.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:31 smithi094 ceph-mon[31053]: pgmap v203: 97 pgs: 2 activating, 95 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:32.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:31 smithi070 ceph-mon[27495]: pgmap v203: 97 pgs: 2 activating, 95 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:32.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:31 smithi070 ceph-mon[39288]: pgmap v203: 97 pgs: 2 activating, 95 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:32.318 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:44:33.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:32 smithi070 ceph-mon[27495]: from='client.14502 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:44:33.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:32 smithi070 ceph-mon[39288]: from='client.14502 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:44:33.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:32 smithi094 ceph-mon[31053]: from='client.14502 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:44:34.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:33 smithi070 ceph-mon[27495]: pgmap v204: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:34.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:33 smithi070 ceph-mon[39288]: pgmap v204: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:34.231 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:44:34.232 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"b6380d42-724a-11ef-bceb-c7b262605968","health":{"status":"HEALTH_OK","checks":{},"mutes":[]},"election_epoch":14,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":260,"monmap":{"epoch":3,"min_mon_release_name":"quincy","num_mons":3},"osdmap":{"epoch":60,"num_osds":8,"num_up_osds":8,"osd_up_since":1726285414,"num_in_osds":8,"osd_in_since":1726285397,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":97}],"num_pgs":97,"num_pools":3,"num_objects":24,"data_bytes":592706,"bytes_used":2442596352,"bytes_avail":765517688832,"bytes_total":767960285184},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"a","status":"up:active","gid":24323}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","restful"],"services":{"dashboard":"https://172.21.15.70:8443/"}},"servicemap":{"epoch":12,"modified":"2024-09-14T03:44:12.499673+0000","services":{"mds":{"daemons":{"summary":"","a":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}},"b":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}},"osd":{"daemons":{"summary":"","7":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}} 2024-09-14T03:44:34.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:33 smithi094 ceph-mon[31053]: pgmap v204: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:35.026 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2024-09-14T03:44:35.026 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T03:44:35.269 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:44:35.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:34 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/140284713' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2024-09-14T03:44:35.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:34 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/140284713' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2024-09-14T03:44:35.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:34 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/140284713' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2024-09-14T03:44:36.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:35 smithi094 ceph-mon[31053]: pgmap v205: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:36.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:35 smithi070 ceph-mon[27495]: pgmap v205: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:36.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:35 smithi070 ceph-mon[39288]: pgmap v205: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:37.158 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:44:37.159 INFO:teuthology.orchestra.run.smithi070.stderr:dumped all 2024-09-14T03:44:37.848 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":206,"stamp":"2024-09-14T03:44:36.505331+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":146,"num_write_kb":1641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111,"ondisk_log_size":111,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":291,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385348,"kb_used_data":6476,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575868,"statfs":{"total":767960285184,"available":765517688832,"internally_reserved":0,"allocated":6631424,"data_stored":3806118,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.579377"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933508+0000","last_change":"2024-09-14T03:44:01.882774+0000","last_active":"2024-09-14T03:44:24.933508+0000","last_peered":"2024-09-14T03:44:24.933508+0000","last_clean":"2024-09-14T03:44:24.933508+0000","last_became_active":"2024-09-14T03:44:01.882481+0000","last_became_peered":"2024-09-14T03:44:01.882481+0000","last_unstale":"2024-09-14T03:44:24.933508+0000","last_undegraded":"2024-09-14T03:44:24.933508+0000","last_fullsized":"2024-09-14T03:44:24.933508+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:49:25.209573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929696+0000","last_change":"2024-09-14T03:44:01.868415+0000","last_active":"2024-09-14T03:44:24.929696+0000","last_peered":"2024-09-14T03:44:24.929696+0000","last_clean":"2024-09-14T03:44:24.929696+0000","last_became_active":"2024-09-14T03:44:01.868152+0000","last_became_peered":"2024-09-14T03:44:01.868152+0000","last_unstale":"2024-09-14T03:44:24.929696+0000","last_undegraded":"2024-09-14T03:44:24.929696+0000","last_fullsized":"2024-09-14T03:44:24.929696+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:56:00.270070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929890+0000","last_change":"2024-09-14T03:44:01.871459+0000","last_active":"2024-09-14T03:44:24.929890+0000","last_peered":"2024-09-14T03:44:24.929890+0000","last_clean":"2024-09-14T03:44:24.929890+0000","last_became_active":"2024-09-14T03:44:01.870749+0000","last_became_peered":"2024-09-14T03:44:01.870749+0000","last_unstale":"2024-09-14T03:44:24.929890+0000","last_undegraded":"2024-09-14T03:44:24.929890+0000","last_fullsized":"2024-09-14T03:44:24.929890+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:54.391363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928911+0000","last_change":"2024-09-14T03:44:01.882503+0000","last_active":"2024-09-14T03:44:24.928911+0000","last_peered":"2024-09-14T03:44:24.928911+0000","last_clean":"2024-09-14T03:44:24.928911+0000","last_became_active":"2024-09-14T03:44:01.882250+0000","last_became_peered":"2024-09-14T03:44:01.882250+0000","last_unstale":"2024-09-14T03:44:24.928911+0000","last_undegraded":"2024-09-14T03:44:24.928911+0000","last_fullsized":"2024-09-14T03:44:24.928911+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:46:29.551285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.932873+0000","last_change":"2024-09-14T03:44:01.880333+0000","last_active":"2024-09-14T03:44:24.932873+0000","last_peered":"2024-09-14T03:44:24.932873+0000","last_clean":"2024-09-14T03:44:24.932873+0000","last_became_active":"2024-09-14T03:44:01.880010+0000","last_became_peered":"2024-09-14T03:44:01.880010+0000","last_unstale":"2024-09-14T03:44:24.932873+0000","last_undegraded":"2024-09-14T03:44:24.932873+0000","last_fullsized":"2024-09-14T03:44:24.932873+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:45:02.741425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928684+0000","last_change":"2024-09-14T03:44:01.880710+0000","last_active":"2024-09-14T03:44:24.928684+0000","last_peered":"2024-09-14T03:44:24.928684+0000","last_clean":"2024-09-14T03:44:24.928684+0000","last_became_active":"2024-09-14T03:44:01.880237+0000","last_became_peered":"2024-09-14T03:44:01.880237+0000","last_unstale":"2024-09-14T03:44:24.928684+0000","last_undegraded":"2024-09-14T03:44:24.928684+0000","last_fullsized":"2024-09-14T03:44:24.928684+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:56:49.808707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.932581+0000","last_change":"2024-09-14T03:44:01.878590+0000","last_active":"2024-09-14T03:44:24.932581+0000","last_peered":"2024-09-14T03:44:24.932581+0000","last_clean":"2024-09-14T03:44:24.932581+0000","last_became_active":"2024-09-14T03:44:01.878281+0000","last_became_peered":"2024-09-14T03:44:01.878281+0000","last_unstale":"2024-09-14T03:44:24.932581+0000","last_undegraded":"2024-09-14T03:44:24.932581+0000","last_fullsized":"2024-09-14T03:44:24.932581+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:41:57.164227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.925754+0000","last_change":"2024-09-14T03:44:01.877057+0000","last_active":"2024-09-14T03:44:24.925754+0000","last_peered":"2024-09-14T03:44:24.925754+0000","last_clean":"2024-09-14T03:44:24.925754+0000","last_became_active":"2024-09-14T03:44:01.876727+0000","last_became_peered":"2024-09-14T03:44:01.876727+0000","last_unstale":"2024-09-14T03:44:24.925754+0000","last_undegraded":"2024-09-14T03:44:24.925754+0000","last_fullsized":"2024-09-14T03:44:24.925754+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:31:24.716316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933832+0000","last_change":"2024-09-14T03:44:01.884865+0000","last_active":"2024-09-14T03:44:24.933832+0000","last_peered":"2024-09-14T03:44:24.933832+0000","last_clean":"2024-09-14T03:44:24.933832+0000","last_became_active":"2024-09-14T03:44:01.884136+0000","last_became_peered":"2024-09-14T03:44:01.884136+0000","last_unstale":"2024-09-14T03:44:24.933832+0000","last_undegraded":"2024-09-14T03:44:24.933832+0000","last_fullsized":"2024-09-14T03:44:24.933832+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:59.903382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":22,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-14T03:44:26.430052+0000","last_change":"2024-09-14T03:44:26.430052+0000","last_active":"2024-09-14T03:44:26.430052+0000","last_peered":"2024-09-14T03:44:26.430052+0000","last_clean":"2024-09-14T03:44:26.430052+0000","last_became_active":"2024-09-14T03:44:26.429658+0000","last_became_peered":"2024-09-14T03:44:26.429658+0000","last_unstale":"2024-09-14T03:44:26.430052+0000","last_undegraded":"2024-09-14T03:44:26.430052+0000","last_fullsized":"2024-09-14T03:44:26.430052+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:34.312039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929430+0000","last_change":"2024-09-14T03:44:01.875017+0000","last_active":"2024-09-14T03:44:24.929430+0000","last_peered":"2024-09-14T03:44:24.929430+0000","last_clean":"2024-09-14T03:44:24.929430+0000","last_became_active":"2024-09-14T03:44:01.874687+0000","last_became_peered":"2024-09-14T03:44:01.874687+0000","last_unstale":"2024-09-14T03:44:24.929430+0000","last_undegraded":"2024-09-14T03:44:24.929430+0000","last_fullsized":"2024-09-14T03:44:24.929430+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:29:09.035342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926391+0000","last_change":"2024-09-14T03:44:01.883313+0000","last_active":"2024-09-14T03:44:24.926391+0000","last_peered":"2024-09-14T03:44:24.926391+0000","last_clean":"2024-09-14T03:44:24.926391+0000","last_became_active":"2024-09-14T03:44:01.882770+0000","last_became_peered":"2024-09-14T03:44:01.882770+0000","last_unstale":"2024-09-14T03:44:24.926391+0000","last_undegraded":"2024-09-14T03:44:24.926391+0000","last_fullsized":"2024-09-14T03:44:24.926391+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:57:19.402319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926070+0000","last_change":"2024-09-14T03:44:01.878245+0000","last_active":"2024-09-14T03:44:24.926070+0000","last_peered":"2024-09-14T03:44:24.926070+0000","last_clean":"2024-09-14T03:44:24.926070+0000","last_became_active":"2024-09-14T03:44:01.877861+0000","last_became_peered":"2024-09-14T03:44:01.877861+0000","last_unstale":"2024-09-14T03:44:24.926070+0000","last_undegraded":"2024-09-14T03:44:24.926070+0000","last_fullsized":"2024-09-14T03:44:24.926070+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:08:29.902090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930473+0000","last_change":"2024-09-14T03:44:01.877391+0000","last_active":"2024-09-14T03:44:24.930473+0000","last_peered":"2024-09-14T03:44:24.930473+0000","last_clean":"2024-09-14T03:44:24.930473+0000","last_became_active":"2024-09-14T03:44:01.876797+0000","last_became_peered":"2024-09-14T03:44:01.876797+0000","last_unstale":"2024-09-14T03:44:24.930473+0000","last_undegraded":"2024-09-14T03:44:24.930473+0000","last_fullsized":"2024-09-14T03:44:24.930473+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:06:02.891762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928601+0000","last_change":"2024-09-14T03:44:01.876700+0000","last_active":"2024-09-14T03:44:24.928601+0000","last_peered":"2024-09-14T03:44:24.928601+0000","last_clean":"2024-09-14T03:44:24.928601+0000","last_became_active":"2024-09-14T03:44:01.876091+0000","last_became_peered":"2024-09-14T03:44:01.876091+0000","last_unstale":"2024-09-14T03:44:24.928601+0000","last_undegraded":"2024-09-14T03:44:24.928601+0000","last_fullsized":"2024-09-14T03:44:24.928601+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:25:41.750127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930443+0000","last_change":"2024-09-14T03:44:01.877120+0000","last_active":"2024-09-14T03:44:24.930443+0000","last_peered":"2024-09-14T03:44:24.930443+0000","last_clean":"2024-09-14T03:44:24.930443+0000","last_became_active":"2024-09-14T03:44:01.876793+0000","last_became_peered":"2024-09-14T03:44:01.876793+0000","last_unstale":"2024-09-14T03:44:24.930443+0000","last_undegraded":"2024-09-14T03:44:24.930443+0000","last_fullsized":"2024-09-14T03:44:24.930443+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:23:56.450728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930751+0000","last_change":"2024-09-14T03:44:01.885108+0000","last_active":"2024-09-14T03:44:24.930751+0000","last_peered":"2024-09-14T03:44:24.930751+0000","last_clean":"2024-09-14T03:44:24.930751+0000","last_became_active":"2024-09-14T03:44:01.884857+0000","last_became_peered":"2024-09-14T03:44:01.884857+0000","last_unstale":"2024-09-14T03:44:24.930751+0000","last_undegraded":"2024-09-14T03:44:24.930751+0000","last_fullsized":"2024-09-14T03:44:24.930751+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:31.448617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928709+0000","last_change":"2024-09-14T03:44:01.876503+0000","last_active":"2024-09-14T03:44:24.928709+0000","last_peered":"2024-09-14T03:44:24.928709+0000","last_clean":"2024-09-14T03:44:24.928709+0000","last_became_active":"2024-09-14T03:44:01.876227+0000","last_became_peered":"2024-09-14T03:44:01.876227+0000","last_unstale":"2024-09-14T03:44:24.928709+0000","last_undegraded":"2024-09-14T03:44:24.928709+0000","last_fullsized":"2024-09-14T03:44:24.928709+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:44:50.132152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933287+0000","last_change":"2024-09-14T03:44:01.883708+0000","last_active":"2024-09-14T03:44:24.933287+0000","last_peered":"2024-09-14T03:44:24.933287+0000","last_clean":"2024-09-14T03:44:24.933287+0000","last_became_active":"2024-09-14T03:44:01.883440+0000","last_became_peered":"2024-09-14T03:44:01.883440+0000","last_unstale":"2024-09-14T03:44:24.933287+0000","last_undegraded":"2024-09-14T03:44:24.933287+0000","last_fullsized":"2024-09-14T03:44:24.933287+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:10:34.658062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929540+0000","last_change":"2024-09-14T03:44:01.888565+0000","last_active":"2024-09-14T03:44:24.929540+0000","last_peered":"2024-09-14T03:44:24.929540+0000","last_clean":"2024-09-14T03:44:24.929540+0000","last_became_active":"2024-09-14T03:44:01.888191+0000","last_became_peered":"2024-09-14T03:44:01.888191+0000","last_unstale":"2024-09-14T03:44:24.929540+0000","last_undegraded":"2024-09-14T03:44:24.929540+0000","last_fullsized":"2024-09-14T03:44:24.929540+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:52:23.782664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930632+0000","last_change":"2024-09-14T03:44:01.874642+0000","last_active":"2024-09-14T03:44:24.930632+0000","last_peered":"2024-09-14T03:44:24.930632+0000","last_clean":"2024-09-14T03:44:24.930632+0000","last_became_active":"2024-09-14T03:44:01.874384+0000","last_became_peered":"2024-09-14T03:44:01.874384+0000","last_unstale":"2024-09-14T03:44:24.930632+0000","last_undegraded":"2024-09-14T03:44:24.930632+0000","last_fullsized":"2024-09-14T03:44:24.930632+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:35:05.535173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928767+0000","last_change":"2024-09-14T03:44:01.886893+0000","last_active":"2024-09-14T03:44:24.928767+0000","last_peered":"2024-09-14T03:44:24.928767+0000","last_clean":"2024-09-14T03:44:24.928767+0000","last_became_active":"2024-09-14T03:44:01.886594+0000","last_became_peered":"2024-09-14T03:44:01.886594+0000","last_unstale":"2024-09-14T03:44:24.928767+0000","last_undegraded":"2024-09-14T03:44:24.928767+0000","last_fullsized":"2024-09-14T03:44:24.928767+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:07:37.154358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.925870+0000","last_change":"2024-09-14T03:44:01.868698+0000","last_active":"2024-09-14T03:44:24.925870+0000","last_peered":"2024-09-14T03:44:24.925870+0000","last_clean":"2024-09-14T03:44:24.925870+0000","last_became_active":"2024-09-14T03:44:01.868442+0000","last_became_peered":"2024-09-14T03:44:01.868442+0000","last_unstale":"2024-09-14T03:44:24.925870+0000","last_undegraded":"2024-09-14T03:44:24.925870+0000","last_fullsized":"2024-09-14T03:44:24.925870+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:41:18.296424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930530+0000","last_change":"2024-09-14T03:44:01.882708+0000","last_active":"2024-09-14T03:44:24.930530+0000","last_peered":"2024-09-14T03:44:24.930530+0000","last_clean":"2024-09-14T03:44:24.930530+0000","last_became_active":"2024-09-14T03:44:01.882443+0000","last_became_peered":"2024-09-14T03:44:01.882443+0000","last_unstale":"2024-09-14T03:44:24.930530+0000","last_undegraded":"2024-09-14T03:44:24.930530+0000","last_fullsized":"2024-09-14T03:44:24.930530+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:46:48.306263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926452+0000","last_change":"2024-09-14T03:44:01.884940+0000","last_active":"2024-09-14T03:44:24.926452+0000","last_peered":"2024-09-14T03:44:24.926452+0000","last_clean":"2024-09-14T03:44:24.926452+0000","last_became_active":"2024-09-14T03:44:01.884304+0000","last_became_peered":"2024-09-14T03:44:01.884304+0000","last_unstale":"2024-09-14T03:44:24.926452+0000","last_undegraded":"2024-09-14T03:44:24.926452+0000","last_fullsized":"2024-09-14T03:44:24.926452+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:01:49.061637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":22,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-14T03:44:26.430891+0000","last_change":"2024-09-14T03:44:26.430891+0000","last_active":"2024-09-14T03:44:26.430891+0000","last_peered":"2024-09-14T03:44:26.430891+0000","last_clean":"2024-09-14T03:44:26.430891+0000","last_became_active":"2024-09-14T03:44:26.430426+0000","last_became_peered":"2024-09-14T03:44:26.430426+0000","last_unstale":"2024-09-14T03:44:26.430891+0000","last_undegraded":"2024-09-14T03:44:26.430891+0000","last_fullsized":"2024-09-14T03:44:26.430891+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:00:53.758328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928051+0000","last_change":"2024-09-14T03:44:01.874564+0000","last_active":"2024-09-14T03:44:24.928051+0000","last_peered":"2024-09-14T03:44:24.928051+0000","last_clean":"2024-09-14T03:44:24.928051+0000","last_became_active":"2024-09-14T03:44:01.874297+0000","last_became_peered":"2024-09-14T03:44:01.874297+0000","last_unstale":"2024-09-14T03:44:24.928051+0000","last_undegraded":"2024-09-14T03:44:24.928051+0000","last_fullsized":"2024-09-14T03:44:24.928051+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:07.027610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926133+0000","last_change":"2024-09-14T03:44:01.884486+0000","last_active":"2024-09-14T03:44:24.926133+0000","last_peered":"2024-09-14T03:44:24.926133+0000","last_clean":"2024-09-14T03:44:24.926133+0000","last_became_active":"2024-09-14T03:44:01.884218+0000","last_became_peered":"2024-09-14T03:44:01.884218+0000","last_unstale":"2024-09-14T03:44:24.926133+0000","last_undegraded":"2024-09-14T03:44:24.926133+0000","last_fullsized":"2024-09-14T03:44:24.926133+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:59:18.153239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930230+0000","last_change":"2024-09-14T03:44:01.878737+0000","last_active":"2024-09-14T03:44:24.930230+0000","last_peered":"2024-09-14T03:44:24.930230+0000","last_clean":"2024-09-14T03:44:24.930230+0000","last_became_active":"2024-09-14T03:44:01.878504+0000","last_became_peered":"2024-09-14T03:44:01.878504+0000","last_unstale":"2024-09-14T03:44:24.930230+0000","last_undegraded":"2024-09-14T03:44:24.930230+0000","last_fullsized":"2024-09-14T03:44:24.930230+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:41:35.420957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928860+0000","last_change":"2024-09-14T03:44:01.880509+0000","last_active":"2024-09-14T03:44:24.928860+0000","last_peered":"2024-09-14T03:44:24.928860+0000","last_clean":"2024-09-14T03:44:24.928860+0000","last_became_active":"2024-09-14T03:44:01.879855+0000","last_became_peered":"2024-09-14T03:44:01.879855+0000","last_unstale":"2024-09-14T03:44:24.928860+0000","last_undegraded":"2024-09-14T03:44:24.928860+0000","last_fullsized":"2024-09-14T03:44:24.928860+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:05:31.166902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928290+0000","last_change":"2024-09-14T03:44:01.869523+0000","last_active":"2024-09-14T03:44:24.928290+0000","last_peered":"2024-09-14T03:44:24.928290+0000","last_clean":"2024-09-14T03:44:24.928290+0000","last_became_active":"2024-09-14T03:44:01.869225+0000","last_became_peered":"2024-09-14T03:44:01.869225+0000","last_unstale":"2024-09-14T03:44:24.928290+0000","last_undegraded":"2024-09-14T03:44:24.928290+0000","last_fullsized":"2024-09-14T03:44:24.928290+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:43:15.638187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929040+0000","last_change":"2024-09-14T03:44:01.876014+0000","last_active":"2024-09-14T03:44:24.929040+0000","last_peered":"2024-09-14T03:44:24.929040+0000","last_clean":"2024-09-14T03:44:24.929040+0000","last_became_active":"2024-09-14T03:44:01.875496+0000","last_became_peered":"2024-09-14T03:44:01.875496+0000","last_unstale":"2024-09-14T03:44:24.929040+0000","last_undegraded":"2024-09-14T03:44:24.929040+0000","last_fullsized":"2024-09-14T03:44:24.929040+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:03:45.344199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926322+0000","last_change":"2024-09-14T03:44:01.865496+0000","last_active":"2024-09-14T03:44:24.926322+0000","last_peered":"2024-09-14T03:44:24.926322+0000","last_clean":"2024-09-14T03:44:24.926322+0000","last_became_active":"2024-09-14T03:44:01.865190+0000","last_became_peered":"2024-09-14T03:44:01.865190+0000","last_unstale":"2024-09-14T03:44:24.926322+0000","last_undegraded":"2024-09-14T03:44:24.926322+0000","last_fullsized":"2024-09-14T03:44:24.926322+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:18:50.037675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"58'1","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926154+0000","last_change":"2024-09-14T03:43:58.729188+0000","last_active":"2024-09-14T03:44:24.926154+0000","last_peered":"2024-09-14T03:44:24.926154+0000","last_clean":"2024-09-14T03:44:24.926154+0000","last_became_active":"2024-09-14T03:43:58.728927+0000","last_became_peered":"2024-09-14T03:43:58.728927+0000","last_unstale":"2024-09-14T03:44:24.926154+0000","last_undegraded":"2024-09-14T03:44:24.926154+0000","last_fullsized":"2024-09-14T03:44:24.926154+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:13:16.419701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930296+0000","last_change":"2024-09-14T03:44:01.879101+0000","last_active":"2024-09-14T03:44:24.930296+0000","last_peered":"2024-09-14T03:44:24.930296+0000","last_clean":"2024-09-14T03:44:24.930296+0000","last_became_active":"2024-09-14T03:44:01.878801+0000","last_became_peered":"2024-09-14T03:44:01.878801+0000","last_unstale":"2024-09-14T03:44:24.930296+0000","last_undegraded":"2024-09-14T03:44:24.930296+0000","last_fullsized":"2024-09-14T03:44:24.930296+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:13.447251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"58'1","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930120+0000","last_change":"2024-09-14T03:43:58.730444+0000","last_active":"2024-09-14T03:44:24.930120+0000","last_peered":"2024-09-14T03:44:24.930120+0000","last_clean":"2024-09-14T03:44:24.930120+0000","last_became_active":"2024-09-14T03:43:58.730172+0000","last_became_peered":"2024-09-14T03:43:58.730172+0000","last_unstale":"2024-09-14T03:44:24.930120+0000","last_undegraded":"2024-09-14T03:44:24.930120+0000","last_fullsized":"2024-09-14T03:44:24.930120+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:03:44.810562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.925963+0000","last_change":"2024-09-14T03:43:58.731458+0000","last_active":"2024-09-14T03:44:24.925963+0000","last_peered":"2024-09-14T03:44:24.925963+0000","last_clean":"2024-09-14T03:44:24.925963+0000","last_became_active":"2024-09-14T03:43:58.731206+0000","last_became_peered":"2024-09-14T03:43:58.731206+0000","last_unstale":"2024-09-14T03:44:24.925963+0000","last_undegraded":"2024-09-14T03:44:24.925963+0000","last_fullsized":"2024-09-14T03:44:24.925963+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:37:06.670539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.925827+0000","last_change":"2024-09-14T03:44:01.869781+0000","last_active":"2024-09-14T03:44:24.925827+0000","last_peered":"2024-09-14T03:44:24.925827+0000","last_clean":"2024-09-14T03:44:24.925827+0000","last_became_active":"2024-09-14T03:44:01.869521+0000","last_became_peered":"2024-09-14T03:44:01.869521+0000","last_unstale":"2024-09-14T03:44:24.925827+0000","last_undegraded":"2024-09-14T03:44:24.925827+0000","last_fullsized":"2024-09-14T03:44:24.925827+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:35:18.565206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":25,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-14T03:44:25.932578+0000","last_change":"2024-09-14T03:44:25.932578+0000","last_active":"2024-09-14T03:44:25.932578+0000","last_peered":"2024-09-14T03:44:25.932578+0000","last_clean":"2024-09-14T03:44:25.932578+0000","last_became_active":"2024-09-14T03:44:25.932288+0000","last_became_peered":"2024-09-14T03:44:25.932288+0000","last_unstale":"2024-09-14T03:44:25.932578+0000","last_undegraded":"2024-09-14T03:44:25.932578+0000","last_fullsized":"2024-09-14T03:44:25.932578+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:24:41.091353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928474+0000","last_change":"2024-09-14T03:44:01.873748+0000","last_active":"2024-09-14T03:44:24.928474+0000","last_peered":"2024-09-14T03:44:24.928474+0000","last_clean":"2024-09-14T03:44:24.928474+0000","last_became_active":"2024-09-14T03:44:01.873473+0000","last_became_peered":"2024-09-14T03:44:01.873473+0000","last_unstale":"2024-09-14T03:44:24.928474+0000","last_undegraded":"2024-09-14T03:44:24.928474+0000","last_fullsized":"2024-09-14T03:44:24.928474+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:04:39.289903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929850+0000","last_change":"2024-09-14T03:43:58.735608+0000","last_active":"2024-09-14T03:44:24.929850+0000","last_peered":"2024-09-14T03:44:24.929850+0000","last_clean":"2024-09-14T03:44:24.929850+0000","last_became_active":"2024-09-14T03:43:58.735301+0000","last_became_peered":"2024-09-14T03:43:58.735301+0000","last_unstale":"2024-09-14T03:44:24.929850+0000","last_undegraded":"2024-09-14T03:44:24.929850+0000","last_fullsized":"2024-09-14T03:44:24.929850+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:44:56.523073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930382+0000","last_change":"2024-09-14T03:44:01.874777+0000","last_active":"2024-09-14T03:44:24.930382+0000","last_peered":"2024-09-14T03:44:24.930382+0000","last_clean":"2024-09-14T03:44:24.930382+0000","last_became_active":"2024-09-14T03:44:01.874473+0000","last_became_peered":"2024-09-14T03:44:01.874473+0000","last_unstale":"2024-09-14T03:44:24.930382+0000","last_undegraded":"2024-09-14T03:44:24.930382+0000","last_fullsized":"2024-09-14T03:44:24.930382+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:49:02.042635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930539+0000","last_change":"2024-09-14T03:43:58.732969+0000","last_active":"2024-09-14T03:44:24.930539+0000","last_peered":"2024-09-14T03:44:24.930539+0000","last_clean":"2024-09-14T03:44:24.930539+0000","last_became_active":"2024-09-14T03:43:58.732492+0000","last_became_peered":"2024-09-14T03:43:58.732492+0000","last_unstale":"2024-09-14T03:44:24.930539+0000","last_undegraded":"2024-09-14T03:44:24.930539+0000","last_fullsized":"2024-09-14T03:44:24.930539+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:50:28.719447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930714+0000","last_change":"2024-09-14T03:44:01.875679+0000","last_active":"2024-09-14T03:44:24.930714+0000","last_peered":"2024-09-14T03:44:24.930714+0000","last_clean":"2024-09-14T03:44:24.930714+0000","last_became_active":"2024-09-14T03:44:01.875422+0000","last_became_peered":"2024-09-14T03:44:01.875422+0000","last_unstale":"2024-09-14T03:44:24.930714+0000","last_undegraded":"2024-09-14T03:44:24.930714+0000","last_fullsized":"2024-09-14T03:44:24.930714+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:13:00.349373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"58'3","reported_seq":18,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929910+0000","last_change":"2024-09-14T03:43:58.735066+0000","last_active":"2024-09-14T03:44:24.929910+0000","last_peered":"2024-09-14T03:44:24.929910+0000","last_clean":"2024-09-14T03:44:24.929910+0000","last_became_active":"2024-09-14T03:43:58.734514+0000","last_became_peered":"2024-09-14T03:43:58.734514+0000","last_unstale":"2024-09-14T03:44:24.929910+0000","last_undegraded":"2024-09-14T03:44:24.929910+0000","last_fullsized":"2024-09-14T03:44:24.929910+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:41:55.478295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926573+0000","last_change":"2024-09-14T03:44:01.881319+0000","last_active":"2024-09-14T03:44:24.926573+0000","last_peered":"2024-09-14T03:44:24.926573+0000","last_clean":"2024-09-14T03:44:24.926573+0000","last_became_active":"2024-09-14T03:44:01.881032+0000","last_became_peered":"2024-09-14T03:44:01.881032+0000","last_unstale":"2024-09-14T03:44:24.926573+0000","last_undegraded":"2024-09-14T03:44:24.926573+0000","last_fullsized":"2024-09-14T03:44:24.926573+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:35:41.762014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"58'6","reported_seq":21,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929594+0000","last_change":"2024-09-14T03:43:58.734778+0000","last_active":"2024-09-14T03:44:24.929594+0000","last_peered":"2024-09-14T03:44:24.929594+0000","last_clean":"2024-09-14T03:44:24.929594+0000","last_became_active":"2024-09-14T03:43:58.734380+0000","last_became_peered":"2024-09-14T03:43:58.734380+0000","last_unstale":"2024-09-14T03:44:24.929594+0000","last_undegraded":"2024-09-14T03:44:24.929594+0000","last_fullsized":"2024-09-14T03:44:24.929594+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:55:13.561065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933222+0000","last_change":"2024-09-14T03:44:01.880139+0000","last_active":"2024-09-14T03:44:24.933222+0000","last_peered":"2024-09-14T03:44:24.933222+0000","last_clean":"2024-09-14T03:44:24.933222+0000","last_became_active":"2024-09-14T03:44:01.879840+0000","last_became_peered":"2024-09-14T03:44:01.879840+0000","last_unstale":"2024-09-14T03:44:24.933222+0000","last_undegraded":"2024-09-14T03:44:24.933222+0000","last_fullsized":"2024-09-14T03:44:24.933222+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:31:34.003676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"58'2","reported_seq":17,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930377+0000","last_change":"2024-09-14T03:43:58.728768+0000","last_active":"2024-09-14T03:44:24.930377+0000","last_peered":"2024-09-14T03:44:24.930377+0000","last_clean":"2024-09-14T03:44:24.930377+0000","last_became_active":"2024-09-14T03:43:58.727893+0000","last_became_peered":"2024-09-14T03:43:58.727893+0000","last_unstale":"2024-09-14T03:44:24.930377+0000","last_undegraded":"2024-09-14T03:44:24.930377+0000","last_fullsized":"2024-09-14T03:44:24.930377+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:54:48.554486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930509+0000","last_change":"2024-09-14T03:44:01.877540+0000","last_active":"2024-09-14T03:44:24.930509+0000","last_peered":"2024-09-14T03:44:24.930509+0000","last_clean":"2024-09-14T03:44:24.930509+0000","last_became_active":"2024-09-14T03:44:01.877147+0000","last_became_peered":"2024-09-14T03:44:01.877147+0000","last_unstale":"2024-09-14T03:44:24.930509+0000","last_undegraded":"2024-09-14T03:44:24.930509+0000","last_fullsized":"2024-09-14T03:44:24.930509+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:17:57.724648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928948+0000","last_change":"2024-09-14T03:43:58.732812+0000","last_active":"2024-09-14T03:44:24.928948+0000","last_peered":"2024-09-14T03:44:24.928948+0000","last_clean":"2024-09-14T03:44:24.928948+0000","last_became_active":"2024-09-14T03:43:58.732537+0000","last_became_peered":"2024-09-14T03:43:58.732537+0000","last_unstale":"2024-09-14T03:44:24.928948+0000","last_undegraded":"2024-09-14T03:44:24.928948+0000","last_fullsized":"2024-09-14T03:44:24.928948+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:26:11.551182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.932936+0000","last_change":"2024-09-14T03:44:01.881140+0000","last_active":"2024-09-14T03:44:24.932936+0000","last_peered":"2024-09-14T03:44:24.932936+0000","last_clean":"2024-09-14T03:44:24.932936+0000","last_became_active":"2024-09-14T03:44:01.880868+0000","last_became_peered":"2024-09-14T03:44:01.880868+0000","last_unstale":"2024-09-14T03:44:24.932936+0000","last_undegraded":"2024-09-14T03:44:24.932936+0000","last_fullsized":"2024-09-14T03:44:24.932936+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:29:05.729804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929350+0000","last_change":"2024-09-14T03:43:58.730505+0000","last_active":"2024-09-14T03:44:24.929350+0000","last_peered":"2024-09-14T03:44:24.929350+0000","last_clean":"2024-09-14T03:44:24.929350+0000","last_became_active":"2024-09-14T03:43:58.729821+0000","last_became_peered":"2024-09-14T03:43:58.729821+0000","last_unstale":"2024-09-14T03:44:24.929350+0000","last_undegraded":"2024-09-14T03:44:24.929350+0000","last_fullsized":"2024-09-14T03:44:24.929350+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:17:44.091667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929155+0000","last_change":"2024-09-14T03:44:01.874477+0000","last_active":"2024-09-14T03:44:24.929155+0000","last_peered":"2024-09-14T03:44:24.929155+0000","last_clean":"2024-09-14T03:44:24.929155+0000","last_became_active":"2024-09-14T03:44:01.873714+0000","last_became_peered":"2024-09-14T03:44:01.873714+0000","last_unstale":"2024-09-14T03:44:24.929155+0000","last_undegraded":"2024-09-14T03:44:24.929155+0000","last_fullsized":"2024-09-14T03:44:24.929155+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:54:21.601768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"58'8","reported_seq":23,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928144+0000","last_change":"2024-09-14T03:43:58.726387+0000","last_active":"2024-09-14T03:44:24.928144+0000","last_peered":"2024-09-14T03:44:24.928144+0000","last_clean":"2024-09-14T03:44:24.928144+0000","last_became_active":"2024-09-14T03:43:58.726115+0000","last_became_peered":"2024-09-14T03:43:58.726115+0000","last_unstale":"2024-09-14T03:44:24.928144+0000","last_undegraded":"2024-09-14T03:44:24.928144+0000","last_fullsized":"2024-09-14T03:44:24.928144+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:18:04.176970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.932701+0000","last_change":"2024-09-14T03:44:01.878624+0000","last_active":"2024-09-14T03:44:24.932701+0000","last_peered":"2024-09-14T03:44:24.932701+0000","last_clean":"2024-09-14T03:44:24.932701+0000","last_became_active":"2024-09-14T03:44:01.878343+0000","last_became_peered":"2024-09-14T03:44:01.878343+0000","last_unstale":"2024-09-14T03:44:24.932701+0000","last_undegraded":"2024-09-14T03:44:24.932701+0000","last_fullsized":"2024-09-14T03:44:24.932701+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:44:05.205671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"58'1","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926865+0000","last_change":"2024-09-14T03:43:58.728489+0000","last_active":"2024-09-14T03:44:24.926865+0000","last_peered":"2024-09-14T03:44:24.926865+0000","last_clean":"2024-09-14T03:44:24.926865+0000","last_became_active":"2024-09-14T03:43:58.728182+0000","last_became_peered":"2024-09-14T03:43:58.728182+0000","last_unstale":"2024-09-14T03:44:24.926865+0000","last_undegraded":"2024-09-14T03:44:24.926865+0000","last_fullsized":"2024-09-14T03:44:24.926865+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:56.310710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933262+0000","last_change":"2024-09-14T03:44:01.880754+0000","last_active":"2024-09-14T03:44:24.933262+0000","last_peered":"2024-09-14T03:44:24.933262+0000","last_clean":"2024-09-14T03:44:24.933262+0000","last_became_active":"2024-09-14T03:44:01.880483+0000","last_became_peered":"2024-09-14T03:44:01.880483+0000","last_unstale":"2024-09-14T03:44:24.933262+0000","last_undegraded":"2024-09-14T03:44:24.933262+0000","last_fullsized":"2024-09-14T03:44:24.933262+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:32:53.245007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"58'2","reported_seq":17,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928841+0000","last_change":"2024-09-14T03:43:58.726366+0000","last_active":"2024-09-14T03:44:24.928841+0000","last_peered":"2024-09-14T03:44:24.928841+0000","last_clean":"2024-09-14T03:44:24.928841+0000","last_became_active":"2024-09-14T03:43:58.726090+0000","last_became_peered":"2024-09-14T03:43:58.726090+0000","last_unstale":"2024-09-14T03:44:24.928841+0000","last_undegraded":"2024-09-14T03:44:24.928841+0000","last_fullsized":"2024-09-14T03:44:24.928841+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:45:33.572820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933584+0000","last_change":"2024-09-14T03:44:01.883704+0000","last_active":"2024-09-14T03:44:24.933584+0000","last_peered":"2024-09-14T03:44:24.933584+0000","last_clean":"2024-09-14T03:44:24.933584+0000","last_became_active":"2024-09-14T03:44:01.883448+0000","last_became_peered":"2024-09-14T03:44:01.883448+0000","last_unstale":"2024-09-14T03:44:24.933584+0000","last_undegraded":"2024-09-14T03:44:24.933584+0000","last_fullsized":"2024-09-14T03:44:24.933584+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:16:27.946770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"58'1","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929063+0000","last_change":"2024-09-14T03:43:58.733205+0000","last_active":"2024-09-14T03:44:24.929063+0000","last_peered":"2024-09-14T03:44:24.929063+0000","last_clean":"2024-09-14T03:44:24.929063+0000","last_became_active":"2024-09-14T03:43:58.732929+0000","last_became_peered":"2024-09-14T03:43:58.732929+0000","last_unstale":"2024-09-14T03:44:24.929063+0000","last_undegraded":"2024-09-14T03:44:24.929063+0000","last_fullsized":"2024-09-14T03:44:24.929063+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:31:13.660965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929096+0000","last_change":"2024-09-14T03:44:01.886158+0000","last_active":"2024-09-14T03:44:24.929096+0000","last_peered":"2024-09-14T03:44:24.929096+0000","last_clean":"2024-09-14T03:44:24.929096+0000","last_became_active":"2024-09-14T03:44:01.884747+0000","last_became_peered":"2024-09-14T03:44:01.884747+0000","last_unstale":"2024-09-14T03:44:24.929096+0000","last_undegraded":"2024-09-14T03:44:24.929096+0000","last_fullsized":"2024-09-14T03:44:24.929096+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:49:56.265783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928891+0000","last_change":"2024-09-14T03:43:58.732401+0000","last_active":"2024-09-14T03:44:24.928891+0000","last_peered":"2024-09-14T03:44:24.928891+0000","last_clean":"2024-09-14T03:44:24.928891+0000","last_became_active":"2024-09-14T03:43:58.732067+0000","last_became_peered":"2024-09-14T03:43:58.732067+0000","last_unstale":"2024-09-14T03:44:24.928891+0000","last_undegraded":"2024-09-14T03:44:24.928891+0000","last_fullsized":"2024-09-14T03:44:24.928891+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:34:00.566245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933105+0000","last_change":"2024-09-14T03:44:01.881736+0000","last_active":"2024-09-14T03:44:24.933105+0000","last_peered":"2024-09-14T03:44:24.933105+0000","last_clean":"2024-09-14T03:44:24.933105+0000","last_became_active":"2024-09-14T03:44:01.881219+0000","last_became_peered":"2024-09-14T03:44:01.881219+0000","last_unstale":"2024-09-14T03:44:24.933105+0000","last_undegraded":"2024-09-14T03:44:24.933105+0000","last_fullsized":"2024-09-14T03:44:24.933105+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:24:23.342066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"22'72","reported_seq":25,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933249+0000","last_change":"2024-09-14T03:43:36.147227+0000","last_active":"2024-09-14T03:44:24.933249+0000","last_peered":"2024-09-14T03:44:24.933249+0000","last_clean":"2024-09-14T03:44:24.933249+0000","last_became_active":"2024-09-14T03:43:36.141953+0000","last_became_peered":"2024-09-14T03:43:36.141953+0000","last_unstale":"2024-09-14T03:44:24.933249+0000","last_undegraded":"2024-09-14T03:44:24.933249+0000","last_fullsized":"2024-09-14T03:44:24.933249+0000","mapping_epoch":50,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":51,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:39.458815+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:39.458815+0000","last_clean_scrub_stamp":"2024-09-14T03:41:39.458815+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:38:43.056328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930279+0000","last_change":"2024-09-14T03:43:58.733557+0000","last_active":"2024-09-14T03:44:24.930279+0000","last_peered":"2024-09-14T03:44:24.930279+0000","last_clean":"2024-09-14T03:44:24.930279+0000","last_became_active":"2024-09-14T03:43:58.733256+0000","last_became_peered":"2024-09-14T03:43:58.733256+0000","last_unstale":"2024-09-14T03:44:24.930279+0000","last_undegraded":"2024-09-14T03:44:24.930279+0000","last_fullsized":"2024-09-14T03:44:24.930279+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:09:12.987156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.925983+0000","last_change":"2024-09-14T03:44:01.875525+0000","last_active":"2024-09-14T03:44:24.925983+0000","last_peered":"2024-09-14T03:44:24.925983+0000","last_clean":"2024-09-14T03:44:24.925983+0000","last_became_active":"2024-09-14T03:44:01.875241+0000","last_became_peered":"2024-09-14T03:44:01.875241+0000","last_unstale":"2024-09-14T03:44:24.925983+0000","last_undegraded":"2024-09-14T03:44:24.925983+0000","last_fullsized":"2024-09-14T03:44:24.925983+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:34:47.346963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933143+0000","last_change":"2024-09-14T03:43:58.732376+0000","last_active":"2024-09-14T03:44:24.933143+0000","last_peered":"2024-09-14T03:44:24.933143+0000","last_clean":"2024-09-14T03:44:24.933143+0000","last_became_active":"2024-09-14T03:43:58.731812+0000","last_became_peered":"2024-09-14T03:43:58.731812+0000","last_unstale":"2024-09-14T03:44:24.933143+0000","last_undegraded":"2024-09-14T03:44:24.933143+0000","last_fullsized":"2024-09-14T03:44:24.933143+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:35:26.262383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930295+0000","last_change":"2024-09-14T03:44:01.876511+0000","last_active":"2024-09-14T03:44:24.930295+0000","last_peered":"2024-09-14T03:44:24.930295+0000","last_clean":"2024-09-14T03:44:24.930295+0000","last_became_active":"2024-09-14T03:44:01.876225+0000","last_became_peered":"2024-09-14T03:44:01.876225+0000","last_unstale":"2024-09-14T03:44:24.930295+0000","last_undegraded":"2024-09-14T03:44:24.930295+0000","last_fullsized":"2024-09-14T03:44:24.930295+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:36:40.283622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928369+0000","last_change":"2024-09-14T03:43:58.727129+0000","last_active":"2024-09-14T03:44:24.928369+0000","last_peered":"2024-09-14T03:44:24.928369+0000","last_clean":"2024-09-14T03:44:24.928369+0000","last_became_active":"2024-09-14T03:43:58.726879+0000","last_became_peered":"2024-09-14T03:43:58.726879+0000","last_unstale":"2024-09-14T03:44:24.928369+0000","last_undegraded":"2024-09-14T03:44:24.928369+0000","last_fullsized":"2024-09-14T03:44:24.928369+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:54:41.748651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928166+0000","last_change":"2024-09-14T03:44:01.871005+0000","last_active":"2024-09-14T03:44:24.928166+0000","last_peered":"2024-09-14T03:44:24.928166+0000","last_clean":"2024-09-14T03:44:24.928166+0000","last_became_active":"2024-09-14T03:44:01.870707+0000","last_became_peered":"2024-09-14T03:44:01.870707+0000","last_unstale":"2024-09-14T03:44:24.928166+0000","last_undegraded":"2024-09-14T03:44:24.928166+0000","last_fullsized":"2024-09-14T03:44:24.928166+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:30:22.758581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929035+0000","last_change":"2024-09-14T03:43:58.733038+0000","last_active":"2024-09-14T03:44:24.929035+0000","last_peered":"2024-09-14T03:44:24.929035+0000","last_clean":"2024-09-14T03:44:24.929035+0000","last_became_active":"2024-09-14T03:43:58.732753+0000","last_became_peered":"2024-09-14T03:43:58.732753+0000","last_unstale":"2024-09-14T03:44:24.929035+0000","last_undegraded":"2024-09-14T03:44:24.929035+0000","last_fullsized":"2024-09-14T03:44:24.929035+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:40:49.754211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":22,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-14T03:44:26.430320+0000","last_change":"2024-09-14T03:44:26.430320+0000","last_active":"2024-09-14T03:44:26.430320+0000","last_peered":"2024-09-14T03:44:26.430320+0000","last_clean":"2024-09-14T03:44:26.430320+0000","last_became_active":"2024-09-14T03:44:26.429935+0000","last_became_peered":"2024-09-14T03:44:26.429935+0000","last_unstale":"2024-09-14T03:44:26.430320+0000","last_undegraded":"2024-09-14T03:44:26.430320+0000","last_fullsized":"2024-09-14T03:44:26.430320+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:12:35.577538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929025+0000","last_change":"2024-09-14T03:43:58.733621+0000","last_active":"2024-09-14T03:44:24.929025+0000","last_peered":"2024-09-14T03:44:24.929025+0000","last_clean":"2024-09-14T03:44:24.929025+0000","last_became_active":"2024-09-14T03:43:58.733357+0000","last_became_peered":"2024-09-14T03:43:58.733357+0000","last_unstale":"2024-09-14T03:44:24.929025+0000","last_undegraded":"2024-09-14T03:44:24.929025+0000","last_fullsized":"2024-09-14T03:44:24.929025+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:25:53.791926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929608+0000","last_change":"2024-09-14T03:44:01.880200+0000","last_active":"2024-09-14T03:44:24.929608+0000","last_peered":"2024-09-14T03:44:24.929608+0000","last_clean":"2024-09-14T03:44:24.929608+0000","last_became_active":"2024-09-14T03:44:01.879925+0000","last_became_peered":"2024-09-14T03:44:01.879925+0000","last_unstale":"2024-09-14T03:44:24.929608+0000","last_undegraded":"2024-09-14T03:44:24.929608+0000","last_fullsized":"2024-09-14T03:44:24.929608+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:55:24.738215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933352+0000","last_change":"2024-09-14T03:43:58.733230+0000","last_active":"2024-09-14T03:44:24.933352+0000","last_peered":"2024-09-14T03:44:24.933352+0000","last_clean":"2024-09-14T03:44:24.933352+0000","last_became_active":"2024-09-14T03:43:58.732868+0000","last_became_peered":"2024-09-14T03:43:58.732868+0000","last_unstale":"2024-09-14T03:44:24.933352+0000","last_undegraded":"2024-09-14T03:44:24.933352+0000","last_fullsized":"2024-09-14T03:44:24.933352+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:15:12.184851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929025+0000","last_change":"2024-09-14T03:44:01.878736+0000","last_active":"2024-09-14T03:44:24.929025+0000","last_peered":"2024-09-14T03:44:24.929025+0000","last_clean":"2024-09-14T03:44:24.929025+0000","last_became_active":"2024-09-14T03:44:01.877390+0000","last_became_peered":"2024-09-14T03:44:01.877390+0000","last_unstale":"2024-09-14T03:44:24.929025+0000","last_undegraded":"2024-09-14T03:44:24.929025+0000","last_fullsized":"2024-09-14T03:44:24.929025+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:17:42.812575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"58'1","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929778+0000","last_change":"2024-09-14T03:43:58.733688+0000","last_active":"2024-09-14T03:44:24.929778+0000","last_peered":"2024-09-14T03:44:24.929778+0000","last_clean":"2024-09-14T03:44:24.929778+0000","last_became_active":"2024-09-14T03:43:58.733457+0000","last_became_peered":"2024-09-14T03:43:58.733457+0000","last_unstale":"2024-09-14T03:44:24.929778+0000","last_undegraded":"2024-09-14T03:44:24.929778+0000","last_fullsized":"2024-09-14T03:44:24.929778+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:09:01.069988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926470+0000","last_change":"2024-09-14T03:44:01.870045+0000","last_active":"2024-09-14T03:44:24.926470+0000","last_peered":"2024-09-14T03:44:24.926470+0000","last_clean":"2024-09-14T03:44:24.926470+0000","last_became_active":"2024-09-14T03:44:01.869780+0000","last_became_peered":"2024-09-14T03:44:01.869780+0000","last_unstale":"2024-09-14T03:44:24.926470+0000","last_undegraded":"2024-09-14T03:44:24.926470+0000","last_fullsized":"2024-09-14T03:44:24.926470+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:04:58.634704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"58'2","reported_seq":17,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929736+0000","last_change":"2024-09-14T03:43:58.735364+0000","last_active":"2024-09-14T03:44:24.929736+0000","last_peered":"2024-09-14T03:44:24.929736+0000","last_clean":"2024-09-14T03:44:24.929736+0000","last_became_active":"2024-09-14T03:43:58.734833+0000","last_became_peered":"2024-09-14T03:43:58.734833+0000","last_unstale":"2024-09-14T03:44:24.929736+0000","last_undegraded":"2024-09-14T03:44:24.929736+0000","last_fullsized":"2024-09-14T03:44:24.929736+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:58:37.428713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930534+0000","last_change":"2024-09-14T03:44:01.878573+0000","last_active":"2024-09-14T03:44:24.930534+0000","last_peered":"2024-09-14T03:44:24.930534+0000","last_clean":"2024-09-14T03:44:24.930534+0000","last_became_active":"2024-09-14T03:44:01.878316+0000","last_became_peered":"2024-09-14T03:44:01.878316+0000","last_unstale":"2024-09-14T03:44:24.930534+0000","last_undegraded":"2024-09-14T03:44:24.930534+0000","last_fullsized":"2024-09-14T03:44:24.930534+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:45:21.129163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933129+0000","last_change":"2024-09-14T03:43:58.730843+0000","last_active":"2024-09-14T03:44:24.933129+0000","last_peered":"2024-09-14T03:44:24.933129+0000","last_clean":"2024-09-14T03:44:24.933129+0000","last_became_active":"2024-09-14T03:43:58.730538+0000","last_became_peered":"2024-09-14T03:43:58.730538+0000","last_unstale":"2024-09-14T03:44:24.933129+0000","last_undegraded":"2024-09-14T03:44:24.933129+0000","last_fullsized":"2024-09-14T03:44:24.933129+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:04:57.059345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926481+0000","last_change":"2024-09-14T03:44:01.874355+0000","last_active":"2024-09-14T03:44:24.926481+0000","last_peered":"2024-09-14T03:44:24.926481+0000","last_clean":"2024-09-14T03:44:24.926481+0000","last_became_active":"2024-09-14T03:44:01.874049+0000","last_became_peered":"2024-09-14T03:44:01.874049+0000","last_unstale":"2024-09-14T03:44:24.926481+0000","last_undegraded":"2024-09-14T03:44:24.926481+0000","last_fullsized":"2024-09-14T03:44:24.926481+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:53:30.306175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928677+0000","last_change":"2024-09-14T03:43:58.728080+0000","last_active":"2024-09-14T03:44:24.928677+0000","last_peered":"2024-09-14T03:44:24.928677+0000","last_clean":"2024-09-14T03:44:24.928677+0000","last_became_active":"2024-09-14T03:43:58.727820+0000","last_became_peered":"2024-09-14T03:43:58.727820+0000","last_unstale":"2024-09-14T03:44:24.928677+0000","last_undegraded":"2024-09-14T03:44:24.928677+0000","last_fullsized":"2024-09-14T03:44:24.928677+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:37:58.166430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.925857+0000","last_change":"2024-09-14T03:44:01.869311+0000","last_active":"2024-09-14T03:44:24.925857+0000","last_peered":"2024-09-14T03:44:24.925857+0000","last_clean":"2024-09-14T03:44:24.925857+0000","last_became_active":"2024-09-14T03:44:01.869065+0000","last_became_peered":"2024-09-14T03:44:01.869065+0000","last_unstale":"2024-09-14T03:44:24.925857+0000","last_undegraded":"2024-09-14T03:44:24.925857+0000","last_fullsized":"2024-09-14T03:44:24.925857+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:45:09.508019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929475+0000","last_change":"2024-09-14T03:43:58.728931+0000","last_active":"2024-09-14T03:44:24.929475+0000","last_peered":"2024-09-14T03:44:24.929475+0000","last_clean":"2024-09-14T03:44:24.929475+0000","last_became_active":"2024-09-14T03:43:58.728647+0000","last_became_peered":"2024-09-14T03:43:58.728647+0000","last_unstale":"2024-09-14T03:44:24.929475+0000","last_undegraded":"2024-09-14T03:44:24.929475+0000","last_fullsized":"2024-09-14T03:44:24.929475+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:09:38.769523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":23,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-14T03:44:26.430230+0000","last_change":"2024-09-14T03:44:26.430230+0000","last_active":"2024-09-14T03:44:26.430230+0000","last_peered":"2024-09-14T03:44:26.430230+0000","last_clean":"2024-09-14T03:44:26.430230+0000","last_became_active":"2024-09-14T03:44:26.429880+0000","last_became_peered":"2024-09-14T03:44:26.429880+0000","last_unstale":"2024-09-14T03:44:26.430230+0000","last_undegraded":"2024-09-14T03:44:26.430230+0000","last_fullsized":"2024-09-14T03:44:26.430230+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:39.132572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"58'2","reported_seq":18,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933131+0000","last_change":"2024-09-14T03:43:58.737705+0000","last_active":"2024-09-14T03:44:24.933131+0000","last_peered":"2024-09-14T03:44:24.933131+0000","last_clean":"2024-09-14T03:44:24.933131+0000","last_became_active":"2024-09-14T03:43:58.736546+0000","last_became_peered":"2024-09-14T03:43:58.736546+0000","last_unstale":"2024-09-14T03:44:24.933131+0000","last_undegraded":"2024-09-14T03:44:24.933131+0000","last_fullsized":"2024-09-14T03:44:24.933131+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:27:12.542706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":10,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-14T03:44:26.429678+0000","last_change":"2024-09-14T03:44:26.429678+0000","last_active":"2024-09-14T03:44:26.429678+0000","last_peered":"2024-09-14T03:44:26.429678+0000","last_clean":"2024-09-14T03:44:26.429678+0000","last_became_active":"2024-09-14T03:44:26.429364+0000","last_became_peered":"2024-09-14T03:44:26.429364+0000","last_unstale":"2024-09-14T03:44:26.429678+0000","last_undegraded":"2024-09-14T03:44:26.429678+0000","last_fullsized":"2024-09-14T03:44:26.429678+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:29:31.230108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929526+0000","last_change":"2024-09-14T03:44:01.888591+0000","last_active":"2024-09-14T03:44:24.929526+0000","last_peered":"2024-09-14T03:44:24.929526+0000","last_clean":"2024-09-14T03:44:24.929526+0000","last_became_active":"2024-09-14T03:44:01.888257+0000","last_became_peered":"2024-09-14T03:44:01.888257+0000","last_unstale":"2024-09-14T03:44:24.929526+0000","last_undegraded":"2024-09-14T03:44:24.929526+0000","last_fullsized":"2024-09-14T03:44:24.929526+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:24:40.992358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"58'2","reported_seq":18,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933222+0000","last_change":"2024-09-14T03:43:58.734680+0000","last_active":"2024-09-14T03:44:24.933222+0000","last_peered":"2024-09-14T03:44:24.933222+0000","last_clean":"2024-09-14T03:44:24.933222+0000","last_became_active":"2024-09-14T03:43:58.734408+0000","last_became_peered":"2024-09-14T03:43:58.734408+0000","last_unstale":"2024-09-14T03:44:24.933222+0000","last_undegraded":"2024-09-14T03:44:24.933222+0000","last_fullsized":"2024-09-14T03:44:24.933222+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:49:02.536404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929006+0000","last_change":"2024-09-14T03:44:01.885572+0000","last_active":"2024-09-14T03:44:24.929006+0000","last_peered":"2024-09-14T03:44:24.929006+0000","last_clean":"2024-09-14T03:44:24.929006+0000","last_became_active":"2024-09-14T03:44:01.885110+0000","last_became_peered":"2024-09-14T03:44:01.885110+0000","last_unstale":"2024-09-14T03:44:24.929006+0000","last_undegraded":"2024-09-14T03:44:24.929006+0000","last_fullsized":"2024-09-14T03:44:24.929006+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:10.439239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1d","version":"58'2","reported_seq":18,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933206+0000","last_change":"2024-09-14T03:43:58.731296+0000","last_active":"2024-09-14T03:44:24.933206+0000","last_peered":"2024-09-14T03:44:24.933206+0000","last_clean":"2024-09-14T03:44:24.933206+0000","last_became_active":"2024-09-14T03:43:58.730951+0000","last_became_peered":"2024-09-14T03:43:58.730951+0000","last_unstale":"2024-09-14T03:44:24.933206+0000","last_undegraded":"2024-09-14T03:44:24.933206+0000","last_fullsized":"2024-09-14T03:44:24.933206+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:26:14.801414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930670+0000","last_change":"2024-09-14T03:44:01.874935+0000","last_active":"2024-09-14T03:44:24.930670+0000","last_peered":"2024-09-14T03:44:24.930670+0000","last_clean":"2024-09-14T03:44:24.930670+0000","last_became_active":"2024-09-14T03:44:01.874668+0000","last_became_peered":"2024-09-14T03:44:01.874668+0000","last_unstale":"2024-09-14T03:44:24.930670+0000","last_undegraded":"2024-09-14T03:44:24.930670+0000","last_fullsized":"2024-09-14T03:44:24.930670+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:24:31.404958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"58'1","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926437+0000","last_change":"2024-09-14T03:43:58.734206+0000","last_active":"2024-09-14T03:44:24.926437+0000","last_peered":"2024-09-14T03:44:24.926437+0000","last_clean":"2024-09-14T03:44:24.926437+0000","last_became_active":"2024-09-14T03:43:58.733960+0000","last_became_peered":"2024-09-14T03:43:58.733960+0000","last_unstale":"2024-09-14T03:44:24.926437+0000","last_undegraded":"2024-09-14T03:44:24.926437+0000","last_fullsized":"2024-09-14T03:44:24.926437+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:44.073503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926323+0000","last_change":"2024-09-14T03:44:01.883550+0000","last_active":"2024-09-14T03:44:24.926323+0000","last_peered":"2024-09-14T03:44:24.926323+0000","last_clean":"2024-09-14T03:44:24.926323+0000","last_became_active":"2024-09-14T03:44:01.883265+0000","last_became_peered":"2024-09-14T03:44:01.883265+0000","last_unstale":"2024-09-14T03:44:24.926323+0000","last_undegraded":"2024-09-14T03:44:24.926323+0000","last_fullsized":"2024-09-14T03:44:24.926323+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:52:04.512990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"58'4","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930891+0000","last_change":"2024-09-14T03:43:58.731954+0000","last_active":"2024-09-14T03:44:24.930891+0000","last_peered":"2024-09-14T03:44:24.930891+0000","last_clean":"2024-09-14T03:44:24.930891+0000","last_became_active":"2024-09-14T03:43:58.731688+0000","last_became_peered":"2024-09-14T03:43:58.731688+0000","last_unstale":"2024-09-14T03:44:24.930891+0000","last_undegraded":"2024-09-14T03:44:24.930891+0000","last_fullsized":"2024-09-14T03:44:24.930891+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:30:10.811609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3563520,"data_stored":3542208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397519,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298528,"kb_used_data":1172,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446624,"statfs":{"total":95995035648,"available":95689342976,"internally_reserved":0,"allocated":1200128,"data_stored":844812,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33700000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.498}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46899999999999997}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39900000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36399999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56000000000000005}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48899999999999999}]}]},{"osd":6,"up_from":43,"seq":184683593747,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298532,"kb_used_data":1172,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446620,"statfs":{"total":95995035648,"available":95689338880,"internally_reserved":0,"allocated":1200128,"data_stored":845457,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.39200000000000002}]},{"osd":1,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.47499999999999998}]},{"osd":2,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.56399999999999995}]},{"osd":3,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.51100000000000001}]},{"osd":4,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.60599999999999998}]},{"osd":5,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.58299999999999996}]},{"osd":7,"last update":"Sat Sep 14 03:44:34 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.63200000000000001}]}]},{"osd":5,"up_from":37,"seq":158913789976,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297956,"kb_used_data":596,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447196,"statfs":{"total":95995035648,"available":95689928704,"internally_reserved":0,"allocated":610304,"data_stored":255179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:43:47 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.57399999999999995}]},{"osd":1,"last update":"Sat Sep 14 03:43:47 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.54500000000000004}]},{"osd":2,"last update":"Sat Sep 14 03:43:47 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.59799999999999998}]},{"osd":3,"last update":"Sat Sep 14 03:43:47 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.53200000000000003}]},{"osd":4,"last update":"Sat Sep 14 03:43:47 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.379,"15min":0.379},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.56000000000000005}]},{"osd":6,"last update":"Sat Sep 14 03:44:11 2024","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.626}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49299999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986204,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297960,"kb_used_data":600,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447192,"statfs":{"total":95995035648,"available":95689924608,"internally_reserved":0,"allocated":614400,"data_stored":254548,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:44:26 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.51000000000000001,"15min":0.51200000000000001},"min":{"1min":0.35499999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.54500000000000004,"15min":0.54700000000000004},"min":{"1min":0.40500000000000003,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.60899999999999999}]},{"osd":1,"last update":"Sat Sep 14 03:44:26 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.48399999999999999,"15min":0.47599999999999998},"min":{"1min":0.34100000000000003,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.39100000000000001,"5min":0.377,"15min":0.377},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.56699999999999995}]},{"osd":2,"last update":"Sat Sep 14 03:44:26 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.495,"15min":0.48999999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.629},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.51100000000000001,"15min":0.505},"min":{"1min":0.38,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.52000000000000002}]},{"osd":3,"last update":"Sat Sep 14 03:44:26 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.505,"15min":0.49299999999999999},"min":{"1min":0.35599999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.50700000000000001,"15min":0.501},"min":{"1min":0.35899999999999999,"5min":0.318,"15min":0.318},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.54100000000000004}]},{"osd":5,"last update":"Sat Sep 14 03:43:49 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.64700000000000002}]},{"osd":6,"last update":"Sat Sep 14 03:44:11 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.627,"5min":0.627,"15min":0.627},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.66000000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68500000000000005}]}]},{"osd":3,"up_from":26,"seq":111669149728,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297960,"kb_used_data":600,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447192,"statfs":{"total":95995035648,"available":95689924608,"internally_reserved":0,"allocated":614400,"data_stored":255193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:44:03 2024","interfaces":[{"interface":"back","average":{"1min":0.34599999999999997,"5min":0.38400000000000001,"15min":0.39000000000000001},"min":{"1min":0.245,"5min":0.185,"15min":0.185},"max":{"1min":0.43099999999999999,"5min":0.48199999999999998,"15min":0.48199999999999998},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.40200000000000002,"15min":0.40799999999999997},"min":{"1min":0.254,"5min":0.218,"15min":0.218},"max":{"1min":0.48099999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.247}]},{"osd":1,"last update":"Sat Sep 14 03:44:03 2024","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.41599999999999998,"15min":0.41999999999999998},"min":{"1min":0.28799999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.5,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.27700000000000002},{"interface":"front","average":{"1min":0.36599999999999999,"5min":0.39300000000000002,"15min":0.39700000000000002},"min":{"1min":0.30499999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.47199999999999998,"5min":0.53700000000000003,"15min":0.53700000000000003},"last":0.307}]},{"osd":2,"last update":"Sat Sep 14 03:44:03 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.434,"15min":0.441},"min":{"1min":0.28999999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.52300000000000002,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.42299999999999999,"15min":0.42799999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.54400000000000004,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.28999999999999998}]},{"osd":4,"last update":"Sat Sep 14 03:44:31 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.53300000000000003,"15min":0.53900000000000003},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.69099999999999995,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.55700000000000005,"15min":0.56100000000000005},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.745,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.44900000000000001}]},{"osd":5,"last update":"Sat Sep 14 03:43:52 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.501}]},{"osd":6,"last update":"Sat Sep 14 03:44:11 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.47099999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41399999999999998}]}]},{"osd":2,"up_from":18,"seq":77309411365,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297940,"kb_used_data":580,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447212,"statfs":{"total":95995035648,"available":95689945088,"internally_reserved":0,"allocated":593920,"data_stored":253241,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:43:45 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.36299999999999999,"15min":0.35599999999999998},"min":{"1min":0.26200000000000001,"5min":0.22,"15min":0.22},"max":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.371,"15min":0.36599999999999999},"min":{"1min":0.27800000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.56200000000000006,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.29199999999999998}]},{"osd":1,"last update":"Sat Sep 14 03:43:45 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.40000000000000002,"15min":0.39800000000000002},"min":{"1min":0.32800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.441,"5min":0.39200000000000002,"15min":0.38400000000000001},"min":{"1min":0.29199999999999998,"5min":0.245,"15min":0.245},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.35899999999999999}]},{"osd":3,"last update":"Sat Sep 14 03:44:08 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.44800000000000001,"15min":0.45100000000000001},"min":{"1min":0.34999999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.53300000000000003,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.44800000000000001,"15min":0.45800000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.56000000000000005,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.32300000000000001}]},{"osd":4,"last update":"Sat Sep 14 03:44:25 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.55400000000000005,"15min":0.55700000000000005},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.78100000000000003,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.59199999999999997,"15min":0.59299999999999997},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.76000000000000001,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.68899999999999995}]},{"osd":5,"last update":"Sat Sep 14 03:43:48 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.65100000000000002}]},{"osd":6,"last update":"Sat Sep 14 03:44:12 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.60499999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58299999999999996}]}]},{"osd":1,"up_from":13,"seq":55834574891,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297936,"kb_used_data":580,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447216,"statfs":{"total":95995035648,"available":95689949184,"internally_reserved":0,"allocated":593920,"data_stored":253067,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:44:20 2024","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.32000000000000001,"15min":0.29599999999999999},"min":{"1min":0.249,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.51900000000000002,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.247},{"interface":"front","average":{"1min":0.374,"5min":0.34399999999999997,"15min":0.31900000000000001},"min":{"1min":0.27000000000000002,"5min":0.189,"15min":0.189},"max":{"1min":0.56799999999999995,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.441}]},{"osd":2,"last update":"Sat Sep 14 03:43:39 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.35499999999999998,"15min":0.34300000000000003},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.36199999999999999,"15min":0.34699999999999998},"min":{"1min":0.29499999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.38100000000000001}]},{"osd":3,"last update":"Sat Sep 14 03:44:07 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.42299999999999999,"15min":0.42199999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.435,"15min":0.437},"min":{"1min":0.27500000000000002,"5min":0.247,"15min":0.247},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.36599999999999999}]},{"osd":4,"last update":"Sat Sep 14 03:44:28 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.77200000000000002,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.64700000000000002}]},{"osd":5,"last update":"Sat Sep 14 03:43:47 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.61099999999999999}]},{"osd":6,"last update":"Sat Sep 14 03:44:10 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.58299999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51600000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738416,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298536,"kb_used_data":1176,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446616,"statfs":{"total":95995035648,"available":95689334784,"internally_reserved":0,"allocated":1204224,"data_stored":844621,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 14 03:44:17 2024","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":0.35699999999999998,"15min":0.34899999999999998},"min":{"1min":0.25700000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.46500000000000002,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.36299999999999999,"15min":0.34799999999999998},"min":{"1min":0.246,"5min":0.216,"15min":0.216},"max":{"1min":0.48999999999999999,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.35399999999999998}]},{"osd":2,"last update":"Sat Sep 14 03:43:43 2024","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.41299999999999998,"15min":0.40999999999999998},"min":{"1min":0.29099999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.54900000000000004,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.308},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.39700000000000002,"15min":0.39500000000000002},"min":{"1min":0.28799999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.505,"5min":0.505,"15min":0.505},"last":0.36899999999999999}]},{"osd":3,"last update":"Sat Sep 14 03:44:04 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.44,"15min":0.44700000000000001},"min":{"1min":0.32500000000000001,"5min":0.307,"15min":0.307},"max":{"1min":0.54300000000000004,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.41499999999999998,"15min":0.42099999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.34100000000000003}]},{"osd":4,"last update":"Sat Sep 14 03:44:26 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.55700000000000005,"15min":0.56499999999999995},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.70099999999999996,"5min":0.754,"15min":0.754},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.55600000000000005,"15min":0.55800000000000005},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.57799999999999996}]},{"osd":5,"last update":"Sat Sep 14 03:43:48 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.64500000000000002}]},{"osd":6,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.65700000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67400000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1642,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T03:44:38.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:37 smithi094 ceph-mon[31053]: pgmap v206: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:38.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:37 smithi070 ceph-mon[39288]: pgmap v206: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:38.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:37 smithi070 ceph-mon[27495]: pgmap v206: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:39.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:38 smithi094 ceph-mon[31053]: from='client.14514 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:44:39.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:38 smithi070 ceph-mon[39288]: from='client.14514 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:44:39.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:38 smithi070 ceph-mon[27495]: from='client.14514 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:44:40.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:39 smithi094 ceph-mon[31053]: pgmap v207: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:40.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:39 smithi070 ceph-mon[27495]: pgmap v207: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:40.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:39 smithi070 ceph-mon[39288]: pgmap v207: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:40.854 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T03:44:41.109 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:44:42.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:41 smithi094 ceph-mon[31053]: pgmap v208: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:42.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:41 smithi070 ceph-mon[27495]: pgmap v208: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:42.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:41 smithi070 ceph-mon[39288]: pgmap v208: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:43.050 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:44:43.051 INFO:teuthology.orchestra.run.smithi070.stderr:dumped all 2024-09-14T03:44:43.713 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":209,"stamp":"2024-09-14T03:44:42.506931+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":146,"num_write_kb":1641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111,"ondisk_log_size":111,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":291,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385348,"kb_used_data":6476,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575868,"statfs":{"total":767960285184,"available":765517688832,"internally_reserved":0,"allocated":6631424,"data_stored":3806118,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003000"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933508+0000","last_change":"2024-09-14T03:44:01.882774+0000","last_active":"2024-09-14T03:44:24.933508+0000","last_peered":"2024-09-14T03:44:24.933508+0000","last_clean":"2024-09-14T03:44:24.933508+0000","last_became_active":"2024-09-14T03:44:01.882481+0000","last_became_peered":"2024-09-14T03:44:01.882481+0000","last_unstale":"2024-09-14T03:44:24.933508+0000","last_undegraded":"2024-09-14T03:44:24.933508+0000","last_fullsized":"2024-09-14T03:44:24.933508+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:49:25.209573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929696+0000","last_change":"2024-09-14T03:44:01.868415+0000","last_active":"2024-09-14T03:44:24.929696+0000","last_peered":"2024-09-14T03:44:24.929696+0000","last_clean":"2024-09-14T03:44:24.929696+0000","last_became_active":"2024-09-14T03:44:01.868152+0000","last_became_peered":"2024-09-14T03:44:01.868152+0000","last_unstale":"2024-09-14T03:44:24.929696+0000","last_undegraded":"2024-09-14T03:44:24.929696+0000","last_fullsized":"2024-09-14T03:44:24.929696+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:56:00.270070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929890+0000","last_change":"2024-09-14T03:44:01.871459+0000","last_active":"2024-09-14T03:44:24.929890+0000","last_peered":"2024-09-14T03:44:24.929890+0000","last_clean":"2024-09-14T03:44:24.929890+0000","last_became_active":"2024-09-14T03:44:01.870749+0000","last_became_peered":"2024-09-14T03:44:01.870749+0000","last_unstale":"2024-09-14T03:44:24.929890+0000","last_undegraded":"2024-09-14T03:44:24.929890+0000","last_fullsized":"2024-09-14T03:44:24.929890+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:54.391363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928911+0000","last_change":"2024-09-14T03:44:01.882503+0000","last_active":"2024-09-14T03:44:24.928911+0000","last_peered":"2024-09-14T03:44:24.928911+0000","last_clean":"2024-09-14T03:44:24.928911+0000","last_became_active":"2024-09-14T03:44:01.882250+0000","last_became_peered":"2024-09-14T03:44:01.882250+0000","last_unstale":"2024-09-14T03:44:24.928911+0000","last_undegraded":"2024-09-14T03:44:24.928911+0000","last_fullsized":"2024-09-14T03:44:24.928911+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:46:29.551285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.932873+0000","last_change":"2024-09-14T03:44:01.880333+0000","last_active":"2024-09-14T03:44:24.932873+0000","last_peered":"2024-09-14T03:44:24.932873+0000","last_clean":"2024-09-14T03:44:24.932873+0000","last_became_active":"2024-09-14T03:44:01.880010+0000","last_became_peered":"2024-09-14T03:44:01.880010+0000","last_unstale":"2024-09-14T03:44:24.932873+0000","last_undegraded":"2024-09-14T03:44:24.932873+0000","last_fullsized":"2024-09-14T03:44:24.932873+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:45:02.741425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928684+0000","last_change":"2024-09-14T03:44:01.880710+0000","last_active":"2024-09-14T03:44:24.928684+0000","last_peered":"2024-09-14T03:44:24.928684+0000","last_clean":"2024-09-14T03:44:24.928684+0000","last_became_active":"2024-09-14T03:44:01.880237+0000","last_became_peered":"2024-09-14T03:44:01.880237+0000","last_unstale":"2024-09-14T03:44:24.928684+0000","last_undegraded":"2024-09-14T03:44:24.928684+0000","last_fullsized":"2024-09-14T03:44:24.928684+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:56:49.808707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.932581+0000","last_change":"2024-09-14T03:44:01.878590+0000","last_active":"2024-09-14T03:44:24.932581+0000","last_peered":"2024-09-14T03:44:24.932581+0000","last_clean":"2024-09-14T03:44:24.932581+0000","last_became_active":"2024-09-14T03:44:01.878281+0000","last_became_peered":"2024-09-14T03:44:01.878281+0000","last_unstale":"2024-09-14T03:44:24.932581+0000","last_undegraded":"2024-09-14T03:44:24.932581+0000","last_fullsized":"2024-09-14T03:44:24.932581+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:41:57.164227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.925754+0000","last_change":"2024-09-14T03:44:01.877057+0000","last_active":"2024-09-14T03:44:24.925754+0000","last_peered":"2024-09-14T03:44:24.925754+0000","last_clean":"2024-09-14T03:44:24.925754+0000","last_became_active":"2024-09-14T03:44:01.876727+0000","last_became_peered":"2024-09-14T03:44:01.876727+0000","last_unstale":"2024-09-14T03:44:24.925754+0000","last_undegraded":"2024-09-14T03:44:24.925754+0000","last_fullsized":"2024-09-14T03:44:24.925754+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:31:24.716316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933832+0000","last_change":"2024-09-14T03:44:01.884865+0000","last_active":"2024-09-14T03:44:24.933832+0000","last_peered":"2024-09-14T03:44:24.933832+0000","last_clean":"2024-09-14T03:44:24.933832+0000","last_became_active":"2024-09-14T03:44:01.884136+0000","last_became_peered":"2024-09-14T03:44:01.884136+0000","last_unstale":"2024-09-14T03:44:24.933832+0000","last_undegraded":"2024-09-14T03:44:24.933832+0000","last_fullsized":"2024-09-14T03:44:24.933832+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:59.903382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":22,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-14T03:44:26.430052+0000","last_change":"2024-09-14T03:44:26.430052+0000","last_active":"2024-09-14T03:44:26.430052+0000","last_peered":"2024-09-14T03:44:26.430052+0000","last_clean":"2024-09-14T03:44:26.430052+0000","last_became_active":"2024-09-14T03:44:26.429658+0000","last_became_peered":"2024-09-14T03:44:26.429658+0000","last_unstale":"2024-09-14T03:44:26.430052+0000","last_undegraded":"2024-09-14T03:44:26.430052+0000","last_fullsized":"2024-09-14T03:44:26.430052+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:34.312039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929430+0000","last_change":"2024-09-14T03:44:01.875017+0000","last_active":"2024-09-14T03:44:24.929430+0000","last_peered":"2024-09-14T03:44:24.929430+0000","last_clean":"2024-09-14T03:44:24.929430+0000","last_became_active":"2024-09-14T03:44:01.874687+0000","last_became_peered":"2024-09-14T03:44:01.874687+0000","last_unstale":"2024-09-14T03:44:24.929430+0000","last_undegraded":"2024-09-14T03:44:24.929430+0000","last_fullsized":"2024-09-14T03:44:24.929430+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:29:09.035342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926391+0000","last_change":"2024-09-14T03:44:01.883313+0000","last_active":"2024-09-14T03:44:24.926391+0000","last_peered":"2024-09-14T03:44:24.926391+0000","last_clean":"2024-09-14T03:44:24.926391+0000","last_became_active":"2024-09-14T03:44:01.882770+0000","last_became_peered":"2024-09-14T03:44:01.882770+0000","last_unstale":"2024-09-14T03:44:24.926391+0000","last_undegraded":"2024-09-14T03:44:24.926391+0000","last_fullsized":"2024-09-14T03:44:24.926391+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:57:19.402319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926070+0000","last_change":"2024-09-14T03:44:01.878245+0000","last_active":"2024-09-14T03:44:24.926070+0000","last_peered":"2024-09-14T03:44:24.926070+0000","last_clean":"2024-09-14T03:44:24.926070+0000","last_became_active":"2024-09-14T03:44:01.877861+0000","last_became_peered":"2024-09-14T03:44:01.877861+0000","last_unstale":"2024-09-14T03:44:24.926070+0000","last_undegraded":"2024-09-14T03:44:24.926070+0000","last_fullsized":"2024-09-14T03:44:24.926070+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:08:29.902090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930473+0000","last_change":"2024-09-14T03:44:01.877391+0000","last_active":"2024-09-14T03:44:24.930473+0000","last_peered":"2024-09-14T03:44:24.930473+0000","last_clean":"2024-09-14T03:44:24.930473+0000","last_became_active":"2024-09-14T03:44:01.876797+0000","last_became_peered":"2024-09-14T03:44:01.876797+0000","last_unstale":"2024-09-14T03:44:24.930473+0000","last_undegraded":"2024-09-14T03:44:24.930473+0000","last_fullsized":"2024-09-14T03:44:24.930473+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:06:02.891762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928601+0000","last_change":"2024-09-14T03:44:01.876700+0000","last_active":"2024-09-14T03:44:24.928601+0000","last_peered":"2024-09-14T03:44:24.928601+0000","last_clean":"2024-09-14T03:44:24.928601+0000","last_became_active":"2024-09-14T03:44:01.876091+0000","last_became_peered":"2024-09-14T03:44:01.876091+0000","last_unstale":"2024-09-14T03:44:24.928601+0000","last_undegraded":"2024-09-14T03:44:24.928601+0000","last_fullsized":"2024-09-14T03:44:24.928601+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:25:41.750127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930443+0000","last_change":"2024-09-14T03:44:01.877120+0000","last_active":"2024-09-14T03:44:24.930443+0000","last_peered":"2024-09-14T03:44:24.930443+0000","last_clean":"2024-09-14T03:44:24.930443+0000","last_became_active":"2024-09-14T03:44:01.876793+0000","last_became_peered":"2024-09-14T03:44:01.876793+0000","last_unstale":"2024-09-14T03:44:24.930443+0000","last_undegraded":"2024-09-14T03:44:24.930443+0000","last_fullsized":"2024-09-14T03:44:24.930443+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:23:56.450728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930751+0000","last_change":"2024-09-14T03:44:01.885108+0000","last_active":"2024-09-14T03:44:24.930751+0000","last_peered":"2024-09-14T03:44:24.930751+0000","last_clean":"2024-09-14T03:44:24.930751+0000","last_became_active":"2024-09-14T03:44:01.884857+0000","last_became_peered":"2024-09-14T03:44:01.884857+0000","last_unstale":"2024-09-14T03:44:24.930751+0000","last_undegraded":"2024-09-14T03:44:24.930751+0000","last_fullsized":"2024-09-14T03:44:24.930751+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:31.448617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928709+0000","last_change":"2024-09-14T03:44:01.876503+0000","last_active":"2024-09-14T03:44:24.928709+0000","last_peered":"2024-09-14T03:44:24.928709+0000","last_clean":"2024-09-14T03:44:24.928709+0000","last_became_active":"2024-09-14T03:44:01.876227+0000","last_became_peered":"2024-09-14T03:44:01.876227+0000","last_unstale":"2024-09-14T03:44:24.928709+0000","last_undegraded":"2024-09-14T03:44:24.928709+0000","last_fullsized":"2024-09-14T03:44:24.928709+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:44:50.132152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933287+0000","last_change":"2024-09-14T03:44:01.883708+0000","last_active":"2024-09-14T03:44:24.933287+0000","last_peered":"2024-09-14T03:44:24.933287+0000","last_clean":"2024-09-14T03:44:24.933287+0000","last_became_active":"2024-09-14T03:44:01.883440+0000","last_became_peered":"2024-09-14T03:44:01.883440+0000","last_unstale":"2024-09-14T03:44:24.933287+0000","last_undegraded":"2024-09-14T03:44:24.933287+0000","last_fullsized":"2024-09-14T03:44:24.933287+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:10:34.658062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929540+0000","last_change":"2024-09-14T03:44:01.888565+0000","last_active":"2024-09-14T03:44:24.929540+0000","last_peered":"2024-09-14T03:44:24.929540+0000","last_clean":"2024-09-14T03:44:24.929540+0000","last_became_active":"2024-09-14T03:44:01.888191+0000","last_became_peered":"2024-09-14T03:44:01.888191+0000","last_unstale":"2024-09-14T03:44:24.929540+0000","last_undegraded":"2024-09-14T03:44:24.929540+0000","last_fullsized":"2024-09-14T03:44:24.929540+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:52:23.782664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930632+0000","last_change":"2024-09-14T03:44:01.874642+0000","last_active":"2024-09-14T03:44:24.930632+0000","last_peered":"2024-09-14T03:44:24.930632+0000","last_clean":"2024-09-14T03:44:24.930632+0000","last_became_active":"2024-09-14T03:44:01.874384+0000","last_became_peered":"2024-09-14T03:44:01.874384+0000","last_unstale":"2024-09-14T03:44:24.930632+0000","last_undegraded":"2024-09-14T03:44:24.930632+0000","last_fullsized":"2024-09-14T03:44:24.930632+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:35:05.535173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928767+0000","last_change":"2024-09-14T03:44:01.886893+0000","last_active":"2024-09-14T03:44:24.928767+0000","last_peered":"2024-09-14T03:44:24.928767+0000","last_clean":"2024-09-14T03:44:24.928767+0000","last_became_active":"2024-09-14T03:44:01.886594+0000","last_became_peered":"2024-09-14T03:44:01.886594+0000","last_unstale":"2024-09-14T03:44:24.928767+0000","last_undegraded":"2024-09-14T03:44:24.928767+0000","last_fullsized":"2024-09-14T03:44:24.928767+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:07:37.154358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.925870+0000","last_change":"2024-09-14T03:44:01.868698+0000","last_active":"2024-09-14T03:44:24.925870+0000","last_peered":"2024-09-14T03:44:24.925870+0000","last_clean":"2024-09-14T03:44:24.925870+0000","last_became_active":"2024-09-14T03:44:01.868442+0000","last_became_peered":"2024-09-14T03:44:01.868442+0000","last_unstale":"2024-09-14T03:44:24.925870+0000","last_undegraded":"2024-09-14T03:44:24.925870+0000","last_fullsized":"2024-09-14T03:44:24.925870+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:41:18.296424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930530+0000","last_change":"2024-09-14T03:44:01.882708+0000","last_active":"2024-09-14T03:44:24.930530+0000","last_peered":"2024-09-14T03:44:24.930530+0000","last_clean":"2024-09-14T03:44:24.930530+0000","last_became_active":"2024-09-14T03:44:01.882443+0000","last_became_peered":"2024-09-14T03:44:01.882443+0000","last_unstale":"2024-09-14T03:44:24.930530+0000","last_undegraded":"2024-09-14T03:44:24.930530+0000","last_fullsized":"2024-09-14T03:44:24.930530+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:46:48.306263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926452+0000","last_change":"2024-09-14T03:44:01.884940+0000","last_active":"2024-09-14T03:44:24.926452+0000","last_peered":"2024-09-14T03:44:24.926452+0000","last_clean":"2024-09-14T03:44:24.926452+0000","last_became_active":"2024-09-14T03:44:01.884304+0000","last_became_peered":"2024-09-14T03:44:01.884304+0000","last_unstale":"2024-09-14T03:44:24.926452+0000","last_undegraded":"2024-09-14T03:44:24.926452+0000","last_fullsized":"2024-09-14T03:44:24.926452+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:01:49.061637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":22,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-14T03:44:26.430891+0000","last_change":"2024-09-14T03:44:26.430891+0000","last_active":"2024-09-14T03:44:26.430891+0000","last_peered":"2024-09-14T03:44:26.430891+0000","last_clean":"2024-09-14T03:44:26.430891+0000","last_became_active":"2024-09-14T03:44:26.430426+0000","last_became_peered":"2024-09-14T03:44:26.430426+0000","last_unstale":"2024-09-14T03:44:26.430891+0000","last_undegraded":"2024-09-14T03:44:26.430891+0000","last_fullsized":"2024-09-14T03:44:26.430891+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:00:53.758328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928051+0000","last_change":"2024-09-14T03:44:01.874564+0000","last_active":"2024-09-14T03:44:24.928051+0000","last_peered":"2024-09-14T03:44:24.928051+0000","last_clean":"2024-09-14T03:44:24.928051+0000","last_became_active":"2024-09-14T03:44:01.874297+0000","last_became_peered":"2024-09-14T03:44:01.874297+0000","last_unstale":"2024-09-14T03:44:24.928051+0000","last_undegraded":"2024-09-14T03:44:24.928051+0000","last_fullsized":"2024-09-14T03:44:24.928051+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:07.027610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926133+0000","last_change":"2024-09-14T03:44:01.884486+0000","last_active":"2024-09-14T03:44:24.926133+0000","last_peered":"2024-09-14T03:44:24.926133+0000","last_clean":"2024-09-14T03:44:24.926133+0000","last_became_active":"2024-09-14T03:44:01.884218+0000","last_became_peered":"2024-09-14T03:44:01.884218+0000","last_unstale":"2024-09-14T03:44:24.926133+0000","last_undegraded":"2024-09-14T03:44:24.926133+0000","last_fullsized":"2024-09-14T03:44:24.926133+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:59:18.153239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930230+0000","last_change":"2024-09-14T03:44:01.878737+0000","last_active":"2024-09-14T03:44:24.930230+0000","last_peered":"2024-09-14T03:44:24.930230+0000","last_clean":"2024-09-14T03:44:24.930230+0000","last_became_active":"2024-09-14T03:44:01.878504+0000","last_became_peered":"2024-09-14T03:44:01.878504+0000","last_unstale":"2024-09-14T03:44:24.930230+0000","last_undegraded":"2024-09-14T03:44:24.930230+0000","last_fullsized":"2024-09-14T03:44:24.930230+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:41:35.420957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928860+0000","last_change":"2024-09-14T03:44:01.880509+0000","last_active":"2024-09-14T03:44:24.928860+0000","last_peered":"2024-09-14T03:44:24.928860+0000","last_clean":"2024-09-14T03:44:24.928860+0000","last_became_active":"2024-09-14T03:44:01.879855+0000","last_became_peered":"2024-09-14T03:44:01.879855+0000","last_unstale":"2024-09-14T03:44:24.928860+0000","last_undegraded":"2024-09-14T03:44:24.928860+0000","last_fullsized":"2024-09-14T03:44:24.928860+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:05:31.166902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928290+0000","last_change":"2024-09-14T03:44:01.869523+0000","last_active":"2024-09-14T03:44:24.928290+0000","last_peered":"2024-09-14T03:44:24.928290+0000","last_clean":"2024-09-14T03:44:24.928290+0000","last_became_active":"2024-09-14T03:44:01.869225+0000","last_became_peered":"2024-09-14T03:44:01.869225+0000","last_unstale":"2024-09-14T03:44:24.928290+0000","last_undegraded":"2024-09-14T03:44:24.928290+0000","last_fullsized":"2024-09-14T03:44:24.928290+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:43:15.638187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929040+0000","last_change":"2024-09-14T03:44:01.876014+0000","last_active":"2024-09-14T03:44:24.929040+0000","last_peered":"2024-09-14T03:44:24.929040+0000","last_clean":"2024-09-14T03:44:24.929040+0000","last_became_active":"2024-09-14T03:44:01.875496+0000","last_became_peered":"2024-09-14T03:44:01.875496+0000","last_unstale":"2024-09-14T03:44:24.929040+0000","last_undegraded":"2024-09-14T03:44:24.929040+0000","last_fullsized":"2024-09-14T03:44:24.929040+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:03:45.344199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926322+0000","last_change":"2024-09-14T03:44:01.865496+0000","last_active":"2024-09-14T03:44:24.926322+0000","last_peered":"2024-09-14T03:44:24.926322+0000","last_clean":"2024-09-14T03:44:24.926322+0000","last_became_active":"2024-09-14T03:44:01.865190+0000","last_became_peered":"2024-09-14T03:44:01.865190+0000","last_unstale":"2024-09-14T03:44:24.926322+0000","last_undegraded":"2024-09-14T03:44:24.926322+0000","last_fullsized":"2024-09-14T03:44:24.926322+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:18:50.037675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"58'1","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926154+0000","last_change":"2024-09-14T03:43:58.729188+0000","last_active":"2024-09-14T03:44:24.926154+0000","last_peered":"2024-09-14T03:44:24.926154+0000","last_clean":"2024-09-14T03:44:24.926154+0000","last_became_active":"2024-09-14T03:43:58.728927+0000","last_became_peered":"2024-09-14T03:43:58.728927+0000","last_unstale":"2024-09-14T03:44:24.926154+0000","last_undegraded":"2024-09-14T03:44:24.926154+0000","last_fullsized":"2024-09-14T03:44:24.926154+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:13:16.419701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930296+0000","last_change":"2024-09-14T03:44:01.879101+0000","last_active":"2024-09-14T03:44:24.930296+0000","last_peered":"2024-09-14T03:44:24.930296+0000","last_clean":"2024-09-14T03:44:24.930296+0000","last_became_active":"2024-09-14T03:44:01.878801+0000","last_became_peered":"2024-09-14T03:44:01.878801+0000","last_unstale":"2024-09-14T03:44:24.930296+0000","last_undegraded":"2024-09-14T03:44:24.930296+0000","last_fullsized":"2024-09-14T03:44:24.930296+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:13.447251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"58'1","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930120+0000","last_change":"2024-09-14T03:43:58.730444+0000","last_active":"2024-09-14T03:44:24.930120+0000","last_peered":"2024-09-14T03:44:24.930120+0000","last_clean":"2024-09-14T03:44:24.930120+0000","last_became_active":"2024-09-14T03:43:58.730172+0000","last_became_peered":"2024-09-14T03:43:58.730172+0000","last_unstale":"2024-09-14T03:44:24.930120+0000","last_undegraded":"2024-09-14T03:44:24.930120+0000","last_fullsized":"2024-09-14T03:44:24.930120+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:03:44.810562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.925963+0000","last_change":"2024-09-14T03:43:58.731458+0000","last_active":"2024-09-14T03:44:24.925963+0000","last_peered":"2024-09-14T03:44:24.925963+0000","last_clean":"2024-09-14T03:44:24.925963+0000","last_became_active":"2024-09-14T03:43:58.731206+0000","last_became_peered":"2024-09-14T03:43:58.731206+0000","last_unstale":"2024-09-14T03:44:24.925963+0000","last_undegraded":"2024-09-14T03:44:24.925963+0000","last_fullsized":"2024-09-14T03:44:24.925963+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:37:06.670539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.925827+0000","last_change":"2024-09-14T03:44:01.869781+0000","last_active":"2024-09-14T03:44:24.925827+0000","last_peered":"2024-09-14T03:44:24.925827+0000","last_clean":"2024-09-14T03:44:24.925827+0000","last_became_active":"2024-09-14T03:44:01.869521+0000","last_became_peered":"2024-09-14T03:44:01.869521+0000","last_unstale":"2024-09-14T03:44:24.925827+0000","last_undegraded":"2024-09-14T03:44:24.925827+0000","last_fullsized":"2024-09-14T03:44:24.925827+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:35:18.565206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":25,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-14T03:44:25.932578+0000","last_change":"2024-09-14T03:44:25.932578+0000","last_active":"2024-09-14T03:44:25.932578+0000","last_peered":"2024-09-14T03:44:25.932578+0000","last_clean":"2024-09-14T03:44:25.932578+0000","last_became_active":"2024-09-14T03:44:25.932288+0000","last_became_peered":"2024-09-14T03:44:25.932288+0000","last_unstale":"2024-09-14T03:44:25.932578+0000","last_undegraded":"2024-09-14T03:44:25.932578+0000","last_fullsized":"2024-09-14T03:44:25.932578+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:24:41.091353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928474+0000","last_change":"2024-09-14T03:44:01.873748+0000","last_active":"2024-09-14T03:44:24.928474+0000","last_peered":"2024-09-14T03:44:24.928474+0000","last_clean":"2024-09-14T03:44:24.928474+0000","last_became_active":"2024-09-14T03:44:01.873473+0000","last_became_peered":"2024-09-14T03:44:01.873473+0000","last_unstale":"2024-09-14T03:44:24.928474+0000","last_undegraded":"2024-09-14T03:44:24.928474+0000","last_fullsized":"2024-09-14T03:44:24.928474+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:04:39.289903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929850+0000","last_change":"2024-09-14T03:43:58.735608+0000","last_active":"2024-09-14T03:44:24.929850+0000","last_peered":"2024-09-14T03:44:24.929850+0000","last_clean":"2024-09-14T03:44:24.929850+0000","last_became_active":"2024-09-14T03:43:58.735301+0000","last_became_peered":"2024-09-14T03:43:58.735301+0000","last_unstale":"2024-09-14T03:44:24.929850+0000","last_undegraded":"2024-09-14T03:44:24.929850+0000","last_fullsized":"2024-09-14T03:44:24.929850+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:44:56.523073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930382+0000","last_change":"2024-09-14T03:44:01.874777+0000","last_active":"2024-09-14T03:44:24.930382+0000","last_peered":"2024-09-14T03:44:24.930382+0000","last_clean":"2024-09-14T03:44:24.930382+0000","last_became_active":"2024-09-14T03:44:01.874473+0000","last_became_peered":"2024-09-14T03:44:01.874473+0000","last_unstale":"2024-09-14T03:44:24.930382+0000","last_undegraded":"2024-09-14T03:44:24.930382+0000","last_fullsized":"2024-09-14T03:44:24.930382+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:49:02.042635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930539+0000","last_change":"2024-09-14T03:43:58.732969+0000","last_active":"2024-09-14T03:44:24.930539+0000","last_peered":"2024-09-14T03:44:24.930539+0000","last_clean":"2024-09-14T03:44:24.930539+0000","last_became_active":"2024-09-14T03:43:58.732492+0000","last_became_peered":"2024-09-14T03:43:58.732492+0000","last_unstale":"2024-09-14T03:44:24.930539+0000","last_undegraded":"2024-09-14T03:44:24.930539+0000","last_fullsized":"2024-09-14T03:44:24.930539+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:50:28.719447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930714+0000","last_change":"2024-09-14T03:44:01.875679+0000","last_active":"2024-09-14T03:44:24.930714+0000","last_peered":"2024-09-14T03:44:24.930714+0000","last_clean":"2024-09-14T03:44:24.930714+0000","last_became_active":"2024-09-14T03:44:01.875422+0000","last_became_peered":"2024-09-14T03:44:01.875422+0000","last_unstale":"2024-09-14T03:44:24.930714+0000","last_undegraded":"2024-09-14T03:44:24.930714+0000","last_fullsized":"2024-09-14T03:44:24.930714+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:13:00.349373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"58'3","reported_seq":18,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929910+0000","last_change":"2024-09-14T03:43:58.735066+0000","last_active":"2024-09-14T03:44:24.929910+0000","last_peered":"2024-09-14T03:44:24.929910+0000","last_clean":"2024-09-14T03:44:24.929910+0000","last_became_active":"2024-09-14T03:43:58.734514+0000","last_became_peered":"2024-09-14T03:43:58.734514+0000","last_unstale":"2024-09-14T03:44:24.929910+0000","last_undegraded":"2024-09-14T03:44:24.929910+0000","last_fullsized":"2024-09-14T03:44:24.929910+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:41:55.478295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926573+0000","last_change":"2024-09-14T03:44:01.881319+0000","last_active":"2024-09-14T03:44:24.926573+0000","last_peered":"2024-09-14T03:44:24.926573+0000","last_clean":"2024-09-14T03:44:24.926573+0000","last_became_active":"2024-09-14T03:44:01.881032+0000","last_became_peered":"2024-09-14T03:44:01.881032+0000","last_unstale":"2024-09-14T03:44:24.926573+0000","last_undegraded":"2024-09-14T03:44:24.926573+0000","last_fullsized":"2024-09-14T03:44:24.926573+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:35:41.762014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"58'6","reported_seq":21,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929594+0000","last_change":"2024-09-14T03:43:58.734778+0000","last_active":"2024-09-14T03:44:24.929594+0000","last_peered":"2024-09-14T03:44:24.929594+0000","last_clean":"2024-09-14T03:44:24.929594+0000","last_became_active":"2024-09-14T03:43:58.734380+0000","last_became_peered":"2024-09-14T03:43:58.734380+0000","last_unstale":"2024-09-14T03:44:24.929594+0000","last_undegraded":"2024-09-14T03:44:24.929594+0000","last_fullsized":"2024-09-14T03:44:24.929594+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:55:13.561065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933222+0000","last_change":"2024-09-14T03:44:01.880139+0000","last_active":"2024-09-14T03:44:24.933222+0000","last_peered":"2024-09-14T03:44:24.933222+0000","last_clean":"2024-09-14T03:44:24.933222+0000","last_became_active":"2024-09-14T03:44:01.879840+0000","last_became_peered":"2024-09-14T03:44:01.879840+0000","last_unstale":"2024-09-14T03:44:24.933222+0000","last_undegraded":"2024-09-14T03:44:24.933222+0000","last_fullsized":"2024-09-14T03:44:24.933222+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:31:34.003676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"58'2","reported_seq":17,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930377+0000","last_change":"2024-09-14T03:43:58.728768+0000","last_active":"2024-09-14T03:44:24.930377+0000","last_peered":"2024-09-14T03:44:24.930377+0000","last_clean":"2024-09-14T03:44:24.930377+0000","last_became_active":"2024-09-14T03:43:58.727893+0000","last_became_peered":"2024-09-14T03:43:58.727893+0000","last_unstale":"2024-09-14T03:44:24.930377+0000","last_undegraded":"2024-09-14T03:44:24.930377+0000","last_fullsized":"2024-09-14T03:44:24.930377+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:54:48.554486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930509+0000","last_change":"2024-09-14T03:44:01.877540+0000","last_active":"2024-09-14T03:44:24.930509+0000","last_peered":"2024-09-14T03:44:24.930509+0000","last_clean":"2024-09-14T03:44:24.930509+0000","last_became_active":"2024-09-14T03:44:01.877147+0000","last_became_peered":"2024-09-14T03:44:01.877147+0000","last_unstale":"2024-09-14T03:44:24.930509+0000","last_undegraded":"2024-09-14T03:44:24.930509+0000","last_fullsized":"2024-09-14T03:44:24.930509+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:17:57.724648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928948+0000","last_change":"2024-09-14T03:43:58.732812+0000","last_active":"2024-09-14T03:44:24.928948+0000","last_peered":"2024-09-14T03:44:24.928948+0000","last_clean":"2024-09-14T03:44:24.928948+0000","last_became_active":"2024-09-14T03:43:58.732537+0000","last_became_peered":"2024-09-14T03:43:58.732537+0000","last_unstale":"2024-09-14T03:44:24.928948+0000","last_undegraded":"2024-09-14T03:44:24.928948+0000","last_fullsized":"2024-09-14T03:44:24.928948+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:26:11.551182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.932936+0000","last_change":"2024-09-14T03:44:01.881140+0000","last_active":"2024-09-14T03:44:24.932936+0000","last_peered":"2024-09-14T03:44:24.932936+0000","last_clean":"2024-09-14T03:44:24.932936+0000","last_became_active":"2024-09-14T03:44:01.880868+0000","last_became_peered":"2024-09-14T03:44:01.880868+0000","last_unstale":"2024-09-14T03:44:24.932936+0000","last_undegraded":"2024-09-14T03:44:24.932936+0000","last_fullsized":"2024-09-14T03:44:24.932936+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:29:05.729804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929350+0000","last_change":"2024-09-14T03:43:58.730505+0000","last_active":"2024-09-14T03:44:24.929350+0000","last_peered":"2024-09-14T03:44:24.929350+0000","last_clean":"2024-09-14T03:44:24.929350+0000","last_became_active":"2024-09-14T03:43:58.729821+0000","last_became_peered":"2024-09-14T03:43:58.729821+0000","last_unstale":"2024-09-14T03:44:24.929350+0000","last_undegraded":"2024-09-14T03:44:24.929350+0000","last_fullsized":"2024-09-14T03:44:24.929350+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:17:44.091667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929155+0000","last_change":"2024-09-14T03:44:01.874477+0000","last_active":"2024-09-14T03:44:24.929155+0000","last_peered":"2024-09-14T03:44:24.929155+0000","last_clean":"2024-09-14T03:44:24.929155+0000","last_became_active":"2024-09-14T03:44:01.873714+0000","last_became_peered":"2024-09-14T03:44:01.873714+0000","last_unstale":"2024-09-14T03:44:24.929155+0000","last_undegraded":"2024-09-14T03:44:24.929155+0000","last_fullsized":"2024-09-14T03:44:24.929155+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:54:21.601768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"58'8","reported_seq":23,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928144+0000","last_change":"2024-09-14T03:43:58.726387+0000","last_active":"2024-09-14T03:44:24.928144+0000","last_peered":"2024-09-14T03:44:24.928144+0000","last_clean":"2024-09-14T03:44:24.928144+0000","last_became_active":"2024-09-14T03:43:58.726115+0000","last_became_peered":"2024-09-14T03:43:58.726115+0000","last_unstale":"2024-09-14T03:44:24.928144+0000","last_undegraded":"2024-09-14T03:44:24.928144+0000","last_fullsized":"2024-09-14T03:44:24.928144+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:18:04.176970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.932701+0000","last_change":"2024-09-14T03:44:01.878624+0000","last_active":"2024-09-14T03:44:24.932701+0000","last_peered":"2024-09-14T03:44:24.932701+0000","last_clean":"2024-09-14T03:44:24.932701+0000","last_became_active":"2024-09-14T03:44:01.878343+0000","last_became_peered":"2024-09-14T03:44:01.878343+0000","last_unstale":"2024-09-14T03:44:24.932701+0000","last_undegraded":"2024-09-14T03:44:24.932701+0000","last_fullsized":"2024-09-14T03:44:24.932701+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:44:05.205671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"58'1","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926865+0000","last_change":"2024-09-14T03:43:58.728489+0000","last_active":"2024-09-14T03:44:24.926865+0000","last_peered":"2024-09-14T03:44:24.926865+0000","last_clean":"2024-09-14T03:44:24.926865+0000","last_became_active":"2024-09-14T03:43:58.728182+0000","last_became_peered":"2024-09-14T03:43:58.728182+0000","last_unstale":"2024-09-14T03:44:24.926865+0000","last_undegraded":"2024-09-14T03:44:24.926865+0000","last_fullsized":"2024-09-14T03:44:24.926865+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:56.310710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933262+0000","last_change":"2024-09-14T03:44:01.880754+0000","last_active":"2024-09-14T03:44:24.933262+0000","last_peered":"2024-09-14T03:44:24.933262+0000","last_clean":"2024-09-14T03:44:24.933262+0000","last_became_active":"2024-09-14T03:44:01.880483+0000","last_became_peered":"2024-09-14T03:44:01.880483+0000","last_unstale":"2024-09-14T03:44:24.933262+0000","last_undegraded":"2024-09-14T03:44:24.933262+0000","last_fullsized":"2024-09-14T03:44:24.933262+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:32:53.245007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"58'2","reported_seq":17,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928841+0000","last_change":"2024-09-14T03:43:58.726366+0000","last_active":"2024-09-14T03:44:24.928841+0000","last_peered":"2024-09-14T03:44:24.928841+0000","last_clean":"2024-09-14T03:44:24.928841+0000","last_became_active":"2024-09-14T03:43:58.726090+0000","last_became_peered":"2024-09-14T03:43:58.726090+0000","last_unstale":"2024-09-14T03:44:24.928841+0000","last_undegraded":"2024-09-14T03:44:24.928841+0000","last_fullsized":"2024-09-14T03:44:24.928841+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:45:33.572820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933584+0000","last_change":"2024-09-14T03:44:01.883704+0000","last_active":"2024-09-14T03:44:24.933584+0000","last_peered":"2024-09-14T03:44:24.933584+0000","last_clean":"2024-09-14T03:44:24.933584+0000","last_became_active":"2024-09-14T03:44:01.883448+0000","last_became_peered":"2024-09-14T03:44:01.883448+0000","last_unstale":"2024-09-14T03:44:24.933584+0000","last_undegraded":"2024-09-14T03:44:24.933584+0000","last_fullsized":"2024-09-14T03:44:24.933584+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:16:27.946770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"58'1","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929063+0000","last_change":"2024-09-14T03:43:58.733205+0000","last_active":"2024-09-14T03:44:24.929063+0000","last_peered":"2024-09-14T03:44:24.929063+0000","last_clean":"2024-09-14T03:44:24.929063+0000","last_became_active":"2024-09-14T03:43:58.732929+0000","last_became_peered":"2024-09-14T03:43:58.732929+0000","last_unstale":"2024-09-14T03:44:24.929063+0000","last_undegraded":"2024-09-14T03:44:24.929063+0000","last_fullsized":"2024-09-14T03:44:24.929063+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:31:13.660965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929096+0000","last_change":"2024-09-14T03:44:01.886158+0000","last_active":"2024-09-14T03:44:24.929096+0000","last_peered":"2024-09-14T03:44:24.929096+0000","last_clean":"2024-09-14T03:44:24.929096+0000","last_became_active":"2024-09-14T03:44:01.884747+0000","last_became_peered":"2024-09-14T03:44:01.884747+0000","last_unstale":"2024-09-14T03:44:24.929096+0000","last_undegraded":"2024-09-14T03:44:24.929096+0000","last_fullsized":"2024-09-14T03:44:24.929096+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:49:56.265783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928891+0000","last_change":"2024-09-14T03:43:58.732401+0000","last_active":"2024-09-14T03:44:24.928891+0000","last_peered":"2024-09-14T03:44:24.928891+0000","last_clean":"2024-09-14T03:44:24.928891+0000","last_became_active":"2024-09-14T03:43:58.732067+0000","last_became_peered":"2024-09-14T03:43:58.732067+0000","last_unstale":"2024-09-14T03:44:24.928891+0000","last_undegraded":"2024-09-14T03:44:24.928891+0000","last_fullsized":"2024-09-14T03:44:24.928891+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:34:00.566245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933105+0000","last_change":"2024-09-14T03:44:01.881736+0000","last_active":"2024-09-14T03:44:24.933105+0000","last_peered":"2024-09-14T03:44:24.933105+0000","last_clean":"2024-09-14T03:44:24.933105+0000","last_became_active":"2024-09-14T03:44:01.881219+0000","last_became_peered":"2024-09-14T03:44:01.881219+0000","last_unstale":"2024-09-14T03:44:24.933105+0000","last_undegraded":"2024-09-14T03:44:24.933105+0000","last_fullsized":"2024-09-14T03:44:24.933105+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:24:23.342066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"22'72","reported_seq":25,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933249+0000","last_change":"2024-09-14T03:43:36.147227+0000","last_active":"2024-09-14T03:44:24.933249+0000","last_peered":"2024-09-14T03:44:24.933249+0000","last_clean":"2024-09-14T03:44:24.933249+0000","last_became_active":"2024-09-14T03:43:36.141953+0000","last_became_peered":"2024-09-14T03:43:36.141953+0000","last_unstale":"2024-09-14T03:44:24.933249+0000","last_undegraded":"2024-09-14T03:44:24.933249+0000","last_fullsized":"2024-09-14T03:44:24.933249+0000","mapping_epoch":50,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":51,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:39.458815+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:39.458815+0000","last_clean_scrub_stamp":"2024-09-14T03:41:39.458815+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:38:43.056328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930279+0000","last_change":"2024-09-14T03:43:58.733557+0000","last_active":"2024-09-14T03:44:24.930279+0000","last_peered":"2024-09-14T03:44:24.930279+0000","last_clean":"2024-09-14T03:44:24.930279+0000","last_became_active":"2024-09-14T03:43:58.733256+0000","last_became_peered":"2024-09-14T03:43:58.733256+0000","last_unstale":"2024-09-14T03:44:24.930279+0000","last_undegraded":"2024-09-14T03:44:24.930279+0000","last_fullsized":"2024-09-14T03:44:24.930279+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:09:12.987156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.925983+0000","last_change":"2024-09-14T03:44:01.875525+0000","last_active":"2024-09-14T03:44:24.925983+0000","last_peered":"2024-09-14T03:44:24.925983+0000","last_clean":"2024-09-14T03:44:24.925983+0000","last_became_active":"2024-09-14T03:44:01.875241+0000","last_became_peered":"2024-09-14T03:44:01.875241+0000","last_unstale":"2024-09-14T03:44:24.925983+0000","last_undegraded":"2024-09-14T03:44:24.925983+0000","last_fullsized":"2024-09-14T03:44:24.925983+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:34:47.346963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933143+0000","last_change":"2024-09-14T03:43:58.732376+0000","last_active":"2024-09-14T03:44:24.933143+0000","last_peered":"2024-09-14T03:44:24.933143+0000","last_clean":"2024-09-14T03:44:24.933143+0000","last_became_active":"2024-09-14T03:43:58.731812+0000","last_became_peered":"2024-09-14T03:43:58.731812+0000","last_unstale":"2024-09-14T03:44:24.933143+0000","last_undegraded":"2024-09-14T03:44:24.933143+0000","last_fullsized":"2024-09-14T03:44:24.933143+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:35:26.262383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930295+0000","last_change":"2024-09-14T03:44:01.876511+0000","last_active":"2024-09-14T03:44:24.930295+0000","last_peered":"2024-09-14T03:44:24.930295+0000","last_clean":"2024-09-14T03:44:24.930295+0000","last_became_active":"2024-09-14T03:44:01.876225+0000","last_became_peered":"2024-09-14T03:44:01.876225+0000","last_unstale":"2024-09-14T03:44:24.930295+0000","last_undegraded":"2024-09-14T03:44:24.930295+0000","last_fullsized":"2024-09-14T03:44:24.930295+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:36:40.283622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928369+0000","last_change":"2024-09-14T03:43:58.727129+0000","last_active":"2024-09-14T03:44:24.928369+0000","last_peered":"2024-09-14T03:44:24.928369+0000","last_clean":"2024-09-14T03:44:24.928369+0000","last_became_active":"2024-09-14T03:43:58.726879+0000","last_became_peered":"2024-09-14T03:43:58.726879+0000","last_unstale":"2024-09-14T03:44:24.928369+0000","last_undegraded":"2024-09-14T03:44:24.928369+0000","last_fullsized":"2024-09-14T03:44:24.928369+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:54:41.748651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928166+0000","last_change":"2024-09-14T03:44:01.871005+0000","last_active":"2024-09-14T03:44:24.928166+0000","last_peered":"2024-09-14T03:44:24.928166+0000","last_clean":"2024-09-14T03:44:24.928166+0000","last_became_active":"2024-09-14T03:44:01.870707+0000","last_became_peered":"2024-09-14T03:44:01.870707+0000","last_unstale":"2024-09-14T03:44:24.928166+0000","last_undegraded":"2024-09-14T03:44:24.928166+0000","last_fullsized":"2024-09-14T03:44:24.928166+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:30:22.758581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929035+0000","last_change":"2024-09-14T03:43:58.733038+0000","last_active":"2024-09-14T03:44:24.929035+0000","last_peered":"2024-09-14T03:44:24.929035+0000","last_clean":"2024-09-14T03:44:24.929035+0000","last_became_active":"2024-09-14T03:43:58.732753+0000","last_became_peered":"2024-09-14T03:43:58.732753+0000","last_unstale":"2024-09-14T03:44:24.929035+0000","last_undegraded":"2024-09-14T03:44:24.929035+0000","last_fullsized":"2024-09-14T03:44:24.929035+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:40:49.754211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":22,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-14T03:44:26.430320+0000","last_change":"2024-09-14T03:44:26.430320+0000","last_active":"2024-09-14T03:44:26.430320+0000","last_peered":"2024-09-14T03:44:26.430320+0000","last_clean":"2024-09-14T03:44:26.430320+0000","last_became_active":"2024-09-14T03:44:26.429935+0000","last_became_peered":"2024-09-14T03:44:26.429935+0000","last_unstale":"2024-09-14T03:44:26.430320+0000","last_undegraded":"2024-09-14T03:44:26.430320+0000","last_fullsized":"2024-09-14T03:44:26.430320+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:12:35.577538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929025+0000","last_change":"2024-09-14T03:43:58.733621+0000","last_active":"2024-09-14T03:44:24.929025+0000","last_peered":"2024-09-14T03:44:24.929025+0000","last_clean":"2024-09-14T03:44:24.929025+0000","last_became_active":"2024-09-14T03:43:58.733357+0000","last_became_peered":"2024-09-14T03:43:58.733357+0000","last_unstale":"2024-09-14T03:44:24.929025+0000","last_undegraded":"2024-09-14T03:44:24.929025+0000","last_fullsized":"2024-09-14T03:44:24.929025+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:25:53.791926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929608+0000","last_change":"2024-09-14T03:44:01.880200+0000","last_active":"2024-09-14T03:44:24.929608+0000","last_peered":"2024-09-14T03:44:24.929608+0000","last_clean":"2024-09-14T03:44:24.929608+0000","last_became_active":"2024-09-14T03:44:01.879925+0000","last_became_peered":"2024-09-14T03:44:01.879925+0000","last_unstale":"2024-09-14T03:44:24.929608+0000","last_undegraded":"2024-09-14T03:44:24.929608+0000","last_fullsized":"2024-09-14T03:44:24.929608+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:55:24.738215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933352+0000","last_change":"2024-09-14T03:43:58.733230+0000","last_active":"2024-09-14T03:44:24.933352+0000","last_peered":"2024-09-14T03:44:24.933352+0000","last_clean":"2024-09-14T03:44:24.933352+0000","last_became_active":"2024-09-14T03:43:58.732868+0000","last_became_peered":"2024-09-14T03:43:58.732868+0000","last_unstale":"2024-09-14T03:44:24.933352+0000","last_undegraded":"2024-09-14T03:44:24.933352+0000","last_fullsized":"2024-09-14T03:44:24.933352+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:15:12.184851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929025+0000","last_change":"2024-09-14T03:44:01.878736+0000","last_active":"2024-09-14T03:44:24.929025+0000","last_peered":"2024-09-14T03:44:24.929025+0000","last_clean":"2024-09-14T03:44:24.929025+0000","last_became_active":"2024-09-14T03:44:01.877390+0000","last_became_peered":"2024-09-14T03:44:01.877390+0000","last_unstale":"2024-09-14T03:44:24.929025+0000","last_undegraded":"2024-09-14T03:44:24.929025+0000","last_fullsized":"2024-09-14T03:44:24.929025+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:17:42.812575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"58'1","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929778+0000","last_change":"2024-09-14T03:43:58.733688+0000","last_active":"2024-09-14T03:44:24.929778+0000","last_peered":"2024-09-14T03:44:24.929778+0000","last_clean":"2024-09-14T03:44:24.929778+0000","last_became_active":"2024-09-14T03:43:58.733457+0000","last_became_peered":"2024-09-14T03:43:58.733457+0000","last_unstale":"2024-09-14T03:44:24.929778+0000","last_undegraded":"2024-09-14T03:44:24.929778+0000","last_fullsized":"2024-09-14T03:44:24.929778+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:09:01.069988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926470+0000","last_change":"2024-09-14T03:44:01.870045+0000","last_active":"2024-09-14T03:44:24.926470+0000","last_peered":"2024-09-14T03:44:24.926470+0000","last_clean":"2024-09-14T03:44:24.926470+0000","last_became_active":"2024-09-14T03:44:01.869780+0000","last_became_peered":"2024-09-14T03:44:01.869780+0000","last_unstale":"2024-09-14T03:44:24.926470+0000","last_undegraded":"2024-09-14T03:44:24.926470+0000","last_fullsized":"2024-09-14T03:44:24.926470+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:04:58.634704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"58'2","reported_seq":17,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929736+0000","last_change":"2024-09-14T03:43:58.735364+0000","last_active":"2024-09-14T03:44:24.929736+0000","last_peered":"2024-09-14T03:44:24.929736+0000","last_clean":"2024-09-14T03:44:24.929736+0000","last_became_active":"2024-09-14T03:43:58.734833+0000","last_became_peered":"2024-09-14T03:43:58.734833+0000","last_unstale":"2024-09-14T03:44:24.929736+0000","last_undegraded":"2024-09-14T03:44:24.929736+0000","last_fullsized":"2024-09-14T03:44:24.929736+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:58:37.428713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930534+0000","last_change":"2024-09-14T03:44:01.878573+0000","last_active":"2024-09-14T03:44:24.930534+0000","last_peered":"2024-09-14T03:44:24.930534+0000","last_clean":"2024-09-14T03:44:24.930534+0000","last_became_active":"2024-09-14T03:44:01.878316+0000","last_became_peered":"2024-09-14T03:44:01.878316+0000","last_unstale":"2024-09-14T03:44:24.930534+0000","last_undegraded":"2024-09-14T03:44:24.930534+0000","last_fullsized":"2024-09-14T03:44:24.930534+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:45:21.129163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933129+0000","last_change":"2024-09-14T03:43:58.730843+0000","last_active":"2024-09-14T03:44:24.933129+0000","last_peered":"2024-09-14T03:44:24.933129+0000","last_clean":"2024-09-14T03:44:24.933129+0000","last_became_active":"2024-09-14T03:43:58.730538+0000","last_became_peered":"2024-09-14T03:43:58.730538+0000","last_unstale":"2024-09-14T03:44:24.933129+0000","last_undegraded":"2024-09-14T03:44:24.933129+0000","last_fullsized":"2024-09-14T03:44:24.933129+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:04:57.059345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926481+0000","last_change":"2024-09-14T03:44:01.874355+0000","last_active":"2024-09-14T03:44:24.926481+0000","last_peered":"2024-09-14T03:44:24.926481+0000","last_clean":"2024-09-14T03:44:24.926481+0000","last_became_active":"2024-09-14T03:44:01.874049+0000","last_became_peered":"2024-09-14T03:44:01.874049+0000","last_unstale":"2024-09-14T03:44:24.926481+0000","last_undegraded":"2024-09-14T03:44:24.926481+0000","last_fullsized":"2024-09-14T03:44:24.926481+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:53:30.306175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.928677+0000","last_change":"2024-09-14T03:43:58.728080+0000","last_active":"2024-09-14T03:44:24.928677+0000","last_peered":"2024-09-14T03:44:24.928677+0000","last_clean":"2024-09-14T03:44:24.928677+0000","last_became_active":"2024-09-14T03:43:58.727820+0000","last_became_peered":"2024-09-14T03:43:58.727820+0000","last_unstale":"2024-09-14T03:44:24.928677+0000","last_undegraded":"2024-09-14T03:44:24.928677+0000","last_fullsized":"2024-09-14T03:44:24.928677+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:37:58.166430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.925857+0000","last_change":"2024-09-14T03:44:01.869311+0000","last_active":"2024-09-14T03:44:24.925857+0000","last_peered":"2024-09-14T03:44:24.925857+0000","last_clean":"2024-09-14T03:44:24.925857+0000","last_became_active":"2024-09-14T03:44:01.869065+0000","last_became_peered":"2024-09-14T03:44:01.869065+0000","last_unstale":"2024-09-14T03:44:24.925857+0000","last_undegraded":"2024-09-14T03:44:24.925857+0000","last_fullsized":"2024-09-14T03:44:24.925857+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:45:09.508019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929475+0000","last_change":"2024-09-14T03:43:58.728931+0000","last_active":"2024-09-14T03:44:24.929475+0000","last_peered":"2024-09-14T03:44:24.929475+0000","last_clean":"2024-09-14T03:44:24.929475+0000","last_became_active":"2024-09-14T03:43:58.728647+0000","last_became_peered":"2024-09-14T03:43:58.728647+0000","last_unstale":"2024-09-14T03:44:24.929475+0000","last_undegraded":"2024-09-14T03:44:24.929475+0000","last_fullsized":"2024-09-14T03:44:24.929475+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:09:38.769523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":23,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-14T03:44:26.430230+0000","last_change":"2024-09-14T03:44:26.430230+0000","last_active":"2024-09-14T03:44:26.430230+0000","last_peered":"2024-09-14T03:44:26.430230+0000","last_clean":"2024-09-14T03:44:26.430230+0000","last_became_active":"2024-09-14T03:44:26.429880+0000","last_became_peered":"2024-09-14T03:44:26.429880+0000","last_unstale":"2024-09-14T03:44:26.430230+0000","last_undegraded":"2024-09-14T03:44:26.430230+0000","last_fullsized":"2024-09-14T03:44:26.430230+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:39.132572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"58'2","reported_seq":18,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933131+0000","last_change":"2024-09-14T03:43:58.737705+0000","last_active":"2024-09-14T03:44:24.933131+0000","last_peered":"2024-09-14T03:44:24.933131+0000","last_clean":"2024-09-14T03:44:24.933131+0000","last_became_active":"2024-09-14T03:43:58.736546+0000","last_became_peered":"2024-09-14T03:43:58.736546+0000","last_unstale":"2024-09-14T03:44:24.933131+0000","last_undegraded":"2024-09-14T03:44:24.933131+0000","last_fullsized":"2024-09-14T03:44:24.933131+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:27:12.542706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":10,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-14T03:44:26.429678+0000","last_change":"2024-09-14T03:44:26.429678+0000","last_active":"2024-09-14T03:44:26.429678+0000","last_peered":"2024-09-14T03:44:26.429678+0000","last_clean":"2024-09-14T03:44:26.429678+0000","last_became_active":"2024-09-14T03:44:26.429364+0000","last_became_peered":"2024-09-14T03:44:26.429364+0000","last_unstale":"2024-09-14T03:44:26.429678+0000","last_undegraded":"2024-09-14T03:44:26.429678+0000","last_fullsized":"2024-09-14T03:44:26.429678+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:29:31.230108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929526+0000","last_change":"2024-09-14T03:44:01.888591+0000","last_active":"2024-09-14T03:44:24.929526+0000","last_peered":"2024-09-14T03:44:24.929526+0000","last_clean":"2024-09-14T03:44:24.929526+0000","last_became_active":"2024-09-14T03:44:01.888257+0000","last_became_peered":"2024-09-14T03:44:01.888257+0000","last_unstale":"2024-09-14T03:44:24.929526+0000","last_undegraded":"2024-09-14T03:44:24.929526+0000","last_fullsized":"2024-09-14T03:44:24.929526+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:24:40.992358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"58'2","reported_seq":18,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933222+0000","last_change":"2024-09-14T03:43:58.734680+0000","last_active":"2024-09-14T03:44:24.933222+0000","last_peered":"2024-09-14T03:44:24.933222+0000","last_clean":"2024-09-14T03:44:24.933222+0000","last_became_active":"2024-09-14T03:43:58.734408+0000","last_became_peered":"2024-09-14T03:43:58.734408+0000","last_unstale":"2024-09-14T03:44:24.933222+0000","last_undegraded":"2024-09-14T03:44:24.933222+0000","last_fullsized":"2024-09-14T03:44:24.933222+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:49:02.536404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.929006+0000","last_change":"2024-09-14T03:44:01.885572+0000","last_active":"2024-09-14T03:44:24.929006+0000","last_peered":"2024-09-14T03:44:24.929006+0000","last_clean":"2024-09-14T03:44:24.929006+0000","last_became_active":"2024-09-14T03:44:01.885110+0000","last_became_peered":"2024-09-14T03:44:01.885110+0000","last_unstale":"2024-09-14T03:44:24.929006+0000","last_undegraded":"2024-09-14T03:44:24.929006+0000","last_fullsized":"2024-09-14T03:44:24.929006+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:10.439239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1d","version":"58'2","reported_seq":18,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.933206+0000","last_change":"2024-09-14T03:43:58.731296+0000","last_active":"2024-09-14T03:44:24.933206+0000","last_peered":"2024-09-14T03:44:24.933206+0000","last_clean":"2024-09-14T03:44:24.933206+0000","last_became_active":"2024-09-14T03:43:58.730951+0000","last_became_peered":"2024-09-14T03:43:58.730951+0000","last_unstale":"2024-09-14T03:44:24.933206+0000","last_undegraded":"2024-09-14T03:44:24.933206+0000","last_fullsized":"2024-09-14T03:44:24.933206+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:26:14.801414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930670+0000","last_change":"2024-09-14T03:44:01.874935+0000","last_active":"2024-09-14T03:44:24.930670+0000","last_peered":"2024-09-14T03:44:24.930670+0000","last_clean":"2024-09-14T03:44:24.930670+0000","last_became_active":"2024-09-14T03:44:01.874668+0000","last_became_peered":"2024-09-14T03:44:01.874668+0000","last_unstale":"2024-09-14T03:44:24.930670+0000","last_undegraded":"2024-09-14T03:44:24.930670+0000","last_fullsized":"2024-09-14T03:44:24.930670+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:24:31.404958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"58'1","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926437+0000","last_change":"2024-09-14T03:43:58.734206+0000","last_active":"2024-09-14T03:44:24.926437+0000","last_peered":"2024-09-14T03:44:24.926437+0000","last_clean":"2024-09-14T03:44:24.926437+0000","last_became_active":"2024-09-14T03:43:58.733960+0000","last_became_peered":"2024-09-14T03:43:58.733960+0000","last_unstale":"2024-09-14T03:44:24.926437+0000","last_undegraded":"2024-09-14T03:44:24.926437+0000","last_fullsized":"2024-09-14T03:44:24.926437+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:44.073503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.926323+0000","last_change":"2024-09-14T03:44:01.883550+0000","last_active":"2024-09-14T03:44:24.926323+0000","last_peered":"2024-09-14T03:44:24.926323+0000","last_clean":"2024-09-14T03:44:24.926323+0000","last_became_active":"2024-09-14T03:44:01.883265+0000","last_became_peered":"2024-09-14T03:44:01.883265+0000","last_unstale":"2024-09-14T03:44:24.926323+0000","last_undegraded":"2024-09-14T03:44:24.926323+0000","last_fullsized":"2024-09-14T03:44:24.926323+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:52:04.512990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"58'4","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-14T03:44:24.930891+0000","last_change":"2024-09-14T03:43:58.731954+0000","last_active":"2024-09-14T03:44:24.930891+0000","last_peered":"2024-09-14T03:44:24.930891+0000","last_clean":"2024-09-14T03:44:24.930891+0000","last_became_active":"2024-09-14T03:43:58.731688+0000","last_became_peered":"2024-09-14T03:43:58.731688+0000","last_unstale":"2024-09-14T03:44:24.930891+0000","last_undegraded":"2024-09-14T03:44:24.930891+0000","last_fullsized":"2024-09-14T03:44:24.930891+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:30:10.811609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3563520,"data_stored":3542208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397520,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298528,"kb_used_data":1172,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446624,"statfs":{"total":95995035648,"available":95689342976,"internally_reserved":0,"allocated":1200128,"data_stored":844812,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:44:36 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.39500000000000002}]},{"osd":1,"last update":"Sat Sep 14 03:44:36 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.432},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.49099999999999999}]},{"osd":2,"last update":"Sat Sep 14 03:44:36 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.379},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.52100000000000002}]},{"osd":3,"last update":"Sat Sep 14 03:44:36 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.47999999999999998}]},{"osd":4,"last update":"Sat Sep 14 03:44:36 2024","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.45500000000000002}]},{"osd":5,"last update":"Sat Sep 14 03:44:36 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.60499999999999998}]},{"osd":6,"last update":"Sat Sep 14 03:44:36 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.621}]}]},{"osd":6,"up_from":43,"seq":184683593748,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298532,"kb_used_data":1172,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446620,"statfs":{"total":95995035648,"available":95689338880,"internally_reserved":0,"allocated":1200128,"data_stored":845457,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.375},{"interface":"front","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.39400000000000002}]},{"osd":1,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.496}]},{"osd":2,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.44},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.52800000000000002}]},{"osd":3,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.55100000000000005}]},{"osd":4,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.45200000000000001}]},{"osd":5,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.51500000000000001}]},{"osd":7,"last update":"Sat Sep 14 03:44:34 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.54200000000000004}]}]},{"osd":5,"up_from":37,"seq":158913789977,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297956,"kb_used_data":596,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447196,"statfs":{"total":95995035648,"available":95689928704,"internally_reserved":0,"allocated":610304,"data_stored":255179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:43:47 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.55200000000000005}]},{"osd":1,"last update":"Sat Sep 14 03:43:47 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.57799999999999996}]},{"osd":2,"last update":"Sat Sep 14 03:43:47 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.59999999999999998}]},{"osd":3,"last update":"Sat Sep 14 03:43:47 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.52200000000000002}]},{"osd":4,"last update":"Sat Sep 14 03:43:47 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.379,"15min":0.379},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.502}]},{"osd":6,"last update":"Sat Sep 14 03:44:11 2024","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.63100000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61699999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986205,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297960,"kb_used_data":600,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447192,"statfs":{"total":95995035648,"available":95689924608,"internally_reserved":0,"allocated":614400,"data_stored":254548,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:44:26 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.51000000000000001,"15min":0.51200000000000001},"min":{"1min":0.35499999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.54500000000000004,"15min":0.54700000000000004},"min":{"1min":0.40500000000000003,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.60899999999999999}]},{"osd":1,"last update":"Sat Sep 14 03:44:26 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.48399999999999999,"15min":0.47599999999999998},"min":{"1min":0.34100000000000003,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.39100000000000001,"5min":0.377,"15min":0.377},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.56699999999999995}]},{"osd":2,"last update":"Sat Sep 14 03:44:26 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.495,"15min":0.48999999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.629},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.51100000000000001,"15min":0.505},"min":{"1min":0.38,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.52000000000000002}]},{"osd":3,"last update":"Sat Sep 14 03:44:26 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.505,"15min":0.49299999999999999},"min":{"1min":0.35599999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.50700000000000001,"15min":0.501},"min":{"1min":0.35899999999999999,"5min":0.318,"15min":0.318},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.54100000000000004}]},{"osd":5,"last update":"Sat Sep 14 03:43:49 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.64700000000000002}]},{"osd":6,"last update":"Sat Sep 14 03:44:11 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.627,"5min":0.627,"15min":0.627},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.66000000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68500000000000005}]}]},{"osd":3,"up_from":26,"seq":111669149729,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297960,"kb_used_data":600,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447192,"statfs":{"total":95995035648,"available":95689924608,"internally_reserved":0,"allocated":614400,"data_stored":255193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:44:03 2024","interfaces":[{"interface":"back","average":{"1min":0.34599999999999997,"5min":0.38400000000000001,"15min":0.39000000000000001},"min":{"1min":0.245,"5min":0.185,"15min":0.185},"max":{"1min":0.43099999999999999,"5min":0.48199999999999998,"15min":0.48199999999999998},"last":0.25900000000000001},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.40200000000000002,"15min":0.40799999999999997},"min":{"1min":0.254,"5min":0.218,"15min":0.218},"max":{"1min":0.48099999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.316}]},{"osd":1,"last update":"Sat Sep 14 03:44:03 2024","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.41599999999999998,"15min":0.41999999999999998},"min":{"1min":0.28799999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.5,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.36599999999999999,"5min":0.39300000000000002,"15min":0.39700000000000002},"min":{"1min":0.30499999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.47199999999999998,"5min":0.53700000000000003,"15min":0.53700000000000003},"last":0.27600000000000002}]},{"osd":2,"last update":"Sat Sep 14 03:44:03 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.434,"15min":0.441},"min":{"1min":0.28999999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.52300000000000002,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.42299999999999999,"15min":0.42799999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.54400000000000004,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.29099999999999998}]},{"osd":4,"last update":"Sat Sep 14 03:44:31 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.53300000000000003,"15min":0.53900000000000003},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.69099999999999995,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.55700000000000005,"15min":0.56100000000000005},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.745,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.66600000000000004}]},{"osd":5,"last update":"Sat Sep 14 03:43:52 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.68600000000000005}]},{"osd":6,"last update":"Sat Sep 14 03:44:11 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.65500000000000003}]},{"osd":7,"last update":"Sat Sep 14 03:44:35 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.624}]}]},{"osd":2,"up_from":18,"seq":77309411367,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297940,"kb_used_data":580,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447212,"statfs":{"total":95995035648,"available":95689945088,"internally_reserved":0,"allocated":593920,"data_stored":253241,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:43:45 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.36299999999999999,"15min":0.35599999999999998},"min":{"1min":0.26200000000000001,"5min":0.22,"15min":0.22},"max":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.371,"15min":0.36599999999999999},"min":{"1min":0.27800000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.56200000000000006,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.38600000000000001}]},{"osd":1,"last update":"Sat Sep 14 03:43:45 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.40000000000000002,"15min":0.39800000000000002},"min":{"1min":0.32800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.441,"5min":0.39200000000000002,"15min":0.38400000000000001},"min":{"1min":0.29199999999999998,"5min":0.245,"15min":0.245},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.30399999999999999}]},{"osd":3,"last update":"Sat Sep 14 03:44:08 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.44800000000000001,"15min":0.45100000000000001},"min":{"1min":0.34999999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.53300000000000003,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.44800000000000001,"15min":0.45800000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.56000000000000005,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.34300000000000003}]},{"osd":4,"last update":"Sat Sep 14 03:44:25 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.55400000000000005,"15min":0.55700000000000005},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.78100000000000003,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.59199999999999997,"15min":0.59299999999999997},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.76000000000000001,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.66100000000000003}]},{"osd":5,"last update":"Sat Sep 14 03:43:48 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.67400000000000004}]},{"osd":6,"last update":"Sat Sep 14 03:44:12 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.54200000000000004}]},{"osd":7,"last update":"Sat Sep 14 03:44:37 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.63700000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574892,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297936,"kb_used_data":580,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447216,"statfs":{"total":95995035648,"available":95689949184,"internally_reserved":0,"allocated":593920,"data_stored":253067,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:44:20 2024","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.32000000000000001,"15min":0.29599999999999999},"min":{"1min":0.249,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.51900000000000002,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.374,"5min":0.34399999999999997,"15min":0.31900000000000001},"min":{"1min":0.27000000000000002,"5min":0.189,"15min":0.189},"max":{"1min":0.56799999999999995,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.36899999999999999}]},{"osd":2,"last update":"Sat Sep 14 03:43:39 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.35499999999999998,"15min":0.34300000000000003},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.36199999999999999,"15min":0.34699999999999998},"min":{"1min":0.29499999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.42699999999999999}]},{"osd":3,"last update":"Sat Sep 14 03:44:07 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.42299999999999999,"15min":0.42199999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.435,"15min":0.437},"min":{"1min":0.27500000000000002,"5min":0.247,"15min":0.247},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.48799999999999999}]},{"osd":4,"last update":"Sat Sep 14 03:44:28 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.77200000000000002,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.78700000000000003},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.64600000000000002}]},{"osd":5,"last update":"Sat Sep 14 03:43:47 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.80800000000000005},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.84799999999999998}]},{"osd":6,"last update":"Sat Sep 14 03:44:10 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.82799999999999996}]},{"osd":7,"last update":"Sat Sep 14 03:44:38 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.77000000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738417,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298536,"kb_used_data":1176,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446616,"statfs":{"total":95995035648,"available":95689334784,"internally_reserved":0,"allocated":1204224,"data_stored":844621,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 14 03:44:17 2024","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":0.35699999999999998,"15min":0.34899999999999998},"min":{"1min":0.25700000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.46500000000000002,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.36299999999999999,"15min":0.34799999999999998},"min":{"1min":0.246,"5min":0.216,"15min":0.216},"max":{"1min":0.48999999999999999,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.56100000000000005}]},{"osd":2,"last update":"Sat Sep 14 03:43:43 2024","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.41299999999999998,"15min":0.40999999999999998},"min":{"1min":0.29099999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.54900000000000004,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.39700000000000002,"15min":0.39500000000000002},"min":{"1min":0.28799999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.505,"5min":0.505,"15min":0.505},"last":0.40899999999999997}]},{"osd":3,"last update":"Sat Sep 14 03:44:04 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.44,"15min":0.44700000000000001},"min":{"1min":0.32500000000000001,"5min":0.307,"15min":0.307},"max":{"1min":0.54300000000000004,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.41499999999999998,"15min":0.42099999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.51600000000000001}]},{"osd":4,"last update":"Sat Sep 14 03:44:26 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.55700000000000005,"15min":0.56499999999999995},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.70099999999999996,"5min":0.754,"15min":0.754},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.55600000000000005,"15min":0.55800000000000005},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.60399999999999998}]},{"osd":5,"last update":"Sat Sep 14 03:43:48 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.629},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.66600000000000004}]},{"osd":6,"last update":"Sat Sep 14 03:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.59199999999999997}]},{"osd":7,"last update":"Sat Sep 14 03:44:39 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.64300000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1642,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T03:44:43.718 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-14T03:44:43.718 INFO:tasks.cephadm:Adding prometheus.a on smithi094 2024-09-14T03:44:43.718 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph orch apply prometheus '1;smithi094=a' 2024-09-14T03:44:43.976 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:44:44.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:43 smithi094 ceph-mon[31053]: pgmap v209: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:44.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:43 smithi094 ceph-mon[31053]: from='client.24428 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:44:44.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:43 smithi070 ceph-mon[39288]: pgmap v209: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:44.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:43 smithi070 ceph-mon[39288]: from='client.24428 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:44:44.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:43 smithi070 ceph-mon[27495]: pgmap v209: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:44.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:43 smithi070 ceph-mon[27495]: from='client.24428 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:44:45.808 INFO:teuthology.orchestra.run.smithi094.stdout:Scheduled prometheus update... 2024-09-14T03:44:46.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:45 smithi094 ceph-mon[31053]: pgmap v210: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:46.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:45 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:44:46.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:45 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:44:46.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:45 smithi070 ceph-mon[27495]: pgmap v210: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:46.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:45 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:44:46.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:45 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:44:46.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:45 smithi070 ceph-mon[39288]: pgmap v210: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:46.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:45 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:44:46.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:45 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:44:46.384 DEBUG:teuthology.orchestra.run.smithi094:prometheus.a> sudo journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@prometheus.a.service 2024-09-14T03:44:46.386 INFO:tasks.cephadm:Adding node-exporter.a on smithi070 2024-09-14T03:44:46.386 INFO:tasks.cephadm:Adding node-exporter.b on smithi094 2024-09-14T03:44:46.386 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph orch apply node-exporter '2;smithi070=a;smithi094=b' 2024-09-14T03:44:46.673 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:44:47.053 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:46 smithi094 ceph-mon[31053]: from='client.24358 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi094=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:44:47.053 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:46 smithi094 ceph-mon[31053]: Saving service prometheus spec with placement smithi094=a;count:1 2024-09-14T03:44:47.053 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:46 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:44:47.053 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:46 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:44:47.053 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:46 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:44:47.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:46 smithi070 ceph-mon[27495]: from='client.24358 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi094=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:44:47.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:46 smithi070 ceph-mon[27495]: Saving service prometheus spec with placement smithi094=a;count:1 2024-09-14T03:44:47.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:46 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:44:47.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:46 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:44:47.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:46 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:44:47.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:46 smithi070 ceph-mon[39288]: from='client.24358 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi094=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:44:47.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:46 smithi070 ceph-mon[39288]: Saving service prometheus spec with placement smithi094=a;count:1 2024-09-14T03:44:47.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:46 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:44:47.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:46 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:44:47.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:46 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:44:47.995 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:47 smithi094 ceph-mon[31053]: pgmap v211: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:47.996 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:47 smithi094 ceph-mon[31053]: Deploying daemon prometheus.a on smithi094 2024-09-14T03:44:48.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:47 smithi070 ceph-mon[27495]: pgmap v211: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:48.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:47 smithi070 ceph-mon[27495]: Deploying daemon prometheus.a on smithi094 2024-09-14T03:44:48.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:47 smithi070 ceph-mon[39288]: pgmap v211: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:48.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:47 smithi070 ceph-mon[39288]: Deploying daemon prometheus.a on smithi094 2024-09-14T03:44:48.380 INFO:teuthology.orchestra.run.smithi094.stdout:Scheduled node-exporter update... 2024-09-14T03:44:49.298 DEBUG:teuthology.orchestra.run.smithi070:node-exporter.a> sudo journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@node-exporter.a.service 2024-09-14T03:44:49.302 DEBUG:teuthology.orchestra.run.smithi094:node-exporter.b> sudo journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@node-exporter.b.service 2024-09-14T03:44:49.305 INFO:tasks.cephadm:Adding alertmanager.a on smithi070 2024-09-14T03:44:49.305 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph orch apply alertmanager '1;smithi070=a' 2024-09-14T03:44:49.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:49 smithi094 ceph-mon[31053]: from='client.24364 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi070=a;smithi094=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:44:49.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:49 smithi094 ceph-mon[31053]: Saving service node-exporter spec with placement smithi070=a;smithi094=b;count:2 2024-09-14T03:44:49.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:49 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:44:49.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:49 smithi094 ceph-mon[31053]: pgmap v212: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:49.642 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:44:49.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:49 smithi070 ceph-mon[27495]: from='client.24364 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi070=a;smithi094=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:44:49.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:49 smithi070 ceph-mon[27495]: Saving service node-exporter spec with placement smithi070=a;smithi094=b;count:2 2024-09-14T03:44:49.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:49 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:44:49.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:49 smithi070 ceph-mon[27495]: pgmap v212: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:49.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:49 smithi070 ceph-mon[39288]: from='client.24364 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi070=a;smithi094=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:44:49.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:49 smithi070 ceph-mon[39288]: Saving service node-exporter spec with placement smithi070=a;smithi094=b;count:2 2024-09-14T03:44:49.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:49 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:44:49.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:49 smithi070 ceph-mon[39288]: pgmap v212: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:51.829 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:51 smithi094 ceph-mon[31053]: pgmap v213: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:51.859 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:51 smithi070 ceph-mon[27495]: pgmap v213: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:51.860 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:51 smithi070 ceph-mon[39288]: pgmap v213: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:52.235 INFO:teuthology.orchestra.run.smithi094.stdout:Scheduled alertmanager update... 2024-09-14T03:44:53.207 DEBUG:teuthology.orchestra.run.smithi070:alertmanager.a> sudo journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@alertmanager.a.service 2024-09-14T03:44:53.210 INFO:tasks.cephadm:Adding grafana.a on smithi094 2024-09-14T03:44:53.210 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph orch apply grafana '1;smithi094=a' 2024-09-14T03:44:53.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:53 smithi094 ceph-mon[31053]: from='client.24370 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi070=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:44:53.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:53 smithi094 ceph-mon[31053]: Saving service alertmanager spec with placement smithi070=a;count:1 2024-09-14T03:44:53.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:53 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:44:53.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:53 smithi094 ceph-mon[31053]: pgmap v214: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:53.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:53 smithi070 ceph-mon[27495]: from='client.24370 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi070=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:44:53.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:53 smithi070 ceph-mon[27495]: Saving service alertmanager spec with placement smithi070=a;count:1 2024-09-14T03:44:53.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:53 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:44:53.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:53 smithi070 ceph-mon[27495]: pgmap v214: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:53.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:53 smithi070 ceph-mon[39288]: from='client.24370 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi070=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:44:53.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:53 smithi070 ceph-mon[39288]: Saving service alertmanager spec with placement smithi070=a;count:1 2024-09-14T03:44:53.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:53 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:44:53.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:53 smithi070 ceph-mon[39288]: pgmap v214: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:54.612 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:44:55.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:55 smithi094 ceph-mon[31053]: pgmap v215: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:56.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:55 smithi070 ceph-mon[27495]: pgmap v215: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:56.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:55 smithi070 ceph-mon[39288]: pgmap v215: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:58.017 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:57 smithi094 ceph-mon[31053]: pgmap v216: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:58.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:57 smithi070 ceph-mon[27495]: pgmap v216: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:58.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:57 smithi070 ceph-mon[39288]: pgmap v216: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:58.486 INFO:teuthology.orchestra.run.smithi094.stdout:Scheduled grafana update... 2024-09-14T03:44:59.761 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:59 smithi094 ceph-mon[31053]: from='client.24376 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi094=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:44:59.761 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:59 smithi094 ceph-mon[31053]: Saving service grafana spec with placement smithi094=a;count:1 2024-09-14T03:44:59.761 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:59 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:44:59.761 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:44:59 smithi094 ceph-mon[31053]: pgmap v217: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:59.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:59 smithi070 ceph-mon[27495]: from='client.24376 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi094=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:44:59.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:59 smithi070 ceph-mon[27495]: Saving service grafana spec with placement smithi094=a;count:1 2024-09-14T03:44:59.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:59 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:44:59.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:44:59 smithi070 ceph-mon[27495]: pgmap v217: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:59.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:59 smithi070 ceph-mon[39288]: from='client.24376 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi094=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:44:59.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:59 smithi070 ceph-mon[39288]: Saving service grafana spec with placement smithi094=a;count:1 2024-09-14T03:44:59.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:59 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:44:59.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:44:59 smithi070 ceph-mon[39288]: pgmap v217: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:44:59.789 DEBUG:teuthology.orchestra.run.smithi094:grafana.a> sudo journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@grafana.a.service 2024-09-14T03:44:59.793 INFO:tasks.cephadm:Setting up client nodes... 2024-09-14T03:44:59.793 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-14T03:45:00.048 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:45:00.777 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:45:00 smithi094 systemd[1]: Starting Ceph prometheus.a for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:45:01.082 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:45:00 smithi094 podman[61373]: 2024-09-14 03:45:00.725354864 +0000 UTC m=+0.016821182 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-14T03:45:01.082 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:45:00 smithi094 podman[61373]: 2024-09-14 03:45:00.822370054 +0000 UTC m=+0.113836360 container create 5d8e345869aa12e9df696951ddcb74223c3b51b1df2a2d9ce286eae79f1fa464 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:45:01.083 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:45:01 smithi094 podman[61373]: 2024-09-14 03:45:01.032684984 +0000 UTC m=+0.324151290 container init 5d8e345869aa12e9df696951ddcb74223c3b51b1df2a2d9ce286eae79f1fa464 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:45:01.083 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:45:01 smithi094 podman[61373]: 2024-09-14 03:45:01.037447414 +0000 UTC m=+0.328913722 container start 5d8e345869aa12e9df696951ddcb74223c3b51b1df2a2d9ce286eae79f1fa464 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:45:01.083 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:45:01 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[61388]: ts=2024-09-14T03:45:01.061Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-14T03:45:01.083 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:45:01 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[61388]: ts=2024-09-14T03:45:01.061Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-14T03:45:01.083 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:45:01 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[61388]: ts=2024-09-14T03:45:01.061Z caller=main.go:570 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi094 (none))" 2024-09-14T03:45:01.083 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:45:01 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[61388]: ts=2024-09-14T03:45:01.061Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-14T03:45:01.083 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:45:01 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[61388]: ts=2024-09-14T03:45:01.061Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-14T03:45:01.083 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:45:01 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[61388]: ts=2024-09-14T03:45:01.062Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-14T03:45:01.083 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:45:01 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[61388]: ts=2024-09-14T03:45:01.063Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-14T03:45:01.083 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:45:01 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[61388]: ts=2024-09-14T03:45:01.063Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-14T03:45:01.083 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:45:01 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[61388]: ts=2024-09-14T03:45:01.063Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-14T03:45:01.083 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:45:01 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[61388]: ts=2024-09-14T03:45:01.066Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-14T03:45:01.084 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:45:01 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[61388]: ts=2024-09-14T03:45:01.066Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.119µs 2024-09-14T03:45:01.084 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:45:01 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[61388]: ts=2024-09-14T03:45:01.066Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-14T03:45:01.084 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:45:01 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[61388]: ts=2024-09-14T03:45:01.067Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-09-14T03:45:01.084 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:45:01 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[61388]: ts=2024-09-14T03:45:01.067Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=25.137µs wal_replay_duration=357.794µs wbl_replay_duration=126ns total_replay_duration=407.063µs 2024-09-14T03:45:01.084 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:45:01 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[61388]: ts=2024-09-14T03:45:01.068Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-14T03:45:01.084 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:45:01 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[61388]: ts=2024-09-14T03:45:01.068Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-14T03:45:01.085 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:45:01 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[61388]: ts=2024-09-14T03:45:01.068Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-14T03:45:01.085 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:45:01 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[61388]: ts=2024-09-14T03:45:01.081Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=13.309262ms db_storage=782ns remote_storage=982ns web_handler=310ns query_engine=523ns scrape=378.754µs scrape_sd=16.225µs notify=632ns notify_sd=1.49µs rules=12.745422ms tracing=5.263µs 2024-09-14T03:45:01.085 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:45:01 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[61388]: ts=2024-09-14T03:45:01.081Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-14T03:45:01.085 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:45:01 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[61388]: ts=2024-09-14T03:45:01.081Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-14T03:45:01.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:45:01 smithi094 bash[61373]: 5d8e345869aa12e9df696951ddcb74223c3b51b1df2a2d9ce286eae79f1fa464 2024-09-14T03:45:01.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:45:01 smithi094 systemd[1]: Started Ceph prometheus.a for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:45:01.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:01 smithi070 ceph-mon[27495]: pgmap v218: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:01.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:01 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:45:01.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:01 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:45:01.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:01 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:45:01.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:01 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-14T03:45:01.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:01 smithi070 ceph-mon[39288]: pgmap v218: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:01.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:01 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:45:01.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:01 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:45:01.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:01 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:45:01.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:01 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-14T03:45:01.966 INFO:teuthology.orchestra.run.smithi070.stdout:[client.0] 2024-09-14T03:45:01.966 INFO:teuthology.orchestra.run.smithi070.stdout: key = AQC9BuVmhSpSORAA9E7wL/4t9PWBf0encGPZlQ== 2024-09-14T03:45:02.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:01 smithi094 ceph-mon[31053]: pgmap v218: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:02.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:01 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:45:02.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:01 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:45:02.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:01 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' 2024-09-14T03:45:02.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:01 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-14T03:45:02.285 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:02 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: ignoring --setuser ceph since I am not root 2024-09-14T03:45:02.285 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:02 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: ignoring --setgroup ceph since I am not root 2024-09-14T03:45:02.286 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:02 smithi070 ceph-mgr[27770]: -- 172.21.15.70:0/962499621 <== mon.0 v2:172.21.15.70:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55c6f9f10340 con 0x55c6f9f1d000 2024-09-14T03:45:02.526 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:02 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: ignoring --setuser ceph since I am not root 2024-09-14T03:45:02.527 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:02 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: ignoring --setgroup ceph since I am not root 2024-09-14T03:45:02.527 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:02 smithi094 ceph-mgr[33054]: -- 172.21.15.94:0/593157077 <== mon.1 v2:172.21.15.94:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x560d9e118340 con 0x560d9e125000 2024-09-14T03:45:02.527 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:02 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:45:02.305+0000 7f8d373e1ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-14T03:45:02.527 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:02 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:45:02.358+0000 7f8d373e1ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-14T03:45:02.527 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:02 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:45:02.423+0000 7f8d373e1ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-14T03:45:02.553 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:02 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:02.306+0000 7fe7b3295ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-14T03:45:02.553 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:02 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:02.363+0000 7fe7b3295ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-14T03:45:02.553 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:02 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:02.428+0000 7fe7b3295ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-14T03:45:02.553 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:02 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:02.551+0000 7fe7b3295ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:45:02.626 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2024-09-14T03:45:02.626 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-14T03:45:02.626 DEBUG:teuthology.orchestra.run.smithi070:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-14T03:45:02.664 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-14T03:45:02.872 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:02 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:02.696+0000 7fe7b3295ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-14T03:45:02.872 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:02 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/485497937' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:45:02.872 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:02 smithi070 ceph-mon[27495]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:45:02.872 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:02 smithi070 ceph-mon[27495]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-14T03:45:02.872 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:02 smithi070 ceph-mon[27495]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-14T03:45:02.873 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:02 smithi070 ceph-mon[27495]: mgrmap e16: y(active, since 5m), standbys: x 2024-09-14T03:45:02.873 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:02 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/485497937' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:45:02.873 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:02 smithi070 ceph-mon[39288]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:45:02.873 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:02 smithi070 ceph-mon[39288]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-14T03:45:02.873 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:02 smithi070 ceph-mon[39288]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-14T03:45:02.873 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:02 smithi070 ceph-mon[39288]: mgrmap e16: y(active, since 5m), standbys: x 2024-09-14T03:45:02.920 INFO:teuthology.orchestra.run.smithi094.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:45:02.960 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:02 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/485497937' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:45:02.960 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:02 smithi094 ceph-mon[31053]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:45:02.960 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:02 smithi094 ceph-mon[31053]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-14T03:45:02.960 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:02 smithi094 ceph-mon[31053]: from='mgr.14150 172.21.15.70:0/2577180687' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-14T03:45:02.960 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:02 smithi094 ceph-mon[31053]: mgrmap e16: y(active, since 5m), standbys: x 2024-09-14T03:45:02.960 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:02 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:45:02.546+0000 7f8d373e1ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:45:02.960 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:02 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:45:02.684+0000 7f8d373e1ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-14T03:45:02.961 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:02 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:45:02.958+0000 7f8d373e1ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-14T03:45:03.176 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:02 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:02.959+0000 7fe7b3295ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-14T03:45:03.176 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:03.017+0000 7fe7b3295ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-14T03:45:03.277 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:03 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:45:03.019+0000 7f8d373e1ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-14T03:45:03.277 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:03 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:45:03.191+0000 7f8d373e1ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-14T03:45:03.535 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:03.175+0000 7fe7b3295ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-14T03:45:04.285 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:04 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:04.006+0000 7fe7b3295ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-14T03:45:04.285 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:04 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:04.069+0000 7fe7b3295ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-14T03:45:04.328 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:04 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:45:04.045+0000 7f8d373e1ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-14T03:45:04.328 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:04 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:45:04.113+0000 7f8d373e1ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-14T03:45:04.931 INFO:teuthology.orchestra.run.smithi094.stdout:[client.1] 2024-09-14T03:45:04.932 INFO:teuthology.orchestra.run.smithi094.stdout: key = AQDABuVmg/dDNxAAFPWcBlE782Xm5sxkKCeobg== 2024-09-14T03:45:04.932 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:04 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:45:04.837+0000 7f8d373e1ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-14T03:45:05.035 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:04 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:04.760+0000 7fe7b3295ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-14T03:45:05.035 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:04 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:04.869+0000 7fe7b3295ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-14T03:45:05.036 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:04 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:04.990+0000 7fe7b3295ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:45:05.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:04 smithi070 ceph-mon[27495]: from='client.? 172.21.15.94:0/2113444988' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:45:05.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:04 smithi070 ceph-mon[27495]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:45:05.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:04 smithi070 ceph-mon[27495]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-14T03:45:05.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:04 smithi070 ceph-mon[39288]: from='client.? 172.21.15.94:0/2113444988' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:45:05.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:04 smithi070 ceph-mon[39288]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:45:05.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:04 smithi070 ceph-mon[39288]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-14T03:45:05.255 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:04 smithi094 ceph-mon[31053]: from='client.? 172.21.15.94:0/2113444988' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:45:05.256 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:04 smithi094 ceph-mon[31053]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:45:05.256 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:04 smithi094 ceph-mon[31053]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-14T03:45:05.256 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:04 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:45:04.955+0000 7f8d373e1ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-14T03:45:05.256 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:45:05.078+0000 7f8d373e1ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:45:05.256 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:45:05.137+0000 7f8d373e1ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-14T03:45:05.431 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:05 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:05.050+0000 7fe7b3295ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-14T03:45:05.431 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:05 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:05.174+0000 7fe7b3295ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-14T03:45:05.432 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:05 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:05.241+0000 7fe7b3295ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-14T03:45:05.432 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:05 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:05.298+0000 7fe7b3295ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-14T03:45:05.432 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:05 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:05.430+0000 7fe7b3295ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-14T03:45:05.526 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:45:05.266+0000 7f8d373e1ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-14T03:45:05.527 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:45:05.333+0000 7f8d373e1ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-14T03:45:05.527 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:45:05.390+0000 7f8d373e1ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-14T03:45:05.605 DEBUG:teuthology.orchestra.run.smithi094:> set -ex 2024-09-14T03:45:05.605 DEBUG:teuthology.orchestra.run.smithi094:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-09-14T03:45:05.605 DEBUG:teuthology.orchestra.run.smithi094:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-09-14T03:45:05.643 INFO:tasks.cephadm:Creating RBD pool 2024-09-14T03:45:05.643 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph osd pool create rbd 8 2024-09-14T03:45:05.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:05 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:05.493+0000 7fe7b3295ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-14T03:45:05.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:05 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:05.557+0000 7fe7b3295ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-14T03:45:05.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:05 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:05.636+0000 7fe7b3295ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-14T03:45:05.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:05 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:05.697+0000 7fe7b3295ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-14T03:45:05.800 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:45:05.527+0000 7f8d373e1ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-14T03:45:05.800 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:45:05.591+0000 7f8d373e1ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-14T03:45:05.800 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:45:05.661+0000 7f8d373e1ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-14T03:45:05.800 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:45:05.741+0000 7f8d373e1ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-14T03:45:05.800 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:45:05.797+0000 7f8d373e1ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-14T03:45:05.895 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:45:06.017 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:05 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:05.758+0000 7fe7b3295ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-14T03:45:06.276 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:45:05.856+0000 7f8d373e1ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-14T03:45:06.590 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:06.532+0000 7fe7b3295ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-14T03:45:06.861 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:06 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:45:06.603+0000 7f8d373e1ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-14T03:45:06.862 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:06 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: [14/Sep/2024:03:45:06] ENGINE Bus STARTING 2024-09-14T03:45:06.862 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:06 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: CherryPy Checker: 2024-09-14T03:45:06.862 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:06 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: The Application mounted at '' has an empty config. 2024-09-14T03:45:06.862 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:06 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: 2024-09-14T03:45:06.862 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:06 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: [14/Sep/2024:03:45:06] ENGINE Serving on http://:::9283 2024-09-14T03:45:06.862 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:06 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: [14/Sep/2024:03:45:06] ENGINE Bus STARTED 2024-09-14T03:45:06.862 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:06 smithi094 ceph-mon[31053]: Active manager daemon y restarted 2024-09-14T03:45:06.862 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:06 smithi094 ceph-mon[31053]: Activating manager daemon y 2024-09-14T03:45:06.862 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:06 smithi094 ceph-mon[31053]: osdmap e61: 8 total, 8 up, 8 in 2024-09-14T03:45:06.863 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:06 smithi094 ceph-mon[31053]: mgrmap e17: y(active, starting, since 0.00660432s), standbys: x 2024-09-14T03:45:06.863 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:06 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:45:06.863 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:06 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:45:06.863 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:06 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:45:06.863 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:06 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-14T03:45:06.863 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:06 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-14T03:45:06.863 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:06 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:45:06.863 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:06 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:45:06.863 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:06 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:45:06.863 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:06 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:45:06.863 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:06 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:45:06.863 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:06 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:45:06.863 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:06 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:45:06.863 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:06 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:45:06.864 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:06 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:45:06.864 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:06 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:45:06.864 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:06 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:45:06.864 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:06 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:45:06.864 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:06 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:45:06.864 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:06 smithi094 ceph-mon[31053]: Manager daemon y is now available 2024-09-14T03:45:06.864 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:06 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:45:06.865 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[27495]: Active manager daemon y restarted 2024-09-14T03:45:06.866 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[27495]: Activating manager daemon y 2024-09-14T03:45:06.866 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[27495]: osdmap e61: 8 total, 8 up, 8 in 2024-09-14T03:45:06.866 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[27495]: mgrmap e17: y(active, starting, since 0.00660432s), standbys: x 2024-09-14T03:45:06.866 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:45:06.866 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:45:06.866 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:45:06.866 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-14T03:45:06.866 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-14T03:45:06.866 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:45:06.866 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:45:06.866 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:45:06.866 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:45:06.866 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:45:06.866 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:45:06.867 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:45:06.867 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:45:06.867 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:45:06.868 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:45:06.868 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:45:06.868 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:45:06.868 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:45:06.868 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[27495]: Manager daemon y is now available 2024-09-14T03:45:06.868 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:45:06.868 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:45:06] ENGINE Bus STARTING 2024-09-14T03:45:06.868 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: CherryPy Checker: 2024-09-14T03:45:06.868 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: The Application mounted at '' has an empty config. 2024-09-14T03:45:06.868 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:06.869 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:06.638+0000 7fe760c6d640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:45:06.869 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:06.638+0000 7fe760c6d640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:45:06.869 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:06.638+0000 7fe760c6d640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:45:06.869 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:06.638+0000 7fe760c6d640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:45:06.869 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:06.638+0000 7fe76747a640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:45:06.869 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:06.638+0000 7fe760c6d640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:45:06.869 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:06.638+0000 7fe76747a640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:45:06.869 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:06.638+0000 7fe76747a640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:45:06.869 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:06.638+0000 7fe76747a640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:45:06.869 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:06.638+0000 7fe76747a640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:45:06.869 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:45:06] ENGINE Serving on http://:::9283 2024-09-14T03:45:06.869 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:45:06] ENGINE Bus STARTED 2024-09-14T03:45:06.870 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[39288]: Active manager daemon y restarted 2024-09-14T03:45:06.870 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[39288]: Activating manager daemon y 2024-09-14T03:45:06.870 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[39288]: osdmap e61: 8 total, 8 up, 8 in 2024-09-14T03:45:06.870 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[39288]: mgrmap e17: y(active, starting, since 0.00660432s), standbys: x 2024-09-14T03:45:06.870 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:45:06.870 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:45:06.870 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:45:06.870 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-14T03:45:06.870 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-14T03:45:06.870 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:45:06.870 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:45:06.870 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:45:06.870 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:45:06.870 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:45:06.871 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:45:06.871 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:45:06.871 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:45:06.871 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:45:06.871 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:45:06.871 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:45:06.871 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:45:06.871 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:45:06.871 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[39288]: Manager daemon y is now available 2024-09-14T03:45:06.871 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:06 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:45:07.693 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:07.693 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:45:07.693 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:45:07.693 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-mon[27495]: Standby manager daemon x restarted 2024-09-14T03:45:07.693 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-mon[27495]: Standby manager daemon x started 2024-09-14T03:45:07.693 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-mon[27495]: from='mgr.? 172.21.15.94:0/3795974762' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:45:07.693 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-mon[27495]: from='mgr.? 172.21.15.94:0/3795974762' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:45:07.693 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-mon[27495]: from='mgr.? 172.21.15.94:0/3795974762' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:45:07.693 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:45:07.693 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-mon[27495]: from='mgr.? 172.21.15.94:0/3795974762' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:45:07.693 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-mon[27495]: mgrmap e18: y(active, since 1.01159s), standbys: x 2024-09-14T03:45:07.694 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:07.694 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:45:07.694 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:45:07.694 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-mon[39288]: Standby manager daemon x restarted 2024-09-14T03:45:07.694 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-mon[39288]: Standby manager daemon x started 2024-09-14T03:45:07.694 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-mon[39288]: from='mgr.? 172.21.15.94:0/3795974762' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:45:07.694 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-mon[39288]: from='mgr.? 172.21.15.94:0/3795974762' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:45:07.694 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-mon[39288]: from='mgr.? 172.21.15.94:0/3795974762' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:45:07.694 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:45:07.694 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-mon[39288]: from='mgr.? 172.21.15.94:0/3795974762' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:45:07.695 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-mon[39288]: mgrmap e18: y(active, since 1.01159s), standbys: x 2024-09-14T03:45:07.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:07 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:07.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:07 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:45:07.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:07 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:45:07.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:07 smithi094 ceph-mon[31053]: Standby manager daemon x restarted 2024-09-14T03:45:07.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:07 smithi094 ceph-mon[31053]: Standby manager daemon x started 2024-09-14T03:45:07.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:07 smithi094 ceph-mon[31053]: from='mgr.? 172.21.15.94:0/3795974762' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:45:07.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:07 smithi094 ceph-mon[31053]: from='mgr.? 172.21.15.94:0/3795974762' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:45:07.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:07 smithi094 ceph-mon[31053]: from='mgr.? 172.21.15.94:0/3795974762' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:45:07.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:07 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:45:07.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:07 smithi094 ceph-mon[31053]: from='mgr.? 172.21.15.94:0/3795974762' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:45:07.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:07 smithi094 ceph-mon[31053]: mgrmap e18: y(active, since 1.01159s), standbys: x 2024-09-14T03:45:07.964 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:07.689+0000 7fe74c044640 -1 log_channel(cephadm) log [ERR] : [14/Sep/2024:03:45:07] ENGINE Error in HTTPServer.serve 2024-09-14T03:45:07.964 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: Traceback (most recent call last): 2024-09-14T03:45:07.964 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:45:07.964 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: self._connections.run(self.expiration_interval) 2024-09-14T03:45:07.965 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:45:07.965 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: self._run(expiration_interval) 2024-09-14T03:45:07.965 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:45:07.965 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:45:07.965 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:45:07.965 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:45:07.965 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:45:07.965 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: s = self.context.wrap_socket( 2024-09-14T03:45:07.965 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:45:07.965 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: return self.sslsocket_class._create( 2024-09-14T03:45:07.965 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:45:07.966 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: self.do_handshake() 2024-09-14T03:45:07.966 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:45:07.966 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: self._sslobj.do_handshake() 2024-09-14T03:45:07.966 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:45:07.966 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:07 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:45:08.709 INFO:teuthology.orchestra.run.smithi070.stderr:pool 'rbd' created 2024-09-14T03:45:08.940 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:08 smithi094 ceph-mon[31053]: [14/Sep/2024:03:45:07] ENGINE Bus STARTING 2024-09-14T03:45:08.940 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:08 smithi094 ceph-mon[31053]: [14/Sep/2024:03:45:07] ENGINE Serving on https://172.21.15.70:7150 2024-09-14T03:45:08.940 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:08 smithi094 ceph-mon[31053]: [14/Sep/2024:03:45:07] ENGINE Bus STARTED 2024-09-14T03:45:08.940 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:08 smithi094 ceph-mon[31053]: [14/Sep/2024:03:45:07] ENGINE Error in HTTPServer.serve 2024-09-14T03:45:08.940 INFO:journalctl@ceph.mon.b.smithi094.stdout: Traceback (most recent call last): 2024-09-14T03:45:08.940 INFO:journalctl@ceph.mon.b.smithi094.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:45:08.941 INFO:journalctl@ceph.mon.b.smithi094.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:45:08.941 INFO:journalctl@ceph.mon.b.smithi094.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:45:08.941 INFO:journalctl@ceph.mon.b.smithi094.stdout: self._run(expiration_interval) 2024-09-14T03:45:08.941 INFO:journalctl@ceph.mon.b.smithi094.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:45:08.941 INFO:journalctl@ceph.mon.b.smithi094.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:45:08.941 INFO:journalctl@ceph.mon.b.smithi094.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:45:08.941 INFO:journalctl@ceph.mon.b.smithi094.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:45:08.941 INFO:journalctl@ceph.mon.b.smithi094.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:45:08.941 INFO:journalctl@ceph.mon.b.smithi094.stdout: s = self.context.wrap_socket( 2024-09-14T03:45:08.941 INFO:journalctl@ceph.mon.b.smithi094.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:45:08.941 INFO:journalctl@ceph.mon.b.smithi094.stdout: return self.sslsocket_class._create( 2024-09-14T03:45:08.941 INFO:journalctl@ceph.mon.b.smithi094.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:45:08.941 INFO:journalctl@ceph.mon.b.smithi094.stdout: self.do_handshake() 2024-09-14T03:45:08.942 INFO:journalctl@ceph.mon.b.smithi094.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:45:08.942 INFO:journalctl@ceph.mon.b.smithi094.stdout: self._sslobj.do_handshake() 2024-09-14T03:45:08.942 INFO:journalctl@ceph.mon.b.smithi094.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:45:08.942 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:08 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:08.942 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:08 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/2175536948' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-14T03:45:08.942 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:08 smithi094 ceph-mon[31053]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-14T03:45:09.027 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:08 smithi070 ceph-mon[27495]: [14/Sep/2024:03:45:07] ENGINE Bus STARTING 2024-09-14T03:45:09.027 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:08 smithi070 ceph-mon[27495]: [14/Sep/2024:03:45:07] ENGINE Serving on https://172.21.15.70:7150 2024-09-14T03:45:09.028 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:08 smithi070 ceph-mon[27495]: [14/Sep/2024:03:45:07] ENGINE Bus STARTED 2024-09-14T03:45:09.028 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:08 smithi070 ceph-mon[27495]: [14/Sep/2024:03:45:07] ENGINE Error in HTTPServer.serve 2024-09-14T03:45:09.028 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-09-14T03:45:09.028 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:45:09.028 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:45:09.028 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:45:09.028 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._run(expiration_interval) 2024-09-14T03:45:09.028 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:45:09.028 INFO:journalctl@ceph.mon.a.smithi070.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:45:09.028 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:45:09.028 INFO:journalctl@ceph.mon.a.smithi070.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:45:09.028 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:45:09.028 INFO:journalctl@ceph.mon.a.smithi070.stdout: s = self.context.wrap_socket( 2024-09-14T03:45:09.028 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:45:09.029 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.sslsocket_class._create( 2024-09-14T03:45:09.029 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:45:09.029 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.do_handshake() 2024-09-14T03:45:09.029 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:45:09.029 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._sslobj.do_handshake() 2024-09-14T03:45:09.029 INFO:journalctl@ceph.mon.a.smithi070.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:45:09.029 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:08 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:09.029 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:08 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/2175536948' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-14T03:45:09.029 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:08 smithi070 ceph-mon[27495]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-14T03:45:09.029 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:08 smithi070 ceph-mon[39288]: [14/Sep/2024:03:45:07] ENGINE Bus STARTING 2024-09-14T03:45:09.029 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:08 smithi070 ceph-mon[39288]: [14/Sep/2024:03:45:07] ENGINE Serving on https://172.21.15.70:7150 2024-09-14T03:45:09.030 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:08 smithi070 ceph-mon[39288]: [14/Sep/2024:03:45:07] ENGINE Bus STARTED 2024-09-14T03:45:09.030 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:08 smithi070 ceph-mon[39288]: [14/Sep/2024:03:45:07] ENGINE Error in HTTPServer.serve 2024-09-14T03:45:09.030 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-09-14T03:45:09.030 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:45:09.030 INFO:journalctl@ceph.mon.c.smithi070.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:45:09.030 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:45:09.030 INFO:journalctl@ceph.mon.c.smithi070.stdout: self._run(expiration_interval) 2024-09-14T03:45:09.030 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:45:09.030 INFO:journalctl@ceph.mon.c.smithi070.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:45:09.030 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:45:09.030 INFO:journalctl@ceph.mon.c.smithi070.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:45:09.030 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:45:09.030 INFO:journalctl@ceph.mon.c.smithi070.stdout: s = self.context.wrap_socket( 2024-09-14T03:45:09.030 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:45:09.031 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.sslsocket_class._create( 2024-09-14T03:45:09.031 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:45:09.031 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.do_handshake() 2024-09-14T03:45:09.031 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:45:09.031 INFO:journalctl@ceph.mon.c.smithi070.stdout: self._sslobj.do_handshake() 2024-09-14T03:45:09.031 INFO:journalctl@ceph.mon.c.smithi070.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:45:09.031 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:08 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:09.031 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:08 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/2175536948' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-14T03:45:09.031 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:08 smithi070 ceph-mon[39288]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-14T03:45:09.492 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- rbd pool init rbd 2024-09-14T03:45:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[27495]: pgmap v4: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[27495]: mgrmap e19: y(active, since 2s), standbys: x 2024-09-14T03:45:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[27495]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-14T03:45:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[27495]: osdmap e62: 8 total, 8 up, 8 in 2024-09-14T03:45:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:45:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:45:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:45:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:45:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:45:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:45:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:45:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:45:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:45:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:45:09.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[39288]: pgmap v4: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:09.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:09.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[39288]: mgrmap e19: y(active, since 2s), standbys: x 2024-09-14T03:45:09.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[39288]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-14T03:45:09.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[39288]: osdmap e62: 8 total, 8 up, 8 in 2024-09-14T03:45:09.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:09.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:09.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:09.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:09.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:09.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:45:09.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:45:09.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:45:09.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:45:09.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:09.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:09.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:09.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:45:09.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:45:09.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:45:09.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:45:09.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:09.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:45:09.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:09 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:45:09.756 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:45:09.951 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:09 smithi094 ceph-mon[31053]: pgmap v4: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:09.951 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:09 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:09.952 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:09 smithi094 ceph-mon[31053]: mgrmap e19: y(active, since 2s), standbys: x 2024-09-14T03:45:09.952 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:09 smithi094 ceph-mon[31053]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-14T03:45:09.952 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:09 smithi094 ceph-mon[31053]: osdmap e62: 8 total, 8 up, 8 in 2024-09-14T03:45:09.952 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:09 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:09.952 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:09 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:09.952 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:09 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:09.952 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:09 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:09.952 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:09 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:09.952 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:09 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:45:09.952 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:09 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:45:09.952 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:09 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:45:09.952 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:09 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:45:09.952 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:09 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:09.953 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:09 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:09.953 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:09 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:09.953 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:09 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:45:09.953 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:09 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:45:09.953 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:09 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:45:09.953 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:09 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:45:09.953 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:09 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:09.953 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:09 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:45:09.953 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:09 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:45:11.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:10 smithi094 ceph-mon[31053]: Adjusting osd_memory_target on smithi070 to 2992M 2024-09-14T03:45:11.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:10 smithi094 ceph-mon[31053]: Adjusting osd_memory_target on smithi094 to 2992M 2024-09-14T03:45:11.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:10 smithi094 ceph-mon[31053]: Updating smithi070:/etc/ceph/ceph.conf 2024-09-14T03:45:11.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:10 smithi094 ceph-mon[31053]: Updating smithi094:/etc/ceph/ceph.conf 2024-09-14T03:45:11.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:10 smithi094 ceph-mon[31053]: osdmap e63: 8 total, 8 up, 8 in 2024-09-14T03:45:11.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:10 smithi094 ceph-mon[31053]: Updating smithi070:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:45:11.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:10 smithi094 ceph-mon[31053]: Updating smithi094:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:45:11.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:10 smithi094 ceph-mon[31053]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:45:11.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:10 smithi094 ceph-mon[31053]: Updating smithi094:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:45:11.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:10 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:11.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:10 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:11.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:10 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:11.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:10 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:11.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:10 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:11.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:10 smithi070 ceph-mon[27495]: Adjusting osd_memory_target on smithi070 to 2992M 2024-09-14T03:45:11.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:10 smithi070 ceph-mon[27495]: Adjusting osd_memory_target on smithi094 to 2992M 2024-09-14T03:45:11.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:10 smithi070 ceph-mon[27495]: Updating smithi070:/etc/ceph/ceph.conf 2024-09-14T03:45:11.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:10 smithi070 ceph-mon[27495]: Updating smithi094:/etc/ceph/ceph.conf 2024-09-14T03:45:11.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:10 smithi070 ceph-mon[27495]: osdmap e63: 8 total, 8 up, 8 in 2024-09-14T03:45:11.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:10 smithi070 ceph-mon[27495]: Updating smithi070:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:45:11.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:10 smithi070 ceph-mon[27495]: Updating smithi094:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:45:11.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:10 smithi070 ceph-mon[27495]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:45:11.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:10 smithi070 ceph-mon[27495]: Updating smithi094:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:45:11.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:10 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:11.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:10 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:11.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:10 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:11.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:10 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:11.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:10 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:11.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:10 smithi070 ceph-mon[39288]: Adjusting osd_memory_target on smithi070 to 2992M 2024-09-14T03:45:11.040 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:10 smithi070 ceph-mon[39288]: Adjusting osd_memory_target on smithi094 to 2992M 2024-09-14T03:45:11.040 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:10 smithi070 ceph-mon[39288]: Updating smithi070:/etc/ceph/ceph.conf 2024-09-14T03:45:11.040 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:10 smithi070 ceph-mon[39288]: Updating smithi094:/etc/ceph/ceph.conf 2024-09-14T03:45:11.040 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:10 smithi070 ceph-mon[39288]: osdmap e63: 8 total, 8 up, 8 in 2024-09-14T03:45:11.040 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:10 smithi070 ceph-mon[39288]: Updating smithi070:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:45:11.040 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:10 smithi070 ceph-mon[39288]: Updating smithi094:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:45:11.040 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:10 smithi070 ceph-mon[39288]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:45:11.040 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:10 smithi070 ceph-mon[39288]: Updating smithi094:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:45:11.040 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:10 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:11.040 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:10 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:11.041 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:10 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:11.041 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:10 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:11.041 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:10 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:12.027 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:11 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:45:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:45:12.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:11 smithi094 ceph-mon[31053]: Updating smithi070:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:45:12.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:11 smithi094 ceph-mon[31053]: Updating smithi094:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:45:12.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:11 smithi094 ceph-mon[31053]: pgmap v7: 105 pgs: 8 unknown, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:12.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:11 smithi094 ceph-mon[31053]: Deploying daemon alertmanager.a on smithi070 2024-09-14T03:45:12.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:11 smithi094 ceph-mon[31053]: osdmap e64: 8 total, 8 up, 8 in 2024-09-14T03:45:12.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:11 smithi094 ceph-mon[31053]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:45:12.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:11 smithi094 ceph-mon[31053]: mgrmap e20: y(active, since 4s), standbys: x 2024-09-14T03:45:12.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:11 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/3876338' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-14T03:45:12.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:11 smithi094 ceph-mon[31053]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-14T03:45:12.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:11 smithi070 ceph-mon[39288]: Updating smithi070:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:45:12.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:11 smithi070 ceph-mon[39288]: Updating smithi094:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:45:12.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:11 smithi070 ceph-mon[39288]: pgmap v7: 105 pgs: 8 unknown, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:12.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:11 smithi070 ceph-mon[39288]: Deploying daemon alertmanager.a on smithi070 2024-09-14T03:45:12.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:11 smithi070 ceph-mon[39288]: osdmap e64: 8 total, 8 up, 8 in 2024-09-14T03:45:12.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:11 smithi070 ceph-mon[39288]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:45:12.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:11 smithi070 ceph-mon[39288]: mgrmap e20: y(active, since 4s), standbys: x 2024-09-14T03:45:12.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:11 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/3876338' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-14T03:45:12.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:11 smithi070 ceph-mon[39288]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-14T03:45:12.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:11 smithi070 ceph-mon[27495]: Updating smithi070:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:45:12.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:11 smithi070 ceph-mon[27495]: Updating smithi094:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:45:12.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:11 smithi070 ceph-mon[27495]: pgmap v7: 105 pgs: 8 unknown, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:12.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:11 smithi070 ceph-mon[27495]: Deploying daemon alertmanager.a on smithi070 2024-09-14T03:45:12.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:11 smithi070 ceph-mon[27495]: osdmap e64: 8 total, 8 up, 8 in 2024-09-14T03:45:12.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:11 smithi070 ceph-mon[27495]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:45:12.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:11 smithi070 ceph-mon[27495]: mgrmap e20: y(active, since 4s), standbys: x 2024-09-14T03:45:12.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:11 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/3876338' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-14T03:45:12.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:11 smithi070 ceph-mon[27495]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-14T03:45:13.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:13 smithi094 ceph-mon[31053]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-14T03:45:13.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:13 smithi094 ceph-mon[31053]: osdmap e65: 8 total, 8 up, 8 in 2024-09-14T03:45:13.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:13 smithi094 ceph-mon[31053]: pgmap v10: 105 pgs: 8 unknown, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:13.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:13 smithi070 ceph-mon[39288]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-14T03:45:13.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:13 smithi070 ceph-mon[39288]: osdmap e65: 8 total, 8 up, 8 in 2024-09-14T03:45:13.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:13 smithi070 ceph-mon[39288]: pgmap v10: 105 pgs: 8 unknown, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:13.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:13 smithi070 ceph-mon[27495]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-14T03:45:13.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:13 smithi070 ceph-mon[27495]: osdmap e65: 8 total, 8 up, 8 in 2024-09-14T03:45:13.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:13 smithi070 ceph-mon[27495]: pgmap v10: 105 pgs: 8 unknown, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:14.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:14 smithi094 ceph-mon[31053]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:45:14.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:14 smithi094 ceph-mon[31053]: Cluster is now healthy 2024-09-14T03:45:14.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:14 smithi094 ceph-mon[31053]: osdmap e66: 8 total, 8 up, 8 in 2024-09-14T03:45:14.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:14 smithi070 ceph-mon[39288]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:45:14.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:14 smithi070 ceph-mon[39288]: Cluster is now healthy 2024-09-14T03:45:14.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:14 smithi070 ceph-mon[39288]: osdmap e66: 8 total, 8 up, 8 in 2024-09-14T03:45:14.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:14 smithi070 ceph-mon[27495]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:45:14.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:14 smithi070 ceph-mon[27495]: Cluster is now healthy 2024-09-14T03:45:14.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:14 smithi070 ceph-mon[27495]: osdmap e66: 8 total, 8 up, 8 in 2024-09-14T03:45:15.701 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph config log 1 --format=json 2024-09-14T03:45:15.725 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:15 smithi070 ceph-mon[27495]: osdmap e67: 8 total, 8 up, 8 in 2024-09-14T03:45:15.725 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:15 smithi070 ceph-mon[27495]: pgmap v13: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 44 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-14T03:45:15.725 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:15 smithi070 ceph-mon[39288]: osdmap e67: 8 total, 8 up, 8 in 2024-09-14T03:45:15.726 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:15 smithi070 ceph-mon[39288]: pgmap v13: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 44 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-14T03:45:15.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:15 smithi094 ceph-mon[31053]: osdmap e67: 8 total, 8 up, 8 in 2024-09-14T03:45:15.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:15 smithi094 ceph-mon[31053]: pgmap v13: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 44 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-14T03:45:16.540 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:45:17.934 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:17 smithi070 ceph-mon[39288]: pgmap v14: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 12 op/s 2024-09-14T03:45:17.935 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:17 smithi070 ceph-mon[27495]: pgmap v14: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 12 op/s 2024-09-14T03:45:18.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:17 smithi094 ceph-mon[31053]: pgmap v14: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 12 op/s 2024-09-14T03:45:19.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:19 smithi094 ceph-mon[31053]: pgmap v15: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 335 B/s wr, 12 op/s 2024-09-14T03:45:19.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:19 smithi070 ceph-mon[27495]: pgmap v15: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 335 B/s wr, 12 op/s 2024-09-14T03:45:19.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:19 smithi070 ceph-mon[39288]: pgmap v15: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 335 B/s wr, 12 op/s 2024-09-14T03:45:19.786 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:45:19 smithi070 systemd[1]: Starting Ceph alertmanager.a for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:45:20.749 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:45:21.036 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:45:20 smithi070 podman[74489]: 2024-09-14 03:45:20.740521923 +0000 UTC m=+0.108484122 volume create 1df656c37de47f986fd8aabe8bba91a2866b5b2b836ff80083b2cf8cebb8ed7c 2024-09-14T03:45:21.036 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:45:20 smithi070 podman[74489]: 2024-09-14 03:45:20.650011134 +0000 UTC m=+0.017973352 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-09-14T03:45:21.036 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:45:20 smithi070 podman[74489]: 2024-09-14 03:45:20.841248819 +0000 UTC m=+0.209211017 container create 81996aabcd0d913124aad3014d2ec0d37ea45c45b567b61686a2bc47e7a8cb85 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:45:21.675 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:21 smithi070 ceph-mon[27495]: osdmap e68: 8 total, 8 up, 8 in 2024-09-14T03:45:21.675 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:21 smithi070 ceph-mon[27495]: pgmap v17: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 288 B/s wr, 0 op/s 2024-09-14T03:45:21.676 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:21 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/3294909933' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-14T03:45:21.676 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:21 smithi070 ceph-mon[39288]: osdmap e68: 8 total, 8 up, 8 in 2024-09-14T03:45:21.676 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:21 smithi070 ceph-mon[39288]: pgmap v17: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 288 B/s wr, 0 op/s 2024-09-14T03:45:21.676 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:21 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/3294909933' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-14T03:45:21.676 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:45:21 smithi070 podman[74489]: 2024-09-14 03:45:21.378678397 +0000 UTC m=+0.746640595 container init 81996aabcd0d913124aad3014d2ec0d37ea45c45b567b61686a2bc47e7a8cb85 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:45:21.676 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:45:21 smithi070 podman[74489]: 2024-09-14 03:45:21.383521601 +0000 UTC m=+0.751483805 container start 81996aabcd0d913124aad3014d2ec0d37ea45c45b567b61686a2bc47e7a8cb85 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:45:21.677 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:45:21 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a[74516]: ts=2024-09-14T03:45:21.396Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-14T03:45:21.677 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:45:21 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a[74516]: ts=2024-09-14T03:45:21.396Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-14T03:45:21.677 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:45:21 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a[74516]: ts=2024-09-14T03:45:21.397Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.70 port=9094 2024-09-14T03:45:21.677 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:45:21 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a[74516]: ts=2024-09-14T03:45:21.397Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-14T03:45:21.677 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:45:21 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a[74516]: ts=2024-09-14T03:45:21.421Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-14T03:45:21.677 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:45:21 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a[74516]: ts=2024-09-14T03:45:21.422Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-14T03:45:21.677 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:45:21 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a[74516]: ts=2024-09-14T03:45:21.424Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-14T03:45:21.677 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:45:21 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a[74516]: ts=2024-09-14T03:45:21.424Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-14T03:45:21.677 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:45:21 smithi070 bash[74489]: 81996aabcd0d913124aad3014d2ec0d37ea45c45b567b61686a2bc47e7a8cb85 2024-09-14T03:45:21.677 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:45:21 smithi070 systemd[1]: Started Ceph alertmanager.a for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:45:21.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:21 smithi094 ceph-mon[31053]: osdmap e68: 8 total, 8 up, 8 in 2024-09-14T03:45:21.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:21 smithi094 ceph-mon[31053]: pgmap v17: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 288 B/s wr, 0 op/s 2024-09-14T03:45:21.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:21 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/3294909933' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-14T03:45:21.777 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:21 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:45:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:45:21.926 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:21 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:45:21] ENGINE Bus STOPPING 2024-09-14T03:45:21.926 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:21 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:45:21] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:45:21.926 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:21 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:45:21] ENGINE Bus STOPPED 2024-09-14T03:45:21.926 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:21 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:45:21] ENGINE Bus STARTING 2024-09-14T03:45:21.927 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:21 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:45:21] ENGINE Serving on http://:::9283 2024-09-14T03:45:21.927 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:45:21 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:45:21] ENGINE Bus STARTED 2024-09-14T03:45:21.947 INFO:teuthology.orchestra.run.smithi070.stdout:[{"version":20,"timestamp":"2024-09-14T03:45:09.569911+0000","name":"","changes":[{"name":"osd/host:smithi094/osd_memory_target","previous_value":"4479531110","new_value":"3137353830"}]}] 2024-09-14T03:45:21.948 INFO:tasks.ceph_manager:config epoch is 20 2024-09-14T03:45:21.948 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-14T03:45:21.948 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-14T03:45:21.948 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-14T03:45:22.242 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:45:22.553 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:22 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:22.554 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:22 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:22.554 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:22 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:22.554 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:22 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:22.554 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:22 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:22.554 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:22 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:45:22.554 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:22 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:22.554 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:22 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:22.554 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:22 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-14T03:45:22.554 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:22 smithi070 ceph-mon[27495]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-14T03:45:22.554 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:22 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:22.555 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:22 smithi070 ceph-mon[27495]: Deploying daemon grafana.a on smithi094 2024-09-14T03:45:22.555 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:22 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:22.555 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:22 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:22.555 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:22 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:22.555 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:22 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:22.555 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:22 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:22.555 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:22 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:45:22.555 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:22 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:22.555 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:22 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:22.555 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:22 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-14T03:45:22.555 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:22 smithi070 ceph-mon[39288]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-14T03:45:22.556 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:22 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:22.556 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:22 smithi070 ceph-mon[39288]: Deploying daemon grafana.a on smithi094 2024-09-14T03:45:23.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:22 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:23.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:22 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:23.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:22 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:23.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:22 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:23.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:22 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:23.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:22 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:45:23.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:22 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:23.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:22 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:23.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:22 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-14T03:45:23.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:22 smithi094 ceph-mon[31053]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-14T03:45:23.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:22 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:23.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:22 smithi094 ceph-mon[31053]: Deploying daemon grafana.a on smithi094 2024-09-14T03:45:23.447 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:45:23 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a[74516]: ts=2024-09-14T03:45:23.398Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000195691s 2024-09-14T03:45:23.764 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:23 smithi070 ceph-mon[27495]: pgmap v18: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 253 B/s wr, 0 op/s 2024-09-14T03:45:23.765 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:23 smithi070 ceph-mon[39288]: pgmap v18: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 253 B/s wr, 0 op/s 2024-09-14T03:45:24.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:23 smithi094 ceph-mon[31053]: pgmap v18: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 253 B/s wr, 0 op/s 2024-09-14T03:45:24.187 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:45:24.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:24 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/1219477959' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-14T03:45:24.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:24 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/1219477959' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-14T03:45:24.880 INFO:teuthology.orchestra.run.smithi070.stdout:{"epoch":20,"flags":0,"active_gid":14550,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6800","nonce":2541724886},{"type":"v1","addr":"172.21.15.70:6801","nonce":2541724886}]},"active_addr":"172.21.15.70:6801/2541724886","active_change":"2024-09-14T03:45:06.535421+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":14547,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.70:8443/","prometheus":"http://172.21.15.70:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":61,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.70:0","nonce":2324380307}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.70:0","nonce":869883831}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.70:0","nonce":1654192586}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.70:0","nonce":2892154750}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.70:0","nonce":2041681420}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.70:0","nonce":3984570208}]}]} 2024-09-14T03:45:24.885 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-14T03:45:24.885 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-14T03:45:24.885 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T03:45:25.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:24 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/1219477959' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-14T03:45:25.128 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:45:25.936 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:25 smithi070 ceph-mon[39288]: pgmap v19: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-14T03:45:25.937 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:25 smithi070 ceph-mon[27495]: pgmap v19: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-14T03:45:26.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:25 smithi094 ceph-mon[31053]: pgmap v19: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-14T03:45:26.982 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:45:26.983 INFO:teuthology.orchestra.run.smithi070.stdout:{"epoch":68,"fsid":"b6380d42-724a-11ef-bceb-c7b262605968","created":"2024-09-14T03:38:29.406762+0000","modified":"2024-09-14T03:45:20.460680+0000","last_up_change":"2024-09-14T03:43:34.130380+0000","last_in_change":"2024-09-14T03:43:17.652311+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:41:38.745294+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-14T03:43:57.402183+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-14T03:44:00.803078+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}},{"pool":4,"pool_name":"rbd","create_time":"2024-09-14T03:45:08.059178+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":67,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"a7fabda1-9e60-42f2-af60-eec626181688","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":62,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6802","nonce":3822384296},{"type":"v1","addr":"172.21.15.70:6803","nonce":3822384296}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6804","nonce":3822384296},{"type":"v1","addr":"172.21.15.70:6805","nonce":3822384296}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6808","nonce":3822384296},{"type":"v1","addr":"172.21.15.70:6809","nonce":3822384296}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6806","nonce":3822384296},{"type":"v1","addr":"172.21.15.70:6807","nonce":3822384296}]},"public_addr":"172.21.15.70:6803/3822384296","cluster_addr":"172.21.15.70:6805/3822384296","heartbeat_back_addr":"172.21.15.70:6809/3822384296","heartbeat_front_addr":"172.21.15.70:6807/3822384296","state":["exists","up"]},{"osd":1,"uuid":"f5b9ef91-59af-4ef0-a101-1c00ba6007ad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":62,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6810","nonce":3986345052},{"type":"v1","addr":"172.21.15.70:6811","nonce":3986345052}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6812","nonce":3986345052},{"type":"v1","addr":"172.21.15.70:6813","nonce":3986345052}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6816","nonce":3986345052},{"type":"v1","addr":"172.21.15.70:6817","nonce":3986345052}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6814","nonce":3986345052},{"type":"v1","addr":"172.21.15.70:6815","nonce":3986345052}]},"public_addr":"172.21.15.70:6811/3986345052","cluster_addr":"172.21.15.70:6813/3986345052","heartbeat_back_addr":"172.21.15.70:6817/3986345052","heartbeat_front_addr":"172.21.15.70:6815/3986345052","state":["exists","up"]},{"osd":2,"uuid":"0dc07495-c6a5-4762-8f63-575eb0fe1671","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6818","nonce":2374764370},{"type":"v1","addr":"172.21.15.70:6819","nonce":2374764370}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6820","nonce":2374764370},{"type":"v1","addr":"172.21.15.70:6821","nonce":2374764370}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6824","nonce":2374764370},{"type":"v1","addr":"172.21.15.70:6825","nonce":2374764370}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6822","nonce":2374764370},{"type":"v1","addr":"172.21.15.70:6823","nonce":2374764370}]},"public_addr":"172.21.15.70:6819/2374764370","cluster_addr":"172.21.15.70:6821/2374764370","heartbeat_back_addr":"172.21.15.70:6825/2374764370","heartbeat_front_addr":"172.21.15.70:6823/2374764370","state":["exists","up"]},{"osd":3,"uuid":"583e5165-ea10-4d03-b731-36cfb1108362","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":62,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6826","nonce":2294516176},{"type":"v1","addr":"172.21.15.70:6827","nonce":2294516176}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6828","nonce":2294516176},{"type":"v1","addr":"172.21.15.70:6829","nonce":2294516176}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6832","nonce":2294516176},{"type":"v1","addr":"172.21.15.70:6833","nonce":2294516176}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6830","nonce":2294516176},{"type":"v1","addr":"172.21.15.70:6831","nonce":2294516176}]},"public_addr":"172.21.15.70:6827/2294516176","cluster_addr":"172.21.15.70:6829/2294516176","heartbeat_back_addr":"172.21.15.70:6833/2294516176","heartbeat_front_addr":"172.21.15.70:6831/2294516176","state":["exists","up"]},{"osd":4,"uuid":"33177119-9632-431c-9180-9d27e7bc0c42","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":62,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6800","nonce":887215534},{"type":"v1","addr":"172.21.15.94:6801","nonce":887215534}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6802","nonce":887215534},{"type":"v1","addr":"172.21.15.94:6803","nonce":887215534}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6806","nonce":887215534},{"type":"v1","addr":"172.21.15.94:6807","nonce":887215534}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6804","nonce":887215534},{"type":"v1","addr":"172.21.15.94:6805","nonce":887215534}]},"public_addr":"172.21.15.94:6801/887215534","cluster_addr":"172.21.15.94:6803/887215534","heartbeat_back_addr":"172.21.15.94:6807/887215534","heartbeat_front_addr":"172.21.15.94:6805/887215534","state":["exists","up"]},{"osd":5,"uuid":"eb037130-5a05-42e0-947e-09bfba8fac22","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6808","nonce":1692579524},{"type":"v1","addr":"172.21.15.94:6809","nonce":1692579524}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6810","nonce":1692579524},{"type":"v1","addr":"172.21.15.94:6811","nonce":1692579524}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6814","nonce":1692579524},{"type":"v1","addr":"172.21.15.94:6815","nonce":1692579524}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6812","nonce":1692579524},{"type":"v1","addr":"172.21.15.94:6813","nonce":1692579524}]},"public_addr":"172.21.15.94:6809/1692579524","cluster_addr":"172.21.15.94:6811/1692579524","heartbeat_back_addr":"172.21.15.94:6815/1692579524","heartbeat_front_addr":"172.21.15.94:6813/1692579524","state":["exists","up"]},{"osd":6,"uuid":"f5dca4b5-8906-4eac-84e4-fcb7d1a29674","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":62,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6816","nonce":1107979948},{"type":"v1","addr":"172.21.15.94:6817","nonce":1107979948}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6818","nonce":1107979948},{"type":"v1","addr":"172.21.15.94:6819","nonce":1107979948}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6822","nonce":1107979948},{"type":"v1","addr":"172.21.15.94:6823","nonce":1107979948}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6820","nonce":1107979948},{"type":"v1","addr":"172.21.15.94:6821","nonce":1107979948}]},"public_addr":"172.21.15.94:6817/1107979948","cluster_addr":"172.21.15.94:6819/1107979948","heartbeat_back_addr":"172.21.15.94:6823/1107979948","heartbeat_front_addr":"172.21.15.94:6821/1107979948","state":["exists","up"]},{"osd":7,"uuid":"86e3ef7d-6ab8-47c3-a4bb-db923af2b49b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6824","nonce":2545649733},{"type":"v1","addr":"172.21.15.94:6825","nonce":2545649733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6826","nonce":2545649733},{"type":"v1","addr":"172.21.15.94:6827","nonce":2545649733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6830","nonce":2545649733},{"type":"v1","addr":"172.21.15.94:6831","nonce":2545649733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6828","nonce":2545649733},{"type":"v1","addr":"172.21.15.94:6829","nonce":2545649733}]},"public_addr":"172.21.15.94:6825/2545649733","cluster_addr":"172.21.15.94:6827/2545649733","heartbeat_back_addr":"172.21.15.94:6831/2545649733","heartbeat_front_addr":"172.21.15.94:6829/2545649733","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:40:47.681447+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:41:11.801118+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:41:35.764497+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:42:00.424802+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:42:21.618849+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:42:45.384437+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:43:09.350263+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:43:32.472815+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.18","mappings":[{"from":7,"to":2}]},{"pgid":"3.3","mappings":[{"from":0,"to":1}]},{"pgid":"3.a","mappings":[{"from":6,"to":2}]},{"pgid":"3.b","mappings":[{"from":0,"to":2},{"from":4,"to":1}]},{"pgid":"3.26","mappings":[{"from":4,"to":3}]},{"pgid":"3.36","mappings":[{"from":5,"to":2}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.70:6801/3164926893":"2024-09-15T03:45:06.535242+0000","172.21.15.70:6800/3164926893":"2024-09-15T03:45:06.535242+0000","172.21.15.70:0/4203647763":"2024-09-15T03:39:01.589983+0000","172.21.15.70:6800/2254107694":"2024-09-15T03:39:01.589983+0000","172.21.15.70:6801/2254107694":"2024-09-15T03:39:01.589983+0000","172.21.15.70:0/2447056811":"2024-09-15T03:39:01.589983+0000","172.21.15.70:0/22543101":"2024-09-15T03:39:01.589983+0000","172.21.15.70:0/3134643770":"2024-09-15T03:39:24.448616+0000","172.21.15.70:0/3005166882":"2024-09-15T03:39:24.448616+0000","172.21.15.70:0/2339853520":"2024-09-15T03:45:06.535242+0000","172.21.15.70:0/4275249763":"2024-09-15T03:45:06.535242+0000","172.21.15.70:0/3199751552":"2024-09-15T03:45:06.535242+0000","172.21.15.70:6800/673916264":"2024-09-15T03:39:24.448616+0000","172.21.15.70:6801/673916264":"2024-09-15T03:39:24.448616+0000","172.21.15.70:0/3067035520":"2024-09-15T03:45:06.535242+0000","172.21.15.70:0/2978576061":"2024-09-15T03:39:24.448616+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":4,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:45:27.621 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-14T03:45:27.621 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T03:45:27.866 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:45:27.986 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:27 smithi070 ceph-mon[39288]: pgmap v20: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-14T03:45:27.986 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:27 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/726201002' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:45:27.987 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:27 smithi070 ceph-mon[27495]: pgmap v20: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-14T03:45:27.987 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:27 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/726201002' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:45:28.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:27 smithi094 ceph-mon[31053]: pgmap v20: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-14T03:45:28.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:27 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/726201002' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:45:29.885 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:29 smithi070 ceph-mon[27495]: pgmap v21: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:29.885 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:29 smithi070 ceph-mon[39288]: pgmap v21: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:30.013 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:29 smithi094 ceph-mon[31053]: pgmap v21: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:30.280 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:45:30.280 INFO:teuthology.orchestra.run.smithi070.stdout:{"epoch":68,"fsid":"b6380d42-724a-11ef-bceb-c7b262605968","created":"2024-09-14T03:38:29.406762+0000","modified":"2024-09-14T03:45:20.460680+0000","last_up_change":"2024-09-14T03:43:34.130380+0000","last_in_change":"2024-09-14T03:43:17.652311+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:41:38.745294+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-14T03:43:57.402183+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-14T03:44:00.803078+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}},{"pool":4,"pool_name":"rbd","create_time":"2024-09-14T03:45:08.059178+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":67,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"a7fabda1-9e60-42f2-af60-eec626181688","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":62,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6802","nonce":3822384296},{"type":"v1","addr":"172.21.15.70:6803","nonce":3822384296}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6804","nonce":3822384296},{"type":"v1","addr":"172.21.15.70:6805","nonce":3822384296}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6808","nonce":3822384296},{"type":"v1","addr":"172.21.15.70:6809","nonce":3822384296}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6806","nonce":3822384296},{"type":"v1","addr":"172.21.15.70:6807","nonce":3822384296}]},"public_addr":"172.21.15.70:6803/3822384296","cluster_addr":"172.21.15.70:6805/3822384296","heartbeat_back_addr":"172.21.15.70:6809/3822384296","heartbeat_front_addr":"172.21.15.70:6807/3822384296","state":["exists","up"]},{"osd":1,"uuid":"f5b9ef91-59af-4ef0-a101-1c00ba6007ad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":62,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6810","nonce":3986345052},{"type":"v1","addr":"172.21.15.70:6811","nonce":3986345052}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6812","nonce":3986345052},{"type":"v1","addr":"172.21.15.70:6813","nonce":3986345052}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6816","nonce":3986345052},{"type":"v1","addr":"172.21.15.70:6817","nonce":3986345052}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6814","nonce":3986345052},{"type":"v1","addr":"172.21.15.70:6815","nonce":3986345052}]},"public_addr":"172.21.15.70:6811/3986345052","cluster_addr":"172.21.15.70:6813/3986345052","heartbeat_back_addr":"172.21.15.70:6817/3986345052","heartbeat_front_addr":"172.21.15.70:6815/3986345052","state":["exists","up"]},{"osd":2,"uuid":"0dc07495-c6a5-4762-8f63-575eb0fe1671","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6818","nonce":2374764370},{"type":"v1","addr":"172.21.15.70:6819","nonce":2374764370}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6820","nonce":2374764370},{"type":"v1","addr":"172.21.15.70:6821","nonce":2374764370}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6824","nonce":2374764370},{"type":"v1","addr":"172.21.15.70:6825","nonce":2374764370}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6822","nonce":2374764370},{"type":"v1","addr":"172.21.15.70:6823","nonce":2374764370}]},"public_addr":"172.21.15.70:6819/2374764370","cluster_addr":"172.21.15.70:6821/2374764370","heartbeat_back_addr":"172.21.15.70:6825/2374764370","heartbeat_front_addr":"172.21.15.70:6823/2374764370","state":["exists","up"]},{"osd":3,"uuid":"583e5165-ea10-4d03-b731-36cfb1108362","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":62,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6826","nonce":2294516176},{"type":"v1","addr":"172.21.15.70:6827","nonce":2294516176}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6828","nonce":2294516176},{"type":"v1","addr":"172.21.15.70:6829","nonce":2294516176}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6832","nonce":2294516176},{"type":"v1","addr":"172.21.15.70:6833","nonce":2294516176}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6830","nonce":2294516176},{"type":"v1","addr":"172.21.15.70:6831","nonce":2294516176}]},"public_addr":"172.21.15.70:6827/2294516176","cluster_addr":"172.21.15.70:6829/2294516176","heartbeat_back_addr":"172.21.15.70:6833/2294516176","heartbeat_front_addr":"172.21.15.70:6831/2294516176","state":["exists","up"]},{"osd":4,"uuid":"33177119-9632-431c-9180-9d27e7bc0c42","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":62,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6800","nonce":887215534},{"type":"v1","addr":"172.21.15.94:6801","nonce":887215534}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6802","nonce":887215534},{"type":"v1","addr":"172.21.15.94:6803","nonce":887215534}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6806","nonce":887215534},{"type":"v1","addr":"172.21.15.94:6807","nonce":887215534}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6804","nonce":887215534},{"type":"v1","addr":"172.21.15.94:6805","nonce":887215534}]},"public_addr":"172.21.15.94:6801/887215534","cluster_addr":"172.21.15.94:6803/887215534","heartbeat_back_addr":"172.21.15.94:6807/887215534","heartbeat_front_addr":"172.21.15.94:6805/887215534","state":["exists","up"]},{"osd":5,"uuid":"eb037130-5a05-42e0-947e-09bfba8fac22","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6808","nonce":1692579524},{"type":"v1","addr":"172.21.15.94:6809","nonce":1692579524}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6810","nonce":1692579524},{"type":"v1","addr":"172.21.15.94:6811","nonce":1692579524}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6814","nonce":1692579524},{"type":"v1","addr":"172.21.15.94:6815","nonce":1692579524}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6812","nonce":1692579524},{"type":"v1","addr":"172.21.15.94:6813","nonce":1692579524}]},"public_addr":"172.21.15.94:6809/1692579524","cluster_addr":"172.21.15.94:6811/1692579524","heartbeat_back_addr":"172.21.15.94:6815/1692579524","heartbeat_front_addr":"172.21.15.94:6813/1692579524","state":["exists","up"]},{"osd":6,"uuid":"f5dca4b5-8906-4eac-84e4-fcb7d1a29674","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":62,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6816","nonce":1107979948},{"type":"v1","addr":"172.21.15.94:6817","nonce":1107979948}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6818","nonce":1107979948},{"type":"v1","addr":"172.21.15.94:6819","nonce":1107979948}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6822","nonce":1107979948},{"type":"v1","addr":"172.21.15.94:6823","nonce":1107979948}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6820","nonce":1107979948},{"type":"v1","addr":"172.21.15.94:6821","nonce":1107979948}]},"public_addr":"172.21.15.94:6817/1107979948","cluster_addr":"172.21.15.94:6819/1107979948","heartbeat_back_addr":"172.21.15.94:6823/1107979948","heartbeat_front_addr":"172.21.15.94:6821/1107979948","state":["exists","up"]},{"osd":7,"uuid":"86e3ef7d-6ab8-47c3-a4bb-db923af2b49b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6824","nonce":2545649733},{"type":"v1","addr":"172.21.15.94:6825","nonce":2545649733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6826","nonce":2545649733},{"type":"v1","addr":"172.21.15.94:6827","nonce":2545649733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6830","nonce":2545649733},{"type":"v1","addr":"172.21.15.94:6831","nonce":2545649733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6828","nonce":2545649733},{"type":"v1","addr":"172.21.15.94:6829","nonce":2545649733}]},"public_addr":"172.21.15.94:6825/2545649733","cluster_addr":"172.21.15.94:6827/2545649733","heartbeat_back_addr":"172.21.15.94:6831/2545649733","heartbeat_front_addr":"172.21.15.94:6829/2545649733","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:40:47.681447+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:41:11.801118+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:41:35.764497+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:42:00.424802+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:42:21.618849+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:42:45.384437+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:43:09.350263+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:43:32.472815+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.18","mappings":[{"from":7,"to":2}]},{"pgid":"3.3","mappings":[{"from":0,"to":1}]},{"pgid":"3.a","mappings":[{"from":6,"to":2}]},{"pgid":"3.b","mappings":[{"from":0,"to":2},{"from":4,"to":1}]},{"pgid":"3.26","mappings":[{"from":4,"to":3}]},{"pgid":"3.36","mappings":[{"from":5,"to":2}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.70:6801/3164926893":"2024-09-15T03:45:06.535242+0000","172.21.15.70:6800/3164926893":"2024-09-15T03:45:06.535242+0000","172.21.15.70:0/4203647763":"2024-09-15T03:39:01.589983+0000","172.21.15.70:6800/2254107694":"2024-09-15T03:39:01.589983+0000","172.21.15.70:6801/2254107694":"2024-09-15T03:39:01.589983+0000","172.21.15.70:0/2447056811":"2024-09-15T03:39:01.589983+0000","172.21.15.70:0/22543101":"2024-09-15T03:39:01.589983+0000","172.21.15.70:0/3134643770":"2024-09-15T03:39:24.448616+0000","172.21.15.70:0/3005166882":"2024-09-15T03:39:24.448616+0000","172.21.15.70:0/2339853520":"2024-09-15T03:45:06.535242+0000","172.21.15.70:0/4275249763":"2024-09-15T03:45:06.535242+0000","172.21.15.70:0/3199751552":"2024-09-15T03:45:06.535242+0000","172.21.15.70:6800/673916264":"2024-09-15T03:39:24.448616+0000","172.21.15.70:6801/673916264":"2024-09-15T03:39:24.448616+0000","172.21.15.70:0/3067035520":"2024-09-15T03:45:06.535242+0000","172.21.15.70:0/2978576061":"2024-09-15T03:39:24.448616+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":4,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:45:30.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:30 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/936366782' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:45:30.911 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:30 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/936366782' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:45:30.911 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:30 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/936366782' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:45:30.936 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-14T03:45:30.937 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-14T03:45:30.937 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-14T03:45:30.938 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-14T03:45:30.938 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-14T03:45:30.939 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-14T03:45:30.939 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-14T03:45:30.939 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-14T03:45:31.430 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:45:31.434 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:45:31.434 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:45:31.435 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:45:31.437 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:45:31.442 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:45:31.442 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:45:31.447 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:45:31.738 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:31 smithi070 ceph-mon[27495]: pgmap v22: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:31.739 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:45:31 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a[74516]: ts=2024-09-14T03:45:31.400Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002898088s 2024-09-14T03:45:31.739 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:31 smithi070 ceph-mon[39288]: pgmap v22: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:31.895 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:31 smithi094 ceph-mon[31053]: pgmap v22: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:31.895 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:31 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:45:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:45:33.165 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 podman[63491]: 2024-09-14 03:45:33.057386213 +0000 UTC m=+0.113956135 container create 846770f482574e93cdbf220736667e7b7f4dc882cb27614e62108cfefa8a003a (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, architecture=x86_64, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, name=ubi8, io.openshift.tags=base rhel8, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., distribution-scope=public) 2024-09-14T03:45:33.527 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 podman[63491]: 2024-09-14 03:45:33.276348118 +0000 UTC m=+0.332918037 container init 846770f482574e93cdbf220736667e7b7f4dc882cb27614e62108cfefa8a003a (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, io.openshift.expose-services=, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, release=239.1651231664, com.redhat.component=ubi8-container, vendor=Red Hat, Inc., description=Ceph Grafana Container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, architecture=x86_64, vcs-type=git, build-date=2022-04-29T11:48:11.055897, distribution-scope=public, name=ubi8, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8) 2024-09-14T03:45:33.527 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 podman[63491]: 2024-09-14 03:45:33.28118403 +0000 UTC m=+0.337753950 container start 846770f482574e93cdbf220736667e7b7f4dc882cb27614e62108cfefa8a003a (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, io.buildah.version=1.32.0, maintainer=Paul Cuzner , architecture=x86_64, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, io.openshift.expose-services=, name=ubi8, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-09-14T03:45:33.527 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 bash[63491]: 846770f482574e93cdbf220736667e7b7f4dc882cb27614e62108cfefa8a003a 2024-09-14T03:45:33.527 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 systemd[1]: Started Ceph grafana.a for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:45:33.528 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=settings t=2024-09-14T03:45:33.431851829Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-09-14T03:45:33.528 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=settings t=2024-09-14T03:45:33.432270112Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-09-14T03:45:33.528 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=settings t=2024-09-14T03:45:33.432297734Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-14T03:45:33.528 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=settings t=2024-09-14T03:45:33.432310749Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-14T03:45:33.528 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=settings t=2024-09-14T03:45:33.432322074Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-14T03:45:33.529 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=settings t=2024-09-14T03:45:33.432333618Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-14T03:45:33.529 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=settings t=2024-09-14T03:45:33.432343894Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-14T03:45:33.529 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=settings t=2024-09-14T03:45:33.432354112Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-14T03:45:33.529 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=settings t=2024-09-14T03:45:33.432363179Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-14T03:45:33.529 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=settings t=2024-09-14T03:45:33.432373308Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-14T03:45:33.529 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=settings t=2024-09-14T03:45:33.432383364Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-14T03:45:33.529 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=settings t=2024-09-14T03:45:33.432393924Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-14T03:45:33.529 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=settings t=2024-09-14T03:45:33.432404053Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-14T03:45:33.529 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=settings t=2024-09-14T03:45:33.432413923Z level=info msg="App mode production" 2024-09-14T03:45:33.529 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=sqlstore t=2024-09-14T03:45:33.43248346Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-14T03:45:33.529 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=sqlstore t=2024-09-14T03:45:33.43250944Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-14T03:45:33.530 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.433625924Z level=info msg="Starting DB migrations" 2024-09-14T03:45:33.530 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.433729413Z level=info msg="Executing migration" id="create migration_log table" 2024-09-14T03:45:33.530 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.434330662Z level=info msg="Executing migration" id="create user table" 2024-09-14T03:45:33.530 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.434791605Z level=info msg="Executing migration" id="add unique index user.login" 2024-09-14T03:45:33.530 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.435260845Z level=info msg="Executing migration" id="add unique index user.email" 2024-09-14T03:45:33.530 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.435670439Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-09-14T03:45:33.530 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.436086111Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-09-14T03:45:33.530 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.436475021Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-09-14T03:45:33.530 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.437234283Z level=info msg="Executing migration" id="create user table v2" 2024-09-14T03:45:33.530 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.437671001Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-09-14T03:45:33.530 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.438059861Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-09-14T03:45:33.530 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.438472458Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-09-14T03:45:33.530 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.438778525Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-09-14T03:45:33.531 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.439120716Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-09-14T03:45:33.531 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.439466096Z level=info msg="Executing migration" id="Update user table charset" 2024-09-14T03:45:33.531 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.439702836Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-09-14T03:45:33.531 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.440180882Z level=info msg="Executing migration" id="Add missing user data" 2024-09-14T03:45:33.531 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.440497736Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-09-14T03:45:33.531 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.440865401Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-09-14T03:45:33.531 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.441255524Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-09-14T03:45:33.531 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.441597241Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-09-14T03:45:33.531 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.442664094Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-09-14T03:45:33.531 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.443126327Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-09-14T03:45:33.531 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.44354285Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-09-14T03:45:33.531 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.443973581Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-09-14T03:45:33.531 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.444380936Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-09-14T03:45:33.531 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.444828695Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-09-14T03:45:33.532 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.445092645Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-09-14T03:45:33.532 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.445427624Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-09-14T03:45:33.532 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.445734078Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-09-14T03:45:33.532 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.446085053Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-09-14T03:45:33.532 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.44643916Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-09-14T03:45:33.532 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.447088069Z level=info msg="Executing migration" id="create temp_user v2" 2024-09-14T03:45:33.532 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.447469383Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-09-14T03:45:33.532 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.447837518Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-09-14T03:45:33.532 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.448207924Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-09-14T03:45:33.532 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.448554143Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-09-14T03:45:33.532 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.448989842Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-09-14T03:45:33.532 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.449289432Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-09-14T03:45:33.532 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.449567676Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-09-14T03:45:33.532 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.449862346Z level=info msg="Executing migration" id="create star table" 2024-09-14T03:45:33.533 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.450203348Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-09-14T03:45:33.533 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.450599336Z level=info msg="Executing migration" id="create org table v1" 2024-09-14T03:45:33.533 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.451008506Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-09-14T03:45:33.533 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.451384221Z level=info msg="Executing migration" id="create org_user table v1" 2024-09-14T03:45:33.533 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.45172359Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-09-14T03:45:33.534 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.452096823Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-09-14T03:45:33.535 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.452470928Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-09-14T03:45:33.535 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.452840011Z level=info msg="Executing migration" id="Update org table charset" 2024-09-14T03:45:33.535 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.453120238Z level=info msg="Executing migration" id="Update org_user table charset" 2024-09-14T03:45:33.535 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.453360436Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-09-14T03:45:33.535 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.453610556Z level=info msg="Executing migration" id="create dashboard table" 2024-09-14T03:45:33.535 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.45396665Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-09-14T03:45:33.535 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.454349214Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-09-14T03:45:33.535 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.454754808Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-09-14T03:45:33.535 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.455148416Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-09-14T03:45:33.535 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.455600947Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-09-14T03:45:33.535 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.455978511Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-09-14T03:45:33.535 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.456794528Z level=info msg="Executing migration" id="create dashboard v2" 2024-09-14T03:45:33.536 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.457160864Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-09-14T03:45:33.536 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.457559426Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-09-14T03:45:33.536 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.457992964Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-09-14T03:45:33.536 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.458292434Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-09-14T03:45:33.536 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.458621769Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-09-14T03:45:33.536 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.458870221Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-09-14T03:45:33.536 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.459344638Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-09-14T03:45:33.536 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.459759572Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-09-14T03:45:33.536 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.460233369Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-09-14T03:45:33.536 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.460594443Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-09-14T03:45:33.536 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.461059478Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-09-14T03:45:33.536 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.461449998Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-09-14T03:45:33.537 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.461792902Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-09-14T03:45:33.537 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.462073203Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-09-14T03:45:33.537 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.462301127Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-09-14T03:45:33.537 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.462792821Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-09-14T03:45:33.537 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.463336643Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-09-14T03:45:33.537 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.463765944Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-09-14T03:45:33.537 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.464298214Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-09-14T03:45:33.537 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.464557149Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-09-14T03:45:33.537 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.464977823Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-09-14T03:45:33.537 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.465390267Z level=info msg="Executing migration" id="Update dashboard title length" 2024-09-14T03:45:33.537 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.46561904Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-09-14T03:45:33.537 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.466027482Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-09-14T03:45:33.537 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.466435566Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-09-14T03:45:33.537 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.467359227Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-09-14T03:45:33.538 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.467775907Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-09-14T03:45:33.538 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.46822689Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-09-14T03:45:33.538 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.468667588Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-09-14T03:45:33.538 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.468994564Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-09-14T03:45:33.538 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.469322754Z level=info msg="Executing migration" id="Add check_sum column" 2024-09-14T03:45:33.538 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.469759365Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-09-14T03:45:33.538 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.470184224Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-09-14T03:45:33.538 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.470456216Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-09-14T03:45:33.538 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.470696589Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-09-14T03:45:33.538 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.471082343Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-09-14T03:45:33.538 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.471595816Z level=info msg="Executing migration" id="create data_source table" 2024-09-14T03:45:33.538 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.472048437Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-09-14T03:45:33.538 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.472482486Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-09-14T03:45:33.538 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.4728736Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-09-14T03:45:33.539 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.473254719Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-09-14T03:45:33.539 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.473576162Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-09-14T03:45:33.539 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.474627251Z level=info msg="Executing migration" id="create data_source table v2" 2024-09-14T03:45:33.539 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.475048513Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-09-14T03:45:33.539 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.47543826Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-09-14T03:45:33.539 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.475806935Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-09-14T03:45:33.539 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.476182433Z level=info msg="Executing migration" id="Add column with_credentials" 2024-09-14T03:45:33.539 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.476662855Z level=info msg="Executing migration" id="Add secure json data column" 2024-09-14T03:45:33.539 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.477170348Z level=info msg="Executing migration" id="Update data_source table charset" 2024-09-14T03:45:33.539 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.477422874Z level=info msg="Executing migration" id="Update initial version to 1" 2024-09-14T03:45:33.539 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.477656577Z level=info msg="Executing migration" id="Add read_only data column" 2024-09-14T03:45:33.539 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.478176218Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-09-14T03:45:33.539 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.478491045Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-09-14T03:45:33.539 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.478727969Z level=info msg="Executing migration" id="Add uid column" 2024-09-14T03:45:33.540 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.479272468Z level=info msg="Executing migration" id="Update uid value" 2024-09-14T03:45:33.540 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.479590785Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-09-14T03:45:33.540 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.479982131Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-09-14T03:45:33.540 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.480380126Z level=info msg="Executing migration" id="create api_key table" 2024-09-14T03:45:33.540 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.480763313Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-09-14T03:45:33.540 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.481160163Z level=info msg="Executing migration" id="add index api_key.key" 2024-09-14T03:45:33.540 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.481535745Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-09-14T03:45:33.540 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.481969732Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-09-14T03:45:33.540 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.482341801Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-09-14T03:45:33.540 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.482695437Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-09-14T03:45:33.540 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.483089839Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-09-14T03:45:33.540 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.484074697Z level=info msg="Executing migration" id="create api_key table v2" 2024-09-14T03:45:33.540 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.484421361Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-09-14T03:45:33.541 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.484754507Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-09-14T03:45:33.541 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.485160441Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-09-14T03:45:33.541 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.485537056Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-09-14T03:45:33.541 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.48585082Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-09-14T03:45:33.541 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.486188023Z level=info msg="Executing migration" id="Update api_key table charset" 2024-09-14T03:45:33.541 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.486402035Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-09-14T03:45:33.541 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.486860386Z level=info msg="Executing migration" id="Add service account foreign key" 2024-09-14T03:45:33.541 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.487347919Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-09-14T03:45:33.541 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.487584066Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-09-14T03:45:33.541 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.488079187Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-09-14T03:45:33.541 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.488573646Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-09-14T03:45:33.541 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.488929272Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-09-14T03:45:33.541 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.489281847Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-09-14T03:45:33.542 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.489596711Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-09-14T03:45:33.542 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.489967221Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-09-14T03:45:33.542 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.490366538Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-09-14T03:45:33.542 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.490732736Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-09-14T03:45:33.542 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.49099567Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-09-14T03:45:33.542 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.491238956Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-09-14T03:45:33.542 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.491706861Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-09-14T03:45:33.542 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.492252355Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-09-14T03:45:33.542 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.492514736Z level=info msg="Executing migration" id="create quota table v1" 2024-09-14T03:45:33.542 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.492857772Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-09-14T03:45:33.542 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.493234437Z level=info msg="Executing migration" id="Update quota table charset" 2024-09-14T03:45:33.542 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.493474113Z level=info msg="Executing migration" id="create plugin_setting table" 2024-09-14T03:45:33.542 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.493862384Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-09-14T03:45:33.543 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.494233766Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-09-14T03:45:33.543 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.494751493Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-09-14T03:45:33.543 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.495018244Z level=info msg="Executing migration" id="create session table" 2024-09-14T03:45:33.543 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.495390364Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-09-14T03:45:33.543 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.495617085Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-09-14T03:45:33.543 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.495848951Z level=info msg="Executing migration" id="create playlist table v2" 2024-09-14T03:45:33.543 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.49624002Z level=info msg="Executing migration" id="create playlist item table v2" 2024-09-14T03:45:33.543 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.49660542Z level=info msg="Executing migration" id="Update playlist table charset" 2024-09-14T03:45:33.543 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.496824523Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-09-14T03:45:33.543 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.497069198Z level=info msg="Executing migration" id="drop preferences table v2" 2024-09-14T03:45:33.543 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.497310037Z level=info msg="Executing migration" id="drop preferences table v3" 2024-09-14T03:45:33.543 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.497532402Z level=info msg="Executing migration" id="create preferences table v3" 2024-09-14T03:45:33.543 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.497890697Z level=info msg="Executing migration" id="Update preferences table charset" 2024-09-14T03:45:33.544 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.498139174Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-09-14T03:45:33.544 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.498638615Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-09-14T03:45:33.544 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.498955651Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-09-14T03:45:33.544 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.499505311Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-09-14T03:45:33.544 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.500069296Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-09-14T03:45:33.544 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.500323332Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-09-14T03:45:33.544 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.500688568Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-09-14T03:45:33.544 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.501094848Z level=info msg="Executing migration" id="create alert table v1" 2024-09-14T03:45:33.544 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.50151472Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-09-14T03:45:33.544 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.501917019Z level=info msg="Executing migration" id="add index alert state" 2024-09-14T03:45:33.544 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.502278961Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-09-14T03:45:33.544 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.502644637Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-09-14T03:45:33.544 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.503026322Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-09-14T03:45:33.544 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.503402184Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-09-14T03:45:33.545 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.503720991Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-09-14T03:45:33.545 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.505071427Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-09-14T03:45:33.545 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.505409416Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-09-14T03:45:33.545 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.505783718Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-09-14T03:45:33.545 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.50609983Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-09-14T03:45:33.545 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.506451184Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-09-14T03:45:33.545 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.506817588Z level=info msg="Executing migration" id="Add column is_default" 2024-09-14T03:45:33.545 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.507477721Z level=info msg="Executing migration" id="Add column frequency" 2024-09-14T03:45:33.545 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.508117984Z level=info msg="Executing migration" id="Add column send_reminder" 2024-09-14T03:45:33.545 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.508749885Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-09-14T03:45:33.545 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.509427539Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-09-14T03:45:33.545 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.509843313Z level=info msg="Executing migration" id="Update alert table charset" 2024-09-14T03:45:33.545 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.510134134Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-09-14T03:45:33.545 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.51038208Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-09-14T03:45:33.546 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.510755831Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-09-14T03:45:33.546 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.511194163Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-09-14T03:45:33.546 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.511564972Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-09-14T03:45:33.546 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.511979846Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-09-14T03:45:33.546 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.512382119Z level=info msg="Executing migration" id="Add for to alert table" 2024-09-14T03:45:33.546 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.51305094Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-09-14T03:45:33.546 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.51368473Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-09-14T03:45:33.546 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.513994562Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-09-14T03:45:33.546 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.514438786Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-09-14T03:45:33.546 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.514790089Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-09-14T03:45:33.546 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.515418412Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-09-14T03:45:33.546 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.515672277Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-09-14T03:45:33.546 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.516077495Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-09-14T03:45:33.547 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.516493303Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-09-14T03:45:33.547 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.516765244Z level=info msg="Executing migration" id="create annotation table v5" 2024-09-14T03:45:33.547 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.51725975Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-09-14T03:45:33.547 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.517662004Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-09-14T03:45:33.547 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.51807575Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-09-14T03:45:33.547 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.518503454Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-09-14T03:45:33.547 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.518944491Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-09-14T03:45:33.547 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.519373614Z level=info msg="Executing migration" id="Update annotation table charset" 2024-09-14T03:45:33.547 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.51962499Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-09-14T03:45:33.547 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.520338017Z level=info msg="Executing migration" id="Drop category_id index" 2024-09-14T03:45:33.547 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.520731304Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-09-14T03:45:33.547 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.52143022Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-09-14T03:45:33.547 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.521850352Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-09-14T03:45:33.548 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.522265096Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-09-14T03:45:33.548 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.522621475Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-09-14T03:45:33.548 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.524196271Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-09-14T03:45:33.548 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.524560702Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-09-14T03:45:33.548 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.524976295Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-09-14T03:45:33.548 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.525280158Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-09-14T03:45:33.548 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.525619895Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-09-14T03:45:33.548 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.525926123Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-09-14T03:45:33.548 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.526593814Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-09-14T03:45:33.548 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.527273423Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-09-14T03:45:33.548 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.527683709Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-09-14T03:45:33.548 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.528094551Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-09-14T03:45:33.548 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.528399185Z level=info msg="Executing migration" id="Add epoch_end column" 2024-09-14T03:45:33.549 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.529123246Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-09-14T03:45:33.549 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.529539139Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-09-14T03:45:33.549 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.529831503Z level=info msg="Executing migration" id="Move region to single row" 2024-09-14T03:45:33.549 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.530194848Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-09-14T03:45:33.549 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.530596807Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-09-14T03:45:33.549 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.531006645Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-09-14T03:45:33.549 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.531416242Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-09-14T03:45:33.549 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.53178725Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-09-14T03:45:33.549 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.532227815Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-09-14T03:45:33.549 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.532592369Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-09-14T03:45:33.549 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.532863627Z level=info msg="Executing migration" id="create test_data table" 2024-09-14T03:45:33.549 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.533263619Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-09-14T03:45:33.549 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.533664364Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-09-14T03:45:33.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:33 smithi070 ceph-mon[27495]: pgmap v23: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:33.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:33 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:33.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:33 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:33.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:33 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:33.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:33 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:33.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:33 smithi070 ceph-mon[39288]: pgmap v23: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:33.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:33 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:33.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:33 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:33.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:33 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:33.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:33 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:34.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-mon[31053]: pgmap v23: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:34.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:34.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:34.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:34.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:34.027 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.534086461Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-09-14T03:45:34.027 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.534497881Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-09-14T03:45:34.028 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.534762824Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-09-14T03:45:34.028 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.535145565Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-09-14T03:45:34.028 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.535408356Z level=info msg="Executing migration" id="create team table" 2024-09-14T03:45:34.028 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.535780603Z level=info msg="Executing migration" id="add index team.org_id" 2024-09-14T03:45:34.028 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.536248206Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-09-14T03:45:34.028 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.536656206Z level=info msg="Executing migration" id="create team member table" 2024-09-14T03:45:34.028 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.537048534Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-09-14T03:45:34.028 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.537454945Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-09-14T03:45:34.028 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.537874812Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-09-14T03:45:34.028 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.538279564Z level=info msg="Executing migration" id="Add column email to team table" 2024-09-14T03:45:34.028 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.538986033Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-09-14T03:45:34.028 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.539698791Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-09-14T03:45:34.028 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.540425522Z level=info msg="Executing migration" id="create dashboard acl table" 2024-09-14T03:45:34.029 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.540834792Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-09-14T03:45:34.029 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.541334078Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-09-14T03:45:34.029 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.541816591Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-09-14T03:45:34.029 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.542262674Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-09-14T03:45:34.029 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.542741846Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-09-14T03:45:34.029 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.543215139Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-09-14T03:45:34.029 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.543656962Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-09-14T03:45:34.029 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.544109774Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-09-14T03:45:34.029 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.544555117Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-09-14T03:45:34.029 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.544843948Z level=info msg="Executing migration" id="create tag table" 2024-09-14T03:45:34.029 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.545221171Z level=info msg="Executing migration" id="add index tag.key_value" 2024-09-14T03:45:34.029 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.545629226Z level=info msg="Executing migration" id="create login attempt table" 2024-09-14T03:45:34.029 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.546057405Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-09-14T03:45:34.030 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.546509863Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-09-14T03:45:34.030 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.546929344Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-09-14T03:45:34.030 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.548779728Z level=info msg="Executing migration" id="create login_attempt v2" 2024-09-14T03:45:34.030 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.549196912Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-09-14T03:45:34.030 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.549601839Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-09-14T03:45:34.030 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.54994803Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-09-14T03:45:34.030 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.550338244Z level=info msg="Executing migration" id="create user auth table" 2024-09-14T03:45:34.030 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.550680429Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-09-14T03:45:34.030 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.551103373Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-09-14T03:45:34.030 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.551403715Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-09-14T03:45:34.030 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.552180785Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-09-14T03:45:34.030 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.552903893Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-09-14T03:45:34.030 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.553616003Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-09-14T03:45:34.030 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.554372482Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-09-14T03:45:34.031 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.554832793Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-09-14T03:45:34.031 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.555588578Z level=info msg="Executing migration" id="create server_lock table" 2024-09-14T03:45:34.031 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.555990635Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-09-14T03:45:34.031 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.556413519Z level=info msg="Executing migration" id="create user auth token table" 2024-09-14T03:45:34.031 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.556868015Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-09-14T03:45:34.031 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.557290453Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-09-14T03:45:34.031 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.557700246Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-09-14T03:45:34.031 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.558142967Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-09-14T03:45:34.031 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.558901018Z level=info msg="Executing migration" id="create cache_data table" 2024-09-14T03:45:34.031 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.559311489Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-09-14T03:45:34.031 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.559733152Z level=info msg="Executing migration" id="create short_url table v1" 2024-09-14T03:45:34.031 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.560153994Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-09-14T03:45:34.031 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.560563333Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-09-14T03:45:34.031 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.56084854Z level=info msg="Executing migration" id="delete alert_definition table" 2024-09-14T03:45:34.032 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.561150258Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-09-14T03:45:34.033 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.56161495Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-09-14T03:45:34.033 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.562046113Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-09-14T03:45:34.033 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.562458449Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-09-14T03:45:34.033 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.562718432Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-09-14T03:45:34.033 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.563122146Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-09-14T03:45:34.033 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.563521462Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-09-14T03:45:34.034 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.563956377Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-09-14T03:45:34.034 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.564385901Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-09-14T03:45:34.034 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.565165758Z level=info msg="Executing migration" id="drop alert_definition table" 2024-09-14T03:45:34.034 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.565576218Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-09-14T03:45:34.034 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.566078829Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-09-14T03:45:34.034 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.566547976Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-09-14T03:45:34.034 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.566999651Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-09-14T03:45:34.034 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.567414516Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-09-14T03:45:34.034 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.56766978Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-09-14T03:45:34.034 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.568093504Z level=info msg="Executing migration" id="create alert_instance table" 2024-09-14T03:45:34.034 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.568504213Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-09-14T03:45:34.034 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.568947456Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-09-14T03:45:34.034 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.56939613Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-09-14T03:45:34.034 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.57025097Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-09-14T03:45:34.035 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.5708554Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-09-14T03:45:34.035 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.571262605Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-09-14T03:45:34.035 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.574692261Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-09-14T03:45:34.035 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.578058591Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-09-14T03:45:34.035 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.578482946Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-09-14T03:45:34.035 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.57887651Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-09-14T03:45:34.035 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.579663288Z level=info msg="Executing migration" id="create alert_rule table" 2024-09-14T03:45:34.035 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.580097419Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-09-14T03:45:34.035 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.580514163Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-09-14T03:45:34.035 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.580939789Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-09-14T03:45:34.035 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.581370604Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-09-14T03:45:34.035 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.581634424Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-09-14T03:45:34.035 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.582501046Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-09-14T03:45:34.035 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.583386789Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-09-14T03:45:34.036 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.584198965Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-09-14T03:45:34.036 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.584635181Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-09-14T03:45:34.036 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.585059907Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-09-14T03:45:34.036 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.585846416Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-09-14T03:45:34.036 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.586669178Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-09-14T03:45:34.036 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.587101882Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-09-14T03:45:34.036 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.587929611Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-09-14T03:45:34.036 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.588732588Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-09-14T03:45:34.036 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.589052048Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-09-14T03:45:34.037 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.589481344Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-09-14T03:45:34.037 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.58992989Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-09-14T03:45:34.037 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.590348521Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-09-14T03:45:34.037 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.590618352Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-09-14T03:45:34.037 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.591495419Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-09-14T03:45:34.037 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.592337366Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-09-14T03:45:34.037 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.593170835Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-09-14T03:45:34.037 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.594005311Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-09-14T03:45:34.037 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.594885818Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-09-14T03:45:34.037 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.595228397Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-09-14T03:45:34.037 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.595625457Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-09-14T03:45:34.037 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.596505499Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-09-14T03:45:34.037 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.596776537Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-09-14T03:45:34.038 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.597639301Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-09-14T03:45:34.038 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.598060916Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-09-14T03:45:34.038 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.598894214Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-09-14T03:45:34.038 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.599277931Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-09-14T03:45:34.038 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.599691183Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-09-14T03:45:34.038 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.600554208Z level=info msg="Executing migration" id="create provenance_type table" 2024-09-14T03:45:34.038 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.600927605Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-09-14T03:45:34.038 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.601344717Z level=info msg="Executing migration" id="create alert_image table" 2024-09-14T03:45:34.038 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.60170708Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-09-14T03:45:34.038 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.602117379Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-09-14T03:45:34.038 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.602393292Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-09-14T03:45:34.038 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.602763517Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-09-14T03:45:34.038 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.60318734Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-09-14T03:45:34.038 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.603285808Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-09-14T03:45:34.039 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.603503593Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-09-14T03:45:34.039 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.603819747Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-09-14T03:45:34.039 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.604238476Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-09-14T03:45:34.039 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.604372783Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-09-14T03:45:34.039 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.604485969Z level=info msg="alerts found to migrate" alerts=0 2024-09-14T03:45:34.039 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.604829792Z level=info msg="Executing migration" id="create library_element table v1" 2024-09-14T03:45:34.039 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.605277101Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-09-14T03:45:34.039 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.605691859Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-09-14T03:45:34.039 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.606087854Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-09-14T03:45:34.039 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.606500237Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-09-14T03:45:34.039 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.606953734Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-09-14T03:45:34.039 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.60721373Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-09-14T03:45:34.039 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.607541892Z level=info msg="Executing migration" id="create data_keys table" 2024-09-14T03:45:34.039 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.607949629Z level=info msg="Executing migration" id="create secrets table" 2024-09-14T03:45:34.040 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.608406419Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-09-14T03:45:34.040 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.612624454Z level=info msg="Executing migration" id="add name column into data_keys" 2024-09-14T03:45:34.040 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.613542224Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-09-14T03:45:34.040 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.613861989Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-09-14T03:45:34.040 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.618027295Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-09-14T03:45:34.040 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.622269239Z level=info msg="Executing migration" id="create kv_store table v1" 2024-09-14T03:45:34.040 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.622664163Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-09-14T03:45:34.040 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.623100166Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-09-14T03:45:34.040 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.623425472Z level=info msg="Executing migration" id="create permission table" 2024-09-14T03:45:34.040 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.62378659Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-09-14T03:45:34.040 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.624210159Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-09-14T03:45:34.040 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.624643713Z level=info msg="Executing migration" id="create role table" 2024-09-14T03:45:34.040 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.625031461Z level=info msg="Executing migration" id="add column display_name" 2024-09-14T03:45:34.040 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.625940503Z level=info msg="Executing migration" id="add column group_name" 2024-09-14T03:45:34.041 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.626850079Z level=info msg="Executing migration" id="add index role.org_id" 2024-09-14T03:45:34.041 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.627261122Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-09-14T03:45:34.041 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.627664215Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-09-14T03:45:34.041 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.628110525Z level=info msg="Executing migration" id="create team role table" 2024-09-14T03:45:34.041 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.628501813Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-09-14T03:45:34.041 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.628915687Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-09-14T03:45:34.041 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.62935396Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-09-14T03:45:34.041 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.629782209Z level=info msg="Executing migration" id="create user role table" 2024-09-14T03:45:34.041 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.630153654Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-09-14T03:45:34.041 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.630556887Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-09-14T03:45:34.041 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.63099675Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-09-14T03:45:34.041 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.631431043Z level=info msg="Executing migration" id="create builtin role table" 2024-09-14T03:45:34.041 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.631822356Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-09-14T03:45:34.041 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.632235526Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-09-14T03:45:34.042 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.632634254Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-09-14T03:45:34.042 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.633623602Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-09-14T03:45:34.042 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.634089908Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-09-14T03:45:34.042 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.634531925Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-09-14T03:45:34.042 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.634940185Z level=info msg="Executing migration" id="add unique index role.uid" 2024-09-14T03:45:34.042 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.635343521Z level=info msg="Executing migration" id="create seed assignment table" 2024-09-14T03:45:34.042 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.635730209Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-09-14T03:45:34.042 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.63617616Z level=info msg="Executing migration" id="add column hidden to role table" 2024-09-14T03:45:34.042 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.637184257Z level=info msg="Executing migration" id="create query_history table v1" 2024-09-14T03:45:34.042 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.637551532Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-09-14T03:45:34.042 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.638015905Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-09-14T03:45:34.042 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.63828954Z level=info msg="Executing migration" id="teams permissions migration" 2024-09-14T03:45:34.042 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.638640462Z level=info msg="Executing migration" id="dashboard permissions" 2024-09-14T03:45:34.042 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.639088718Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-09-14T03:45:34.043 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.639453061Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-09-14T03:45:34.043 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.63970516Z level=info msg="Executing migration" id="alerting notification permissions" 2024-09-14T03:45:34.043 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.640038425Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-09-14T03:45:34.043 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.640402235Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-09-14T03:45:34.043 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.640850972Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-09-14T03:45:34.043 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.641838244Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-09-14T03:45:34.043 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.642114141Z level=info msg="Executing migration" id="create correlation table v1" 2024-09-14T03:45:34.043 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.642490577Z level=info msg="Executing migration" id="add index correlations.uid" 2024-09-14T03:45:34.043 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.642936795Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-09-14T03:45:34.043 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.643349419Z level=info msg="Executing migration" id="add correlation config column" 2024-09-14T03:45:34.043 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.644323816Z level=info msg="Executing migration" id="create entity_events table" 2024-09-14T03:45:34.043 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.644736003Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-09-14T03:45:34.043 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.645149005Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-14T03:45:34.043 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.645240416Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-14T03:45:34.044 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.645499725Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-14T03:45:34.044 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.645580661Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-14T03:45:34.044 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.645830522Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-09-14T03:45:34.044 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.646219594Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-09-14T03:45:34.044 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.646577554Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-09-14T03:45:34.044 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.647001292Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-14T03:45:34.044 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.647411837Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-09-14T03:45:34.044 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.647772407Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-14T03:45:34.044 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.648221571Z level=info msg="Executing migration" id="Drop public config table" 2024-09-14T03:45:34.044 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.648575563Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-09-14T03:45:34.044 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.648985809Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-09-14T03:45:34.044 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.649402917Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-14T03:45:34.044 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.649871586Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-09-14T03:45:34.045 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.650295467Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-09-14T03:45:34.045 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.653279918Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-09-14T03:45:34.045 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.654263442Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-09-14T03:45:34.045 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.65532792Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-09-14T03:45:34.045 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.655618973Z level=info msg="Executing migration" id="add share column" 2024-09-14T03:45:34.045 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.656627009Z level=info msg="Executing migration" id="create default alerting folders" 2024-09-14T03:45:34.045 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.657053373Z level=info msg="Executing migration" id="create file table" 2024-09-14T03:45:34.045 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.657452807Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-09-14T03:45:34.045 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.65789444Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-09-14T03:45:34.045 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.658347005Z level=info msg="Executing migration" id="create file_meta table" 2024-09-14T03:45:34.045 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.658703545Z level=info msg="Executing migration" id="file table idx: path key" 2024-09-14T03:45:34.045 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.659182353Z level=info msg="Executing migration" id="set path collation in file table" 2024-09-14T03:45:34.045 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.65944694Z level=info msg="Executing migration" id="managed permissions migration" 2024-09-14T03:45:34.046 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.659753053Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-09-14T03:45:34.046 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.660092291Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-09-14T03:45:34.046 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.660478675Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-09-14T03:45:34.046 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.661536935Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-09-14T03:45:34.046 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.661845853Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-09-14T03:45:34.046 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.662299974Z level=info msg="Executing migration" id="update group index for alert rules" 2024-09-14T03:45:34.046 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.66263904Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-09-14T03:45:34.046 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.662954025Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-09-14T03:45:34.046 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.663294297Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-09-14T03:45:34.046 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.664290421Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-09-14T03:45:34.046 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.665320175Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-09-14T03:45:34.046 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.665738332Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-09-14T03:45:34.046 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.675477022Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-09-14T03:45:34.047 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.675915208Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-09-14T03:45:34.047 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.676385969Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-09-14T03:45:34.047 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.679704987Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-09-14T03:45:34.047 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.680016728Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-09-14T03:45:34.047 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.680361646Z level=info msg="Executing migration" id="create folder table" 2024-09-14T03:45:34.047 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.680718151Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-09-14T03:45:34.047 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.681202635Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-09-14T03:45:34.047 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.681631647Z level=info msg="Executing migration" id="Update folder title length" 2024-09-14T03:45:34.047 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.681918296Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-09-14T03:45:34.047 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=migrator t=2024-09-14T03:45:33.682350398Z level=info msg="migrations completed" performed=485 skipped=0 duration=248.663308ms 2024-09-14T03:45:34.047 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=sqlstore t=2024-09-14T03:45:33.682747696Z level=info msg="Created default organization" 2024-09-14T03:45:34.047 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=plugin.loader t=2024-09-14T03:45:33.737105545Z level=info msg="Plugin registered" pluginID=input 2024-09-14T03:45:34.047 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=plugin.loader t=2024-09-14T03:45:33.76389762Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-09-14T03:45:34.047 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=plugin.loader t=2024-09-14T03:45:33.763918314Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-09-14T03:45:34.048 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=secrets t=2024-09-14T03:45:33.763951931Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-14T03:45:34.048 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=query_data t=2024-09-14T03:45:33.767363346Z level=info msg="Query Service initialization" 2024-09-14T03:45:34.048 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=live.push_http t=2024-09-14T03:45:33.769839501Z level=info msg="Live Push Gateway initialization" 2024-09-14T03:45:34.684 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:34 smithi070 ceph-mon[39288]: Deploying daemon node-exporter.a on smithi070 2024-09-14T03:45:34.684 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:34 smithi070 ceph-mon[27495]: Deploying daemon node-exporter.a on smithi070 2024-09-14T03:45:34.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:34 smithi094 ceph-mon[31053]: Deploying daemon node-exporter.a on smithi070 2024-09-14T03:45:34.777 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:34 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=infra.usagestats.collector t=2024-09-14T03:45:34.476428294Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-14T03:45:34.777 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:34 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=provisioning.datasources t=2024-09-14T03:45:34.476723766Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-09-14T03:45:34.777 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:34 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=provisioning.datasources t=2024-09-14T03:45:34.509260153Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-14T03:45:34.777 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:34 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=provisioning.datasources t=2024-09-14T03:45:34.540892291Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-09-14T03:45:34.777 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:34 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=provisioning.alerting t=2024-09-14T03:45:34.562185297Z level=info msg="starting to provision alerting" 2024-09-14T03:45:34.777 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:34 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=provisioning.alerting t=2024-09-14T03:45:34.56220858Z level=info msg="finished to provision alerting" 2024-09-14T03:45:34.778 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:34 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=grafanaStorageLogger t=2024-09-14T03:45:34.562377517Z level=info msg="storage starting" 2024-09-14T03:45:34.778 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:34 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=ngalert.state.manager t=2024-09-14T03:45:34.562402632Z level=info msg="Warming state cache for startup" 2024-09-14T03:45:34.778 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:34 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=http.server t=2024-09-14T03:45:34.564117211Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-14T03:45:34.778 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:34 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=sqlstore.transactions t=2024-09-14T03:45:34.573454227Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-14T03:45:34.778 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:34 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=ngalert.state.manager t=2024-09-14T03:45:34.573685932Z level=info msg="State cache has been initialized" states=0 duration=11.282773ms 2024-09-14T03:45:34.778 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:34 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=ticker t=2024-09-14T03:45:34.573742145Z level=info msg=starting first_tick=2024-09-14T03:45:40Z 2024-09-14T03:45:34.778 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:34 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=ngalert.multiorg.alertmanager t=2024-09-14T03:45:34.573760594Z level=info msg="starting MultiOrg Alertmanager" 2024-09-14T03:45:34.778 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:34 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=sqlstore.transactions t=2024-09-14T03:45:34.624776751Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-14T03:45:34.778 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:34 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=sqlstore.transactions t=2024-09-14T03:45:34.635991025Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-09-14T03:45:34.778 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:45:34 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[63506]: logger=sqlstore.transactions t=2024-09-14T03:45:34.64793464Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-14T03:45:35.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:35 smithi070 ceph-mon[39288]: pgmap v24: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:35.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:35 smithi070 ceph-mon[27495]: pgmap v24: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:36.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:35 smithi094 ceph-mon[31053]: pgmap v24: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:37.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:37 smithi070 ceph-mon[27495]: pgmap v25: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:37.789 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:37 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:37.789 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:37 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:45:37.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:37 smithi070 ceph-mon[39288]: pgmap v25: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:37.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:37 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:37.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:37 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:45:38.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:37 smithi094 ceph-mon[31053]: pgmap v25: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:38.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:37 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:38.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:37 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:45:38.694 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:38 smithi070 systemd[1]: Starting Ceph node-exporter.a for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:45:39.694 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:39 smithi070 ceph-mon[27495]: pgmap v26: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:39.777 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:39 smithi070 ceph-mon[39288]: pgmap v26: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:39.854 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:39 smithi094 ceph-mon[31053]: pgmap v26: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:41.904 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:41 smithi070 ceph-mon[27495]: pgmap v27: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:41.904 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:41 smithi070 ceph-mon[39288]: pgmap v27: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:42.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:41 smithi094 ceph-mon[31053]: pgmap v27: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:42.027 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:45:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:45:43.540 INFO:teuthology.orchestra.run.smithi070.stdout:55834574905 2024-09-14T03:45:43.540 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-14T03:45:43.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:43 smithi070 ceph-mon[27495]: pgmap v28: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:43.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:43 smithi070 ceph-mon[39288]: pgmap v28: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:44.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:43 smithi094 ceph-mon[31053]: pgmap v28: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:44.392 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:44 smithi070 bash[76613]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-09-14T03:45:44.393 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:44 smithi070 bash[76613]: Getting image source signatures 2024-09-14T03:45:44.393 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:44 smithi070 bash[76613]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-09-14T03:45:44.393 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:44 smithi070 bash[76613]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-09-14T03:45:44.393 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:44 smithi070 bash[76613]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-09-14T03:45:44.485 INFO:teuthology.orchestra.run.smithi070.stdout:210453397533 2024-09-14T03:45:44.486 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-14T03:45:45.922 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:45 smithi070 ceph-mon[27495]: pgmap v29: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:45.923 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:45 smithi070 ceph-mon[39288]: pgmap v29: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:45.946 INFO:teuthology.orchestra.run.smithi070.stdout:34359738430 2024-09-14T03:45:45.946 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-14T03:45:46.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:45 smithi094 ceph-mon[31053]: pgmap v29: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:46.535 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:46 smithi070 bash[76613]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-09-14T03:45:46.649 INFO:teuthology.orchestra.run.smithi070.stdout:77309411378 2024-09-14T03:45:46.649 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-14T03:45:47.035 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:46 smithi070 bash[76613]: Writing manifest to image destination 2024-09-14T03:45:47.535 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:47 smithi070 podman[76613]: 2024-09-14 03:45:47.175954203 +0000 UTC m=+5.122279904 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-09-14T03:45:47.904 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:47 smithi070 ceph-mon[27495]: pgmap v30: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:47.904 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:47 smithi070 ceph-mon[39288]: pgmap v30: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:47.927 INFO:teuthology.orchestra.run.smithi070.stdout:133143986218 2024-09-14T03:45:47.927 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-14T03:45:48.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:47 smithi094 ceph-mon[31053]: pgmap v30: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:48.239 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 podman[76613]: 2024-09-14 03:45:48.0201693 +0000 UTC m=+5.966495011 container create 315c3f32399febe38ae8d87f5be4faf5d0f66b9f7f27cd6b1ae5582404a2ba04 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:45:48.536 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 podman[76613]: 2024-09-14 03:45:48.38171326 +0000 UTC m=+6.328038962 container init 315c3f32399febe38ae8d87f5be4faf5d0f66b9f7f27cd6b1ae5582404a2ba04 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:45:48.536 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 podman[76613]: 2024-09-14 03:45:48.386721085 +0000 UTC m=+6.333046788 container start 315c3f32399febe38ae8d87f5be4faf5d0f66b9f7f27cd6b1ae5582404a2ba04 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:45:48.536 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.390Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-14T03:45:48.536 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.390Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-14T03:45:48.536 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.390Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-14T03:45:48.536 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.391Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-14T03:45:48.537 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.391Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-14T03:45:48.537 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.391Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-14T03:45:48.537 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.391Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-14T03:45:48.537 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.391Z caller=node_exporter.go:117 level=info collector=arp 2024-09-14T03:45:48.537 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.391Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-14T03:45:48.537 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.391Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-14T03:45:48.537 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.391Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-14T03:45:48.537 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.391Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-14T03:45:48.537 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.391Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-14T03:45:48.537 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.391Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-14T03:45:48.538 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.391Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-14T03:45:48.538 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.391Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-14T03:45:48.538 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.391Z caller=node_exporter.go:117 level=info collector=edac 2024-09-14T03:45:48.538 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.391Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-14T03:45:48.538 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.391Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-14T03:45:48.538 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.391Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-14T03:45:48.538 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.391Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-14T03:45:48.538 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.391Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-14T03:45:48.539 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.391Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-14T03:45:48.539 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.391Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-14T03:45:48.539 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.391Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-14T03:45:48.539 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.391Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-14T03:45:48.539 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.391Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-14T03:45:48.539 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.391Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-14T03:45:48.539 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.391Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-14T03:45:48.539 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.392Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-14T03:45:48.539 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.392Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-14T03:45:48.539 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.392Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-14T03:45:48.540 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.392Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-14T03:45:48.540 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.392Z caller=node_exporter.go:117 level=info collector=os 2024-09-14T03:45:48.540 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.392Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-14T03:45:48.540 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.392Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-14T03:45:48.540 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.392Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-14T03:45:48.540 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.392Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-14T03:45:48.540 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.392Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-14T03:45:48.540 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.392Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-14T03:45:48.540 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.392Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-14T03:45:48.540 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.392Z caller=node_exporter.go:117 level=info collector=stat 2024-09-14T03:45:48.540 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.392Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-14T03:45:48.540 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.392Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-14T03:45:48.540 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.392Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-14T03:45:48.541 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.392Z caller=node_exporter.go:117 level=info collector=time 2024-09-14T03:45:48.541 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.392Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-14T03:45:48.541 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.392Z caller=node_exporter.go:117 level=info collector=uname 2024-09-14T03:45:48.541 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.392Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-14T03:45:48.541 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.392Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-14T03:45:48.541 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.392Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-14T03:45:48.541 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.392Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-14T03:45:48.541 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[77146]: ts=2024-09-14T03:45:48.392Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-14T03:45:48.541 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 bash[76613]: 315c3f32399febe38ae8d87f5be4faf5d0f66b9f7f27cd6b1ae5582404a2ba04 2024-09-14T03:45:48.541 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:45:48 smithi070 systemd[1]: Started Ceph node-exporter.a for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:45:49.527 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:49 smithi094 bash[63789]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-09-14T03:45:50.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:50.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:50.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:50.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:49 smithi094 ceph-mon[31053]: Deploying daemon node-exporter.b on smithi094 2024-09-14T03:45:50.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:49 smithi094 ceph-mon[31053]: pgmap v31: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:50.027 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:49 smithi094 bash[63789]: Getting image source signatures 2024-09-14T03:45:50.027 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:49 smithi094 bash[63789]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-09-14T03:45:50.027 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:49 smithi094 bash[63789]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-09-14T03:45:50.027 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:49 smithi094 bash[63789]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-09-14T03:45:50.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:50.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:50.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:50.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:49 smithi070 ceph-mon[27495]: Deploying daemon node-exporter.b on smithi094 2024-09-14T03:45:50.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:49 smithi070 ceph-mon[27495]: pgmap v31: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:50.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:50.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:50.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:50.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:49 smithi070 ceph-mon[39288]: Deploying daemon node-exporter.b on smithi094 2024-09-14T03:45:50.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:49 smithi070 ceph-mon[39288]: pgmap v31: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:50.085 INFO:teuthology.orchestra.run.smithi070.stdout:184683593760 2024-09-14T03:45:50.085 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-14T03:45:51.034 INFO:teuthology.orchestra.run.smithi070.stdout:111669149743 2024-09-14T03:45:51.034 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-14T03:45:51.088 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:45:51.114 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:45:51.114 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:45:51.526 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 bash[63789]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-09-14T03:45:51.527 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 bash[63789]: Writing manifest to image destination 2024-09-14T03:45:51.587 INFO:teuthology.orchestra.run.smithi070.stdout:158913789991 2024-09-14T03:45:51.588 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-14T03:45:51.823 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:51 smithi070 ceph-mon[27495]: pgmap v32: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:51.824 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:51 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:45:51.824 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:51 smithi070 ceph-mon[39288]: pgmap v32: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:51.824 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:51 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:45:51.850 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-mon[31053]: pgmap v32: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:51.850 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:45:51.850 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:45:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:45:51.851 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 podman[63789]: 2024-09-14 03:45:51.473228761 +0000 UTC m=+2.156216796 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-09-14T03:45:51.851 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 podman[63789]: 2024-09-14 03:45:51.622304985 +0000 UTC m=+2.305293062 container create 3dcf1723ea3558eeb4a4360245d68a3f318ff0adad4cdebcd061821d743f0686 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:45:51.851 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 podman[63789]: 2024-09-14 03:45:51.849210811 +0000 UTC m=+2.532198851 container init 3dcf1723ea3558eeb4a4360245d68a3f318ff0adad4cdebcd061821d743f0686 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:45:52.277 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 podman[63789]: 2024-09-14 03:45:51.854373414 +0000 UTC m=+2.537361450 container start 3dcf1723ea3558eeb4a4360245d68a3f318ff0adad4cdebcd061821d743f0686 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:45:52.277 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.858Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-14T03:45:52.277 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.858Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-14T03:45:52.277 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.858Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-14T03:45:52.277 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.858Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-14T03:45:52.277 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.858Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-14T03:45:52.277 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.858Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-14T03:45:52.278 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-14T03:45:52.278 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=arp 2024-09-14T03:45:52.278 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-14T03:45:52.278 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-14T03:45:52.278 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-14T03:45:52.278 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-14T03:45:52.278 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-14T03:45:52.278 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-14T03:45:52.278 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-14T03:45:52.278 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-14T03:45:52.279 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=edac 2024-09-14T03:45:52.279 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-14T03:45:52.279 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-14T03:45:52.279 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-14T03:45:52.279 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-14T03:45:52.279 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-14T03:45:52.279 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-14T03:45:52.279 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-14T03:45:52.279 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-14T03:45:52.280 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-14T03:45:52.280 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-14T03:45:52.280 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-14T03:45:52.280 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-14T03:45:52.280 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-14T03:45:52.280 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-14T03:45:52.280 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-14T03:45:52.280 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-14T03:45:52.280 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=os 2024-09-14T03:45:52.280 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-14T03:45:52.280 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-14T03:45:52.280 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-14T03:45:52.280 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-14T03:45:52.281 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-14T03:45:52.281 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-14T03:45:52.281 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-14T03:45:52.281 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=stat 2024-09-14T03:45:52.281 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-14T03:45:52.281 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-14T03:45:52.281 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-14T03:45:52.281 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=time 2024-09-14T03:45:52.281 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-14T03:45:52.281 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=uname 2024-09-14T03:45:52.281 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-14T03:45:52.281 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-14T03:45:52.282 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-14T03:45:52.282 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-14T03:45:52.282 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[63866]: ts=2024-09-14T03:45:51.859Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-14T03:45:52.282 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 bash[63789]: 3dcf1723ea3558eeb4a4360245d68a3f318ff0adad4cdebcd061821d743f0686 2024-09-14T03:45:52.282 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:45:51 smithi094 systemd[1]: Started Ceph node-exporter.b for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:45:52.970 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:45:52.971 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:45:53.276 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:52 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:53.277 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:52 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:53.277 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:52 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:53.277 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:52 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:53.277 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:52 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:53.277 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:52 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:45:53.277 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:52 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:53.277 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:52 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:53.278 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:52 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:53.278 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:52 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:53.278 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:52 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:53.278 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:52 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:45:53.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:52 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:53.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:52 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:53.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:52 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:53.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:52 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:53.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:52 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:53.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:52 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:45:54.134 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:54 smithi070 ceph-mon[27495]: pgmap v33: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:54.135 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:54 smithi070 ceph-mon[39288]: pgmap v33: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:54.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:54 smithi094 ceph-mon[31053]: pgmap v33: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:55.743 INFO:teuthology.orchestra.run.smithi070.stdout:34359738432 2024-09-14T03:45:56.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:56 smithi094 ceph-mon[31053]: pgmap v34: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:56.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:56 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/659289160' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-14T03:45:56.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:56 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:56.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:56 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:56.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:56 smithi070 ceph-mon[27495]: pgmap v34: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:56.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:56 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/659289160' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-14T03:45:56.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:56 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:56.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:56 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:56.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:56 smithi070 ceph-mon[39288]: pgmap v34: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:56.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:56 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/659289160' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-14T03:45:56.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:56 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:56.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:56 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:56.994 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:45:58.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:57 smithi094 ceph-mon[31053]: pgmap v35: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:58.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:57 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:58.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:57 smithi070 ceph-mon[27495]: pgmap v35: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:58.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:57 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:58.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:57 smithi070 ceph-mon[39288]: pgmap v35: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:58.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:57 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:45:58.658 INFO:teuthology.orchestra.run.smithi070.stdout:210453397536 2024-09-14T03:45:59.054 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738430 got 34359738432 for osd.0 2024-09-14T03:45:59.054 DEBUG:teuthology.parallel:result is None 2024-09-14T03:45:59.400 INFO:teuthology.orchestra.run.smithi070.stdout:77309411382 2024-09-14T03:45:59.684 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:59 smithi070 ceph-mon[27495]: pgmap v36: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:59.684 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:59 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/3313771186' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-14T03:45:59.684 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:45:59 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/595174113' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-14T03:45:59.684 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:59 smithi070 ceph-mon[39288]: pgmap v36: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:45:59.685 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:59 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/3313771186' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-14T03:45:59.685 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:45:59 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/595174113' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-14T03:46:00.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:59 smithi094 ceph-mon[31053]: pgmap v36: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:00.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:59 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/3313771186' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-14T03:46:00.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:45:59 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/595174113' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-14T03:46:00.333 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:46:00.334 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:46:00.911 INFO:teuthology.orchestra.run.smithi070.stdout:55834574908 2024-09-14T03:46:01.233 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411378 got 77309411382 for osd.2 2024-09-14T03:46:01.234 DEBUG:teuthology.parallel:result is None 2024-09-14T03:46:01.997 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:01 smithi070 ceph-mon[39288]: pgmap v37: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:01.997 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:01 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/1389542244' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-14T03:46:01.997 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:01 smithi070 ceph-mon[27495]: pgmap v37: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:01.998 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:01 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/1389542244' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-14T03:46:02.025 INFO:tasks.cephadm.ceph_manager.ceph:need seq 210453397533 got 210453397536 for osd.7 2024-09-14T03:46:02.025 DEBUG:teuthology.parallel:result is None 2024-09-14T03:46:02.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:01 smithi094 ceph-mon[31053]: pgmap v37: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:02.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:01 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/1389542244' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-14T03:46:02.027 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:46:01 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:46:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:46:03.954 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:03 smithi070 ceph-mon[27495]: pgmap v38: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:03.954 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:03 smithi070 ceph-mon[39288]: pgmap v38: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:04.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:03 smithi094 ceph-mon[31053]: pgmap v38: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:04.635 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574905 got 55834574908 for osd.1 2024-09-14T03:46:04.635 DEBUG:teuthology.parallel:result is None 2024-09-14T03:46:05.404 INFO:teuthology.orchestra.run.smithi070.stdout:184683593765 2024-09-14T03:46:05.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:05 smithi070 ceph-mon[27495]: pgmap v39: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:05.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:05 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/2261084962' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-14T03:46:05.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:05 smithi070 ceph-mon[39288]: pgmap v39: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:05.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:05 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/2261084962' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-14T03:46:06.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:05 smithi094 ceph-mon[31053]: pgmap v39: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:06.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:05 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/2261084962' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-14T03:46:06.836 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:06 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:46:06.836 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:06 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:46:07.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:06 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:46:07.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:07 smithi070 ceph-mon[27495]: pgmap v40: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:07.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:07 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/1634801067' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-14T03:46:07.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:07 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/1898876053' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-14T03:46:07.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:07 smithi070 ceph-mon[39288]: pgmap v40: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:07.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:07 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/1634801067' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-14T03:46:07.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:07 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/1898876053' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-14T03:46:07.932 INFO:tasks.cephadm.ceph_manager.ceph:need seq 184683593760 got 184683593765 for osd.6 2024-09-14T03:46:07.933 DEBUG:teuthology.parallel:result is None 2024-09-14T03:46:08.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:07 smithi094 ceph-mon[31053]: pgmap v40: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:08.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:07 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/1634801067' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-14T03:46:08.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:07 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/1898876053' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-14T03:46:08.116 INFO:teuthology.orchestra.run.smithi070.stdout:158913789994 2024-09-14T03:46:08.802 INFO:tasks.cephadm.ceph_manager.ceph:need seq 158913789991 got 158913789994 for osd.5 2024-09-14T03:46:08.803 DEBUG:teuthology.parallel:result is None 2024-09-14T03:46:08.948 INFO:teuthology.orchestra.run.smithi070.stdout:133143986222 2024-09-14T03:46:09.547 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986218 got 133143986222 for osd.4 2024-09-14T03:46:09.547 DEBUG:teuthology.parallel:result is None 2024-09-14T03:46:09.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:09 smithi070 ceph-mon[27495]: pgmap v41: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:09.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:09 smithi070 ceph-mon[39288]: pgmap v41: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:10.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:09 smithi094 ceph-mon[31053]: pgmap v41: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:10.231 INFO:teuthology.orchestra.run.smithi070.stdout:111669149747 2024-09-14T03:46:11.009 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:10 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/3826278492' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-14T03:46:11.009 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:10 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/3826278492' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-14T03:46:11.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:10 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/3826278492' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-14T03:46:11.033 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149743 got 111669149747 for osd.3 2024-09-14T03:46:11.033 DEBUG:teuthology.parallel:result is None 2024-09-14T03:46:11.033 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-14T03:46:11.034 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T03:46:11.793 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:46:11.916 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:11 smithi070 ceph-mon[27495]: pgmap v42: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:11.916 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:11 smithi070 ceph-mon[39288]: pgmap v42: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:12.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:11 smithi094 ceph-mon[31053]: pgmap v42: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:12.027 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:46:11 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:46:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:46:13.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:13 smithi070 ceph-mon[27495]: pgmap v43: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:13.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:13 smithi070 ceph-mon[39288]: pgmap v43: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:14.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:13 smithi094 ceph-mon[31053]: pgmap v43: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:14.336 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:46:14.338 INFO:teuthology.orchestra.run.smithi070.stderr:dumped all 2024-09-14T03:46:15.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:14 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:15.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:14 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:15.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:14 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:46:15.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:14 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:46:15.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:14 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:15.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:14 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:15.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:14 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:15.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:14 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:46:15.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:14 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:46:15.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:14 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:15.190 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":43,"stamp":"2024-09-14T03:46:12.560490+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":593515,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":163,"num_write_kb":1646,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124,"ondisk_log_size":124,"up":315,"acting":315,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":315,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386104,"kb_used_data":7256,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575112,"statfs":{"total":767960285184,"available":765516914688,"internally_reserved":0,"allocated":7430144,"data_stored":4278265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.661024+0000","last_change":"2024-09-14T03:44:01.882774+0000","last_active":"2024-09-14T03:45:14.661024+0000","last_peered":"2024-09-14T03:45:14.661024+0000","last_clean":"2024-09-14T03:45:14.661024+0000","last_became_active":"2024-09-14T03:44:01.882481+0000","last_became_peered":"2024-09-14T03:44:01.882481+0000","last_unstale":"2024-09-14T03:45:14.661024+0000","last_undegraded":"2024-09-14T03:45:14.661024+0000","last_fullsized":"2024-09-14T03:45:14.661024+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:49:25.209573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.458995+0000","last_change":"2024-09-14T03:44:01.868415+0000","last_active":"2024-09-14T03:45:14.458995+0000","last_peered":"2024-09-14T03:45:14.458995+0000","last_clean":"2024-09-14T03:45:14.458995+0000","last_became_active":"2024-09-14T03:44:01.868152+0000","last_became_peered":"2024-09-14T03:44:01.868152+0000","last_unstale":"2024-09-14T03:45:14.458995+0000","last_undegraded":"2024-09-14T03:45:14.458995+0000","last_fullsized":"2024-09-14T03:45:14.458995+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:56:00.270070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.684432+0000","last_change":"2024-09-14T03:44:01.871459+0000","last_active":"2024-09-14T03:45:14.684432+0000","last_peered":"2024-09-14T03:45:14.684432+0000","last_clean":"2024-09-14T03:45:14.684432+0000","last_became_active":"2024-09-14T03:44:01.870749+0000","last_became_peered":"2024-09-14T03:44:01.870749+0000","last_unstale":"2024-09-14T03:45:14.684432+0000","last_undegraded":"2024-09-14T03:45:14.684432+0000","last_fullsized":"2024-09-14T03:45:14.684432+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:54.391363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.461823+0000","last_change":"2024-09-14T03:44:01.882503+0000","last_active":"2024-09-14T03:45:14.461823+0000","last_peered":"2024-09-14T03:45:14.461823+0000","last_clean":"2024-09-14T03:45:14.461823+0000","last_became_active":"2024-09-14T03:44:01.882250+0000","last_became_peered":"2024-09-14T03:44:01.882250+0000","last_unstale":"2024-09-14T03:45:14.461823+0000","last_undegraded":"2024-09-14T03:45:14.461823+0000","last_fullsized":"2024-09-14T03:45:14.461823+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:46:29.551285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.660633+0000","last_change":"2024-09-14T03:44:01.880333+0000","last_active":"2024-09-14T03:45:14.660633+0000","last_peered":"2024-09-14T03:45:14.660633+0000","last_clean":"2024-09-14T03:45:14.660633+0000","last_became_active":"2024-09-14T03:44:01.880010+0000","last_became_peered":"2024-09-14T03:44:01.880010+0000","last_unstale":"2024-09-14T03:45:14.660633+0000","last_undegraded":"2024-09-14T03:45:14.660633+0000","last_fullsized":"2024-09-14T03:45:14.660633+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:45:02.741425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.461594+0000","last_change":"2024-09-14T03:44:01.880710+0000","last_active":"2024-09-14T03:45:14.461594+0000","last_peered":"2024-09-14T03:45:14.461594+0000","last_clean":"2024-09-14T03:45:14.461594+0000","last_became_active":"2024-09-14T03:44:01.880237+0000","last_became_peered":"2024-09-14T03:44:01.880237+0000","last_unstale":"2024-09-14T03:45:14.461594+0000","last_undegraded":"2024-09-14T03:45:14.461594+0000","last_fullsized":"2024-09-14T03:45:14.461594+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:56:49.808707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.660415+0000","last_change":"2024-09-14T03:44:01.878590+0000","last_active":"2024-09-14T03:45:14.660415+0000","last_peered":"2024-09-14T03:45:14.660415+0000","last_clean":"2024-09-14T03:45:14.660415+0000","last_became_active":"2024-09-14T03:44:01.878281+0000","last_became_peered":"2024-09-14T03:44:01.878281+0000","last_unstale":"2024-09-14T03:45:14.660415+0000","last_undegraded":"2024-09-14T03:45:14.660415+0000","last_fullsized":"2024-09-14T03:45:14.660415+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:41:57.164227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.857411+0000","last_change":"2024-09-14T03:44:01.877057+0000","last_active":"2024-09-14T03:45:14.857411+0000","last_peered":"2024-09-14T03:45:14.857411+0000","last_clean":"2024-09-14T03:45:14.857411+0000","last_became_active":"2024-09-14T03:44:01.876727+0000","last_became_peered":"2024-09-14T03:44:01.876727+0000","last_unstale":"2024-09-14T03:45:14.857411+0000","last_undegraded":"2024-09-14T03:45:14.857411+0000","last_fullsized":"2024-09-14T03:45:14.857411+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:31:24.716316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.661211+0000","last_change":"2024-09-14T03:44:01.884865+0000","last_active":"2024-09-14T03:45:14.661211+0000","last_peered":"2024-09-14T03:45:14.661211+0000","last_clean":"2024-09-14T03:45:14.661211+0000","last_became_active":"2024-09-14T03:44:01.884136+0000","last_became_peered":"2024-09-14T03:44:01.884136+0000","last_unstale":"2024-09-14T03:45:14.661211+0000","last_undegraded":"2024-09-14T03:45:14.661211+0000","last_fullsized":"2024-09-14T03:45:14.661211+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:59.903382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":30,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.544938+0000","last_change":"2024-09-14T03:44:26.430052+0000","last_active":"2024-09-14T03:45:14.544938+0000","last_peered":"2024-09-14T03:45:14.544938+0000","last_clean":"2024-09-14T03:45:14.544938+0000","last_became_active":"2024-09-14T03:44:26.429658+0000","last_became_peered":"2024-09-14T03:44:26.429658+0000","last_unstale":"2024-09-14T03:45:14.544938+0000","last_undegraded":"2024-09-14T03:45:14.544938+0000","last_fullsized":"2024-09-14T03:45:14.544938+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:34.312039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.544941+0000","last_change":"2024-09-14T03:44:01.875017+0000","last_active":"2024-09-14T03:45:14.544941+0000","last_peered":"2024-09-14T03:45:14.544941+0000","last_clean":"2024-09-14T03:45:14.544941+0000","last_became_active":"2024-09-14T03:44:01.874687+0000","last_became_peered":"2024-09-14T03:44:01.874687+0000","last_unstale":"2024-09-14T03:45:14.544941+0000","last_undegraded":"2024-09-14T03:45:14.544941+0000","last_fullsized":"2024-09-14T03:45:14.544941+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:29:09.035342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.458577+0000","last_change":"2024-09-14T03:44:01.883313+0000","last_active":"2024-09-14T03:45:14.458577+0000","last_peered":"2024-09-14T03:45:14.458577+0000","last_clean":"2024-09-14T03:45:14.458577+0000","last_became_active":"2024-09-14T03:44:01.882770+0000","last_became_peered":"2024-09-14T03:44:01.882770+0000","last_unstale":"2024-09-14T03:45:14.458577+0000","last_undegraded":"2024-09-14T03:45:14.458577+0000","last_fullsized":"2024-09-14T03:45:14.458577+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:57:19.402319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.857631+0000","last_change":"2024-09-14T03:44:01.878245+0000","last_active":"2024-09-14T03:45:14.857631+0000","last_peered":"2024-09-14T03:45:14.857631+0000","last_clean":"2024-09-14T03:45:14.857631+0000","last_became_active":"2024-09-14T03:44:01.877861+0000","last_became_peered":"2024-09-14T03:44:01.877861+0000","last_unstale":"2024-09-14T03:45:14.857631+0000","last_undegraded":"2024-09-14T03:45:14.857631+0000","last_fullsized":"2024-09-14T03:45:14.857631+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:08:29.902090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.684034+0000","last_change":"2024-09-14T03:44:01.877391+0000","last_active":"2024-09-14T03:45:14.684034+0000","last_peered":"2024-09-14T03:45:14.684034+0000","last_clean":"2024-09-14T03:45:14.684034+0000","last_became_active":"2024-09-14T03:44:01.876797+0000","last_became_peered":"2024-09-14T03:44:01.876797+0000","last_unstale":"2024-09-14T03:45:14.684034+0000","last_undegraded":"2024-09-14T03:45:14.684034+0000","last_fullsized":"2024-09-14T03:45:14.684034+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:06:02.891762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.458588+0000","last_change":"2024-09-14T03:44:01.876700+0000","last_active":"2024-09-14T03:45:14.458588+0000","last_peered":"2024-09-14T03:45:14.458588+0000","last_clean":"2024-09-14T03:45:14.458588+0000","last_became_active":"2024-09-14T03:44:01.876091+0000","last_became_peered":"2024-09-14T03:44:01.876091+0000","last_unstale":"2024-09-14T03:45:14.458588+0000","last_undegraded":"2024-09-14T03:45:14.458588+0000","last_fullsized":"2024-09-14T03:45:14.458588+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:25:41.750127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.461481+0000","last_change":"2024-09-14T03:44:01.877120+0000","last_active":"2024-09-14T03:45:14.461481+0000","last_peered":"2024-09-14T03:45:14.461481+0000","last_clean":"2024-09-14T03:45:14.461481+0000","last_became_active":"2024-09-14T03:44:01.876793+0000","last_became_peered":"2024-09-14T03:44:01.876793+0000","last_unstale":"2024-09-14T03:45:14.461481+0000","last_undegraded":"2024-09-14T03:45:14.461481+0000","last_fullsized":"2024-09-14T03:45:14.461481+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:23:56.450728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.462251+0000","last_change":"2024-09-14T03:44:01.885108+0000","last_active":"2024-09-14T03:45:14.462251+0000","last_peered":"2024-09-14T03:45:14.462251+0000","last_clean":"2024-09-14T03:45:14.462251+0000","last_became_active":"2024-09-14T03:44:01.884857+0000","last_became_peered":"2024-09-14T03:44:01.884857+0000","last_unstale":"2024-09-14T03:45:14.462251+0000","last_undegraded":"2024-09-14T03:45:14.462251+0000","last_fullsized":"2024-09-14T03:45:14.462251+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:31.448617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:15.135790+0000","last_change":"2024-09-14T03:44:01.876503+0000","last_active":"2024-09-14T03:45:15.135790+0000","last_peered":"2024-09-14T03:45:15.135790+0000","last_clean":"2024-09-14T03:45:15.135790+0000","last_became_active":"2024-09-14T03:44:01.876227+0000","last_became_peered":"2024-09-14T03:44:01.876227+0000","last_unstale":"2024-09-14T03:45:15.135790+0000","last_undegraded":"2024-09-14T03:45:15.135790+0000","last_fullsized":"2024-09-14T03:45:15.135790+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:44:50.132152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.661079+0000","last_change":"2024-09-14T03:44:01.883708+0000","last_active":"2024-09-14T03:45:14.661079+0000","last_peered":"2024-09-14T03:45:14.661079+0000","last_clean":"2024-09-14T03:45:14.661079+0000","last_became_active":"2024-09-14T03:44:01.883440+0000","last_became_peered":"2024-09-14T03:44:01.883440+0000","last_unstale":"2024-09-14T03:45:14.661079+0000","last_undegraded":"2024-09-14T03:45:14.661079+0000","last_fullsized":"2024-09-14T03:45:14.661079+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:10:34.658062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.544854+0000","last_change":"2024-09-14T03:44:01.888565+0000","last_active":"2024-09-14T03:45:14.544854+0000","last_peered":"2024-09-14T03:45:14.544854+0000","last_clean":"2024-09-14T03:45:14.544854+0000","last_became_active":"2024-09-14T03:44:01.888191+0000","last_became_peered":"2024-09-14T03:44:01.888191+0000","last_unstale":"2024-09-14T03:45:14.544854+0000","last_undegraded":"2024-09-14T03:45:14.544854+0000","last_fullsized":"2024-09-14T03:45:14.544854+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:52:23.782664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.684144+0000","last_change":"2024-09-14T03:44:01.874642+0000","last_active":"2024-09-14T03:45:14.684144+0000","last_peered":"2024-09-14T03:45:14.684144+0000","last_clean":"2024-09-14T03:45:14.684144+0000","last_became_active":"2024-09-14T03:44:01.874384+0000","last_became_peered":"2024-09-14T03:44:01.874384+0000","last_unstale":"2024-09-14T03:45:14.684144+0000","last_undegraded":"2024-09-14T03:45:14.684144+0000","last_fullsized":"2024-09-14T03:45:14.684144+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:35:05.535173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.461689+0000","last_change":"2024-09-14T03:44:01.886893+0000","last_active":"2024-09-14T03:45:14.461689+0000","last_peered":"2024-09-14T03:45:14.461689+0000","last_clean":"2024-09-14T03:45:14.461689+0000","last_became_active":"2024-09-14T03:44:01.886594+0000","last_became_peered":"2024-09-14T03:44:01.886594+0000","last_unstale":"2024-09-14T03:45:14.461689+0000","last_undegraded":"2024-09-14T03:45:14.461689+0000","last_fullsized":"2024-09-14T03:45:14.461689+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:07:37.154358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.857420+0000","last_change":"2024-09-14T03:44:01.868698+0000","last_active":"2024-09-14T03:45:14.857420+0000","last_peered":"2024-09-14T03:45:14.857420+0000","last_clean":"2024-09-14T03:45:14.857420+0000","last_became_active":"2024-09-14T03:44:01.868442+0000","last_became_peered":"2024-09-14T03:44:01.868442+0000","last_unstale":"2024-09-14T03:45:14.857420+0000","last_undegraded":"2024-09-14T03:45:14.857420+0000","last_fullsized":"2024-09-14T03:45:14.857420+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:41:18.296424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.461524+0000","last_change":"2024-09-14T03:44:01.882708+0000","last_active":"2024-09-14T03:45:14.461524+0000","last_peered":"2024-09-14T03:45:14.461524+0000","last_clean":"2024-09-14T03:45:14.461524+0000","last_became_active":"2024-09-14T03:44:01.882443+0000","last_became_peered":"2024-09-14T03:44:01.882443+0000","last_unstale":"2024-09-14T03:45:14.461524+0000","last_undegraded":"2024-09-14T03:45:14.461524+0000","last_fullsized":"2024-09-14T03:45:14.461524+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:46:48.306263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.857790+0000","last_change":"2024-09-14T03:44:01.884940+0000","last_active":"2024-09-14T03:45:14.857790+0000","last_peered":"2024-09-14T03:45:14.857790+0000","last_clean":"2024-09-14T03:45:14.857790+0000","last_became_active":"2024-09-14T03:44:01.884304+0000","last_became_peered":"2024-09-14T03:44:01.884304+0000","last_unstale":"2024-09-14T03:45:14.857790+0000","last_undegraded":"2024-09-14T03:45:14.857790+0000","last_fullsized":"2024-09-14T03:45:14.857790+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:01:49.061637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":30,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.544999+0000","last_change":"2024-09-14T03:44:26.430891+0000","last_active":"2024-09-14T03:45:14.544999+0000","last_peered":"2024-09-14T03:45:14.544999+0000","last_clean":"2024-09-14T03:45:14.544999+0000","last_became_active":"2024-09-14T03:44:26.430426+0000","last_became_peered":"2024-09-14T03:44:26.430426+0000","last_unstale":"2024-09-14T03:45:14.544999+0000","last_undegraded":"2024-09-14T03:45:14.544999+0000","last_fullsized":"2024-09-14T03:45:14.544999+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:00:53.758328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.458426+0000","last_change":"2024-09-14T03:44:01.874564+0000","last_active":"2024-09-14T03:45:14.458426+0000","last_peered":"2024-09-14T03:45:14.458426+0000","last_clean":"2024-09-14T03:45:14.458426+0000","last_became_active":"2024-09-14T03:44:01.874297+0000","last_became_peered":"2024-09-14T03:44:01.874297+0000","last_unstale":"2024-09-14T03:45:14.458426+0000","last_undegraded":"2024-09-14T03:45:14.458426+0000","last_fullsized":"2024-09-14T03:45:14.458426+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:07.027610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.857969+0000","last_change":"2024-09-14T03:44:01.884486+0000","last_active":"2024-09-14T03:45:14.857969+0000","last_peered":"2024-09-14T03:45:14.857969+0000","last_clean":"2024-09-14T03:45:14.857969+0000","last_became_active":"2024-09-14T03:44:01.884218+0000","last_became_peered":"2024-09-14T03:44:01.884218+0000","last_unstale":"2024-09-14T03:45:14.857969+0000","last_undegraded":"2024-09-14T03:45:14.857969+0000","last_fullsized":"2024-09-14T03:45:14.857969+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:59:18.153239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.684242+0000","last_change":"2024-09-14T03:44:01.878737+0000","last_active":"2024-09-14T03:45:14.684242+0000","last_peered":"2024-09-14T03:45:14.684242+0000","last_clean":"2024-09-14T03:45:14.684242+0000","last_became_active":"2024-09-14T03:44:01.878504+0000","last_became_peered":"2024-09-14T03:44:01.878504+0000","last_unstale":"2024-09-14T03:45:14.684242+0000","last_undegraded":"2024-09-14T03:45:14.684242+0000","last_fullsized":"2024-09-14T03:45:14.684242+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:41:35.420957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.461691+0000","last_change":"2024-09-14T03:44:01.880509+0000","last_active":"2024-09-14T03:45:14.461691+0000","last_peered":"2024-09-14T03:45:14.461691+0000","last_clean":"2024-09-14T03:45:14.461691+0000","last_became_active":"2024-09-14T03:44:01.879855+0000","last_became_peered":"2024-09-14T03:44:01.879855+0000","last_unstale":"2024-09-14T03:45:14.461691+0000","last_undegraded":"2024-09-14T03:45:14.461691+0000","last_fullsized":"2024-09-14T03:45:14.461691+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:05:31.166902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:15.135594+0000","last_change":"2024-09-14T03:44:01.869523+0000","last_active":"2024-09-14T03:45:15.135594+0000","last_peered":"2024-09-14T03:45:15.135594+0000","last_clean":"2024-09-14T03:45:15.135594+0000","last_became_active":"2024-09-14T03:44:01.869225+0000","last_became_peered":"2024-09-14T03:44:01.869225+0000","last_unstale":"2024-09-14T03:45:15.135594+0000","last_undegraded":"2024-09-14T03:45:15.135594+0000","last_fullsized":"2024-09-14T03:45:15.135594+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:43:15.638187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.544409+0000","last_change":"2024-09-14T03:44:01.876014+0000","last_active":"2024-09-14T03:45:14.544409+0000","last_peered":"2024-09-14T03:45:14.544409+0000","last_clean":"2024-09-14T03:45:14.544409+0000","last_became_active":"2024-09-14T03:44:01.875496+0000","last_became_peered":"2024-09-14T03:44:01.875496+0000","last_unstale":"2024-09-14T03:45:14.544409+0000","last_undegraded":"2024-09-14T03:45:14.544409+0000","last_fullsized":"2024-09-14T03:45:14.544409+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:03:45.344199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.458305+0000","last_change":"2024-09-14T03:44:01.865496+0000","last_active":"2024-09-14T03:45:14.458305+0000","last_peered":"2024-09-14T03:45:14.458305+0000","last_clean":"2024-09-14T03:45:14.458305+0000","last_became_active":"2024-09-14T03:44:01.865190+0000","last_became_peered":"2024-09-14T03:44:01.865190+0000","last_unstale":"2024-09-14T03:45:14.458305+0000","last_undegraded":"2024-09-14T03:45:14.458305+0000","last_fullsized":"2024-09-14T03:45:14.458305+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:18:50.037675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"58'1","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.857788+0000","last_change":"2024-09-14T03:43:58.729188+0000","last_active":"2024-09-14T03:45:14.857788+0000","last_peered":"2024-09-14T03:45:14.857788+0000","last_clean":"2024-09-14T03:45:14.857788+0000","last_became_active":"2024-09-14T03:43:58.728927+0000","last_became_peered":"2024-09-14T03:43:58.728927+0000","last_unstale":"2024-09-14T03:45:14.857788+0000","last_undegraded":"2024-09-14T03:45:14.857788+0000","last_fullsized":"2024-09-14T03:45:14.857788+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:13:16.419701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.684318+0000","last_change":"2024-09-14T03:44:01.879101+0000","last_active":"2024-09-14T03:45:14.684318+0000","last_peered":"2024-09-14T03:45:14.684318+0000","last_clean":"2024-09-14T03:45:14.684318+0000","last_became_active":"2024-09-14T03:44:01.878801+0000","last_became_peered":"2024-09-14T03:44:01.878801+0000","last_unstale":"2024-09-14T03:45:14.684318+0000","last_undegraded":"2024-09-14T03:45:14.684318+0000","last_fullsized":"2024-09-14T03:45:14.684318+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:13.447251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"58'1","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.544625+0000","last_change":"2024-09-14T03:43:58.730444+0000","last_active":"2024-09-14T03:45:14.544625+0000","last_peered":"2024-09-14T03:45:14.544625+0000","last_clean":"2024-09-14T03:45:14.544625+0000","last_became_active":"2024-09-14T03:43:58.730172+0000","last_became_peered":"2024-09-14T03:43:58.730172+0000","last_unstale":"2024-09-14T03:45:14.544625+0000","last_undegraded":"2024-09-14T03:45:14.544625+0000","last_fullsized":"2024-09-14T03:45:14.544625+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:03:44.810562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":23,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.857500+0000","last_change":"2024-09-14T03:43:58.731458+0000","last_active":"2024-09-14T03:45:14.857500+0000","last_peered":"2024-09-14T03:45:14.857500+0000","last_clean":"2024-09-14T03:45:14.857500+0000","last_became_active":"2024-09-14T03:43:58.731206+0000","last_became_peered":"2024-09-14T03:43:58.731206+0000","last_unstale":"2024-09-14T03:45:14.857500+0000","last_undegraded":"2024-09-14T03:45:14.857500+0000","last_fullsized":"2024-09-14T03:45:14.857500+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:37:06.670539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.857453+0000","last_change":"2024-09-14T03:44:01.869781+0000","last_active":"2024-09-14T03:45:14.857453+0000","last_peered":"2024-09-14T03:45:14.857453+0000","last_clean":"2024-09-14T03:45:14.857453+0000","last_became_active":"2024-09-14T03:44:01.869521+0000","last_became_peered":"2024-09-14T03:44:01.869521+0000","last_unstale":"2024-09-14T03:45:14.857453+0000","last_undegraded":"2024-09-14T03:45:14.857453+0000","last_fullsized":"2024-09-14T03:45:14.857453+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:35:18.565206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":33,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.461579+0000","last_change":"2024-09-14T03:44:25.932578+0000","last_active":"2024-09-14T03:45:14.461579+0000","last_peered":"2024-09-14T03:45:14.461579+0000","last_clean":"2024-09-14T03:45:14.461579+0000","last_became_active":"2024-09-14T03:44:25.932288+0000","last_became_peered":"2024-09-14T03:44:25.932288+0000","last_unstale":"2024-09-14T03:45:14.461579+0000","last_undegraded":"2024-09-14T03:45:14.461579+0000","last_fullsized":"2024-09-14T03:45:14.461579+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:24:41.091353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.457866+0000","last_change":"2024-09-14T03:44:01.873748+0000","last_active":"2024-09-14T03:45:14.457866+0000","last_peered":"2024-09-14T03:45:14.457866+0000","last_clean":"2024-09-14T03:45:14.457866+0000","last_became_active":"2024-09-14T03:44:01.873473+0000","last_became_peered":"2024-09-14T03:44:01.873473+0000","last_unstale":"2024-09-14T03:45:14.457866+0000","last_undegraded":"2024-09-14T03:45:14.457866+0000","last_fullsized":"2024-09-14T03:45:14.457866+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:04:39.289903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":23,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.544953+0000","last_change":"2024-09-14T03:43:58.735608+0000","last_active":"2024-09-14T03:45:14.544953+0000","last_peered":"2024-09-14T03:45:14.544953+0000","last_clean":"2024-09-14T03:45:14.544953+0000","last_became_active":"2024-09-14T03:43:58.735301+0000","last_became_peered":"2024-09-14T03:43:58.735301+0000","last_unstale":"2024-09-14T03:45:14.544953+0000","last_undegraded":"2024-09-14T03:45:14.544953+0000","last_fullsized":"2024-09-14T03:45:14.544953+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:44:56.523073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.462078+0000","last_change":"2024-09-14T03:44:01.874777+0000","last_active":"2024-09-14T03:45:14.462078+0000","last_peered":"2024-09-14T03:45:14.462078+0000","last_clean":"2024-09-14T03:45:14.462078+0000","last_became_active":"2024-09-14T03:44:01.874473+0000","last_became_peered":"2024-09-14T03:44:01.874473+0000","last_unstale":"2024-09-14T03:45:14.462078+0000","last_undegraded":"2024-09-14T03:45:14.462078+0000","last_fullsized":"2024-09-14T03:45:14.462078+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:49:02.042635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":23,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.462172+0000","last_change":"2024-09-14T03:43:58.732969+0000","last_active":"2024-09-14T03:45:14.462172+0000","last_peered":"2024-09-14T03:45:14.462172+0000","last_clean":"2024-09-14T03:45:14.462172+0000","last_became_active":"2024-09-14T03:43:58.732492+0000","last_became_peered":"2024-09-14T03:43:58.732492+0000","last_unstale":"2024-09-14T03:45:14.462172+0000","last_undegraded":"2024-09-14T03:45:14.462172+0000","last_fullsized":"2024-09-14T03:45:14.462172+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:50:28.719447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.684384+0000","last_change":"2024-09-14T03:44:01.875679+0000","last_active":"2024-09-14T03:45:14.684384+0000","last_peered":"2024-09-14T03:45:14.684384+0000","last_clean":"2024-09-14T03:45:14.684384+0000","last_became_active":"2024-09-14T03:44:01.875422+0000","last_became_peered":"2024-09-14T03:44:01.875422+0000","last_unstale":"2024-09-14T03:45:14.684384+0000","last_undegraded":"2024-09-14T03:45:14.684384+0000","last_fullsized":"2024-09-14T03:45:14.684384+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:13:00.349373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"58'3","reported_seq":26,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.458567+0000","last_change":"2024-09-14T03:43:58.735066+0000","last_active":"2024-09-14T03:45:14.458567+0000","last_peered":"2024-09-14T03:45:14.458567+0000","last_clean":"2024-09-14T03:45:14.458567+0000","last_became_active":"2024-09-14T03:43:58.734514+0000","last_became_peered":"2024-09-14T03:43:58.734514+0000","last_unstale":"2024-09-14T03:45:14.458567+0000","last_undegraded":"2024-09-14T03:45:14.458567+0000","last_fullsized":"2024-09-14T03:45:14.458567+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:41:55.478295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.458619+0000","last_change":"2024-09-14T03:44:01.881319+0000","last_active":"2024-09-14T03:45:14.458619+0000","last_peered":"2024-09-14T03:45:14.458619+0000","last_clean":"2024-09-14T03:45:14.458619+0000","last_became_active":"2024-09-14T03:44:01.881032+0000","last_became_peered":"2024-09-14T03:44:01.881032+0000","last_unstale":"2024-09-14T03:45:14.458619+0000","last_undegraded":"2024-09-14T03:45:14.458619+0000","last_fullsized":"2024-09-14T03:45:14.458619+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:35:41.762014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"61'9","reported_seq":32,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.544859+0000","last_change":"2024-09-14T03:43:58.734778+0000","last_active":"2024-09-14T03:45:14.544859+0000","last_peered":"2024-09-14T03:45:14.544859+0000","last_clean":"2024-09-14T03:45:14.544859+0000","last_became_active":"2024-09-14T03:43:58.734380+0000","last_became_peered":"2024-09-14T03:43:58.734380+0000","last_unstale":"2024-09-14T03:45:14.544859+0000","last_undegraded":"2024-09-14T03:45:14.544859+0000","last_fullsized":"2024-09-14T03:45:14.544859+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:55:13.561065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2342,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.660824+0000","last_change":"2024-09-14T03:44:01.880139+0000","last_active":"2024-09-14T03:45:14.660824+0000","last_peered":"2024-09-14T03:45:14.660824+0000","last_clean":"2024-09-14T03:45:14.660824+0000","last_became_active":"2024-09-14T03:44:01.879840+0000","last_became_peered":"2024-09-14T03:44:01.879840+0000","last_unstale":"2024-09-14T03:45:14.660824+0000","last_undegraded":"2024-09-14T03:45:14.660824+0000","last_fullsized":"2024-09-14T03:45:14.660824+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:31:34.003676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"58'2","reported_seq":25,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.684403+0000","last_change":"2024-09-14T03:43:58.728768+0000","last_active":"2024-09-14T03:45:14.684403+0000","last_peered":"2024-09-14T03:45:14.684403+0000","last_clean":"2024-09-14T03:45:14.684403+0000","last_became_active":"2024-09-14T03:43:58.727893+0000","last_became_peered":"2024-09-14T03:43:58.727893+0000","last_unstale":"2024-09-14T03:45:14.684403+0000","last_undegraded":"2024-09-14T03:45:14.684403+0000","last_fullsized":"2024-09-14T03:45:14.684403+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:54:48.554486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.684119+0000","last_change":"2024-09-14T03:44:01.877540+0000","last_active":"2024-09-14T03:45:14.684119+0000","last_peered":"2024-09-14T03:45:14.684119+0000","last_clean":"2024-09-14T03:45:14.684119+0000","last_became_active":"2024-09-14T03:44:01.877147+0000","last_became_peered":"2024-09-14T03:44:01.877147+0000","last_unstale":"2024-09-14T03:45:14.684119+0000","last_undegraded":"2024-09-14T03:45:14.684119+0000","last_fullsized":"2024-09-14T03:45:14.684119+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:17:57.724648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":23,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.461815+0000","last_change":"2024-09-14T03:43:58.732812+0000","last_active":"2024-09-14T03:45:14.461815+0000","last_peered":"2024-09-14T03:45:14.461815+0000","last_clean":"2024-09-14T03:45:14.461815+0000","last_became_active":"2024-09-14T03:43:58.732537+0000","last_became_peered":"2024-09-14T03:43:58.732537+0000","last_unstale":"2024-09-14T03:45:14.461815+0000","last_undegraded":"2024-09-14T03:45:14.461815+0000","last_fullsized":"2024-09-14T03:45:14.461815+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:26:11.551182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.660730+0000","last_change":"2024-09-14T03:44:01.881140+0000","last_active":"2024-09-14T03:45:14.660730+0000","last_peered":"2024-09-14T03:45:14.660730+0000","last_clean":"2024-09-14T03:45:14.660730+0000","last_became_active":"2024-09-14T03:44:01.880868+0000","last_became_peered":"2024-09-14T03:44:01.880868+0000","last_unstale":"2024-09-14T03:45:14.660730+0000","last_undegraded":"2024-09-14T03:45:14.660730+0000","last_fullsized":"2024-09-14T03:45:14.660730+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:29:05.729804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":23,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.544699+0000","last_change":"2024-09-14T03:43:58.730505+0000","last_active":"2024-09-14T03:45:14.544699+0000","last_peered":"2024-09-14T03:45:14.544699+0000","last_clean":"2024-09-14T03:45:14.544699+0000","last_became_active":"2024-09-14T03:43:58.729821+0000","last_became_peered":"2024-09-14T03:43:58.729821+0000","last_unstale":"2024-09-14T03:45:14.544699+0000","last_undegraded":"2024-09-14T03:45:14.544699+0000","last_fullsized":"2024-09-14T03:45:14.544699+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:17:44.091667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.544408+0000","last_change":"2024-09-14T03:44:01.874477+0000","last_active":"2024-09-14T03:45:14.544408+0000","last_peered":"2024-09-14T03:45:14.544408+0000","last_clean":"2024-09-14T03:45:14.544408+0000","last_became_active":"2024-09-14T03:44:01.873714+0000","last_became_peered":"2024-09-14T03:44:01.873714+0000","last_unstale":"2024-09-14T03:45:14.544408+0000","last_undegraded":"2024-09-14T03:45:14.544408+0000","last_fullsized":"2024-09-14T03:45:14.544408+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:54:21.601768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"58'8","reported_seq":31,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:15.135490+0000","last_change":"2024-09-14T03:43:58.726387+0000","last_active":"2024-09-14T03:45:15.135490+0000","last_peered":"2024-09-14T03:45:15.135490+0000","last_clean":"2024-09-14T03:45:15.135490+0000","last_became_active":"2024-09-14T03:43:58.726115+0000","last_became_peered":"2024-09-14T03:43:58.726115+0000","last_unstale":"2024-09-14T03:45:15.135490+0000","last_undegraded":"2024-09-14T03:45:15.135490+0000","last_fullsized":"2024-09-14T03:45:15.135490+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:18:04.176970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.660478+0000","last_change":"2024-09-14T03:44:01.878624+0000","last_active":"2024-09-14T03:45:14.660478+0000","last_peered":"2024-09-14T03:45:14.660478+0000","last_clean":"2024-09-14T03:45:14.660478+0000","last_became_active":"2024-09-14T03:44:01.878343+0000","last_became_peered":"2024-09-14T03:44:01.878343+0000","last_unstale":"2024-09-14T03:45:14.660478+0000","last_undegraded":"2024-09-14T03:45:14.660478+0000","last_fullsized":"2024-09-14T03:45:14.660478+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:44:05.205671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"58'1","reported_seq":23,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.458928+0000","last_change":"2024-09-14T03:43:58.728489+0000","last_active":"2024-09-14T03:45:14.458928+0000","last_peered":"2024-09-14T03:45:14.458928+0000","last_clean":"2024-09-14T03:45:14.458928+0000","last_became_active":"2024-09-14T03:43:58.728182+0000","last_became_peered":"2024-09-14T03:43:58.728182+0000","last_unstale":"2024-09-14T03:45:14.458928+0000","last_undegraded":"2024-09-14T03:45:14.458928+0000","last_fullsized":"2024-09-14T03:45:14.458928+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:56.310710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.661108+0000","last_change":"2024-09-14T03:44:01.880754+0000","last_active":"2024-09-14T03:45:14.661108+0000","last_peered":"2024-09-14T03:45:14.661108+0000","last_clean":"2024-09-14T03:45:14.661108+0000","last_became_active":"2024-09-14T03:44:01.880483+0000","last_became_peered":"2024-09-14T03:44:01.880483+0000","last_unstale":"2024-09-14T03:45:14.661108+0000","last_undegraded":"2024-09-14T03:45:14.661108+0000","last_fullsized":"2024-09-14T03:45:14.661108+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:32:53.245007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"58'2","reported_seq":25,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:15.135841+0000","last_change":"2024-09-14T03:43:58.726366+0000","last_active":"2024-09-14T03:45:15.135841+0000","last_peered":"2024-09-14T03:45:15.135841+0000","last_clean":"2024-09-14T03:45:15.135841+0000","last_became_active":"2024-09-14T03:43:58.726090+0000","last_became_peered":"2024-09-14T03:43:58.726090+0000","last_unstale":"2024-09-14T03:45:15.135841+0000","last_undegraded":"2024-09-14T03:45:15.135841+0000","last_fullsized":"2024-09-14T03:45:15.135841+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:45:33.572820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.661089+0000","last_change":"2024-09-14T03:44:01.883704+0000","last_active":"2024-09-14T03:45:14.661089+0000","last_peered":"2024-09-14T03:45:14.661089+0000","last_clean":"2024-09-14T03:45:14.661089+0000","last_became_active":"2024-09-14T03:44:01.883448+0000","last_became_peered":"2024-09-14T03:44:01.883448+0000","last_unstale":"2024-09-14T03:45:14.661089+0000","last_undegraded":"2024-09-14T03:45:14.661089+0000","last_fullsized":"2024-09-14T03:45:14.661089+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:16:27.946770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"58'1","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.458573+0000","last_change":"2024-09-14T03:43:58.733205+0000","last_active":"2024-09-14T03:45:14.458573+0000","last_peered":"2024-09-14T03:45:14.458573+0000","last_clean":"2024-09-14T03:45:14.458573+0000","last_became_active":"2024-09-14T03:43:58.732929+0000","last_became_peered":"2024-09-14T03:43:58.732929+0000","last_unstale":"2024-09-14T03:45:14.458573+0000","last_undegraded":"2024-09-14T03:45:14.458573+0000","last_fullsized":"2024-09-14T03:45:14.458573+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:31:13.660965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.461995+0000","last_change":"2024-09-14T03:44:01.886158+0000","last_active":"2024-09-14T03:45:14.461995+0000","last_peered":"2024-09-14T03:45:14.461995+0000","last_clean":"2024-09-14T03:45:14.461995+0000","last_became_active":"2024-09-14T03:44:01.884747+0000","last_became_peered":"2024-09-14T03:44:01.884747+0000","last_unstale":"2024-09-14T03:45:14.461995+0000","last_undegraded":"2024-09-14T03:45:14.461995+0000","last_fullsized":"2024-09-14T03:45:14.461995+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:49:56.265783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":23,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:15.135633+0000","last_change":"2024-09-14T03:43:58.732401+0000","last_active":"2024-09-14T03:45:15.135633+0000","last_peered":"2024-09-14T03:45:15.135633+0000","last_clean":"2024-09-14T03:45:15.135633+0000","last_became_active":"2024-09-14T03:43:58.732067+0000","last_became_peered":"2024-09-14T03:43:58.732067+0000","last_unstale":"2024-09-14T03:45:15.135633+0000","last_undegraded":"2024-09-14T03:45:15.135633+0000","last_fullsized":"2024-09-14T03:45:15.135633+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:34:00.566245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.660858+0000","last_change":"2024-09-14T03:44:01.881736+0000","last_active":"2024-09-14T03:45:14.660858+0000","last_peered":"2024-09-14T03:45:14.660858+0000","last_clean":"2024-09-14T03:45:14.660858+0000","last_became_active":"2024-09-14T03:44:01.881219+0000","last_became_peered":"2024-09-14T03:44:01.881219+0000","last_unstale":"2024-09-14T03:45:14.660858+0000","last_undegraded":"2024-09-14T03:45:14.660858+0000","last_fullsized":"2024-09-14T03:45:14.660858+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:24:23.342066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-09-14T03:45:20.503002+0000","last_change":"2024-09-14T03:45:14.545163+0000","last_active":"2024-09-14T03:45:20.503002+0000","last_peered":"2024-09-14T03:45:20.503002+0000","last_clean":"2024-09-14T03:45:20.503002+0000","last_became_active":"2024-09-14T03:45:09.755484+0000","last_became_peered":"2024-09-14T03:45:09.755484+0000","last_unstale":"2024-09-14T03:45:20.503002+0000","last_undegraded":"2024-09-14T03:45:20.503002+0000","last_fullsized":"2024-09-14T03:45:20.503002+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_clean_scrub_stamp":"2024-09-14T03:45:08.693776+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:18:49.100636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039339699999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"61'79","reported_seq":58,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.660571+0000","last_change":"2024-09-14T03:43:36.147227+0000","last_active":"2024-09-14T03:45:14.660571+0000","last_peered":"2024-09-14T03:45:14.660571+0000","last_clean":"2024-09-14T03:45:14.660571+0000","last_became_active":"2024-09-14T03:43:36.141953+0000","last_became_peered":"2024-09-14T03:43:36.141953+0000","last_unstale":"2024-09-14T03:45:14.660571+0000","last_undegraded":"2024-09-14T03:45:14.660571+0000","last_fullsized":"2024-09-14T03:45:14.660571+0000","mapping_epoch":50,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":51,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:39.458815+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:39.458815+0000","last_clean_scrub_stamp":"2024-09-14T03:41:39.458815+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:38:43.056328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":23,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.461867+0000","last_change":"2024-09-14T03:43:58.733557+0000","last_active":"2024-09-14T03:45:14.461867+0000","last_peered":"2024-09-14T03:45:14.461867+0000","last_clean":"2024-09-14T03:45:14.461867+0000","last_became_active":"2024-09-14T03:43:58.733256+0000","last_became_peered":"2024-09-14T03:43:58.733256+0000","last_unstale":"2024-09-14T03:45:14.461867+0000","last_undegraded":"2024-09-14T03:45:14.461867+0000","last_fullsized":"2024-09-14T03:45:14.461867+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:09:12.987156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.857593+0000","last_change":"2024-09-14T03:44:01.875525+0000","last_active":"2024-09-14T03:45:14.857593+0000","last_peered":"2024-09-14T03:45:14.857593+0000","last_clean":"2024-09-14T03:45:14.857593+0000","last_became_active":"2024-09-14T03:44:01.875241+0000","last_became_peered":"2024-09-14T03:44:01.875241+0000","last_unstale":"2024-09-14T03:45:14.857593+0000","last_undegraded":"2024-09-14T03:45:14.857593+0000","last_fullsized":"2024-09-14T03:45:14.857593+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:34:47.346963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-09-14T03:45:20.503141+0000","last_change":"2024-09-14T03:45:14.684739+0000","last_active":"2024-09-14T03:45:20.503141+0000","last_peered":"2024-09-14T03:45:20.503141+0000","last_clean":"2024-09-14T03:45:20.503141+0000","last_became_active":"2024-09-14T03:45:09.711005+0000","last_became_peered":"2024-09-14T03:45:09.711005+0000","last_unstale":"2024-09-14T03:45:20.503141+0000","last_undegraded":"2024-09-14T03:45:20.503141+0000","last_fullsized":"2024-09-14T03:45:20.503141+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_clean_scrub_stamp":"2024-09-14T03:45:08.693776+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:35.312720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00019552,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.660472+0000","last_change":"2024-09-14T03:43:58.732376+0000","last_active":"2024-09-14T03:45:14.660472+0000","last_peered":"2024-09-14T03:45:14.660472+0000","last_clean":"2024-09-14T03:45:14.660472+0000","last_became_active":"2024-09-14T03:43:58.731812+0000","last_became_peered":"2024-09-14T03:43:58.731812+0000","last_unstale":"2024-09-14T03:45:14.660472+0000","last_undegraded":"2024-09-14T03:45:14.660472+0000","last_fullsized":"2024-09-14T03:45:14.660472+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:35:26.262383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.683969+0000","last_change":"2024-09-14T03:44:01.876511+0000","last_active":"2024-09-14T03:45:14.683969+0000","last_peered":"2024-09-14T03:45:14.683969+0000","last_clean":"2024-09-14T03:45:14.683969+0000","last_became_active":"2024-09-14T03:44:01.876225+0000","last_became_peered":"2024-09-14T03:44:01.876225+0000","last_unstale":"2024-09-14T03:45:14.683969+0000","last_undegraded":"2024-09-14T03:45:14.683969+0000","last_fullsized":"2024-09-14T03:45:14.683969+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:36:40.283622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-09-14T03:45:20.464110+0000","last_change":"2024-09-14T03:45:14.458990+0000","last_active":"2024-09-14T03:45:20.464110+0000","last_peered":"2024-09-14T03:45:20.464110+0000","last_clean":"2024-09-14T03:45:20.464110+0000","last_became_active":"2024-09-14T03:45:09.879190+0000","last_became_peered":"2024-09-14T03:45:09.879190+0000","last_unstale":"2024-09-14T03:45:20.464110+0000","last_undegraded":"2024-09-14T03:45:20.464110+0000","last_fullsized":"2024-09-14T03:45:20.464110+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_clean_scrub_stamp":"2024-09-14T03:45:08.693776+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:23:17.328924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057164100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":23,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:15.135684+0000","last_change":"2024-09-14T03:43:58.727129+0000","last_active":"2024-09-14T03:45:15.135684+0000","last_peered":"2024-09-14T03:45:15.135684+0000","last_clean":"2024-09-14T03:45:15.135684+0000","last_became_active":"2024-09-14T03:43:58.726879+0000","last_became_peered":"2024-09-14T03:43:58.726879+0000","last_unstale":"2024-09-14T03:45:15.135684+0000","last_undegraded":"2024-09-14T03:45:15.135684+0000","last_fullsized":"2024-09-14T03:45:15.135684+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:54:41.748651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.457983+0000","last_change":"2024-09-14T03:44:01.871005+0000","last_active":"2024-09-14T03:45:14.457983+0000","last_peered":"2024-09-14T03:45:14.457983+0000","last_clean":"2024-09-14T03:45:14.457983+0000","last_became_active":"2024-09-14T03:44:01.870707+0000","last_became_peered":"2024-09-14T03:44:01.870707+0000","last_unstale":"2024-09-14T03:45:14.457983+0000","last_undegraded":"2024-09-14T03:45:14.457983+0000","last_fullsized":"2024-09-14T03:45:14.457983+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:30:22.758581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-09-14T03:45:20.464062+0000","last_change":"2024-09-14T03:45:14.458757+0000","last_active":"2024-09-14T03:45:20.464062+0000","last_peered":"2024-09-14T03:45:20.464062+0000","last_clean":"2024-09-14T03:45:20.464062+0000","last_became_active":"2024-09-14T03:45:09.711634+0000","last_became_peered":"2024-09-14T03:45:09.711634+0000","last_unstale":"2024-09-14T03:45:20.464062+0000","last_undegraded":"2024-09-14T03:45:20.464062+0000","last_fullsized":"2024-09-14T03:45:20.464062+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_clean_scrub_stamp":"2024-09-14T03:45:08.693776+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:42:58.495982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054748100000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":23,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.461874+0000","last_change":"2024-09-14T03:43:58.733038+0000","last_active":"2024-09-14T03:45:14.461874+0000","last_peered":"2024-09-14T03:45:14.461874+0000","last_clean":"2024-09-14T03:45:14.461874+0000","last_became_active":"2024-09-14T03:43:58.732753+0000","last_became_peered":"2024-09-14T03:43:58.732753+0000","last_unstale":"2024-09-14T03:45:14.461874+0000","last_undegraded":"2024-09-14T03:45:14.461874+0000","last_fullsized":"2024-09-14T03:45:14.461874+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:40:49.754211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":30,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.458431+0000","last_change":"2024-09-14T03:44:26.430320+0000","last_active":"2024-09-14T03:45:14.458431+0000","last_peered":"2024-09-14T03:45:14.458431+0000","last_clean":"2024-09-14T03:45:14.458431+0000","last_became_active":"2024-09-14T03:44:26.429935+0000","last_became_peered":"2024-09-14T03:44:26.429935+0000","last_unstale":"2024-09-14T03:45:14.458431+0000","last_undegraded":"2024-09-14T03:45:14.458431+0000","last_fullsized":"2024-09-14T03:45:14.458431+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:12:35.577538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.2","version":"67'2","reported_seq":21,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-09-14T03:45:20.464076+0000","last_change":"2024-09-14T03:45:14.458846+0000","last_active":"2024-09-14T03:45:20.464076+0000","last_peered":"2024-09-14T03:45:20.464076+0000","last_clean":"2024-09-14T03:45:20.464076+0000","last_became_active":"2024-09-14T03:45:09.879248+0000","last_became_peered":"2024-09-14T03:45:09.879248+0000","last_unstale":"2024-09-14T03:45:20.464076+0000","last_undegraded":"2024-09-14T03:45:20.464076+0000","last_fullsized":"2024-09-14T03:45:20.464076+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_clean_scrub_stamp":"2024-09-14T03:45:08.693776+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:02:23.634325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000682786,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":23,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.458338+0000","last_change":"2024-09-14T03:43:58.733621+0000","last_active":"2024-09-14T03:45:14.458338+0000","last_peered":"2024-09-14T03:45:14.458338+0000","last_clean":"2024-09-14T03:45:14.458338+0000","last_became_active":"2024-09-14T03:43:58.733357+0000","last_became_peered":"2024-09-14T03:43:58.733357+0000","last_unstale":"2024-09-14T03:45:14.458338+0000","last_undegraded":"2024-09-14T03:45:14.458338+0000","last_fullsized":"2024-09-14T03:45:14.458338+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:25:53.791926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.462075+0000","last_change":"2024-09-14T03:44:01.880200+0000","last_active":"2024-09-14T03:45:14.462075+0000","last_peered":"2024-09-14T03:45:14.462075+0000","last_clean":"2024-09-14T03:45:14.462075+0000","last_became_active":"2024-09-14T03:44:01.879925+0000","last_became_peered":"2024-09-14T03:44:01.879925+0000","last_unstale":"2024-09-14T03:45:14.462075+0000","last_undegraded":"2024-09-14T03:45:14.462075+0000","last_fullsized":"2024-09-14T03:45:14.462075+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:55:24.738215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"65'1","reported_seq":20,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-09-14T03:45:20.502304+0000","last_change":"2024-09-14T03:45:14.684595+0000","last_active":"2024-09-14T03:45:20.502304+0000","last_peered":"2024-09-14T03:45:20.502304+0000","last_clean":"2024-09-14T03:45:20.502304+0000","last_became_active":"2024-09-14T03:45:09.879229+0000","last_became_peered":"2024-09-14T03:45:09.879229+0000","last_unstale":"2024-09-14T03:45:20.502304+0000","last_undegraded":"2024-09-14T03:45:20.502304+0000","last_fullsized":"2024-09-14T03:45:20.502304+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_clean_scrub_stamp":"2024-09-14T03:45:08.693776+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:49:58.555610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040600400000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.661082+0000","last_change":"2024-09-14T03:43:58.733230+0000","last_active":"2024-09-14T03:45:14.661082+0000","last_peered":"2024-09-14T03:45:14.661082+0000","last_clean":"2024-09-14T03:45:14.661082+0000","last_became_active":"2024-09-14T03:43:58.732868+0000","last_became_peered":"2024-09-14T03:43:58.732868+0000","last_unstale":"2024-09-14T03:45:14.661082+0000","last_undegraded":"2024-09-14T03:45:14.661082+0000","last_fullsized":"2024-09-14T03:45:14.661082+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:15:12.184851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.458306+0000","last_change":"2024-09-14T03:44:01.878736+0000","last_active":"2024-09-14T03:45:14.458306+0000","last_peered":"2024-09-14T03:45:14.458306+0000","last_clean":"2024-09-14T03:45:14.458306+0000","last_became_active":"2024-09-14T03:44:01.877390+0000","last_became_peered":"2024-09-14T03:44:01.877390+0000","last_unstale":"2024-09-14T03:45:14.458306+0000","last_undegraded":"2024-09-14T03:45:14.458306+0000","last_fullsized":"2024-09-14T03:45:14.458306+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:17:42.812575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-09-14T03:45:20.501909+0000","last_change":"2024-09-14T03:45:14.857657+0000","last_active":"2024-09-14T03:45:20.501909+0000","last_peered":"2024-09-14T03:45:20.501909+0000","last_clean":"2024-09-14T03:45:20.501909+0000","last_became_active":"2024-09-14T03:45:09.710318+0000","last_became_peered":"2024-09-14T03:45:09.710318+0000","last_unstale":"2024-09-14T03:45:20.501909+0000","last_undegraded":"2024-09-14T03:45:20.501909+0000","last_fullsized":"2024-09-14T03:45:20.501909+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_clean_scrub_stamp":"2024-09-14T03:45:08.693776+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:37:23.130384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031572099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"58'1","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.459070+0000","last_change":"2024-09-14T03:43:58.733688+0000","last_active":"2024-09-14T03:45:14.459070+0000","last_peered":"2024-09-14T03:45:14.459070+0000","last_clean":"2024-09-14T03:45:14.459070+0000","last_became_active":"2024-09-14T03:43:58.733457+0000","last_became_peered":"2024-09-14T03:43:58.733457+0000","last_unstale":"2024-09-14T03:45:14.459070+0000","last_undegraded":"2024-09-14T03:45:14.459070+0000","last_fullsized":"2024-09-14T03:45:14.459070+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:09:01.069988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.857916+0000","last_change":"2024-09-14T03:44:01.870045+0000","last_active":"2024-09-14T03:45:14.857916+0000","last_peered":"2024-09-14T03:45:14.857916+0000","last_clean":"2024-09-14T03:45:14.857916+0000","last_became_active":"2024-09-14T03:44:01.869780+0000","last_became_peered":"2024-09-14T03:44:01.869780+0000","last_unstale":"2024-09-14T03:45:14.857916+0000","last_undegraded":"2024-09-14T03:45:14.857916+0000","last_fullsized":"2024-09-14T03:45:14.857916+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:04:58.634704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-09-14T03:45:20.502067+0000","last_change":"2024-09-14T03:45:14.459017+0000","last_active":"2024-09-14T03:45:20.502067+0000","last_peered":"2024-09-14T03:45:20.502067+0000","last_clean":"2024-09-14T03:45:20.502067+0000","last_became_active":"2024-09-14T03:45:09.879281+0000","last_became_peered":"2024-09-14T03:45:09.879281+0000","last_unstale":"2024-09-14T03:45:20.502067+0000","last_undegraded":"2024-09-14T03:45:20.502067+0000","last_fullsized":"2024-09-14T03:45:20.502067+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_clean_scrub_stamp":"2024-09-14T03:45:08.693776+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:10:03.506895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082715299999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"58'2","reported_seq":25,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.545051+0000","last_change":"2024-09-14T03:43:58.735364+0000","last_active":"2024-09-14T03:45:14.545051+0000","last_peered":"2024-09-14T03:45:14.545051+0000","last_clean":"2024-09-14T03:45:14.545051+0000","last_became_active":"2024-09-14T03:43:58.734833+0000","last_became_peered":"2024-09-14T03:43:58.734833+0000","last_unstale":"2024-09-14T03:45:14.545051+0000","last_undegraded":"2024-09-14T03:45:14.545051+0000","last_fullsized":"2024-09-14T03:45:14.545051+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:58:37.428713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.684514+0000","last_change":"2024-09-14T03:44:01.878573+0000","last_active":"2024-09-14T03:45:14.684514+0000","last_peered":"2024-09-14T03:45:14.684514+0000","last_clean":"2024-09-14T03:45:14.684514+0000","last_became_active":"2024-09-14T03:44:01.878316+0000","last_became_peered":"2024-09-14T03:44:01.878316+0000","last_unstale":"2024-09-14T03:45:14.684514+0000","last_undegraded":"2024-09-14T03:45:14.684514+0000","last_fullsized":"2024-09-14T03:45:14.684514+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:45:21.129163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.660416+0000","last_change":"2024-09-14T03:43:58.730843+0000","last_active":"2024-09-14T03:45:14.660416+0000","last_peered":"2024-09-14T03:45:14.660416+0000","last_clean":"2024-09-14T03:45:14.660416+0000","last_became_active":"2024-09-14T03:43:58.730538+0000","last_became_peered":"2024-09-14T03:43:58.730538+0000","last_unstale":"2024-09-14T03:45:14.660416+0000","last_undegraded":"2024-09-14T03:45:14.660416+0000","last_fullsized":"2024-09-14T03:45:14.660416+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:04:57.059345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.458548+0000","last_change":"2024-09-14T03:44:01.874355+0000","last_active":"2024-09-14T03:45:14.458548+0000","last_peered":"2024-09-14T03:45:14.458548+0000","last_clean":"2024-09-14T03:45:14.458548+0000","last_became_active":"2024-09-14T03:44:01.874049+0000","last_became_peered":"2024-09-14T03:44:01.874049+0000","last_unstale":"2024-09-14T03:45:14.458548+0000","last_undegraded":"2024-09-14T03:45:14.458548+0000","last_fullsized":"2024-09-14T03:45:14.458548+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:53:30.306175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":23,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.458659+0000","last_change":"2024-09-14T03:43:58.728080+0000","last_active":"2024-09-14T03:45:14.458659+0000","last_peered":"2024-09-14T03:45:14.458659+0000","last_clean":"2024-09-14T03:45:14.458659+0000","last_became_active":"2024-09-14T03:43:58.727820+0000","last_became_peered":"2024-09-14T03:43:58.727820+0000","last_unstale":"2024-09-14T03:45:14.458659+0000","last_undegraded":"2024-09-14T03:45:14.458659+0000","last_fullsized":"2024-09-14T03:45:14.458659+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:37:58.166430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.857492+0000","last_change":"2024-09-14T03:44:01.869311+0000","last_active":"2024-09-14T03:45:14.857492+0000","last_peered":"2024-09-14T03:45:14.857492+0000","last_clean":"2024-09-14T03:45:14.857492+0000","last_became_active":"2024-09-14T03:44:01.869065+0000","last_became_peered":"2024-09-14T03:44:01.869065+0000","last_unstale":"2024-09-14T03:45:14.857492+0000","last_undegraded":"2024-09-14T03:45:14.857492+0000","last_fullsized":"2024-09-14T03:45:14.857492+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:45:09.508019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":23,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.459148+0000","last_change":"2024-09-14T03:43:58.728931+0000","last_active":"2024-09-14T03:45:14.459148+0000","last_peered":"2024-09-14T03:45:14.459148+0000","last_clean":"2024-09-14T03:45:14.459148+0000","last_became_active":"2024-09-14T03:43:58.728647+0000","last_became_peered":"2024-09-14T03:43:58.728647+0000","last_unstale":"2024-09-14T03:45:14.459148+0000","last_undegraded":"2024-09-14T03:45:14.459148+0000","last_fullsized":"2024-09-14T03:45:14.459148+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:09:38.769523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":31,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.857689+0000","last_change":"2024-09-14T03:44:26.430230+0000","last_active":"2024-09-14T03:45:14.857689+0000","last_peered":"2024-09-14T03:45:14.857689+0000","last_clean":"2024-09-14T03:45:14.857689+0000","last_became_active":"2024-09-14T03:44:26.429880+0000","last_became_peered":"2024-09-14T03:44:26.429880+0000","last_unstale":"2024-09-14T03:45:14.857689+0000","last_undegraded":"2024-09-14T03:45:14.857689+0000","last_fullsized":"2024-09-14T03:45:14.857689+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:39.132572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"58'2","reported_seq":26,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.660988+0000","last_change":"2024-09-14T03:43:58.737705+0000","last_active":"2024-09-14T03:45:14.660988+0000","last_peered":"2024-09-14T03:45:14.660988+0000","last_clean":"2024-09-14T03:45:14.660988+0000","last_became_active":"2024-09-14T03:43:58.736546+0000","last_became_peered":"2024-09-14T03:43:58.736546+0000","last_unstale":"2024-09-14T03:45:14.660988+0000","last_undegraded":"2024-09-14T03:45:14.660988+0000","last_fullsized":"2024-09-14T03:45:14.660988+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:27:12.542706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":18,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:15.135501+0000","last_change":"2024-09-14T03:44:26.429678+0000","last_active":"2024-09-14T03:45:15.135501+0000","last_peered":"2024-09-14T03:45:15.135501+0000","last_clean":"2024-09-14T03:45:15.135501+0000","last_became_active":"2024-09-14T03:44:26.429364+0000","last_became_peered":"2024-09-14T03:44:26.429364+0000","last_unstale":"2024-09-14T03:45:15.135501+0000","last_undegraded":"2024-09-14T03:45:15.135501+0000","last_fullsized":"2024-09-14T03:45:15.135501+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:29:31.230108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.462051+0000","last_change":"2024-09-14T03:44:01.888591+0000","last_active":"2024-09-14T03:45:14.462051+0000","last_peered":"2024-09-14T03:45:14.462051+0000","last_clean":"2024-09-14T03:45:14.462051+0000","last_became_active":"2024-09-14T03:44:01.888257+0000","last_became_peered":"2024-09-14T03:44:01.888257+0000","last_unstale":"2024-09-14T03:45:14.462051+0000","last_undegraded":"2024-09-14T03:45:14.462051+0000","last_fullsized":"2024-09-14T03:45:14.462051+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:24:40.992358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"58'2","reported_seq":26,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.660896+0000","last_change":"2024-09-14T03:43:58.734680+0000","last_active":"2024-09-14T03:45:14.660896+0000","last_peered":"2024-09-14T03:45:14.660896+0000","last_clean":"2024-09-14T03:45:14.660896+0000","last_became_active":"2024-09-14T03:43:58.734408+0000","last_became_peered":"2024-09-14T03:43:58.734408+0000","last_unstale":"2024-09-14T03:45:14.660896+0000","last_undegraded":"2024-09-14T03:45:14.660896+0000","last_fullsized":"2024-09-14T03:45:14.660896+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:49:02.536404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.461880+0000","last_change":"2024-09-14T03:44:01.885572+0000","last_active":"2024-09-14T03:45:14.461880+0000","last_peered":"2024-09-14T03:45:14.461880+0000","last_clean":"2024-09-14T03:45:14.461880+0000","last_became_active":"2024-09-14T03:44:01.885110+0000","last_became_peered":"2024-09-14T03:44:01.885110+0000","last_unstale":"2024-09-14T03:45:14.461880+0000","last_undegraded":"2024-09-14T03:45:14.461880+0000","last_fullsized":"2024-09-14T03:45:14.461880+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:10.439239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1d","version":"58'2","reported_seq":26,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.660935+0000","last_change":"2024-09-14T03:43:58.731296+0000","last_active":"2024-09-14T03:45:14.660935+0000","last_peered":"2024-09-14T03:45:14.660935+0000","last_clean":"2024-09-14T03:45:14.660935+0000","last_became_active":"2024-09-14T03:43:58.730951+0000","last_became_peered":"2024-09-14T03:43:58.730951+0000","last_unstale":"2024-09-14T03:45:14.660935+0000","last_undegraded":"2024-09-14T03:45:14.660935+0000","last_fullsized":"2024-09-14T03:45:14.660935+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:26:14.801414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.684378+0000","last_change":"2024-09-14T03:44:01.874935+0000","last_active":"2024-09-14T03:45:14.684378+0000","last_peered":"2024-09-14T03:45:14.684378+0000","last_clean":"2024-09-14T03:45:14.684378+0000","last_became_active":"2024-09-14T03:44:01.874668+0000","last_became_peered":"2024-09-14T03:44:01.874668+0000","last_unstale":"2024-09-14T03:45:14.684378+0000","last_undegraded":"2024-09-14T03:45:14.684378+0000","last_fullsized":"2024-09-14T03:45:14.684378+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:24:31.404958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"58'1","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.857877+0000","last_change":"2024-09-14T03:43:58.734206+0000","last_active":"2024-09-14T03:45:14.857877+0000","last_peered":"2024-09-14T03:45:14.857877+0000","last_clean":"2024-09-14T03:45:14.857877+0000","last_became_active":"2024-09-14T03:43:58.733960+0000","last_became_peered":"2024-09-14T03:43:58.733960+0000","last_unstale":"2024-09-14T03:45:14.857877+0000","last_undegraded":"2024-09-14T03:45:14.857877+0000","last_fullsized":"2024-09-14T03:45:14.857877+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:44.073503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.857768+0000","last_change":"2024-09-14T03:44:01.883550+0000","last_active":"2024-09-14T03:45:14.857768+0000","last_peered":"2024-09-14T03:45:14.857768+0000","last_clean":"2024-09-14T03:45:14.857768+0000","last_became_active":"2024-09-14T03:44:01.883265+0000","last_became_peered":"2024-09-14T03:44:01.883265+0000","last_unstale":"2024-09-14T03:45:14.857768+0000","last_undegraded":"2024-09-14T03:45:14.857768+0000","last_fullsized":"2024-09-14T03:45:14.857768+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:52:04.512990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"58'4","reported_seq":27,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.684569+0000","last_change":"2024-09-14T03:43:58.731954+0000","last_active":"2024-09-14T03:45:14.684569+0000","last_peered":"2024-09-14T03:45:14.684569+0000","last_clean":"2024-09-14T03:45:14.684569+0000","last_became_active":"2024-09-14T03:43:58.731688+0000","last_became_peered":"2024-09-14T03:43:58.731688+0000","last_unstale":"2024-09-14T03:45:14.684569+0000","last_undegraded":"2024-09-14T03:45:14.684569+0000","last_fullsized":"2024-09-14T03:45:14.684569+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:30:10.811609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":3128,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":48,"num_write_kb":16,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":9384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":42,"ondisk_log_size":42,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397539,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298624,"kb_used_data":1268,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446528,"statfs":{"total":95995035648,"available":95689244672,"internally_reserved":0,"allocated":1298432,"data_stored":903527,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:45:39 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.50800000000000001,"15min":0.51000000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.51800000000000002,"15min":0.51700000000000002},"min":{"1min":0.38100000000000001,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.49199999999999999}]},{"osd":1,"last update":"Sat Sep 14 03:45:39 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.50600000000000001,"15min":0.51000000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.622,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54100000000000004,"15min":0.54000000000000004},"min":{"1min":0.41799999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.44600000000000001}]},{"osd":2,"last update":"Sat Sep 14 03:45:39 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.53500000000000003,"15min":0.54100000000000004},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.66500000000000004,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.499,"5min":0.52400000000000002,"15min":0.52800000000000002},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.57899999999999996}]},{"osd":3,"last update":"Sat Sep 14 03:45:39 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.53300000000000003,"15min":0.53900000000000003},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.629,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.61399999999999999}]},{"osd":4,"last update":"Sat Sep 14 03:45:39 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.47599999999999998,"15min":0.48899999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.56999999999999995,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.47399999999999998,"15min":0.48499999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.628}]},{"osd":5,"last update":"Sat Sep 14 03:45:39 2024","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.46100000000000002,"15min":0.46899999999999997},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.65300000000000002,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.441,"5min":0.51300000000000001,"15min":0.52500000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.55800000000000005}]},{"osd":6,"last update":"Sat Sep 14 03:45:39 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.53800000000000003,"15min":0.55200000000000005},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.60999999999999999,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.52800000000000002,"15min":0.54100000000000004},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.65100000000000002}]}]},{"osd":6,"up_from":43,"seq":184683593767,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298624,"kb_used_data":1268,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446528,"statfs":{"total":95995035648,"available":95689244672,"internally_reserved":0,"allocated":1298432,"data_stored":904962,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:45:15 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.502,"15min":0.501},"min":{"1min":0.35699999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.496,"15min":0.5},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.65100000000000002,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.36199999999999999}]},{"osd":1,"last update":"Sat Sep 14 03:45:15 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.65600000000000003,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.54300000000000004,"15min":0.54500000000000004},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.45800000000000002}]},{"osd":2,"last update":"Sat Sep 14 03:45:15 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.47699999999999998,"15min":0.47199999999999998},"min":{"1min":0.39700000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.52200000000000002,"15min":0.52400000000000002},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.47599999999999998}]},{"osd":3,"last update":"Sat Sep 14 03:45:15 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.52300000000000002,"15min":0.52800000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.40400000000000003}]},{"osd":4,"last update":"Sat Sep 14 03:45:15 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.41599999999999998,"15min":0.41199999999999998},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.314},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.45200000000000001,"15min":0.45600000000000002},"min":{"1min":0.29499999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.374}]},{"osd":5,"last update":"Sat Sep 14 03:45:15 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.47399999999999998,"15min":0.47199999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.48399999999999999,"15min":0.48599999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.70199999999999996,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.34699999999999998}]},{"osd":7,"last update":"Sat Sep 14 03:45:34 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.52600000000000002,"15min":0.52900000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.66800000000000004,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.56200000000000006,"15min":0.56999999999999995},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.73299999999999998,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.46400000000000002}]}]},{"osd":5,"up_from":37,"seq":158913789996,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298052,"kb_used_data":696,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447100,"statfs":{"total":95995035648,"available":95689830400,"internally_reserved":0,"allocated":712704,"data_stored":314703,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:45:49 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53800000000000003,"15min":0.54000000000000004},"min":{"1min":0.38700000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.79900000000000004,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.52800000000000002,"15min":0.51700000000000002},"min":{"1min":0.41499999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.38300000000000001}]},{"osd":1,"last update":"Sat Sep 14 03:45:49 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.52000000000000002,"15min":0.51200000000000001},"min":{"1min":0.36899999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.73399999999999999,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.51600000000000001,"15min":0.51000000000000001},"min":{"1min":0.40699999999999997,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.78000000000000003,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.39600000000000002}]},{"osd":2,"last update":"Sat Sep 14 03:45:49 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52300000000000002,"15min":0.51900000000000002},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.72099999999999997,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.53100000000000003,"15min":0.51600000000000001},"min":{"1min":0.42399999999999999,"5min":0.376,"15min":0.376},"max":{"1min":0.76100000000000001,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.60399999999999998}]},{"osd":3,"last update":"Sat Sep 14 03:45:49 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.53200000000000003,"15min":0.52100000000000002},"min":{"1min":0.41699999999999998,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.54000000000000004,"15min":0.54200000000000004},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.77800000000000002,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.56399999999999995}]},{"osd":4,"last update":"Sat Sep 14 03:45:49 2024","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":0.39300000000000002,"15min":0.38300000000000001},"min":{"1min":0.28499999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.40699999999999997,"15min":0.39500000000000002},"min":{"1min":0.313,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.57499999999999996,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.52300000000000002}]},{"osd":6,"last update":"Sat Sep 14 03:45:13 2024","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.435,"15min":0.433},"min":{"1min":0.34699999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.43099999999999999,"15min":0.42399999999999999},"min":{"1min":0.32200000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.496}]},{"osd":7,"last update":"Sat Sep 14 03:45:43 2024","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.48099999999999998,"15min":0.48199999999999998},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.49399999999999999,"15min":0.496},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.58999999999999997}]}]},{"osd":4,"up_from":31,"seq":133143986224,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298056,"kb_used_data":700,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447096,"statfs":{"total":95995035648,"available":95689826304,"internally_reserved":0,"allocated":716800,"data_stored":313282,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:45:27 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51100000000000001,"15min":0.51200000000000001},"min":{"1min":0.372,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.66000000000000003,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.68899999999999995,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.38300000000000001}]},{"osd":1,"last update":"Sat Sep 14 03:45:27 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.496,"15min":0.47999999999999998},"min":{"1min":0.34100000000000003,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.64100000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.51400000000000001,"15min":0.51100000000000001},"min":{"1min":0.44400000000000001,"5min":0.377,"15min":0.377},"max":{"1min":0.69999999999999996,"5min":0.753,"15min":0.753},"last":0.442}]},{"osd":2,"last update":"Sat Sep 14 03:45:27 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.50600000000000001,"15min":0.49399999999999999},"min":{"1min":0.35799999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.69699999999999995,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.51500000000000001,"15min":0.50600000000000001},"min":{"1min":0.45100000000000001,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.59199999999999997,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.47299999999999998}]},{"osd":3,"last update":"Sat Sep 14 03:45:27 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.51500000000000001,"15min":0.497},"min":{"1min":0.42599999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.68000000000000005,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.51200000000000001,"15min":0.502},"min":{"1min":0.41499999999999998,"5min":0.318,"15min":0.318},"max":{"1min":0.72699999999999998,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.29899999999999999}]},{"osd":5,"last update":"Sat Sep 14 03:45:52 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.435,"15min":0.41099999999999998},"min":{"1min":0.30099999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.57699999999999996,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.442,"15min":0.41999999999999998},"min":{"1min":0.28299999999999997,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.59799999999999998,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.253}]},{"osd":6,"last update":"Sat Sep 14 03:45:11 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.44900000000000001,"15min":0.44500000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.28299999999999997},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.48999999999999999,"15min":0.48499999999999999},"min":{"1min":0.32100000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.48599999999999999}]},{"osd":7,"last update":"Sat Sep 14 03:45:41 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.57099999999999995,"15min":0.58399999999999996},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.66300000000000003,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.55700000000000005,"15min":0.56999999999999995},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.66100000000000003,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.36599999999999999}]}]},{"osd":3,"up_from":26,"seq":111669149748,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298052,"kb_used_data":696,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447100,"statfs":{"total":95995035648,"available":95689830400,"internally_reserved":0,"allocated":712704,"data_stored":314698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:46:06 2024","interfaces":[{"interface":"back","average":{"1min":0.34100000000000003,"5min":0.36399999999999999,"15min":0.38400000000000001},"min":{"1min":0.16800000000000001,"5min":0.16800000000000001,"15min":0.16800000000000001},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.33000000000000002,"5min":0.376,"15min":0.40000000000000002},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.41499999999999998}]},{"osd":1,"last update":"Sat Sep 14 03:46:06 2024","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.39700000000000002,"15min":0.41399999999999998},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.35999999999999999,"5min":0.376,"15min":0.39200000000000002},"min":{"1min":0.17999999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.28899999999999998}]},{"osd":2,"last update":"Sat Sep 14 03:46:06 2024","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.40200000000000002,"15min":0.42999999999999999},"min":{"1min":0.25900000000000001,"5min":0.251,"15min":0.251},"max":{"1min":0.52000000000000002,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.378},{"interface":"front","average":{"1min":0.35899999999999999,"5min":0.39700000000000002,"15min":0.41899999999999998},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.36499999999999999}]},{"osd":4,"last update":"Sat Sep 14 03:45:32 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53200000000000003,"15min":0.53900000000000003},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.441},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55500000000000005,"15min":0.56000000000000005},"min":{"1min":0.39100000000000001,"5min":0.379,"15min":0.379},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.52300000000000002}]},{"osd":5,"last update":"Sat Sep 14 03:45:53 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.57999999999999996,"15min":0.59099999999999997},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":1.161,"5min":1.161,"15min":1.161},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.56000000000000005,"15min":0.56699999999999995},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.42899999999999999}]},{"osd":6,"last update":"Sat Sep 14 03:45:13 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.57299999999999995,"15min":0.57799999999999996},"min":{"1min":0.41799999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.70999999999999996,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55800000000000005,"15min":0.55900000000000005},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.45300000000000001}]},{"osd":7,"last update":"Sat Sep 14 03:45:36 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.53300000000000003,"15min":0.52800000000000002},"min":{"1min":0.38,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.53800000000000003,"15min":0.53300000000000003},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.47499999999999998}]}]},{"osd":2,"up_from":18,"seq":77309411385,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298032,"kb_used_data":676,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447120,"statfs":{"total":95995035648,"available":95689850880,"internally_reserved":0,"allocated":692224,"data_stored":311956,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:45:51 2024","interfaces":[{"interface":"back","average":{"1min":0.318,"5min":0.34699999999999998,"15min":0.35099999999999998},"min":{"1min":0.19400000000000001,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.40600000000000003,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.34000000000000002,"5min":0.35699999999999998,"15min":0.36099999999999999},"min":{"1min":0.252,"5min":0.193,"15min":0.193},"max":{"1min":0.48899999999999999,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.36899999999999999}]},{"osd":1,"last update":"Sat Sep 14 03:45:51 2024","interfaces":[{"interface":"back","average":{"1min":0.35099999999999998,"5min":0.38200000000000001,"15min":0.39200000000000002},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.51200000000000001,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.34000000000000002,"5min":0.373,"15min":0.377},"min":{"1min":0.24399999999999999,"5min":0.216,"15min":0.216},"max":{"1min":0.50900000000000001,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.35399999999999998}]},{"osd":3,"last update":"Sat Sep 14 03:45:11 2024","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.43099999999999999,"15min":0.44500000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.496,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.42599999999999999,"15min":0.45000000000000001},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.50600000000000001,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.31}]},{"osd":4,"last update":"Sat Sep 14 03:45:28 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.54900000000000004,"15min":0.55500000000000005},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.76500000000000001,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.58699999999999997,"15min":0.59099999999999997},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.45300000000000001}]},{"osd":5,"last update":"Sat Sep 14 03:45:51 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.58099999999999996,"15min":0.59999999999999998},"min":{"1min":0.39200000000000002,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.56499999999999995,"15min":0.57999999999999996},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.73399999999999999,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.39300000000000002}]},{"osd":6,"last update":"Sat Sep 14 03:45:15 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.56799999999999995,"15min":0.56999999999999995},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.76000000000000001,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.57199999999999995,"15min":0.57499999999999996},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.71799999999999997,"5min":0.875,"15min":0.875},"last":0.44500000000000001}]},{"osd":7,"last update":"Sat Sep 14 03:45:38 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.56799999999999995,"15min":0.57799999999999996},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.72899999999999998,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.57999999999999996,"15min":0.58599999999999997},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.42899999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574911,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298036,"kb_used_data":680,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447116,"statfs":{"total":95995035648,"available":95689846784,"internally_reserved":0,"allocated":696320,"data_stored":311801,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:45:24 2024","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.33400000000000002,"15min":0.30099999999999999},"min":{"1min":0.247,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.56599999999999995,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.34200000000000003,"5min":0.35099999999999998,"15min":0.32100000000000001},"min":{"1min":0.23400000000000001,"5min":0.189,"15min":0.189},"max":{"1min":0.49399999999999999,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.246}]},{"osd":2,"last update":"Sat Sep 14 03:45:44 2024","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.35799999999999998,"15min":0.34399999999999997},"min":{"1min":0.23899999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.25900000000000001},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.375,"15min":0.35099999999999998},"min":{"1min":0.245,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.53000000000000003,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.23000000000000001}]},{"osd":3,"last update":"Sat Sep 14 03:45:09 2024","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.40999999999999998,"15min":0.41799999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.60899999999999999,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.42699999999999999,"15min":0.434},"min":{"1min":0.27100000000000002,"5min":0.247,"15min":0.247},"max":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.33100000000000002}]},{"osd":4,"last update":"Sat Sep 14 03:45:31 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.54300000000000004,"15min":0.54700000000000004},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.78700000000000003,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.54400000000000004,"15min":0.54900000000000004},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.67800000000000005,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.42599999999999999}]},{"osd":5,"last update":"Sat Sep 14 03:45:49 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.56799999999999995,"15min":0.58399999999999996},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.627,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.59099999999999997,"15min":0.60299999999999998},"min":{"1min":0.39100000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.70999999999999996,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.441}]},{"osd":6,"last update":"Sat Sep 14 03:45:12 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.57999999999999996,"15min":0.58799999999999997},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.58499999999999996,"15min":0.58899999999999997},"min":{"1min":0.376,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.46400000000000002}]},{"osd":7,"last update":"Sat Sep 14 03:45:39 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.56899999999999995,"15min":0.57299999999999995},"min":{"1min":0.41699999999999998,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.82599999999999996,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.55700000000000005,"15min":0.56299999999999994},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.68700000000000006,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.45500000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738436,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298628,"kb_used_data":1272,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446524,"statfs":{"total":95995035648,"available":95689240576,"internally_reserved":0,"allocated":1302528,"data_stored":903336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 14 03:45:20 2024","interfaces":[{"interface":"back","average":{"1min":0.33100000000000002,"5min":0.35399999999999998,"15min":0.34799999999999998},"min":{"1min":0.24299999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.49099999999999999,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.33000000000000002,"5min":0.35999999999999999,"15min":0.34699999999999998},"min":{"1min":0.23400000000000001,"5min":0.216,"15min":0.216},"max":{"1min":0.56100000000000005,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.36899999999999999}]},{"osd":2,"last update":"Sat Sep 14 03:45:45 2024","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.39100000000000001,"15min":0.40300000000000002},"min":{"1min":0.23799999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.33900000000000002,"5min":0.38,"15min":0.39000000000000001},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":0.496,"5min":0.505,"15min":0.505},"last":0.38100000000000001}]},{"osd":3,"last update":"Sat Sep 14 03:45:08 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.42799999999999999,"15min":0.443},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.50600000000000001,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.36599999999999999,"5min":0.40400000000000003,"15min":0.41699999999999998},"min":{"1min":0.26100000000000001,"5min":0.253,"15min":0.253},"max":{"1min":0.51600000000000001,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.34499999999999997}]},{"osd":4,"last update":"Sat Sep 14 03:45:29 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.54800000000000004,"15min":0.56200000000000006},"min":{"1min":0.36699999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.503,"5min":0.54500000000000004,"15min":0.55500000000000005},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.67600000000000005,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.48299999999999998}]},{"osd":5,"last update":"Sat Sep 14 03:45:51 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.82899999999999996,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.57499999999999996,"15min":0.57299999999999995},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.56799999999999995}]},{"osd":6,"last update":"Sat Sep 14 03:45:13 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.56000000000000005,"15min":0.56599999999999995},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.69699999999999995,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.55900000000000005,"15min":0.56100000000000005},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.77100000000000002,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.53700000000000003}]},{"osd":7,"last update":"Sat Sep 14 03:45:41 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.55200000000000005,"15min":0.55000000000000004},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.57999999999999996,"15min":0.58299999999999996},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.58599999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":2446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":2432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":2342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T03:46:15.196 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T03:46:15.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:14 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:15.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:14 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:15.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:14 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:46:15.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:14 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:46:15.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:14 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:16.093 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:46:16.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:15 smithi070 ceph-mon[39288]: from='client.24632 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:46:16.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:15 smithi070 ceph-mon[39288]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-14T03:46:16.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:15 smithi070 ceph-mon[39288]: Reconfiguring daemon alertmanager.a on smithi070 2024-09-14T03:46:16.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:15 smithi070 ceph-mon[39288]: pgmap v44: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:16.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:15 smithi070 ceph-mon[27495]: from='client.24632 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:46:16.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:15 smithi070 ceph-mon[27495]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-14T03:46:16.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:15 smithi070 ceph-mon[27495]: Reconfiguring daemon alertmanager.a on smithi070 2024-09-14T03:46:16.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:15 smithi070 ceph-mon[27495]: pgmap v44: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:16.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:15 smithi094 ceph-mon[31053]: from='client.24632 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:46:16.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:15 smithi094 ceph-mon[31053]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-14T03:46:16.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:15 smithi094 ceph-mon[31053]: Reconfiguring daemon alertmanager.a on smithi070 2024-09-14T03:46:16.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:15 smithi094 ceph-mon[31053]: pgmap v44: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:18.107 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:17 smithi070 ceph-mon[27495]: pgmap v45: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:18.108 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:17 smithi070 ceph-mon[39288]: pgmap v45: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:18.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:17 smithi094 ceph-mon[31053]: pgmap v45: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:18.876 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:46:18 smithi070 systemd[1]: Stopping Ceph alertmanager.a for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:46:20.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:19 smithi070 ceph-mon[27495]: pgmap v46: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:20.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:19 smithi070 ceph-mon[39288]: pgmap v46: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:20.181 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:46:20.182 INFO:teuthology.orchestra.run.smithi070.stderr:dumped all 2024-09-14T03:46:20.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:19 smithi094 ceph-mon[31053]: pgmap v46: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:20.430 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:46:20 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a[74516]: ts=2024-09-14T03:46:20.122Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-14T03:46:20.430 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:46:20 smithi070 podman[80057]: 2024-09-14 03:46:20.142669138 +0000 UTC m=+0.113026342 container died 81996aabcd0d913124aad3014d2ec0d37ea45c45b567b61686a2bc47e7a8cb85 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:46:20.772 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:46:20 smithi070 podman[80057]: 2024-09-14 03:46:20.561964179 +0000 UTC m=+0.532321383 container cleanup 81996aabcd0d913124aad3014d2ec0d37ea45c45b567b61686a2bc47e7a8cb85 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:46:20.772 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:46:20 smithi070 bash[80057]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a 2024-09-14T03:46:21.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:20 smithi070 ceph-mon[27495]: from='client.14730 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:46:21.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:20 smithi070 ceph-mon[39288]: from='client.14730 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:46:21.036 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:46:20 smithi070 podman[80075]: 2024-09-14 03:46:20.771396162 +0000 UTC m=+0.630897213 container remove 81996aabcd0d913124aad3014d2ec0d37ea45c45b567b61686a2bc47e7a8cb85 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:46:21.036 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:46:20 smithi070 podman[80075]: 2024-09-14 03:46:20.871947134 +0000 UTC m=+0.731448188 volume remove 1df656c37de47f986fd8aabe8bba91a2866b5b2b836ff80083b2cf8cebb8ed7c 2024-09-14T03:46:21.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:20 smithi094 ceph-mon[31053]: from='client.14730 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:46:21.541 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":46,"stamp":"2024-09-14T03:46:18.561760+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":593515,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":163,"num_write_kb":1646,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124,"ondisk_log_size":124,"up":315,"acting":315,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":315,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386104,"kb_used_data":7256,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575112,"statfs":{"total":767960285184,"available":765516914688,"internally_reserved":0,"allocated":7430144,"data_stored":4278265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002520"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.661024+0000","last_change":"2024-09-14T03:44:01.882774+0000","last_active":"2024-09-14T03:45:14.661024+0000","last_peered":"2024-09-14T03:45:14.661024+0000","last_clean":"2024-09-14T03:45:14.661024+0000","last_became_active":"2024-09-14T03:44:01.882481+0000","last_became_peered":"2024-09-14T03:44:01.882481+0000","last_unstale":"2024-09-14T03:45:14.661024+0000","last_undegraded":"2024-09-14T03:45:14.661024+0000","last_fullsized":"2024-09-14T03:45:14.661024+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:49:25.209573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.458995+0000","last_change":"2024-09-14T03:44:01.868415+0000","last_active":"2024-09-14T03:45:14.458995+0000","last_peered":"2024-09-14T03:45:14.458995+0000","last_clean":"2024-09-14T03:45:14.458995+0000","last_became_active":"2024-09-14T03:44:01.868152+0000","last_became_peered":"2024-09-14T03:44:01.868152+0000","last_unstale":"2024-09-14T03:45:14.458995+0000","last_undegraded":"2024-09-14T03:45:14.458995+0000","last_fullsized":"2024-09-14T03:45:14.458995+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:56:00.270070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.684432+0000","last_change":"2024-09-14T03:44:01.871459+0000","last_active":"2024-09-14T03:45:14.684432+0000","last_peered":"2024-09-14T03:45:14.684432+0000","last_clean":"2024-09-14T03:45:14.684432+0000","last_became_active":"2024-09-14T03:44:01.870749+0000","last_became_peered":"2024-09-14T03:44:01.870749+0000","last_unstale":"2024-09-14T03:45:14.684432+0000","last_undegraded":"2024-09-14T03:45:14.684432+0000","last_fullsized":"2024-09-14T03:45:14.684432+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:54.391363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.461823+0000","last_change":"2024-09-14T03:44:01.882503+0000","last_active":"2024-09-14T03:45:14.461823+0000","last_peered":"2024-09-14T03:45:14.461823+0000","last_clean":"2024-09-14T03:45:14.461823+0000","last_became_active":"2024-09-14T03:44:01.882250+0000","last_became_peered":"2024-09-14T03:44:01.882250+0000","last_unstale":"2024-09-14T03:45:14.461823+0000","last_undegraded":"2024-09-14T03:45:14.461823+0000","last_fullsized":"2024-09-14T03:45:14.461823+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:46:29.551285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.660633+0000","last_change":"2024-09-14T03:44:01.880333+0000","last_active":"2024-09-14T03:45:14.660633+0000","last_peered":"2024-09-14T03:45:14.660633+0000","last_clean":"2024-09-14T03:45:14.660633+0000","last_became_active":"2024-09-14T03:44:01.880010+0000","last_became_peered":"2024-09-14T03:44:01.880010+0000","last_unstale":"2024-09-14T03:45:14.660633+0000","last_undegraded":"2024-09-14T03:45:14.660633+0000","last_fullsized":"2024-09-14T03:45:14.660633+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:45:02.741425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.461594+0000","last_change":"2024-09-14T03:44:01.880710+0000","last_active":"2024-09-14T03:45:14.461594+0000","last_peered":"2024-09-14T03:45:14.461594+0000","last_clean":"2024-09-14T03:45:14.461594+0000","last_became_active":"2024-09-14T03:44:01.880237+0000","last_became_peered":"2024-09-14T03:44:01.880237+0000","last_unstale":"2024-09-14T03:45:14.461594+0000","last_undegraded":"2024-09-14T03:45:14.461594+0000","last_fullsized":"2024-09-14T03:45:14.461594+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:56:49.808707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.660415+0000","last_change":"2024-09-14T03:44:01.878590+0000","last_active":"2024-09-14T03:45:14.660415+0000","last_peered":"2024-09-14T03:45:14.660415+0000","last_clean":"2024-09-14T03:45:14.660415+0000","last_became_active":"2024-09-14T03:44:01.878281+0000","last_became_peered":"2024-09-14T03:44:01.878281+0000","last_unstale":"2024-09-14T03:45:14.660415+0000","last_undegraded":"2024-09-14T03:45:14.660415+0000","last_fullsized":"2024-09-14T03:45:14.660415+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:41:57.164227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.857411+0000","last_change":"2024-09-14T03:44:01.877057+0000","last_active":"2024-09-14T03:45:14.857411+0000","last_peered":"2024-09-14T03:45:14.857411+0000","last_clean":"2024-09-14T03:45:14.857411+0000","last_became_active":"2024-09-14T03:44:01.876727+0000","last_became_peered":"2024-09-14T03:44:01.876727+0000","last_unstale":"2024-09-14T03:45:14.857411+0000","last_undegraded":"2024-09-14T03:45:14.857411+0000","last_fullsized":"2024-09-14T03:45:14.857411+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:31:24.716316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.661211+0000","last_change":"2024-09-14T03:44:01.884865+0000","last_active":"2024-09-14T03:45:14.661211+0000","last_peered":"2024-09-14T03:45:14.661211+0000","last_clean":"2024-09-14T03:45:14.661211+0000","last_became_active":"2024-09-14T03:44:01.884136+0000","last_became_peered":"2024-09-14T03:44:01.884136+0000","last_unstale":"2024-09-14T03:45:14.661211+0000","last_undegraded":"2024-09-14T03:45:14.661211+0000","last_fullsized":"2024-09-14T03:45:14.661211+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:59.903382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":30,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.544938+0000","last_change":"2024-09-14T03:44:26.430052+0000","last_active":"2024-09-14T03:45:14.544938+0000","last_peered":"2024-09-14T03:45:14.544938+0000","last_clean":"2024-09-14T03:45:14.544938+0000","last_became_active":"2024-09-14T03:44:26.429658+0000","last_became_peered":"2024-09-14T03:44:26.429658+0000","last_unstale":"2024-09-14T03:45:14.544938+0000","last_undegraded":"2024-09-14T03:45:14.544938+0000","last_fullsized":"2024-09-14T03:45:14.544938+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:34.312039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.544941+0000","last_change":"2024-09-14T03:44:01.875017+0000","last_active":"2024-09-14T03:45:14.544941+0000","last_peered":"2024-09-14T03:45:14.544941+0000","last_clean":"2024-09-14T03:45:14.544941+0000","last_became_active":"2024-09-14T03:44:01.874687+0000","last_became_peered":"2024-09-14T03:44:01.874687+0000","last_unstale":"2024-09-14T03:45:14.544941+0000","last_undegraded":"2024-09-14T03:45:14.544941+0000","last_fullsized":"2024-09-14T03:45:14.544941+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:29:09.035342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.458577+0000","last_change":"2024-09-14T03:44:01.883313+0000","last_active":"2024-09-14T03:45:14.458577+0000","last_peered":"2024-09-14T03:45:14.458577+0000","last_clean":"2024-09-14T03:45:14.458577+0000","last_became_active":"2024-09-14T03:44:01.882770+0000","last_became_peered":"2024-09-14T03:44:01.882770+0000","last_unstale":"2024-09-14T03:45:14.458577+0000","last_undegraded":"2024-09-14T03:45:14.458577+0000","last_fullsized":"2024-09-14T03:45:14.458577+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:57:19.402319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.857631+0000","last_change":"2024-09-14T03:44:01.878245+0000","last_active":"2024-09-14T03:45:14.857631+0000","last_peered":"2024-09-14T03:45:14.857631+0000","last_clean":"2024-09-14T03:45:14.857631+0000","last_became_active":"2024-09-14T03:44:01.877861+0000","last_became_peered":"2024-09-14T03:44:01.877861+0000","last_unstale":"2024-09-14T03:45:14.857631+0000","last_undegraded":"2024-09-14T03:45:14.857631+0000","last_fullsized":"2024-09-14T03:45:14.857631+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:08:29.902090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.684034+0000","last_change":"2024-09-14T03:44:01.877391+0000","last_active":"2024-09-14T03:45:14.684034+0000","last_peered":"2024-09-14T03:45:14.684034+0000","last_clean":"2024-09-14T03:45:14.684034+0000","last_became_active":"2024-09-14T03:44:01.876797+0000","last_became_peered":"2024-09-14T03:44:01.876797+0000","last_unstale":"2024-09-14T03:45:14.684034+0000","last_undegraded":"2024-09-14T03:45:14.684034+0000","last_fullsized":"2024-09-14T03:45:14.684034+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:06:02.891762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.458588+0000","last_change":"2024-09-14T03:44:01.876700+0000","last_active":"2024-09-14T03:45:14.458588+0000","last_peered":"2024-09-14T03:45:14.458588+0000","last_clean":"2024-09-14T03:45:14.458588+0000","last_became_active":"2024-09-14T03:44:01.876091+0000","last_became_peered":"2024-09-14T03:44:01.876091+0000","last_unstale":"2024-09-14T03:45:14.458588+0000","last_undegraded":"2024-09-14T03:45:14.458588+0000","last_fullsized":"2024-09-14T03:45:14.458588+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:25:41.750127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.461481+0000","last_change":"2024-09-14T03:44:01.877120+0000","last_active":"2024-09-14T03:45:14.461481+0000","last_peered":"2024-09-14T03:45:14.461481+0000","last_clean":"2024-09-14T03:45:14.461481+0000","last_became_active":"2024-09-14T03:44:01.876793+0000","last_became_peered":"2024-09-14T03:44:01.876793+0000","last_unstale":"2024-09-14T03:45:14.461481+0000","last_undegraded":"2024-09-14T03:45:14.461481+0000","last_fullsized":"2024-09-14T03:45:14.461481+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:23:56.450728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.462251+0000","last_change":"2024-09-14T03:44:01.885108+0000","last_active":"2024-09-14T03:45:14.462251+0000","last_peered":"2024-09-14T03:45:14.462251+0000","last_clean":"2024-09-14T03:45:14.462251+0000","last_became_active":"2024-09-14T03:44:01.884857+0000","last_became_peered":"2024-09-14T03:44:01.884857+0000","last_unstale":"2024-09-14T03:45:14.462251+0000","last_undegraded":"2024-09-14T03:45:14.462251+0000","last_fullsized":"2024-09-14T03:45:14.462251+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:31.448617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:15.135790+0000","last_change":"2024-09-14T03:44:01.876503+0000","last_active":"2024-09-14T03:45:15.135790+0000","last_peered":"2024-09-14T03:45:15.135790+0000","last_clean":"2024-09-14T03:45:15.135790+0000","last_became_active":"2024-09-14T03:44:01.876227+0000","last_became_peered":"2024-09-14T03:44:01.876227+0000","last_unstale":"2024-09-14T03:45:15.135790+0000","last_undegraded":"2024-09-14T03:45:15.135790+0000","last_fullsized":"2024-09-14T03:45:15.135790+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:44:50.132152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.661079+0000","last_change":"2024-09-14T03:44:01.883708+0000","last_active":"2024-09-14T03:45:14.661079+0000","last_peered":"2024-09-14T03:45:14.661079+0000","last_clean":"2024-09-14T03:45:14.661079+0000","last_became_active":"2024-09-14T03:44:01.883440+0000","last_became_peered":"2024-09-14T03:44:01.883440+0000","last_unstale":"2024-09-14T03:45:14.661079+0000","last_undegraded":"2024-09-14T03:45:14.661079+0000","last_fullsized":"2024-09-14T03:45:14.661079+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:10:34.658062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.544854+0000","last_change":"2024-09-14T03:44:01.888565+0000","last_active":"2024-09-14T03:45:14.544854+0000","last_peered":"2024-09-14T03:45:14.544854+0000","last_clean":"2024-09-14T03:45:14.544854+0000","last_became_active":"2024-09-14T03:44:01.888191+0000","last_became_peered":"2024-09-14T03:44:01.888191+0000","last_unstale":"2024-09-14T03:45:14.544854+0000","last_undegraded":"2024-09-14T03:45:14.544854+0000","last_fullsized":"2024-09-14T03:45:14.544854+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:52:23.782664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.684144+0000","last_change":"2024-09-14T03:44:01.874642+0000","last_active":"2024-09-14T03:45:14.684144+0000","last_peered":"2024-09-14T03:45:14.684144+0000","last_clean":"2024-09-14T03:45:14.684144+0000","last_became_active":"2024-09-14T03:44:01.874384+0000","last_became_peered":"2024-09-14T03:44:01.874384+0000","last_unstale":"2024-09-14T03:45:14.684144+0000","last_undegraded":"2024-09-14T03:45:14.684144+0000","last_fullsized":"2024-09-14T03:45:14.684144+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:35:05.535173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.461689+0000","last_change":"2024-09-14T03:44:01.886893+0000","last_active":"2024-09-14T03:45:14.461689+0000","last_peered":"2024-09-14T03:45:14.461689+0000","last_clean":"2024-09-14T03:45:14.461689+0000","last_became_active":"2024-09-14T03:44:01.886594+0000","last_became_peered":"2024-09-14T03:44:01.886594+0000","last_unstale":"2024-09-14T03:45:14.461689+0000","last_undegraded":"2024-09-14T03:45:14.461689+0000","last_fullsized":"2024-09-14T03:45:14.461689+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:07:37.154358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.857420+0000","last_change":"2024-09-14T03:44:01.868698+0000","last_active":"2024-09-14T03:45:14.857420+0000","last_peered":"2024-09-14T03:45:14.857420+0000","last_clean":"2024-09-14T03:45:14.857420+0000","last_became_active":"2024-09-14T03:44:01.868442+0000","last_became_peered":"2024-09-14T03:44:01.868442+0000","last_unstale":"2024-09-14T03:45:14.857420+0000","last_undegraded":"2024-09-14T03:45:14.857420+0000","last_fullsized":"2024-09-14T03:45:14.857420+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:41:18.296424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.461524+0000","last_change":"2024-09-14T03:44:01.882708+0000","last_active":"2024-09-14T03:45:14.461524+0000","last_peered":"2024-09-14T03:45:14.461524+0000","last_clean":"2024-09-14T03:45:14.461524+0000","last_became_active":"2024-09-14T03:44:01.882443+0000","last_became_peered":"2024-09-14T03:44:01.882443+0000","last_unstale":"2024-09-14T03:45:14.461524+0000","last_undegraded":"2024-09-14T03:45:14.461524+0000","last_fullsized":"2024-09-14T03:45:14.461524+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:46:48.306263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.857790+0000","last_change":"2024-09-14T03:44:01.884940+0000","last_active":"2024-09-14T03:45:14.857790+0000","last_peered":"2024-09-14T03:45:14.857790+0000","last_clean":"2024-09-14T03:45:14.857790+0000","last_became_active":"2024-09-14T03:44:01.884304+0000","last_became_peered":"2024-09-14T03:44:01.884304+0000","last_unstale":"2024-09-14T03:45:14.857790+0000","last_undegraded":"2024-09-14T03:45:14.857790+0000","last_fullsized":"2024-09-14T03:45:14.857790+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:01:49.061637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":30,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.544999+0000","last_change":"2024-09-14T03:44:26.430891+0000","last_active":"2024-09-14T03:45:14.544999+0000","last_peered":"2024-09-14T03:45:14.544999+0000","last_clean":"2024-09-14T03:45:14.544999+0000","last_became_active":"2024-09-14T03:44:26.430426+0000","last_became_peered":"2024-09-14T03:44:26.430426+0000","last_unstale":"2024-09-14T03:45:14.544999+0000","last_undegraded":"2024-09-14T03:45:14.544999+0000","last_fullsized":"2024-09-14T03:45:14.544999+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:00:53.758328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.458426+0000","last_change":"2024-09-14T03:44:01.874564+0000","last_active":"2024-09-14T03:45:14.458426+0000","last_peered":"2024-09-14T03:45:14.458426+0000","last_clean":"2024-09-14T03:45:14.458426+0000","last_became_active":"2024-09-14T03:44:01.874297+0000","last_became_peered":"2024-09-14T03:44:01.874297+0000","last_unstale":"2024-09-14T03:45:14.458426+0000","last_undegraded":"2024-09-14T03:45:14.458426+0000","last_fullsized":"2024-09-14T03:45:14.458426+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:07.027610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.857969+0000","last_change":"2024-09-14T03:44:01.884486+0000","last_active":"2024-09-14T03:45:14.857969+0000","last_peered":"2024-09-14T03:45:14.857969+0000","last_clean":"2024-09-14T03:45:14.857969+0000","last_became_active":"2024-09-14T03:44:01.884218+0000","last_became_peered":"2024-09-14T03:44:01.884218+0000","last_unstale":"2024-09-14T03:45:14.857969+0000","last_undegraded":"2024-09-14T03:45:14.857969+0000","last_fullsized":"2024-09-14T03:45:14.857969+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:59:18.153239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.684242+0000","last_change":"2024-09-14T03:44:01.878737+0000","last_active":"2024-09-14T03:45:14.684242+0000","last_peered":"2024-09-14T03:45:14.684242+0000","last_clean":"2024-09-14T03:45:14.684242+0000","last_became_active":"2024-09-14T03:44:01.878504+0000","last_became_peered":"2024-09-14T03:44:01.878504+0000","last_unstale":"2024-09-14T03:45:14.684242+0000","last_undegraded":"2024-09-14T03:45:14.684242+0000","last_fullsized":"2024-09-14T03:45:14.684242+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:41:35.420957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.461691+0000","last_change":"2024-09-14T03:44:01.880509+0000","last_active":"2024-09-14T03:45:14.461691+0000","last_peered":"2024-09-14T03:45:14.461691+0000","last_clean":"2024-09-14T03:45:14.461691+0000","last_became_active":"2024-09-14T03:44:01.879855+0000","last_became_peered":"2024-09-14T03:44:01.879855+0000","last_unstale":"2024-09-14T03:45:14.461691+0000","last_undegraded":"2024-09-14T03:45:14.461691+0000","last_fullsized":"2024-09-14T03:45:14.461691+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:05:31.166902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:15.135594+0000","last_change":"2024-09-14T03:44:01.869523+0000","last_active":"2024-09-14T03:45:15.135594+0000","last_peered":"2024-09-14T03:45:15.135594+0000","last_clean":"2024-09-14T03:45:15.135594+0000","last_became_active":"2024-09-14T03:44:01.869225+0000","last_became_peered":"2024-09-14T03:44:01.869225+0000","last_unstale":"2024-09-14T03:45:15.135594+0000","last_undegraded":"2024-09-14T03:45:15.135594+0000","last_fullsized":"2024-09-14T03:45:15.135594+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:43:15.638187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.544409+0000","last_change":"2024-09-14T03:44:01.876014+0000","last_active":"2024-09-14T03:45:14.544409+0000","last_peered":"2024-09-14T03:45:14.544409+0000","last_clean":"2024-09-14T03:45:14.544409+0000","last_became_active":"2024-09-14T03:44:01.875496+0000","last_became_peered":"2024-09-14T03:44:01.875496+0000","last_unstale":"2024-09-14T03:45:14.544409+0000","last_undegraded":"2024-09-14T03:45:14.544409+0000","last_fullsized":"2024-09-14T03:45:14.544409+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:03:45.344199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.458305+0000","last_change":"2024-09-14T03:44:01.865496+0000","last_active":"2024-09-14T03:45:14.458305+0000","last_peered":"2024-09-14T03:45:14.458305+0000","last_clean":"2024-09-14T03:45:14.458305+0000","last_became_active":"2024-09-14T03:44:01.865190+0000","last_became_peered":"2024-09-14T03:44:01.865190+0000","last_unstale":"2024-09-14T03:45:14.458305+0000","last_undegraded":"2024-09-14T03:45:14.458305+0000","last_fullsized":"2024-09-14T03:45:14.458305+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:18:50.037675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"58'1","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.857788+0000","last_change":"2024-09-14T03:43:58.729188+0000","last_active":"2024-09-14T03:45:14.857788+0000","last_peered":"2024-09-14T03:45:14.857788+0000","last_clean":"2024-09-14T03:45:14.857788+0000","last_became_active":"2024-09-14T03:43:58.728927+0000","last_became_peered":"2024-09-14T03:43:58.728927+0000","last_unstale":"2024-09-14T03:45:14.857788+0000","last_undegraded":"2024-09-14T03:45:14.857788+0000","last_fullsized":"2024-09-14T03:45:14.857788+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:13:16.419701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.684318+0000","last_change":"2024-09-14T03:44:01.879101+0000","last_active":"2024-09-14T03:45:14.684318+0000","last_peered":"2024-09-14T03:45:14.684318+0000","last_clean":"2024-09-14T03:45:14.684318+0000","last_became_active":"2024-09-14T03:44:01.878801+0000","last_became_peered":"2024-09-14T03:44:01.878801+0000","last_unstale":"2024-09-14T03:45:14.684318+0000","last_undegraded":"2024-09-14T03:45:14.684318+0000","last_fullsized":"2024-09-14T03:45:14.684318+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:13.447251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"58'1","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.544625+0000","last_change":"2024-09-14T03:43:58.730444+0000","last_active":"2024-09-14T03:45:14.544625+0000","last_peered":"2024-09-14T03:45:14.544625+0000","last_clean":"2024-09-14T03:45:14.544625+0000","last_became_active":"2024-09-14T03:43:58.730172+0000","last_became_peered":"2024-09-14T03:43:58.730172+0000","last_unstale":"2024-09-14T03:45:14.544625+0000","last_undegraded":"2024-09-14T03:45:14.544625+0000","last_fullsized":"2024-09-14T03:45:14.544625+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:03:44.810562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":23,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.857500+0000","last_change":"2024-09-14T03:43:58.731458+0000","last_active":"2024-09-14T03:45:14.857500+0000","last_peered":"2024-09-14T03:45:14.857500+0000","last_clean":"2024-09-14T03:45:14.857500+0000","last_became_active":"2024-09-14T03:43:58.731206+0000","last_became_peered":"2024-09-14T03:43:58.731206+0000","last_unstale":"2024-09-14T03:45:14.857500+0000","last_undegraded":"2024-09-14T03:45:14.857500+0000","last_fullsized":"2024-09-14T03:45:14.857500+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:37:06.670539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.857453+0000","last_change":"2024-09-14T03:44:01.869781+0000","last_active":"2024-09-14T03:45:14.857453+0000","last_peered":"2024-09-14T03:45:14.857453+0000","last_clean":"2024-09-14T03:45:14.857453+0000","last_became_active":"2024-09-14T03:44:01.869521+0000","last_became_peered":"2024-09-14T03:44:01.869521+0000","last_unstale":"2024-09-14T03:45:14.857453+0000","last_undegraded":"2024-09-14T03:45:14.857453+0000","last_fullsized":"2024-09-14T03:45:14.857453+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:35:18.565206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":33,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.461579+0000","last_change":"2024-09-14T03:44:25.932578+0000","last_active":"2024-09-14T03:45:14.461579+0000","last_peered":"2024-09-14T03:45:14.461579+0000","last_clean":"2024-09-14T03:45:14.461579+0000","last_became_active":"2024-09-14T03:44:25.932288+0000","last_became_peered":"2024-09-14T03:44:25.932288+0000","last_unstale":"2024-09-14T03:45:14.461579+0000","last_undegraded":"2024-09-14T03:45:14.461579+0000","last_fullsized":"2024-09-14T03:45:14.461579+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:24:41.091353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.457866+0000","last_change":"2024-09-14T03:44:01.873748+0000","last_active":"2024-09-14T03:45:14.457866+0000","last_peered":"2024-09-14T03:45:14.457866+0000","last_clean":"2024-09-14T03:45:14.457866+0000","last_became_active":"2024-09-14T03:44:01.873473+0000","last_became_peered":"2024-09-14T03:44:01.873473+0000","last_unstale":"2024-09-14T03:45:14.457866+0000","last_undegraded":"2024-09-14T03:45:14.457866+0000","last_fullsized":"2024-09-14T03:45:14.457866+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:04:39.289903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":23,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.544953+0000","last_change":"2024-09-14T03:43:58.735608+0000","last_active":"2024-09-14T03:45:14.544953+0000","last_peered":"2024-09-14T03:45:14.544953+0000","last_clean":"2024-09-14T03:45:14.544953+0000","last_became_active":"2024-09-14T03:43:58.735301+0000","last_became_peered":"2024-09-14T03:43:58.735301+0000","last_unstale":"2024-09-14T03:45:14.544953+0000","last_undegraded":"2024-09-14T03:45:14.544953+0000","last_fullsized":"2024-09-14T03:45:14.544953+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:44:56.523073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.462078+0000","last_change":"2024-09-14T03:44:01.874777+0000","last_active":"2024-09-14T03:45:14.462078+0000","last_peered":"2024-09-14T03:45:14.462078+0000","last_clean":"2024-09-14T03:45:14.462078+0000","last_became_active":"2024-09-14T03:44:01.874473+0000","last_became_peered":"2024-09-14T03:44:01.874473+0000","last_unstale":"2024-09-14T03:45:14.462078+0000","last_undegraded":"2024-09-14T03:45:14.462078+0000","last_fullsized":"2024-09-14T03:45:14.462078+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:49:02.042635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":23,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.462172+0000","last_change":"2024-09-14T03:43:58.732969+0000","last_active":"2024-09-14T03:45:14.462172+0000","last_peered":"2024-09-14T03:45:14.462172+0000","last_clean":"2024-09-14T03:45:14.462172+0000","last_became_active":"2024-09-14T03:43:58.732492+0000","last_became_peered":"2024-09-14T03:43:58.732492+0000","last_unstale":"2024-09-14T03:45:14.462172+0000","last_undegraded":"2024-09-14T03:45:14.462172+0000","last_fullsized":"2024-09-14T03:45:14.462172+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:50:28.719447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.684384+0000","last_change":"2024-09-14T03:44:01.875679+0000","last_active":"2024-09-14T03:45:14.684384+0000","last_peered":"2024-09-14T03:45:14.684384+0000","last_clean":"2024-09-14T03:45:14.684384+0000","last_became_active":"2024-09-14T03:44:01.875422+0000","last_became_peered":"2024-09-14T03:44:01.875422+0000","last_unstale":"2024-09-14T03:45:14.684384+0000","last_undegraded":"2024-09-14T03:45:14.684384+0000","last_fullsized":"2024-09-14T03:45:14.684384+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:13:00.349373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"58'3","reported_seq":26,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.458567+0000","last_change":"2024-09-14T03:43:58.735066+0000","last_active":"2024-09-14T03:45:14.458567+0000","last_peered":"2024-09-14T03:45:14.458567+0000","last_clean":"2024-09-14T03:45:14.458567+0000","last_became_active":"2024-09-14T03:43:58.734514+0000","last_became_peered":"2024-09-14T03:43:58.734514+0000","last_unstale":"2024-09-14T03:45:14.458567+0000","last_undegraded":"2024-09-14T03:45:14.458567+0000","last_fullsized":"2024-09-14T03:45:14.458567+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:41:55.478295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.458619+0000","last_change":"2024-09-14T03:44:01.881319+0000","last_active":"2024-09-14T03:45:14.458619+0000","last_peered":"2024-09-14T03:45:14.458619+0000","last_clean":"2024-09-14T03:45:14.458619+0000","last_became_active":"2024-09-14T03:44:01.881032+0000","last_became_peered":"2024-09-14T03:44:01.881032+0000","last_unstale":"2024-09-14T03:45:14.458619+0000","last_undegraded":"2024-09-14T03:45:14.458619+0000","last_fullsized":"2024-09-14T03:45:14.458619+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:35:41.762014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"61'9","reported_seq":32,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.544859+0000","last_change":"2024-09-14T03:43:58.734778+0000","last_active":"2024-09-14T03:45:14.544859+0000","last_peered":"2024-09-14T03:45:14.544859+0000","last_clean":"2024-09-14T03:45:14.544859+0000","last_became_active":"2024-09-14T03:43:58.734380+0000","last_became_peered":"2024-09-14T03:43:58.734380+0000","last_unstale":"2024-09-14T03:45:14.544859+0000","last_undegraded":"2024-09-14T03:45:14.544859+0000","last_fullsized":"2024-09-14T03:45:14.544859+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:55:13.561065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2342,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.660824+0000","last_change":"2024-09-14T03:44:01.880139+0000","last_active":"2024-09-14T03:45:14.660824+0000","last_peered":"2024-09-14T03:45:14.660824+0000","last_clean":"2024-09-14T03:45:14.660824+0000","last_became_active":"2024-09-14T03:44:01.879840+0000","last_became_peered":"2024-09-14T03:44:01.879840+0000","last_unstale":"2024-09-14T03:45:14.660824+0000","last_undegraded":"2024-09-14T03:45:14.660824+0000","last_fullsized":"2024-09-14T03:45:14.660824+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:31:34.003676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"58'2","reported_seq":25,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.684403+0000","last_change":"2024-09-14T03:43:58.728768+0000","last_active":"2024-09-14T03:45:14.684403+0000","last_peered":"2024-09-14T03:45:14.684403+0000","last_clean":"2024-09-14T03:45:14.684403+0000","last_became_active":"2024-09-14T03:43:58.727893+0000","last_became_peered":"2024-09-14T03:43:58.727893+0000","last_unstale":"2024-09-14T03:45:14.684403+0000","last_undegraded":"2024-09-14T03:45:14.684403+0000","last_fullsized":"2024-09-14T03:45:14.684403+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:54:48.554486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.684119+0000","last_change":"2024-09-14T03:44:01.877540+0000","last_active":"2024-09-14T03:45:14.684119+0000","last_peered":"2024-09-14T03:45:14.684119+0000","last_clean":"2024-09-14T03:45:14.684119+0000","last_became_active":"2024-09-14T03:44:01.877147+0000","last_became_peered":"2024-09-14T03:44:01.877147+0000","last_unstale":"2024-09-14T03:45:14.684119+0000","last_undegraded":"2024-09-14T03:45:14.684119+0000","last_fullsized":"2024-09-14T03:45:14.684119+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:17:57.724648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":23,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.461815+0000","last_change":"2024-09-14T03:43:58.732812+0000","last_active":"2024-09-14T03:45:14.461815+0000","last_peered":"2024-09-14T03:45:14.461815+0000","last_clean":"2024-09-14T03:45:14.461815+0000","last_became_active":"2024-09-14T03:43:58.732537+0000","last_became_peered":"2024-09-14T03:43:58.732537+0000","last_unstale":"2024-09-14T03:45:14.461815+0000","last_undegraded":"2024-09-14T03:45:14.461815+0000","last_fullsized":"2024-09-14T03:45:14.461815+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:26:11.551182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.660730+0000","last_change":"2024-09-14T03:44:01.881140+0000","last_active":"2024-09-14T03:45:14.660730+0000","last_peered":"2024-09-14T03:45:14.660730+0000","last_clean":"2024-09-14T03:45:14.660730+0000","last_became_active":"2024-09-14T03:44:01.880868+0000","last_became_peered":"2024-09-14T03:44:01.880868+0000","last_unstale":"2024-09-14T03:45:14.660730+0000","last_undegraded":"2024-09-14T03:45:14.660730+0000","last_fullsized":"2024-09-14T03:45:14.660730+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:29:05.729804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":23,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.544699+0000","last_change":"2024-09-14T03:43:58.730505+0000","last_active":"2024-09-14T03:45:14.544699+0000","last_peered":"2024-09-14T03:45:14.544699+0000","last_clean":"2024-09-14T03:45:14.544699+0000","last_became_active":"2024-09-14T03:43:58.729821+0000","last_became_peered":"2024-09-14T03:43:58.729821+0000","last_unstale":"2024-09-14T03:45:14.544699+0000","last_undegraded":"2024-09-14T03:45:14.544699+0000","last_fullsized":"2024-09-14T03:45:14.544699+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:17:44.091667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.544408+0000","last_change":"2024-09-14T03:44:01.874477+0000","last_active":"2024-09-14T03:45:14.544408+0000","last_peered":"2024-09-14T03:45:14.544408+0000","last_clean":"2024-09-14T03:45:14.544408+0000","last_became_active":"2024-09-14T03:44:01.873714+0000","last_became_peered":"2024-09-14T03:44:01.873714+0000","last_unstale":"2024-09-14T03:45:14.544408+0000","last_undegraded":"2024-09-14T03:45:14.544408+0000","last_fullsized":"2024-09-14T03:45:14.544408+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:54:21.601768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"58'8","reported_seq":31,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:15.135490+0000","last_change":"2024-09-14T03:43:58.726387+0000","last_active":"2024-09-14T03:45:15.135490+0000","last_peered":"2024-09-14T03:45:15.135490+0000","last_clean":"2024-09-14T03:45:15.135490+0000","last_became_active":"2024-09-14T03:43:58.726115+0000","last_became_peered":"2024-09-14T03:43:58.726115+0000","last_unstale":"2024-09-14T03:45:15.135490+0000","last_undegraded":"2024-09-14T03:45:15.135490+0000","last_fullsized":"2024-09-14T03:45:15.135490+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:18:04.176970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.660478+0000","last_change":"2024-09-14T03:44:01.878624+0000","last_active":"2024-09-14T03:45:14.660478+0000","last_peered":"2024-09-14T03:45:14.660478+0000","last_clean":"2024-09-14T03:45:14.660478+0000","last_became_active":"2024-09-14T03:44:01.878343+0000","last_became_peered":"2024-09-14T03:44:01.878343+0000","last_unstale":"2024-09-14T03:45:14.660478+0000","last_undegraded":"2024-09-14T03:45:14.660478+0000","last_fullsized":"2024-09-14T03:45:14.660478+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:44:05.205671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"58'1","reported_seq":23,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.458928+0000","last_change":"2024-09-14T03:43:58.728489+0000","last_active":"2024-09-14T03:45:14.458928+0000","last_peered":"2024-09-14T03:45:14.458928+0000","last_clean":"2024-09-14T03:45:14.458928+0000","last_became_active":"2024-09-14T03:43:58.728182+0000","last_became_peered":"2024-09-14T03:43:58.728182+0000","last_unstale":"2024-09-14T03:45:14.458928+0000","last_undegraded":"2024-09-14T03:45:14.458928+0000","last_fullsized":"2024-09-14T03:45:14.458928+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:56.310710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.661108+0000","last_change":"2024-09-14T03:44:01.880754+0000","last_active":"2024-09-14T03:45:14.661108+0000","last_peered":"2024-09-14T03:45:14.661108+0000","last_clean":"2024-09-14T03:45:14.661108+0000","last_became_active":"2024-09-14T03:44:01.880483+0000","last_became_peered":"2024-09-14T03:44:01.880483+0000","last_unstale":"2024-09-14T03:45:14.661108+0000","last_undegraded":"2024-09-14T03:45:14.661108+0000","last_fullsized":"2024-09-14T03:45:14.661108+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:32:53.245007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"58'2","reported_seq":25,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:15.135841+0000","last_change":"2024-09-14T03:43:58.726366+0000","last_active":"2024-09-14T03:45:15.135841+0000","last_peered":"2024-09-14T03:45:15.135841+0000","last_clean":"2024-09-14T03:45:15.135841+0000","last_became_active":"2024-09-14T03:43:58.726090+0000","last_became_peered":"2024-09-14T03:43:58.726090+0000","last_unstale":"2024-09-14T03:45:15.135841+0000","last_undegraded":"2024-09-14T03:45:15.135841+0000","last_fullsized":"2024-09-14T03:45:15.135841+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:45:33.572820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.661089+0000","last_change":"2024-09-14T03:44:01.883704+0000","last_active":"2024-09-14T03:45:14.661089+0000","last_peered":"2024-09-14T03:45:14.661089+0000","last_clean":"2024-09-14T03:45:14.661089+0000","last_became_active":"2024-09-14T03:44:01.883448+0000","last_became_peered":"2024-09-14T03:44:01.883448+0000","last_unstale":"2024-09-14T03:45:14.661089+0000","last_undegraded":"2024-09-14T03:45:14.661089+0000","last_fullsized":"2024-09-14T03:45:14.661089+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:16:27.946770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"58'1","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.458573+0000","last_change":"2024-09-14T03:43:58.733205+0000","last_active":"2024-09-14T03:45:14.458573+0000","last_peered":"2024-09-14T03:45:14.458573+0000","last_clean":"2024-09-14T03:45:14.458573+0000","last_became_active":"2024-09-14T03:43:58.732929+0000","last_became_peered":"2024-09-14T03:43:58.732929+0000","last_unstale":"2024-09-14T03:45:14.458573+0000","last_undegraded":"2024-09-14T03:45:14.458573+0000","last_fullsized":"2024-09-14T03:45:14.458573+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:31:13.660965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.461995+0000","last_change":"2024-09-14T03:44:01.886158+0000","last_active":"2024-09-14T03:45:14.461995+0000","last_peered":"2024-09-14T03:45:14.461995+0000","last_clean":"2024-09-14T03:45:14.461995+0000","last_became_active":"2024-09-14T03:44:01.884747+0000","last_became_peered":"2024-09-14T03:44:01.884747+0000","last_unstale":"2024-09-14T03:45:14.461995+0000","last_undegraded":"2024-09-14T03:45:14.461995+0000","last_fullsized":"2024-09-14T03:45:14.461995+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:49:56.265783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":23,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:15.135633+0000","last_change":"2024-09-14T03:43:58.732401+0000","last_active":"2024-09-14T03:45:15.135633+0000","last_peered":"2024-09-14T03:45:15.135633+0000","last_clean":"2024-09-14T03:45:15.135633+0000","last_became_active":"2024-09-14T03:43:58.732067+0000","last_became_peered":"2024-09-14T03:43:58.732067+0000","last_unstale":"2024-09-14T03:45:15.135633+0000","last_undegraded":"2024-09-14T03:45:15.135633+0000","last_fullsized":"2024-09-14T03:45:15.135633+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:34:00.566245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.660858+0000","last_change":"2024-09-14T03:44:01.881736+0000","last_active":"2024-09-14T03:45:14.660858+0000","last_peered":"2024-09-14T03:45:14.660858+0000","last_clean":"2024-09-14T03:45:14.660858+0000","last_became_active":"2024-09-14T03:44:01.881219+0000","last_became_peered":"2024-09-14T03:44:01.881219+0000","last_unstale":"2024-09-14T03:45:14.660858+0000","last_undegraded":"2024-09-14T03:45:14.660858+0000","last_fullsized":"2024-09-14T03:45:14.660858+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:24:23.342066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-09-14T03:45:20.503002+0000","last_change":"2024-09-14T03:45:14.545163+0000","last_active":"2024-09-14T03:45:20.503002+0000","last_peered":"2024-09-14T03:45:20.503002+0000","last_clean":"2024-09-14T03:45:20.503002+0000","last_became_active":"2024-09-14T03:45:09.755484+0000","last_became_peered":"2024-09-14T03:45:09.755484+0000","last_unstale":"2024-09-14T03:45:20.503002+0000","last_undegraded":"2024-09-14T03:45:20.503002+0000","last_fullsized":"2024-09-14T03:45:20.503002+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_clean_scrub_stamp":"2024-09-14T03:45:08.693776+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:18:49.100636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039339699999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"61'79","reported_seq":58,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.660571+0000","last_change":"2024-09-14T03:43:36.147227+0000","last_active":"2024-09-14T03:45:14.660571+0000","last_peered":"2024-09-14T03:45:14.660571+0000","last_clean":"2024-09-14T03:45:14.660571+0000","last_became_active":"2024-09-14T03:43:36.141953+0000","last_became_peered":"2024-09-14T03:43:36.141953+0000","last_unstale":"2024-09-14T03:45:14.660571+0000","last_undegraded":"2024-09-14T03:45:14.660571+0000","last_fullsized":"2024-09-14T03:45:14.660571+0000","mapping_epoch":50,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":51,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:39.458815+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:39.458815+0000","last_clean_scrub_stamp":"2024-09-14T03:41:39.458815+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:38:43.056328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":23,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.461867+0000","last_change":"2024-09-14T03:43:58.733557+0000","last_active":"2024-09-14T03:45:14.461867+0000","last_peered":"2024-09-14T03:45:14.461867+0000","last_clean":"2024-09-14T03:45:14.461867+0000","last_became_active":"2024-09-14T03:43:58.733256+0000","last_became_peered":"2024-09-14T03:43:58.733256+0000","last_unstale":"2024-09-14T03:45:14.461867+0000","last_undegraded":"2024-09-14T03:45:14.461867+0000","last_fullsized":"2024-09-14T03:45:14.461867+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:09:12.987156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.857593+0000","last_change":"2024-09-14T03:44:01.875525+0000","last_active":"2024-09-14T03:45:14.857593+0000","last_peered":"2024-09-14T03:45:14.857593+0000","last_clean":"2024-09-14T03:45:14.857593+0000","last_became_active":"2024-09-14T03:44:01.875241+0000","last_became_peered":"2024-09-14T03:44:01.875241+0000","last_unstale":"2024-09-14T03:45:14.857593+0000","last_undegraded":"2024-09-14T03:45:14.857593+0000","last_fullsized":"2024-09-14T03:45:14.857593+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:34:47.346963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-09-14T03:45:20.503141+0000","last_change":"2024-09-14T03:45:14.684739+0000","last_active":"2024-09-14T03:45:20.503141+0000","last_peered":"2024-09-14T03:45:20.503141+0000","last_clean":"2024-09-14T03:45:20.503141+0000","last_became_active":"2024-09-14T03:45:09.711005+0000","last_became_peered":"2024-09-14T03:45:09.711005+0000","last_unstale":"2024-09-14T03:45:20.503141+0000","last_undegraded":"2024-09-14T03:45:20.503141+0000","last_fullsized":"2024-09-14T03:45:20.503141+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_clean_scrub_stamp":"2024-09-14T03:45:08.693776+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:35.312720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00019552,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.660472+0000","last_change":"2024-09-14T03:43:58.732376+0000","last_active":"2024-09-14T03:45:14.660472+0000","last_peered":"2024-09-14T03:45:14.660472+0000","last_clean":"2024-09-14T03:45:14.660472+0000","last_became_active":"2024-09-14T03:43:58.731812+0000","last_became_peered":"2024-09-14T03:43:58.731812+0000","last_unstale":"2024-09-14T03:45:14.660472+0000","last_undegraded":"2024-09-14T03:45:14.660472+0000","last_fullsized":"2024-09-14T03:45:14.660472+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:35:26.262383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.683969+0000","last_change":"2024-09-14T03:44:01.876511+0000","last_active":"2024-09-14T03:45:14.683969+0000","last_peered":"2024-09-14T03:45:14.683969+0000","last_clean":"2024-09-14T03:45:14.683969+0000","last_became_active":"2024-09-14T03:44:01.876225+0000","last_became_peered":"2024-09-14T03:44:01.876225+0000","last_unstale":"2024-09-14T03:45:14.683969+0000","last_undegraded":"2024-09-14T03:45:14.683969+0000","last_fullsized":"2024-09-14T03:45:14.683969+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:36:40.283622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-09-14T03:45:20.464110+0000","last_change":"2024-09-14T03:45:14.458990+0000","last_active":"2024-09-14T03:45:20.464110+0000","last_peered":"2024-09-14T03:45:20.464110+0000","last_clean":"2024-09-14T03:45:20.464110+0000","last_became_active":"2024-09-14T03:45:09.879190+0000","last_became_peered":"2024-09-14T03:45:09.879190+0000","last_unstale":"2024-09-14T03:45:20.464110+0000","last_undegraded":"2024-09-14T03:45:20.464110+0000","last_fullsized":"2024-09-14T03:45:20.464110+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_clean_scrub_stamp":"2024-09-14T03:45:08.693776+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:23:17.328924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057164100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":23,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:15.135684+0000","last_change":"2024-09-14T03:43:58.727129+0000","last_active":"2024-09-14T03:45:15.135684+0000","last_peered":"2024-09-14T03:45:15.135684+0000","last_clean":"2024-09-14T03:45:15.135684+0000","last_became_active":"2024-09-14T03:43:58.726879+0000","last_became_peered":"2024-09-14T03:43:58.726879+0000","last_unstale":"2024-09-14T03:45:15.135684+0000","last_undegraded":"2024-09-14T03:45:15.135684+0000","last_fullsized":"2024-09-14T03:45:15.135684+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:54:41.748651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.457983+0000","last_change":"2024-09-14T03:44:01.871005+0000","last_active":"2024-09-14T03:45:14.457983+0000","last_peered":"2024-09-14T03:45:14.457983+0000","last_clean":"2024-09-14T03:45:14.457983+0000","last_became_active":"2024-09-14T03:44:01.870707+0000","last_became_peered":"2024-09-14T03:44:01.870707+0000","last_unstale":"2024-09-14T03:45:14.457983+0000","last_undegraded":"2024-09-14T03:45:14.457983+0000","last_fullsized":"2024-09-14T03:45:14.457983+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:30:22.758581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-09-14T03:45:20.464062+0000","last_change":"2024-09-14T03:45:14.458757+0000","last_active":"2024-09-14T03:45:20.464062+0000","last_peered":"2024-09-14T03:45:20.464062+0000","last_clean":"2024-09-14T03:45:20.464062+0000","last_became_active":"2024-09-14T03:45:09.711634+0000","last_became_peered":"2024-09-14T03:45:09.711634+0000","last_unstale":"2024-09-14T03:45:20.464062+0000","last_undegraded":"2024-09-14T03:45:20.464062+0000","last_fullsized":"2024-09-14T03:45:20.464062+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_clean_scrub_stamp":"2024-09-14T03:45:08.693776+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:42:58.495982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054748100000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":23,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.461874+0000","last_change":"2024-09-14T03:43:58.733038+0000","last_active":"2024-09-14T03:45:14.461874+0000","last_peered":"2024-09-14T03:45:14.461874+0000","last_clean":"2024-09-14T03:45:14.461874+0000","last_became_active":"2024-09-14T03:43:58.732753+0000","last_became_peered":"2024-09-14T03:43:58.732753+0000","last_unstale":"2024-09-14T03:45:14.461874+0000","last_undegraded":"2024-09-14T03:45:14.461874+0000","last_fullsized":"2024-09-14T03:45:14.461874+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:40:49.754211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":30,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.458431+0000","last_change":"2024-09-14T03:44:26.430320+0000","last_active":"2024-09-14T03:45:14.458431+0000","last_peered":"2024-09-14T03:45:14.458431+0000","last_clean":"2024-09-14T03:45:14.458431+0000","last_became_active":"2024-09-14T03:44:26.429935+0000","last_became_peered":"2024-09-14T03:44:26.429935+0000","last_unstale":"2024-09-14T03:45:14.458431+0000","last_undegraded":"2024-09-14T03:45:14.458431+0000","last_fullsized":"2024-09-14T03:45:14.458431+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:12:35.577538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.2","version":"67'2","reported_seq":21,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-09-14T03:45:20.464076+0000","last_change":"2024-09-14T03:45:14.458846+0000","last_active":"2024-09-14T03:45:20.464076+0000","last_peered":"2024-09-14T03:45:20.464076+0000","last_clean":"2024-09-14T03:45:20.464076+0000","last_became_active":"2024-09-14T03:45:09.879248+0000","last_became_peered":"2024-09-14T03:45:09.879248+0000","last_unstale":"2024-09-14T03:45:20.464076+0000","last_undegraded":"2024-09-14T03:45:20.464076+0000","last_fullsized":"2024-09-14T03:45:20.464076+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_clean_scrub_stamp":"2024-09-14T03:45:08.693776+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:02:23.634325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000682786,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":23,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.458338+0000","last_change":"2024-09-14T03:43:58.733621+0000","last_active":"2024-09-14T03:45:14.458338+0000","last_peered":"2024-09-14T03:45:14.458338+0000","last_clean":"2024-09-14T03:45:14.458338+0000","last_became_active":"2024-09-14T03:43:58.733357+0000","last_became_peered":"2024-09-14T03:43:58.733357+0000","last_unstale":"2024-09-14T03:45:14.458338+0000","last_undegraded":"2024-09-14T03:45:14.458338+0000","last_fullsized":"2024-09-14T03:45:14.458338+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:25:53.791926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.462075+0000","last_change":"2024-09-14T03:44:01.880200+0000","last_active":"2024-09-14T03:45:14.462075+0000","last_peered":"2024-09-14T03:45:14.462075+0000","last_clean":"2024-09-14T03:45:14.462075+0000","last_became_active":"2024-09-14T03:44:01.879925+0000","last_became_peered":"2024-09-14T03:44:01.879925+0000","last_unstale":"2024-09-14T03:45:14.462075+0000","last_undegraded":"2024-09-14T03:45:14.462075+0000","last_fullsized":"2024-09-14T03:45:14.462075+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:55:24.738215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"65'1","reported_seq":20,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-09-14T03:45:20.502304+0000","last_change":"2024-09-14T03:45:14.684595+0000","last_active":"2024-09-14T03:45:20.502304+0000","last_peered":"2024-09-14T03:45:20.502304+0000","last_clean":"2024-09-14T03:45:20.502304+0000","last_became_active":"2024-09-14T03:45:09.879229+0000","last_became_peered":"2024-09-14T03:45:09.879229+0000","last_unstale":"2024-09-14T03:45:20.502304+0000","last_undegraded":"2024-09-14T03:45:20.502304+0000","last_fullsized":"2024-09-14T03:45:20.502304+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_clean_scrub_stamp":"2024-09-14T03:45:08.693776+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:49:58.555610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040600400000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.661082+0000","last_change":"2024-09-14T03:43:58.733230+0000","last_active":"2024-09-14T03:45:14.661082+0000","last_peered":"2024-09-14T03:45:14.661082+0000","last_clean":"2024-09-14T03:45:14.661082+0000","last_became_active":"2024-09-14T03:43:58.732868+0000","last_became_peered":"2024-09-14T03:43:58.732868+0000","last_unstale":"2024-09-14T03:45:14.661082+0000","last_undegraded":"2024-09-14T03:45:14.661082+0000","last_fullsized":"2024-09-14T03:45:14.661082+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:15:12.184851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.458306+0000","last_change":"2024-09-14T03:44:01.878736+0000","last_active":"2024-09-14T03:45:14.458306+0000","last_peered":"2024-09-14T03:45:14.458306+0000","last_clean":"2024-09-14T03:45:14.458306+0000","last_became_active":"2024-09-14T03:44:01.877390+0000","last_became_peered":"2024-09-14T03:44:01.877390+0000","last_unstale":"2024-09-14T03:45:14.458306+0000","last_undegraded":"2024-09-14T03:45:14.458306+0000","last_fullsized":"2024-09-14T03:45:14.458306+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:17:42.812575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-09-14T03:45:20.501909+0000","last_change":"2024-09-14T03:45:14.857657+0000","last_active":"2024-09-14T03:45:20.501909+0000","last_peered":"2024-09-14T03:45:20.501909+0000","last_clean":"2024-09-14T03:45:20.501909+0000","last_became_active":"2024-09-14T03:45:09.710318+0000","last_became_peered":"2024-09-14T03:45:09.710318+0000","last_unstale":"2024-09-14T03:45:20.501909+0000","last_undegraded":"2024-09-14T03:45:20.501909+0000","last_fullsized":"2024-09-14T03:45:20.501909+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_clean_scrub_stamp":"2024-09-14T03:45:08.693776+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:37:23.130384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031572099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"58'1","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.459070+0000","last_change":"2024-09-14T03:43:58.733688+0000","last_active":"2024-09-14T03:45:14.459070+0000","last_peered":"2024-09-14T03:45:14.459070+0000","last_clean":"2024-09-14T03:45:14.459070+0000","last_became_active":"2024-09-14T03:43:58.733457+0000","last_became_peered":"2024-09-14T03:43:58.733457+0000","last_unstale":"2024-09-14T03:45:14.459070+0000","last_undegraded":"2024-09-14T03:45:14.459070+0000","last_fullsized":"2024-09-14T03:45:14.459070+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:09:01.069988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.857916+0000","last_change":"2024-09-14T03:44:01.870045+0000","last_active":"2024-09-14T03:45:14.857916+0000","last_peered":"2024-09-14T03:45:14.857916+0000","last_clean":"2024-09-14T03:45:14.857916+0000","last_became_active":"2024-09-14T03:44:01.869780+0000","last_became_peered":"2024-09-14T03:44:01.869780+0000","last_unstale":"2024-09-14T03:45:14.857916+0000","last_undegraded":"2024-09-14T03:45:14.857916+0000","last_fullsized":"2024-09-14T03:45:14.857916+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:04:58.634704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-09-14T03:45:20.502067+0000","last_change":"2024-09-14T03:45:14.459017+0000","last_active":"2024-09-14T03:45:20.502067+0000","last_peered":"2024-09-14T03:45:20.502067+0000","last_clean":"2024-09-14T03:45:20.502067+0000","last_became_active":"2024-09-14T03:45:09.879281+0000","last_became_peered":"2024-09-14T03:45:09.879281+0000","last_unstale":"2024-09-14T03:45:20.502067+0000","last_undegraded":"2024-09-14T03:45:20.502067+0000","last_fullsized":"2024-09-14T03:45:20.502067+0000","mapping_epoch":62,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_clean_scrub_stamp":"2024-09-14T03:45:08.693776+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:10:03.506895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082715299999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"58'2","reported_seq":25,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.545051+0000","last_change":"2024-09-14T03:43:58.735364+0000","last_active":"2024-09-14T03:45:14.545051+0000","last_peered":"2024-09-14T03:45:14.545051+0000","last_clean":"2024-09-14T03:45:14.545051+0000","last_became_active":"2024-09-14T03:43:58.734833+0000","last_became_peered":"2024-09-14T03:43:58.734833+0000","last_unstale":"2024-09-14T03:45:14.545051+0000","last_undegraded":"2024-09-14T03:45:14.545051+0000","last_fullsized":"2024-09-14T03:45:14.545051+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:58:37.428713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.684514+0000","last_change":"2024-09-14T03:44:01.878573+0000","last_active":"2024-09-14T03:45:14.684514+0000","last_peered":"2024-09-14T03:45:14.684514+0000","last_clean":"2024-09-14T03:45:14.684514+0000","last_became_active":"2024-09-14T03:44:01.878316+0000","last_became_peered":"2024-09-14T03:44:01.878316+0000","last_unstale":"2024-09-14T03:45:14.684514+0000","last_undegraded":"2024-09-14T03:45:14.684514+0000","last_fullsized":"2024-09-14T03:45:14.684514+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:45:21.129163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.660416+0000","last_change":"2024-09-14T03:43:58.730843+0000","last_active":"2024-09-14T03:45:14.660416+0000","last_peered":"2024-09-14T03:45:14.660416+0000","last_clean":"2024-09-14T03:45:14.660416+0000","last_became_active":"2024-09-14T03:43:58.730538+0000","last_became_peered":"2024-09-14T03:43:58.730538+0000","last_unstale":"2024-09-14T03:45:14.660416+0000","last_undegraded":"2024-09-14T03:45:14.660416+0000","last_fullsized":"2024-09-14T03:45:14.660416+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:04:57.059345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.458548+0000","last_change":"2024-09-14T03:44:01.874355+0000","last_active":"2024-09-14T03:45:14.458548+0000","last_peered":"2024-09-14T03:45:14.458548+0000","last_clean":"2024-09-14T03:45:14.458548+0000","last_became_active":"2024-09-14T03:44:01.874049+0000","last_became_peered":"2024-09-14T03:44:01.874049+0000","last_unstale":"2024-09-14T03:45:14.458548+0000","last_undegraded":"2024-09-14T03:45:14.458548+0000","last_fullsized":"2024-09-14T03:45:14.458548+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:53:30.306175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":23,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.458659+0000","last_change":"2024-09-14T03:43:58.728080+0000","last_active":"2024-09-14T03:45:14.458659+0000","last_peered":"2024-09-14T03:45:14.458659+0000","last_clean":"2024-09-14T03:45:14.458659+0000","last_became_active":"2024-09-14T03:43:58.727820+0000","last_became_peered":"2024-09-14T03:43:58.727820+0000","last_unstale":"2024-09-14T03:45:14.458659+0000","last_undegraded":"2024-09-14T03:45:14.458659+0000","last_fullsized":"2024-09-14T03:45:14.458659+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:37:58.166430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.857492+0000","last_change":"2024-09-14T03:44:01.869311+0000","last_active":"2024-09-14T03:45:14.857492+0000","last_peered":"2024-09-14T03:45:14.857492+0000","last_clean":"2024-09-14T03:45:14.857492+0000","last_became_active":"2024-09-14T03:44:01.869065+0000","last_became_peered":"2024-09-14T03:44:01.869065+0000","last_unstale":"2024-09-14T03:45:14.857492+0000","last_undegraded":"2024-09-14T03:45:14.857492+0000","last_fullsized":"2024-09-14T03:45:14.857492+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:45:09.508019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":23,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.459148+0000","last_change":"2024-09-14T03:43:58.728931+0000","last_active":"2024-09-14T03:45:14.459148+0000","last_peered":"2024-09-14T03:45:14.459148+0000","last_clean":"2024-09-14T03:45:14.459148+0000","last_became_active":"2024-09-14T03:43:58.728647+0000","last_became_peered":"2024-09-14T03:43:58.728647+0000","last_unstale":"2024-09-14T03:45:14.459148+0000","last_undegraded":"2024-09-14T03:45:14.459148+0000","last_fullsized":"2024-09-14T03:45:14.459148+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:09:38.769523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":31,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.857689+0000","last_change":"2024-09-14T03:44:26.430230+0000","last_active":"2024-09-14T03:45:14.857689+0000","last_peered":"2024-09-14T03:45:14.857689+0000","last_clean":"2024-09-14T03:45:14.857689+0000","last_became_active":"2024-09-14T03:44:26.429880+0000","last_became_peered":"2024-09-14T03:44:26.429880+0000","last_unstale":"2024-09-14T03:45:14.857689+0000","last_undegraded":"2024-09-14T03:45:14.857689+0000","last_fullsized":"2024-09-14T03:45:14.857689+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:39.132572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"58'2","reported_seq":26,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.660988+0000","last_change":"2024-09-14T03:43:58.737705+0000","last_active":"2024-09-14T03:45:14.660988+0000","last_peered":"2024-09-14T03:45:14.660988+0000","last_clean":"2024-09-14T03:45:14.660988+0000","last_became_active":"2024-09-14T03:43:58.736546+0000","last_became_peered":"2024-09-14T03:43:58.736546+0000","last_unstale":"2024-09-14T03:45:14.660988+0000","last_undegraded":"2024-09-14T03:45:14.660988+0000","last_fullsized":"2024-09-14T03:45:14.660988+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:27:12.542706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":18,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:15.135501+0000","last_change":"2024-09-14T03:44:26.429678+0000","last_active":"2024-09-14T03:45:15.135501+0000","last_peered":"2024-09-14T03:45:15.135501+0000","last_clean":"2024-09-14T03:45:15.135501+0000","last_became_active":"2024-09-14T03:44:26.429364+0000","last_became_peered":"2024-09-14T03:44:26.429364+0000","last_unstale":"2024-09-14T03:45:15.135501+0000","last_undegraded":"2024-09-14T03:45:15.135501+0000","last_fullsized":"2024-09-14T03:45:15.135501+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:29:31.230108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.462051+0000","last_change":"2024-09-14T03:44:01.888591+0000","last_active":"2024-09-14T03:45:14.462051+0000","last_peered":"2024-09-14T03:45:14.462051+0000","last_clean":"2024-09-14T03:45:14.462051+0000","last_became_active":"2024-09-14T03:44:01.888257+0000","last_became_peered":"2024-09-14T03:44:01.888257+0000","last_unstale":"2024-09-14T03:45:14.462051+0000","last_undegraded":"2024-09-14T03:45:14.462051+0000","last_fullsized":"2024-09-14T03:45:14.462051+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:24:40.992358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"58'2","reported_seq":26,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.660896+0000","last_change":"2024-09-14T03:43:58.734680+0000","last_active":"2024-09-14T03:45:14.660896+0000","last_peered":"2024-09-14T03:45:14.660896+0000","last_clean":"2024-09-14T03:45:14.660896+0000","last_became_active":"2024-09-14T03:43:58.734408+0000","last_became_peered":"2024-09-14T03:43:58.734408+0000","last_unstale":"2024-09-14T03:45:14.660896+0000","last_undegraded":"2024-09-14T03:45:14.660896+0000","last_fullsized":"2024-09-14T03:45:14.660896+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:49:02.536404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.461880+0000","last_change":"2024-09-14T03:44:01.885572+0000","last_active":"2024-09-14T03:45:14.461880+0000","last_peered":"2024-09-14T03:45:14.461880+0000","last_clean":"2024-09-14T03:45:14.461880+0000","last_became_active":"2024-09-14T03:44:01.885110+0000","last_became_peered":"2024-09-14T03:44:01.885110+0000","last_unstale":"2024-09-14T03:45:14.461880+0000","last_undegraded":"2024-09-14T03:45:14.461880+0000","last_fullsized":"2024-09-14T03:45:14.461880+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:10.439239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1d","version":"58'2","reported_seq":26,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.660935+0000","last_change":"2024-09-14T03:43:58.731296+0000","last_active":"2024-09-14T03:45:14.660935+0000","last_peered":"2024-09-14T03:45:14.660935+0000","last_clean":"2024-09-14T03:45:14.660935+0000","last_became_active":"2024-09-14T03:43:58.730951+0000","last_became_peered":"2024-09-14T03:43:58.730951+0000","last_unstale":"2024-09-14T03:45:14.660935+0000","last_undegraded":"2024-09-14T03:45:14.660935+0000","last_fullsized":"2024-09-14T03:45:14.660935+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:26:14.801414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.684378+0000","last_change":"2024-09-14T03:44:01.874935+0000","last_active":"2024-09-14T03:45:14.684378+0000","last_peered":"2024-09-14T03:45:14.684378+0000","last_clean":"2024-09-14T03:45:14.684378+0000","last_became_active":"2024-09-14T03:44:01.874668+0000","last_became_peered":"2024-09-14T03:44:01.874668+0000","last_unstale":"2024-09-14T03:45:14.684378+0000","last_undegraded":"2024-09-14T03:45:14.684378+0000","last_fullsized":"2024-09-14T03:45:14.684378+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:24:31.404958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"58'1","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.857877+0000","last_change":"2024-09-14T03:43:58.734206+0000","last_active":"2024-09-14T03:45:14.857877+0000","last_peered":"2024-09-14T03:45:14.857877+0000","last_clean":"2024-09-14T03:45:14.857877+0000","last_became_active":"2024-09-14T03:43:58.733960+0000","last_became_peered":"2024-09-14T03:43:58.733960+0000","last_unstale":"2024-09-14T03:45:14.857877+0000","last_undegraded":"2024-09-14T03:45:14.857877+0000","last_fullsized":"2024-09-14T03:45:14.857877+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:36:44.073503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.857768+0000","last_change":"2024-09-14T03:44:01.883550+0000","last_active":"2024-09-14T03:45:14.857768+0000","last_peered":"2024-09-14T03:45:14.857768+0000","last_clean":"2024-09-14T03:45:14.857768+0000","last_became_active":"2024-09-14T03:44:01.883265+0000","last_became_peered":"2024-09-14T03:44:01.883265+0000","last_unstale":"2024-09-14T03:45:14.857768+0000","last_undegraded":"2024-09-14T03:45:14.857768+0000","last_fullsized":"2024-09-14T03:45:14.857768+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:52:04.512990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"58'4","reported_seq":27,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:45:14.684569+0000","last_change":"2024-09-14T03:43:58.731954+0000","last_active":"2024-09-14T03:45:14.684569+0000","last_peered":"2024-09-14T03:45:14.684569+0000","last_clean":"2024-09-14T03:45:14.684569+0000","last_became_active":"2024-09-14T03:43:58.731688+0000","last_became_peered":"2024-09-14T03:43:58.731688+0000","last_unstale":"2024-09-14T03:45:14.684569+0000","last_undegraded":"2024-09-14T03:45:14.684569+0000","last_fullsized":"2024-09-14T03:45:14.684569+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:30:10.811609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":3128,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":48,"num_write_kb":16,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":9384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":42,"ondisk_log_size":42,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397540,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298624,"kb_used_data":1268,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446528,"statfs":{"total":95995035648,"available":95689244672,"internally_reserved":0,"allocated":1298432,"data_stored":903527,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:45:39 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.50800000000000001,"15min":0.51000000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.503},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.51800000000000002,"15min":0.51700000000000002},"min":{"1min":0.38100000000000001,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.40100000000000002}]},{"osd":1,"last update":"Sat Sep 14 03:45:39 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.50600000000000001,"15min":0.51000000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.622,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54100000000000004,"15min":0.54000000000000004},"min":{"1min":0.41799999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.57499999999999996}]},{"osd":2,"last update":"Sat Sep 14 03:45:39 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.53500000000000003,"15min":0.54100000000000004},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.66500000000000004,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.499,"5min":0.52400000000000002,"15min":0.52800000000000002},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.53700000000000003}]},{"osd":3,"last update":"Sat Sep 14 03:45:39 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.53300000000000003,"15min":0.53900000000000003},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.629,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.48799999999999999}]},{"osd":4,"last update":"Sat Sep 14 03:45:39 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.47599999999999998,"15min":0.48899999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.56999999999999995,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.47399999999999998,"15min":0.48499999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.36599999999999999}]},{"osd":5,"last update":"Sat Sep 14 03:45:39 2024","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.46100000000000002,"15min":0.46899999999999997},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.65300000000000002,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.379},{"interface":"front","average":{"1min":0.441,"5min":0.51300000000000001,"15min":0.52500000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.45600000000000002}]},{"osd":6,"last update":"Sat Sep 14 03:45:39 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.53800000000000003,"15min":0.55200000000000005},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.60999999999999999,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.52800000000000002,"15min":0.54100000000000004},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.52800000000000002}]}]},{"osd":6,"up_from":43,"seq":184683593768,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298624,"kb_used_data":1268,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446528,"statfs":{"total":95995035648,"available":95689244672,"internally_reserved":0,"allocated":1298432,"data_stored":904962,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:46:16 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.50600000000000001,"15min":0.502},"min":{"1min":0.38300000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.495},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.502,"15min":0.502},"min":{"1min":0.36199999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.69399999999999995,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.61499999999999999}]},{"osd":1,"last update":"Sat Sep 14 03:46:16 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.51900000000000002,"15min":0.51100000000000001},"min":{"1min":0.375,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.434,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.65300000000000002}]},{"osd":2,"last update":"Sat Sep 14 03:46:16 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.48399999999999999,"15min":0.47399999999999998},"min":{"1min":0.38200000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.65300000000000002,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.73099999999999998,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.47799999999999998}]},{"osd":3,"last update":"Sat Sep 14 03:46:16 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.52500000000000002,"15min":0.52900000000000003},"min":{"1min":0.41299999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.67400000000000004,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.52200000000000002,"15min":0.51900000000000002},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.67700000000000005}]},{"osd":4,"last update":"Sat Sep 14 03:46:16 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.41899999999999998,"15min":0.41299999999999998},"min":{"1min":0.31,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.60099999999999998,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.45000000000000001,"15min":0.45500000000000002},"min":{"1min":0.309,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.55900000000000005,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.38300000000000001}]},{"osd":5,"last update":"Sat Sep 14 03:46:16 2024","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.47299999999999998,"15min":0.47199999999999998},"min":{"1min":0.35099999999999998,"5min":0.315,"15min":0.315},"max":{"1min":0.61699999999999999,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.48199999999999998,"15min":0.48599999999999999},"min":{"1min":0.34100000000000003,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.42599999999999999}]},{"osd":7,"last update":"Sat Sep 14 03:45:34 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.52600000000000002,"15min":0.52900000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.66800000000000004,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.56200000000000006,"15min":0.56999999999999995},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.73299999999999998,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.45100000000000001}]}]},{"osd":5,"up_from":37,"seq":158913789997,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298052,"kb_used_data":696,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447100,"statfs":{"total":95995035648,"available":95689830400,"internally_reserved":0,"allocated":712704,"data_stored":314703,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:45:49 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53800000000000003,"15min":0.54000000000000004},"min":{"1min":0.38700000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.79900000000000004,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.52800000000000002,"15min":0.51700000000000002},"min":{"1min":0.41499999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.50700000000000001}]},{"osd":1,"last update":"Sat Sep 14 03:45:49 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.52000000000000002,"15min":0.51200000000000001},"min":{"1min":0.36899999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.73399999999999999,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.51600000000000001,"15min":0.51000000000000001},"min":{"1min":0.40699999999999997,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.78000000000000003,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.58199999999999996}]},{"osd":2,"last update":"Sat Sep 14 03:45:49 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52300000000000002,"15min":0.51900000000000002},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.72099999999999997,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.53100000000000003,"15min":0.51600000000000001},"min":{"1min":0.42399999999999999,"5min":0.376,"15min":0.376},"max":{"1min":0.76100000000000001,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.64800000000000002}]},{"osd":3,"last update":"Sat Sep 14 03:45:49 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.53200000000000003,"15min":0.52100000000000002},"min":{"1min":0.41699999999999998,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.627},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.54000000000000004,"15min":0.54200000000000004},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.77800000000000002,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.61299999999999999}]},{"osd":4,"last update":"Sat Sep 14 03:45:49 2024","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":0.39300000000000002,"15min":0.38300000000000001},"min":{"1min":0.28499999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.40699999999999997,"15min":0.39500000000000002},"min":{"1min":0.313,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.57499999999999996,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.46400000000000002}]},{"osd":6,"last update":"Sat Sep 14 03:45:13 2024","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.435,"15min":0.433},"min":{"1min":0.34699999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.43099999999999999,"15min":0.42399999999999999},"min":{"1min":0.32200000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.40600000000000003}]},{"osd":7,"last update":"Sat Sep 14 03:45:43 2024","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.48099999999999998,"15min":0.48199999999999998},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.49399999999999999,"15min":0.496},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.495}]}]},{"osd":4,"up_from":31,"seq":133143986225,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298056,"kb_used_data":700,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447096,"statfs":{"total":95995035648,"available":95689826304,"internally_reserved":0,"allocated":716800,"data_stored":313282,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:45:27 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51100000000000001,"15min":0.51200000000000001},"min":{"1min":0.372,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.66000000000000003,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.629},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.68899999999999995,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.60599999999999998}]},{"osd":1,"last update":"Sat Sep 14 03:45:27 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.496,"15min":0.47999999999999998},"min":{"1min":0.34100000000000003,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.64100000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.51400000000000001,"15min":0.51100000000000001},"min":{"1min":0.44400000000000001,"5min":0.377,"15min":0.377},"max":{"1min":0.69999999999999996,"5min":0.753,"15min":0.753},"last":0.61699999999999999}]},{"osd":2,"last update":"Sat Sep 14 03:45:27 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.50600000000000001,"15min":0.49399999999999999},"min":{"1min":0.35799999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.69699999999999995,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.51500000000000001,"15min":0.50600000000000001},"min":{"1min":0.45100000000000001,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.59199999999999997,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.56699999999999995}]},{"osd":3,"last update":"Sat Sep 14 03:45:27 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.51500000000000001,"15min":0.497},"min":{"1min":0.42599999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.68000000000000005,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.51200000000000001,"15min":0.502},"min":{"1min":0.41499999999999998,"5min":0.318,"15min":0.318},"max":{"1min":0.72699999999999998,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.53300000000000003}]},{"osd":5,"last update":"Sat Sep 14 03:45:52 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.435,"15min":0.41099999999999998},"min":{"1min":0.30099999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.57699999999999996,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.442,"15min":0.41999999999999998},"min":{"1min":0.28299999999999997,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.59799999999999998,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.46700000000000003}]},{"osd":6,"last update":"Sat Sep 14 03:46:16 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.44900000000000001,"15min":0.44500000000000001},"min":{"1min":0.28299999999999997,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.67500000000000004,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.436},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.48499999999999999,"15min":0.48299999999999998},"min":{"1min":0.32100000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.63600000000000001,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.39800000000000002}]},{"osd":7,"last update":"Sat Sep 14 03:45:41 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.57099999999999995,"15min":0.58399999999999996},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.66300000000000003,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.5},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.55700000000000005,"15min":0.56999999999999995},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.66100000000000003,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.51300000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149749,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298052,"kb_used_data":696,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447100,"statfs":{"total":95995035648,"available":95689830400,"internally_reserved":0,"allocated":712704,"data_stored":314698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:46:06 2024","interfaces":[{"interface":"back","average":{"1min":0.34100000000000003,"5min":0.36399999999999999,"15min":0.38400000000000001},"min":{"1min":0.16800000000000001,"5min":0.16800000000000001,"15min":0.16800000000000001},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.33000000000000002,"5min":0.376,"15min":0.40000000000000002},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.32600000000000001}]},{"osd":1,"last update":"Sat Sep 14 03:46:06 2024","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.39700000000000002,"15min":0.41399999999999998},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.251},{"interface":"front","average":{"1min":0.35999999999999999,"5min":0.376,"15min":0.39200000000000002},"min":{"1min":0.17999999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.29999999999999999}]},{"osd":2,"last update":"Sat Sep 14 03:46:06 2024","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.40200000000000002,"15min":0.42999999999999999},"min":{"1min":0.25900000000000001,"5min":0.251,"15min":0.251},"max":{"1min":0.52000000000000002,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.35899999999999999,"5min":0.39700000000000002,"15min":0.41899999999999998},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.35299999999999998}]},{"osd":4,"last update":"Sat Sep 14 03:45:32 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53200000000000003,"15min":0.53900000000000003},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55500000000000005,"15min":0.56000000000000005},"min":{"1min":0.39100000000000001,"5min":0.379,"15min":0.379},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.48399999999999999}]},{"osd":5,"last update":"Sat Sep 14 03:45:53 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.57999999999999996,"15min":0.59099999999999997},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":1.161,"5min":1.161,"15min":1.161},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.56000000000000005,"15min":0.56699999999999995},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.439}]},{"osd":6,"last update":"Sat Sep 14 03:46:17 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.56000000000000005,"15min":0.57399999999999995},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.1799999999999999,"5min":1.1799999999999999,"15min":1.1799999999999999},"last":0.432},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.55100000000000005,"15min":0.55700000000000005},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.504}]},{"osd":7,"last update":"Sat Sep 14 03:45:36 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.53300000000000003,"15min":0.52800000000000002},"min":{"1min":0.38,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.53800000000000003,"15min":0.53300000000000003},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.46800000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411386,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298032,"kb_used_data":676,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447120,"statfs":{"total":95995035648,"available":95689850880,"internally_reserved":0,"allocated":692224,"data_stored":311956,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:45:51 2024","interfaces":[{"interface":"back","average":{"1min":0.318,"5min":0.34699999999999998,"15min":0.35099999999999998},"min":{"1min":0.19400000000000001,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.40600000000000003,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.34000000000000002,"5min":0.35699999999999998,"15min":0.36099999999999999},"min":{"1min":0.252,"5min":0.193,"15min":0.193},"max":{"1min":0.48899999999999999,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.27300000000000002}]},{"osd":1,"last update":"Sat Sep 14 03:45:51 2024","interfaces":[{"interface":"back","average":{"1min":0.35099999999999998,"5min":0.38200000000000001,"15min":0.39200000000000002},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.51200000000000001,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.378},{"interface":"front","average":{"1min":0.34000000000000002,"5min":0.373,"15min":0.377},"min":{"1min":0.24399999999999999,"5min":0.216,"15min":0.216},"max":{"1min":0.50900000000000001,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.35099999999999998}]},{"osd":3,"last update":"Sat Sep 14 03:46:13 2024","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.41399999999999998,"15min":0.44},"min":{"1min":0.30099999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.47699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.38200000000000001,"5min":0.40999999999999998,"15min":0.44500000000000001},"min":{"1min":0.27900000000000003,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.38300000000000001}]},{"osd":4,"last update":"Sat Sep 14 03:45:28 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.54900000000000004,"15min":0.55500000000000005},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.76500000000000001,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.58699999999999997,"15min":0.59099999999999997},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.55600000000000005}]},{"osd":5,"last update":"Sat Sep 14 03:45:51 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.58099999999999996,"15min":0.59999999999999998},"min":{"1min":0.39200000000000002,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.56499999999999995,"15min":0.57999999999999996},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.73399999999999999,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.48499999999999999}]},{"osd":6,"last update":"Sat Sep 14 03:45:15 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.56799999999999995,"15min":0.56999999999999995},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.76000000000000001,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.57199999999999995,"15min":0.57499999999999996},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.71799999999999997,"5min":0.875,"15min":0.875},"last":0.56499999999999995}]},{"osd":7,"last update":"Sat Sep 14 03:45:38 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.56799999999999995,"15min":0.57799999999999996},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.72899999999999998,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.57999999999999996,"15min":0.58599999999999997},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.47699999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574912,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298036,"kb_used_data":680,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447116,"statfs":{"total":95995035648,"available":95689846784,"internally_reserved":0,"allocated":696320,"data_stored":311801,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:45:24 2024","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.33400000000000002,"15min":0.30099999999999999},"min":{"1min":0.247,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.56599999999999995,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.34200000000000003,"5min":0.35099999999999998,"15min":0.32100000000000001},"min":{"1min":0.23400000000000001,"5min":0.189,"15min":0.189},"max":{"1min":0.49399999999999999,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.34000000000000002}]},{"osd":2,"last update":"Sat Sep 14 03:45:44 2024","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.35799999999999998,"15min":0.34399999999999997},"min":{"1min":0.23899999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.375,"15min":0.35099999999999998},"min":{"1min":0.245,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.53000000000000003,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.35599999999999998}]},{"osd":3,"last update":"Sat Sep 14 03:46:13 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.40100000000000002,"15min":0.41499999999999998},"min":{"1min":0.27900000000000003,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.54500000000000004,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.41199999999999998,"15min":0.42899999999999999},"min":{"1min":0.27700000000000002,"5min":0.247,"15min":0.247},"max":{"1min":0.55200000000000005,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.41599999999999998}]},{"osd":4,"last update":"Sat Sep 14 03:45:31 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.54300000000000004,"15min":0.54700000000000004},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.78700000000000003,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.499},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.54400000000000004,"15min":0.54900000000000004},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.67800000000000005,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.51600000000000001}]},{"osd":5,"last update":"Sat Sep 14 03:45:49 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.56799999999999995,"15min":0.58399999999999996},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.627,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.59099999999999997,"15min":0.60299999999999998},"min":{"1min":0.39100000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.70999999999999996,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.58499999999999996}]},{"osd":6,"last update":"Sat Sep 14 03:46:13 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.57299999999999995,"15min":0.58599999999999997},"min":{"1min":0.42799999999999999,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.67000000000000004,"5min":0.747,"15min":0.747},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.58499999999999996,"15min":0.58899999999999997},"min":{"1min":0.45900000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.75900000000000001,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.60399999999999998}]},{"osd":7,"last update":"Sat Sep 14 03:45:39 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.56899999999999995,"15min":0.57299999999999995},"min":{"1min":0.41699999999999998,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.82599999999999996,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.55700000000000005,"15min":0.56299999999999994},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.68700000000000006,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.53900000000000003}]}]},{"osd":0,"up_from":8,"seq":34359738437,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298628,"kb_used_data":1272,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446524,"statfs":{"total":95995035648,"available":95689240576,"internally_reserved":0,"allocated":1302528,"data_stored":903336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 14 03:45:20 2024","interfaces":[{"interface":"back","average":{"1min":0.33100000000000002,"5min":0.35399999999999998,"15min":0.34799999999999998},"min":{"1min":0.24299999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.49099999999999999,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.33000000000000002,"5min":0.35999999999999999,"15min":0.34699999999999998},"min":{"1min":0.23400000000000001,"5min":0.216,"15min":0.216},"max":{"1min":0.56100000000000005,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.46999999999999997}]},{"osd":2,"last update":"Sat Sep 14 03:45:45 2024","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.39100000000000001,"15min":0.40300000000000002},"min":{"1min":0.23799999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.33900000000000002,"5min":0.38,"15min":0.39000000000000001},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":0.496,"5min":0.505,"15min":0.505},"last":0.497}]},{"osd":3,"last update":"Sat Sep 14 03:46:12 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.41299999999999998,"15min":0.438},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.52600000000000002,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.39000000000000001,"15min":0.41199999999999998},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.42499999999999999}]},{"osd":4,"last update":"Sat Sep 14 03:45:29 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.54800000000000004,"15min":0.56200000000000006},"min":{"1min":0.36699999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.74099999999999999},{"interface":"front","average":{"1min":0.503,"5min":0.54500000000000004,"15min":0.55500000000000005},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.67600000000000005,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.63900000000000001}]},{"osd":5,"last update":"Sat Sep 14 03:45:51 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.82899999999999996,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.57499999999999996,"15min":0.57299999999999995},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.61499999999999999}]},{"osd":6,"last update":"Sat Sep 14 03:46:14 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.55900000000000005,"15min":0.56599999999999995},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.626},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.55500000000000005,"15min":0.56000000000000005},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.77800000000000002,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.76900000000000002}]},{"osd":7,"last update":"Sat Sep 14 03:45:41 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.55200000000000005,"15min":0.55000000000000004},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.57999999999999996,"15min":0.58299999999999996},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.69099999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":2446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":2432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":2342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T03:46:21.546 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-14T03:46:21.546 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-14T03:46:21.546 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-14T03:46:21.547 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-14T03:46:21.831 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:46:21.924 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:46:21 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:46:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:46:22.035 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:46:21 smithi070 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-14T03:46:22.035 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:46:21 smithi070 systemd[1]: Stopped Ceph alertmanager.a for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:46:22.036 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:46:21 smithi070 systemd[1]: Starting Ceph alertmanager.a for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:46:22.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:21 smithi070 ceph-mon[27495]: pgmap v47: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:22.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:21 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:46:22.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:21 smithi070 ceph-mon[39288]: pgmap v47: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:22.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:21 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:46:22.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:21 smithi094 ceph-mon[31053]: pgmap v47: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:22.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:21 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:46:22.414 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:46:22 smithi070 podman[80265]: 2024-09-14 03:46:22.127501795 +0000 UTC m=+0.018231896 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-09-14T03:46:22.414 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:46:22 smithi070 podman[80265]: 2024-09-14 03:46:22.238046155 +0000 UTC m=+0.128776247 volume create 45362eb6e33738f2094bb877012773b280777f866153f3160f4f375d6346d9e0 2024-09-14T03:46:22.414 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:46:22 smithi070 podman[80265]: 2024-09-14 03:46:22.346776685 +0000 UTC m=+0.237506776 container create fef584d3d5c685dfc50a2bc94f046b8b4893870f003ee756a2381512c7e5c231 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:46:23.029 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:46:22 smithi070 podman[80265]: 2024-09-14 03:46:22.7947259 +0000 UTC m=+0.685455992 container init fef584d3d5c685dfc50a2bc94f046b8b4893870f003ee756a2381512c7e5c231 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:46:23.029 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:46:22 smithi070 podman[80265]: 2024-09-14 03:46:22.800300752 +0000 UTC m=+0.691030844 container start fef584d3d5c685dfc50a2bc94f046b8b4893870f003ee756a2381512c7e5c231 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:46:23.029 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:46:22 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a[80300]: ts=2024-09-14T03:46:22.812Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-14T03:46:23.029 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:46:22 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a[80300]: ts=2024-09-14T03:46:22.812Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-14T03:46:23.029 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:46:22 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a[80300]: ts=2024-09-14T03:46:22.813Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.70 port=9094 2024-09-14T03:46:23.029 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:46:22 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a[80300]: ts=2024-09-14T03:46:22.814Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-14T03:46:23.029 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:46:22 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a[80300]: ts=2024-09-14T03:46:22.838Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-14T03:46:23.029 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:46:22 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a[80300]: ts=2024-09-14T03:46:22.839Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-14T03:46:23.030 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:46:22 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a[80300]: ts=2024-09-14T03:46:22.841Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-14T03:46:23.030 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:46:22 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a[80300]: ts=2024-09-14T03:46:22.841Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-14T03:46:23.030 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:46:22 smithi070 bash[80265]: fef584d3d5c685dfc50a2bc94f046b8b4893870f003ee756a2381512c7e5c231 2024-09-14T03:46:23.030 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:46:22 smithi070 systemd[1]: Started Ceph alertmanager.a for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:46:24.000 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:23 smithi070 ceph-mon[39288]: pgmap v48: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:24.000 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:23 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:24.000 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:23 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:24.000 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:23 smithi070 ceph-mon[39288]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:46:24.000 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:23 smithi070 ceph-mon[39288]: Reconfiguring daemon prometheus.a on smithi094 2024-09-14T03:46:24.000 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:23 smithi070 ceph-mon[27495]: pgmap v48: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:24.001 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:23 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:24.001 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:23 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:24.001 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:23 smithi070 ceph-mon[27495]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:46:24.001 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:23 smithi070 ceph-mon[27495]: Reconfiguring daemon prometheus.a on smithi094 2024-09-14T03:46:24.236 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:23 smithi094 ceph-mon[31053]: pgmap v48: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:24.237 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:23 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:24.237 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:23 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:24.237 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:23 smithi094 ceph-mon[31053]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:46:24.237 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:23 smithi094 ceph-mon[31053]: Reconfiguring daemon prometheus.a on smithi094 2024-09-14T03:46:24.389 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:46:24.390 INFO:teuthology.orchestra.run.smithi070.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-14T03:46:25.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:24 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/2763585554' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-14T03:46:25.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:24 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/2763585554' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-14T03:46:25.036 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:46:24 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a[80300]: ts=2024-09-14T03:46:24.815Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000570065s 2024-09-14T03:46:25.158 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-14T03:46:25.159 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph fs dump --format=json 2024-09-14T03:46:25.199 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:24 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/2763585554' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-14T03:46:25.407 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:46:26.193 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:25 smithi094 ceph-mon[31053]: pgmap v49: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:26.247 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:25 smithi070 ceph-mon[27495]: pgmap v49: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:26.247 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:25 smithi070 ceph-mon[39288]: pgmap v49: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:26.445 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:26 smithi094 systemd[1]: Stopping Ceph prometheus.a for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:46:26.446 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:26 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[61388]: ts=2024-09-14T03:46:26.433Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-14T03:46:26.446 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:26 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[61388]: ts=2024-09-14T03:46:26.433Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-14T03:46:26.446 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:26 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[61388]: ts=2024-09-14T03:46:26.434Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-14T03:46:26.446 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:26 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[61388]: ts=2024-09-14T03:46:26.434Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-14T03:46:26.446 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:26 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[61388]: ts=2024-09-14T03:46:26.434Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-14T03:46:26.446 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:26 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[61388]: ts=2024-09-14T03:46:26.434Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-14T03:46:26.446 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:26 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[61388]: ts=2024-09-14T03:46:26.434Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-14T03:46:26.446 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:26 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[61388]: ts=2024-09-14T03:46:26.434Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-14T03:46:26.446 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:26 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[61388]: ts=2024-09-14T03:46:26.434Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-14T03:46:26.446 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:26 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[61388]: ts=2024-09-14T03:46:26.434Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-14T03:46:26.446 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:26 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[61388]: ts=2024-09-14T03:46:26.434Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-14T03:46:26.446 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:26 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[61388]: ts=2024-09-14T03:46:26.434Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-14T03:46:26.446 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:26 smithi094 podman[64760]: 2024-09-14 03:46:26.444040272 +0000 UTC m=+0.106069989 container died 5d8e345869aa12e9df696951ddcb74223c3b51b1df2a2d9ce286eae79f1fa464 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:46:27.026 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:26 smithi094 podman[64760]: 2024-09-14 03:46:26.805534489 +0000 UTC m=+0.467564194 container cleanup 5d8e345869aa12e9df696951ddcb74223c3b51b1df2a2d9ce286eae79f1fa464 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:46:27.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:26 smithi094 bash[64760]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a 2024-09-14T03:46:27.391 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:27 smithi094 podman[64777]: 2024-09-14 03:46:27.048382531 +0000 UTC m=+0.606434653 container remove 5d8e345869aa12e9df696951ddcb74223c3b51b1df2a2d9ce286eae79f1fa464 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:46:27.391 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:27 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-14T03:46:27.391 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:27 smithi094 systemd[1]: Stopped Ceph prometheus.a for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:46:27.391 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:27 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 1.071s CPU time. 2024-09-14T03:46:27.391 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:27 smithi094 systemd[1]: Starting Ceph prometheus.a for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:46:27.554 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:46:27.554 INFO:teuthology.orchestra.run.smithi070.stdout:{"epoch":8,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":24304,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.94:6833/3620660683","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6832","nonce":3620660683},{"type":"v1","addr":"172.21.15.94:6833","nonce":3620660683}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":8,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-14T03:44:03.568995+0000","modified":"2024-09-14T03:44:16.900718+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":1,"in":[0],"up":{"mds_0":24323},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24323":{"gid":24323,"name":"a","rank":0,"incarnation":4,"state":"up:active","state_seq":6,"addr":"172.21.15.70:6835/421917097","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6834","nonce":421917097},{"type":"v1","addr":"172.21.15.70:6835","nonce":421917097}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","standby_count_wanted":0},"id":1}]} 2024-09-14T03:46:27.554 INFO:teuthology.orchestra.run.smithi070.stderr:dumped fsmap epoch 8 2024-09-14T03:46:27.654 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:27 smithi094 podman[64874]: 2024-09-14 03:46:27.29404625 +0000 UTC m=+0.016434416 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-14T03:46:27.655 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:27 smithi094 podman[64874]: 2024-09-14 03:46:27.416031087 +0000 UTC m=+0.138419285 container create 9c88e2923e9d8ea13213e5b0be68f011c0d04d874b57eab8902d63e16e59c815 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:46:27.655 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:27 smithi094 podman[64874]: 2024-09-14 03:46:27.654107545 +0000 UTC m=+0.376495697 container init 9c88e2923e9d8ea13213e5b0be68f011c0d04d874b57eab8902d63e16e59c815 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:46:27.871 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:46:27 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:46:27] ENGINE Bus STOPPING 2024-09-14T03:46:27.939 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:27 smithi094 podman[64874]: 2024-09-14 03:46:27.65882393 +0000 UTC m=+0.381212083 container start 9c88e2923e9d8ea13213e5b0be68f011c0d04d874b57eab8902d63e16e59c815 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:46:27.940 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[64889]: ts=2024-09-14T03:46:27.682Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-14T03:46:27.940 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[64889]: ts=2024-09-14T03:46:27.682Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-14T03:46:27.940 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[64889]: ts=2024-09-14T03:46:27.682Z caller=main.go:570 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi094 (none))" 2024-09-14T03:46:27.940 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[64889]: ts=2024-09-14T03:46:27.682Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-14T03:46:27.940 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[64889]: ts=2024-09-14T03:46:27.683Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-14T03:46:27.940 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[64889]: ts=2024-09-14T03:46:27.684Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-14T03:46:27.940 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[64889]: ts=2024-09-14T03:46:27.684Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-14T03:46:27.940 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[64889]: ts=2024-09-14T03:46:27.686Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-14T03:46:27.940 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[64889]: ts=2024-09-14T03:46:27.686Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-14T03:46:27.940 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[64889]: ts=2024-09-14T03:46:27.688Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-14T03:46:27.940 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[64889]: ts=2024-09-14T03:46:27.688Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.74µs 2024-09-14T03:46:27.940 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[64889]: ts=2024-09-14T03:46:27.688Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-14T03:46:27.940 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[64889]: ts=2024-09-14T03:46:27.688Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-09-14T03:46:27.940 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[64889]: ts=2024-09-14T03:46:27.688Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-09-14T03:46:27.941 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[64889]: ts=2024-09-14T03:46:27.688Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=18.356µs wal_replay_duration=569.213µs wbl_replay_duration=101ns total_replay_duration=605.263µs 2024-09-14T03:46:27.941 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[64889]: ts=2024-09-14T03:46:27.689Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-14T03:46:27.941 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[64889]: ts=2024-09-14T03:46:27.689Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-14T03:46:27.941 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[64889]: ts=2024-09-14T03:46:27.689Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-14T03:46:27.941 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[64889]: ts=2024-09-14T03:46:27.705Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.116977ms db_storage=811ns remote_storage=1.251µs web_handler=283ns query_engine=554ns scrape=371.286µs scrape_sd=26.728µs notify=24.768µs notify_sd=6.345µs rules=14.451516ms tracing=6.081µs 2024-09-14T03:46:27.941 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[64889]: ts=2024-09-14T03:46:27.705Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-14T03:46:27.941 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[64889]: ts=2024-09-14T03:46:27.705Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-14T03:46:27.941 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:27 smithi094 bash[64874]: 9c88e2923e9d8ea13213e5b0be68f011c0d04d874b57eab8902d63e16e59c815 2024-09-14T03:46:27.941 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:46:27 smithi094 systemd[1]: Started Ceph prometheus.a for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:46:28.264 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:46:27 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:46:27] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:46:28.264 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:46:27 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:46:27] ENGINE Bus STOPPED 2024-09-14T03:46:28.264 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:46:27 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:46:27] ENGINE Bus STARTING 2024-09-14T03:46:28.264 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:46:28 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:46:28] ENGINE Serving on http://:::9283 2024-09-14T03:46:28.265 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:46:28 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:46:28] ENGINE Bus STARTED 2024-09-14T03:46:28.265 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:46:28 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:46:28] ENGINE Bus STOPPING 2024-09-14T03:46:28.265 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:27 smithi070 ceph-mon[39288]: pgmap v50: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:28.265 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:27 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/1857673572' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:46:28.265 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:27 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:28.265 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:27 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:28.265 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:27 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:46:28.265 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:27 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi070.front.sepia.ceph.com:9093"}]: dispatch 2024-09-14T03:46:28.265 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:27 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:28.266 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:27 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:46:28.266 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:27 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi094.front.sepia.ceph.com:9095"}]: dispatch 2024-09-14T03:46:28.266 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:27 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:28.266 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:27 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:46:28.266 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:27 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi094.front.sepia.ceph.com:3000"}]: dispatch 2024-09-14T03:46:28.266 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:27 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:28.266 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:27 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:46:28.266 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:27 smithi070 ceph-mon[27495]: pgmap v50: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:28.266 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:27 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/1857673572' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:46:28.266 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:27 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:28.266 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:27 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:28.267 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:27 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:46:28.267 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:27 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi070.front.sepia.ceph.com:9093"}]: dispatch 2024-09-14T03:46:28.267 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:27 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:28.267 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:27 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:46:28.267 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:27 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi094.front.sepia.ceph.com:9095"}]: dispatch 2024-09-14T03:46:28.267 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:27 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:28.267 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:27 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:46:28.267 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:27 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi094.front.sepia.ceph.com:3000"}]: dispatch 2024-09-14T03:46:28.267 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:27 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:28.267 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:27 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:46:28.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-mon[31053]: pgmap v50: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:28.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/1857673572' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:46:28.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:28.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:28.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:46:28.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi070.front.sepia.ceph.com:9093"}]: dispatch 2024-09-14T03:46:28.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:28.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:46:28.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi094.front.sepia.ceph.com:9095"}]: dispatch 2024-09-14T03:46:28.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:28.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:46:28.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi094.front.sepia.ceph.com:3000"}]: dispatch 2024-09-14T03:46:28.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:28.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:27 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:46:28.292 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph fs dump --format=json 2024-09-14T03:46:28.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:46:28 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:46:28] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:46:28.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:46:28 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:46:28] ENGINE Bus STOPPED 2024-09-14T03:46:28.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:46:28 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:46:28] ENGINE Bus STARTING 2024-09-14T03:46:28.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:46:28 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:46:28] ENGINE Serving on http://:::9283 2024-09-14T03:46:28.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:46:28 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:46:28] ENGINE Bus STARTED 2024-09-14T03:46:28.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:46:28 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:46:28] ENGINE Bus STOPPING 2024-09-14T03:46:29.124 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:46:29.139 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:28 smithi070 ceph-mon[39288]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:46:29.139 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:28 smithi070 ceph-mon[39288]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi070.front.sepia.ceph.com:9093"}]: dispatch 2024-09-14T03:46:29.139 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:28 smithi070 ceph-mon[39288]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:46:29.139 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:28 smithi070 ceph-mon[39288]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi094.front.sepia.ceph.com:9095"}]: dispatch 2024-09-14T03:46:29.140 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:28 smithi070 ceph-mon[39288]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:46:29.140 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:28 smithi070 ceph-mon[39288]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi094.front.sepia.ceph.com:3000"}]: dispatch 2024-09-14T03:46:29.140 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:28 smithi070 ceph-mon[27495]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:46:29.140 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:28 smithi070 ceph-mon[27495]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi070.front.sepia.ceph.com:9093"}]: dispatch 2024-09-14T03:46:29.140 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:28 smithi070 ceph-mon[27495]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:46:29.140 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:28 smithi070 ceph-mon[27495]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi094.front.sepia.ceph.com:9095"}]: dispatch 2024-09-14T03:46:29.140 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:28 smithi070 ceph-mon[27495]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:46:29.140 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:28 smithi070 ceph-mon[27495]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi094.front.sepia.ceph.com:3000"}]: dispatch 2024-09-14T03:46:29.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:28 smithi094 ceph-mon[31053]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:46:29.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:28 smithi094 ceph-mon[31053]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi070.front.sepia.ceph.com:9093"}]: dispatch 2024-09-14T03:46:29.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:28 smithi094 ceph-mon[31053]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:46:29.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:28 smithi094 ceph-mon[31053]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi094.front.sepia.ceph.com:9095"}]: dispatch 2024-09-14T03:46:29.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:28 smithi094 ceph-mon[31053]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:46:29.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:28 smithi094 ceph-mon[31053]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi094.front.sepia.ceph.com:3000"}]: dispatch 2024-09-14T03:46:29.496 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:46:29 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:46:29] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:46:29.496 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:46:29 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:46:29] ENGINE Bus STOPPED 2024-09-14T03:46:29.496 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:46:29 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:46:29] ENGINE Bus STARTING 2024-09-14T03:46:29.496 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:46:29 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:46:29] ENGINE Serving on http://:::9283 2024-09-14T03:46:29.496 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:46:29 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:46:29] ENGINE Bus STARTED 2024-09-14T03:46:30.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:29 smithi094 ceph-mon[31053]: pgmap v51: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:30.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:29 smithi070 ceph-mon[39288]: pgmap v51: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:30.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:29 smithi070 ceph-mon[27495]: pgmap v51: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:31.093 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:46:31.093 INFO:teuthology.orchestra.run.smithi070.stdout:{"epoch":8,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":24304,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.94:6833/3620660683","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6832","nonce":3620660683},{"type":"v1","addr":"172.21.15.94:6833","nonce":3620660683}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":8,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-14T03:44:03.568995+0000","modified":"2024-09-14T03:44:16.900718+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":1,"in":[0],"up":{"mds_0":24323},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24323":{"gid":24323,"name":"a","rank":0,"incarnation":4,"state":"up:active","state_seq":6,"addr":"172.21.15.70:6835/421917097","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6834","nonce":421917097},{"type":"v1","addr":"172.21.15.70:6835","nonce":421917097}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","standby_count_wanted":0},"id":1}]} 2024-09-14T03:46:31.093 INFO:teuthology.orchestra.run.smithi070.stderr:dumped fsmap epoch 8 2024-09-14T03:46:31.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:31 smithi094 ceph-mon[31053]: pgmap v52: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:31.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:31 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/371987593' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:46:31.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:31 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:31.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:31 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:31.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:31 smithi070 ceph-mon[27495]: pgmap v52: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:31.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:31 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/371987593' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:46:31.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:31 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:31.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:31 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:31.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:31 smithi070 ceph-mon[39288]: pgmap v52: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:31.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:31 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/371987593' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:46:31.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:31 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:31.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:31 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:31.941 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T03:46:32.727 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:46:33.108 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:46:32 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a[80300]: ts=2024-09-14T03:46:32.816Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002068606s 2024-09-14T03:46:33.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:33 smithi070 ceph-mon[27495]: pgmap v53: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:33.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:33 smithi070 ceph-mon[39288]: pgmap v53: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:34.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:33 smithi094 ceph-mon[31053]: pgmap v53: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:34.937 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:46:34.937 INFO:teuthology.orchestra.run.smithi070.stdout:{"epoch":68,"fsid":"b6380d42-724a-11ef-bceb-c7b262605968","created":"2024-09-14T03:38:29.406762+0000","modified":"2024-09-14T03:45:20.460680+0000","last_up_change":"2024-09-14T03:43:34.130380+0000","last_in_change":"2024-09-14T03:43:17.652311+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:41:38.745294+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-14T03:43:57.402183+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-14T03:44:00.803078+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}},{"pool":4,"pool_name":"rbd","create_time":"2024-09-14T03:45:08.059178+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":67,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"a7fabda1-9e60-42f2-af60-eec626181688","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":62,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6802","nonce":3822384296},{"type":"v1","addr":"172.21.15.70:6803","nonce":3822384296}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6804","nonce":3822384296},{"type":"v1","addr":"172.21.15.70:6805","nonce":3822384296}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6808","nonce":3822384296},{"type":"v1","addr":"172.21.15.70:6809","nonce":3822384296}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6806","nonce":3822384296},{"type":"v1","addr":"172.21.15.70:6807","nonce":3822384296}]},"public_addr":"172.21.15.70:6803/3822384296","cluster_addr":"172.21.15.70:6805/3822384296","heartbeat_back_addr":"172.21.15.70:6809/3822384296","heartbeat_front_addr":"172.21.15.70:6807/3822384296","state":["exists","up"]},{"osd":1,"uuid":"f5b9ef91-59af-4ef0-a101-1c00ba6007ad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":62,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6810","nonce":3986345052},{"type":"v1","addr":"172.21.15.70:6811","nonce":3986345052}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6812","nonce":3986345052},{"type":"v1","addr":"172.21.15.70:6813","nonce":3986345052}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6816","nonce":3986345052},{"type":"v1","addr":"172.21.15.70:6817","nonce":3986345052}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6814","nonce":3986345052},{"type":"v1","addr":"172.21.15.70:6815","nonce":3986345052}]},"public_addr":"172.21.15.70:6811/3986345052","cluster_addr":"172.21.15.70:6813/3986345052","heartbeat_back_addr":"172.21.15.70:6817/3986345052","heartbeat_front_addr":"172.21.15.70:6815/3986345052","state":["exists","up"]},{"osd":2,"uuid":"0dc07495-c6a5-4762-8f63-575eb0fe1671","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6818","nonce":2374764370},{"type":"v1","addr":"172.21.15.70:6819","nonce":2374764370}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6820","nonce":2374764370},{"type":"v1","addr":"172.21.15.70:6821","nonce":2374764370}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6824","nonce":2374764370},{"type":"v1","addr":"172.21.15.70:6825","nonce":2374764370}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6822","nonce":2374764370},{"type":"v1","addr":"172.21.15.70:6823","nonce":2374764370}]},"public_addr":"172.21.15.70:6819/2374764370","cluster_addr":"172.21.15.70:6821/2374764370","heartbeat_back_addr":"172.21.15.70:6825/2374764370","heartbeat_front_addr":"172.21.15.70:6823/2374764370","state":["exists","up"]},{"osd":3,"uuid":"583e5165-ea10-4d03-b731-36cfb1108362","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":62,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6826","nonce":2294516176},{"type":"v1","addr":"172.21.15.70:6827","nonce":2294516176}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6828","nonce":2294516176},{"type":"v1","addr":"172.21.15.70:6829","nonce":2294516176}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6832","nonce":2294516176},{"type":"v1","addr":"172.21.15.70:6833","nonce":2294516176}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6830","nonce":2294516176},{"type":"v1","addr":"172.21.15.70:6831","nonce":2294516176}]},"public_addr":"172.21.15.70:6827/2294516176","cluster_addr":"172.21.15.70:6829/2294516176","heartbeat_back_addr":"172.21.15.70:6833/2294516176","heartbeat_front_addr":"172.21.15.70:6831/2294516176","state":["exists","up"]},{"osd":4,"uuid":"33177119-9632-431c-9180-9d27e7bc0c42","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":62,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6800","nonce":887215534},{"type":"v1","addr":"172.21.15.94:6801","nonce":887215534}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6802","nonce":887215534},{"type":"v1","addr":"172.21.15.94:6803","nonce":887215534}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6806","nonce":887215534},{"type":"v1","addr":"172.21.15.94:6807","nonce":887215534}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6804","nonce":887215534},{"type":"v1","addr":"172.21.15.94:6805","nonce":887215534}]},"public_addr":"172.21.15.94:6801/887215534","cluster_addr":"172.21.15.94:6803/887215534","heartbeat_back_addr":"172.21.15.94:6807/887215534","heartbeat_front_addr":"172.21.15.94:6805/887215534","state":["exists","up"]},{"osd":5,"uuid":"eb037130-5a05-42e0-947e-09bfba8fac22","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6808","nonce":1692579524},{"type":"v1","addr":"172.21.15.94:6809","nonce":1692579524}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6810","nonce":1692579524},{"type":"v1","addr":"172.21.15.94:6811","nonce":1692579524}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6814","nonce":1692579524},{"type":"v1","addr":"172.21.15.94:6815","nonce":1692579524}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6812","nonce":1692579524},{"type":"v1","addr":"172.21.15.94:6813","nonce":1692579524}]},"public_addr":"172.21.15.94:6809/1692579524","cluster_addr":"172.21.15.94:6811/1692579524","heartbeat_back_addr":"172.21.15.94:6815/1692579524","heartbeat_front_addr":"172.21.15.94:6813/1692579524","state":["exists","up"]},{"osd":6,"uuid":"f5dca4b5-8906-4eac-84e4-fcb7d1a29674","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":62,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6816","nonce":1107979948},{"type":"v1","addr":"172.21.15.94:6817","nonce":1107979948}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6818","nonce":1107979948},{"type":"v1","addr":"172.21.15.94:6819","nonce":1107979948}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6822","nonce":1107979948},{"type":"v1","addr":"172.21.15.94:6823","nonce":1107979948}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6820","nonce":1107979948},{"type":"v1","addr":"172.21.15.94:6821","nonce":1107979948}]},"public_addr":"172.21.15.94:6817/1107979948","cluster_addr":"172.21.15.94:6819/1107979948","heartbeat_back_addr":"172.21.15.94:6823/1107979948","heartbeat_front_addr":"172.21.15.94:6821/1107979948","state":["exists","up"]},{"osd":7,"uuid":"86e3ef7d-6ab8-47c3-a4bb-db923af2b49b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6824","nonce":2545649733},{"type":"v1","addr":"172.21.15.94:6825","nonce":2545649733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6826","nonce":2545649733},{"type":"v1","addr":"172.21.15.94:6827","nonce":2545649733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6830","nonce":2545649733},{"type":"v1","addr":"172.21.15.94:6831","nonce":2545649733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6828","nonce":2545649733},{"type":"v1","addr":"172.21.15.94:6829","nonce":2545649733}]},"public_addr":"172.21.15.94:6825/2545649733","cluster_addr":"172.21.15.94:6827/2545649733","heartbeat_back_addr":"172.21.15.94:6831/2545649733","heartbeat_front_addr":"172.21.15.94:6829/2545649733","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:40:47.681447+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:41:11.801118+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:41:35.764497+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:42:00.424802+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:42:21.618849+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:42:45.384437+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:43:09.350263+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:43:32.472815+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.18","mappings":[{"from":7,"to":2}]},{"pgid":"3.3","mappings":[{"from":0,"to":1}]},{"pgid":"3.a","mappings":[{"from":6,"to":2}]},{"pgid":"3.b","mappings":[{"from":0,"to":2},{"from":4,"to":1}]},{"pgid":"3.26","mappings":[{"from":4,"to":3}]},{"pgid":"3.36","mappings":[{"from":5,"to":2}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.70:6801/3164926893":"2024-09-15T03:45:06.535242+0000","172.21.15.70:6800/3164926893":"2024-09-15T03:45:06.535242+0000","172.21.15.70:0/4203647763":"2024-09-15T03:39:01.589983+0000","172.21.15.70:6800/2254107694":"2024-09-15T03:39:01.589983+0000","172.21.15.70:6801/2254107694":"2024-09-15T03:39:01.589983+0000","172.21.15.70:0/2447056811":"2024-09-15T03:39:01.589983+0000","172.21.15.70:0/22543101":"2024-09-15T03:39:01.589983+0000","172.21.15.70:0/3134643770":"2024-09-15T03:39:24.448616+0000","172.21.15.70:0/3005166882":"2024-09-15T03:39:24.448616+0000","172.21.15.70:0/2339853520":"2024-09-15T03:45:06.535242+0000","172.21.15.70:0/4275249763":"2024-09-15T03:45:06.535242+0000","172.21.15.70:0/3199751552":"2024-09-15T03:45:06.535242+0000","172.21.15.70:6800/673916264":"2024-09-15T03:39:24.448616+0000","172.21.15.70:6801/673916264":"2024-09-15T03:39:24.448616+0000","172.21.15.70:0/3067035520":"2024-09-15T03:45:06.535242+0000","172.21.15.70:0/2978576061":"2024-09-15T03:39:24.448616+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":4,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:46:35.874 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:35 smithi070 ceph-mon[27495]: pgmap v54: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:35.875 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:35 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/3148098315' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:46:35.875 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:35 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:35.875 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:35 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:35.875 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:35 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:46:35.875 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:35 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:46:35.875 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:35 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:35.875 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:35 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:46:35.875 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:35 smithi070 ceph-mon[39288]: pgmap v54: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:35.875 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:35 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/3148098315' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:46:35.875 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:35 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:35.875 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:35 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:35.876 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:35 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:46:35.876 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:35 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:46:35.876 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:35 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:35.876 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:35 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:46:35.906 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph fs dump --format=json 2024-09-14T03:46:35.962 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:35 smithi094 ceph-mon[31053]: pgmap v54: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:35.963 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:35 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/3148098315' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:46:35.963 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:35 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:35.963 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:35 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:35.963 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:35 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:46:35.963 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:35 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:46:35.963 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:35 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:35.963 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:35 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:46:36.535 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:46:36 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:46:36] "GET /metrics HTTP/1.1" 200 236468 "" "Prometheus/2.43.0" 2024-09-14T03:46:36.718 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:46:36.957 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:36 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:46:36.960 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:36 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:46:36.968 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:36 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:46:37.976 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:37 smithi070 ceph-mon[27495]: pgmap v55: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:37.977 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:37 smithi070 ceph-mon[39288]: pgmap v55: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:38.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:37 smithi094 ceph-mon[31053]: pgmap v55: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:38.695 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T03:46:38.695 INFO:teuthology.orchestra.run.smithi070.stdout:{"epoch":8,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":24304,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.94:6833/3620660683","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6832","nonce":3620660683},{"type":"v1","addr":"172.21.15.94:6833","nonce":3620660683}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":8,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-14T03:44:03.568995+0000","modified":"2024-09-14T03:44:16.900718+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":1,"in":[0],"up":{"mds_0":24323},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24323":{"gid":24323,"name":"a","rank":0,"incarnation":4,"state":"up:active","state_seq":6,"addr":"172.21.15.70:6835/421917097","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6834","nonce":421917097},{"type":"v1","addr":"172.21.15.70:6835","nonce":421917097}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","standby_count_wanted":0},"id":1}]} 2024-09-14T03:46:38.695 INFO:teuthology.orchestra.run.smithi070.stderr:dumped fsmap epoch 8 2024-09-14T03:46:39.661 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 8, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-09-14T03:44:03.568995+0000', 'modified': '2024-09-14T03:44:16.900718+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 24323}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_24323': {'gid': 24323, 'name': 'a', 'rank': 0, 'incarnation': 4, 'state': 'up:active', 'state_seq': 6, 'addr': '172.21.15.70:6835/421917097', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.70:6834', 'nonce': 421917097}, {'type': 'v1', 'addr': '172.21.15.70:6835', 'nonce': 421917097}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138320759226367, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}}, 'data_pools': [3], 'metadata_pool': 2, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'standby_count_wanted': 0} 2024-09-14T03:46:39.661 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 1/1 2024-09-14T03:46:39.661 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph tell mds.a status 2024-09-14T03:46:39.919 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:39 smithi070 ceph-mon[39288]: pgmap v56: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:46:39.920 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:39 smithi070 ceph-mon[39288]: mgrmap e21: y(active, since 92s), standbys: x 2024-09-14T03:46:39.920 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:39 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/763579065' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:46:39.920 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:39 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:39.920 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:39 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:39.920 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:39 smithi070 ceph-mon[27495]: pgmap v56: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:46:39.920 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:39 smithi070 ceph-mon[27495]: mgrmap e21: y(active, since 92s), standbys: x 2024-09-14T03:46:39.920 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:39 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/763579065' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:46:39.920 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:39 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:39.920 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:39 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:39.949 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:39 smithi094 ceph-mon[31053]: pgmap v56: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:46:39.949 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:39 smithi094 ceph-mon[31053]: mgrmap e21: y(active, since 92s), standbys: x 2024-09-14T03:46:39.950 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:39 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/763579065' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-14T03:46:39.950 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:39 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:39.950 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:39 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:40.585 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:46:41.902 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:46:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:46:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:46:42.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:41 smithi070 ceph-mon[27495]: pgmap v57: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:46:42.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:41 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:42.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:41 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:42.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:41 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:46:42.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:41 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:46:42.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:41 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:42.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:41 smithi070 ceph-mon[39288]: pgmap v57: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:46:42.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:41 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:42.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:41 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:42.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:41 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:46:42.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:41 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:46:42.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:41 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:42.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:41 smithi094 ceph-mon[31053]: pgmap v57: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:46:42.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:41 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:42.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:41 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:42.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:41 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:46:42.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:41 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:46:42.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:41 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:42.676 INFO:teuthology.orchestra.run.smithi070.stderr:2024-09-14T03:46:42.657+0000 7f2579ffb640 0 client.14772 ms_handle_reset on v2:172.21.15.70:6834/421917097 2024-09-14T03:46:42.676 INFO:teuthology.orchestra.run.smithi070.stderr:2024-09-14T03:46:42.674+0000 7f2579ffb640 0 client.14778 ms_handle_reset on v2:172.21.15.70:6834/421917097 2024-09-14T03:46:42.676 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:46:42.676 INFO:teuthology.orchestra.run.smithi070.stdout: "cluster_fsid": "b6380d42-724a-11ef-bceb-c7b262605968", 2024-09-14T03:46:42.676 INFO:teuthology.orchestra.run.smithi070.stdout: "whoami": 0, 2024-09-14T03:46:42.676 INFO:teuthology.orchestra.run.smithi070.stdout: "id": 24323, 2024-09-14T03:46:42.676 INFO:teuthology.orchestra.run.smithi070.stdout: "want_state": "up:active", 2024-09-14T03:46:42.676 INFO:teuthology.orchestra.run.smithi070.stdout: "state": "up:active", 2024-09-14T03:46:42.677 INFO:teuthology.orchestra.run.smithi070.stdout: "fs_name": "cephfs", 2024-09-14T03:46:42.677 INFO:teuthology.orchestra.run.smithi070.stdout: "rank_uptime": 158.80417706899999, 2024-09-14T03:46:42.677 INFO:teuthology.orchestra.run.smithi070.stdout: "mdsmap_epoch": 8, 2024-09-14T03:46:42.677 INFO:teuthology.orchestra.run.smithi070.stdout: "osdmap_epoch": 68, 2024-09-14T03:46:42.677 INFO:teuthology.orchestra.run.smithi070.stdout: "osdmap_epoch_barrier": 58, 2024-09-14T03:46:42.677 INFO:teuthology.orchestra.run.smithi070.stdout: "uptime": 176.22919250800001 2024-09-14T03:46:42.677 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:46:43.499 INFO:tasks.cephadm:Setup complete, yielding 2024-09-14T03:46:43.500 INFO:teuthology.run_tasks:Running task print... 2024-09-14T03:46:43.508 INFO:teuthology.task.print:**** done end installing quincy cephadm ... 2024-09-14T03:46:43.509 INFO:teuthology.run_tasks:Running task print... 2024-09-14T03:46:43.516 INFO:teuthology.task.print:**** done start cephadm.shell ceph config set mgr... 2024-09-14T03:46:43.516 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-14T03:46:43.525 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi070.front.sepia.ceph.com 2024-09-14T03:46:43.525 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- bash -c 'ceph config set mgr mgr/cephadm/use_repo_digest true --force' 2024-09-14T03:46:43.773 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:46:44.175 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:43 smithi070 ceph-mon[27495]: pgmap v58: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:46:44.175 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:43 smithi070 ceph-mon[39288]: pgmap v58: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:46:44.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:43 smithi094 ceph-mon[31053]: pgmap v58: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:46:45.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:46:45 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:46:45] ENGINE Bus STOPPING 2024-09-14T03:46:46.227 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:45 smithi070 ceph-mon[27495]: pgmap v59: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:46:46.227 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:45 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/99806039' entity='client.admin' 2024-09-14T03:46:46.227 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:45 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:46:46.227 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:45 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:46:46.228 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:45 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:46:46.228 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:45 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:46.228 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:46:45 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:46:45] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:46:46.228 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:46:45 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:46:45] ENGINE Bus STOPPED 2024-09-14T03:46:46.228 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:46:45 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:46:45] ENGINE Bus STARTING 2024-09-14T03:46:46.228 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:46:46 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:46:46] ENGINE Serving on http://:::9283 2024-09-14T03:46:46.228 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:46:46 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:46:46] ENGINE Bus STARTED 2024-09-14T03:46:46.228 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:45 smithi070 ceph-mon[39288]: pgmap v59: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:46:46.229 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:45 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/99806039' entity='client.admin' 2024-09-14T03:46:46.229 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:45 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:46:46.229 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:45 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:46:46.229 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:45 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:46:46.229 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:45 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:46.250 INFO:teuthology.run_tasks:Running task print... 2024-09-14T03:46:46.259 INFO:teuthology.task.print:**** done cephadm.shell ceph config set mgr... 2024-09-14T03:46:46.260 INFO:teuthology.run_tasks:Running task print... 2024-09-14T03:46:46.268 INFO:teuthology.task.print:**** done start telemetry quincy... 2024-09-14T03:46:46.268 INFO:teuthology.run_tasks:Running task workunit... 2024-09-14T03:46:46.282 INFO:tasks.workunit:Pulling workunits from ref 1c941a66c1b420259b8fc5ac19e792dbbf60aded 2024-09-14T03:46:46.282 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-14T03:46:46.283 DEBUG:teuthology.orchestra.run.smithi070:> stat -- /home/ubuntu/cephtest/mnt.0 2024-09-14T03:46:46.283 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:45 smithi094 ceph-mon[31053]: pgmap v59: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:46:46.283 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:45 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/99806039' entity='client.admin' 2024-09-14T03:46:46.284 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:45 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:46:46.284 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:45 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:46:46.284 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:45 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:46:46.284 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:45 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:46:46.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T03:46:46.299 INFO:teuthology.orchestra.run.smithi070.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-09-14T03:46:46.300 DEBUG:teuthology.orchestra.run.smithi070:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-09-14T03:46:46.355 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-09-14T03:46:46.355 DEBUG:teuthology.orchestra.run.smithi070:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-09-14T03:46:46.410 INFO:tasks.workunit:timeout=3h 2024-09-14T03:46:46.410 INFO:tasks.workunit:cleanup=True 2024-09-14T03:46:46.411 DEBUG:teuthology.orchestra.run.smithi070:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 && git checkout 1c941a66c1b420259b8fc5ac19e792dbbf60aded 2024-09-14T03:46:46.467 INFO:tasks.workunit.client.0.smithi070.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-09-14T03:46:46.535 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:46:46 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:46:46] "GET /metrics HTTP/1.1" 200 236464 "" "Prometheus/2.43.0" 2024-09-14T03:46:48.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:47 smithi094 ceph-mon[31053]: pgmap v60: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:46:48.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:47 smithi070 ceph-mon[27495]: pgmap v60: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:46:48.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:47 smithi070 ceph-mon[39288]: pgmap v60: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:46:50.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:49 smithi094 ceph-mon[31053]: pgmap v61: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:46:50.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:49 smithi070 ceph-mon[27495]: pgmap v61: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:46:50.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:49 smithi070 ceph-mon[39288]: pgmap v61: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:46:51.915 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:46:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:46:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:46:52.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:51 smithi094 ceph-mon[31053]: pgmap v62: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:52.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:51 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:46:52.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:51 smithi070 ceph-mon[39288]: pgmap v62: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:52.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:51 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:46:52.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:51 smithi070 ceph-mon[27495]: pgmap v62: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:52.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:51 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:46:54.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:53 smithi094 ceph-mon[31053]: pgmap v63: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:54.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:53 smithi070 ceph-mon[27495]: pgmap v63: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:54.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:53 smithi070 ceph-mon[39288]: pgmap v63: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:56.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:55 smithi094 ceph-mon[31053]: pgmap v64: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:56.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:55 smithi070 ceph-mon[27495]: pgmap v64: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:56.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:55 smithi070 ceph-mon[39288]: pgmap v64: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:56.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:46:56 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:46:56] "GET /metrics HTTP/1.1" 200 236465 "" "Prometheus/2.43.0" 2024-09-14T03:46:58.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:57 smithi094 ceph-mon[31053]: pgmap v65: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:58.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:57 smithi070 ceph-mon[39288]: pgmap v65: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:46:58.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:57 smithi070 ceph-mon[27495]: pgmap v65: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:00.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:46:59 smithi094 ceph-mon[31053]: pgmap v66: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:00.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:46:59 smithi070 ceph-mon[27495]: pgmap v66: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:00.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:46:59 smithi070 ceph-mon[39288]: pgmap v66: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:01.927 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:47:01 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:47:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:47:02.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:01 smithi094 ceph-mon[31053]: pgmap v67: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:02.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:01 smithi070 ceph-mon[27495]: pgmap v67: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:02.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:01 smithi070 ceph-mon[39288]: pgmap v67: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:04.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:03 smithi094 ceph-mon[31053]: pgmap v68: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:04.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:03 smithi070 ceph-mon[27495]: pgmap v68: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:04.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:03 smithi070 ceph-mon[39288]: pgmap v68: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:06.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:05 smithi094 ceph-mon[31053]: pgmap v69: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:06.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:05 smithi070 ceph-mon[27495]: pgmap v69: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:06.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:05 smithi070 ceph-mon[39288]: pgmap v69: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:06.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:47:06] "GET /metrics HTTP/1.1" 200 236465 "" "Prometheus/2.43.0" 2024-09-14T03:47:07.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:06 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:47:07.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:06 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:47:07.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:06 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:47:08.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:07 smithi094 ceph-mon[31053]: pgmap v70: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:08.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:07 smithi070 ceph-mon[39288]: pgmap v70: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:08.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:07 smithi070 ceph-mon[27495]: pgmap v70: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:10.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:09 smithi094 ceph-mon[31053]: pgmap v71: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:10.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:09 smithi070 ceph-mon[27495]: pgmap v71: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:10.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:09 smithi070 ceph-mon[39288]: pgmap v71: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:11.944 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:47:11 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:47:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:47:12.441 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:11 smithi070 ceph-mon[39288]: pgmap v72: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:12.441 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:11 smithi094 ceph-mon[31053]: pgmap v72: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:12.442 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:11 smithi070 ceph-mon[27495]: pgmap v72: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:14.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:13 smithi094 ceph-mon[31053]: pgmap v73: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:14.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:13 smithi070 ceph-mon[27495]: pgmap v73: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:14.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:13 smithi070 ceph-mon[39288]: pgmap v73: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:16.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:15 smithi094 ceph-mon[31053]: pgmap v74: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:16.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:15 smithi070 ceph-mon[27495]: pgmap v74: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:16.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:15 smithi070 ceph-mon[39288]: pgmap v74: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:16.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:16 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:47:16] "GET /metrics HTTP/1.1" 200 236463 "" "Prometheus/2.43.0" 2024-09-14T03:47:18.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:17 smithi094 ceph-mon[31053]: pgmap v75: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:18.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:17 smithi070 ceph-mon[27495]: pgmap v75: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:18.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:17 smithi070 ceph-mon[39288]: pgmap v75: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:20.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:19 smithi094 ceph-mon[31053]: pgmap v76: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:20.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:19 smithi070 ceph-mon[27495]: pgmap v76: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:20.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:19 smithi070 ceph-mon[39288]: pgmap v76: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:21.952 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:47:21 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:47:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:47:22.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:21 smithi094 ceph-mon[31053]: pgmap v77: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:22.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:21 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:47:22.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:21 smithi070 ceph-mon[27495]: pgmap v77: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:22.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:21 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:47:22.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:21 smithi070 ceph-mon[39288]: pgmap v77: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:22.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:21 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:47:24.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:23 smithi070 ceph-mon[27495]: pgmap v78: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:24.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:23 smithi070 ceph-mon[39288]: pgmap v78: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:24.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:23 smithi094 ceph-mon[31053]: pgmap v78: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:26.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:25 smithi094 ceph-mon[31053]: pgmap v79: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:26.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:25 smithi070 ceph-mon[27495]: pgmap v79: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:26.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:25 smithi070 ceph-mon[39288]: pgmap v79: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:26.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:26 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:47:26] "GET /metrics HTTP/1.1" 200 236458 "" "Prometheus/2.43.0" 2024-09-14T03:47:28.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:27 smithi094 ceph-mon[31053]: pgmap v80: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:28.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:27 smithi070 ceph-mon[27495]: pgmap v80: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:28.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:27 smithi070 ceph-mon[39288]: pgmap v80: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:30.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:29 smithi094 ceph-mon[31053]: pgmap v81: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:30.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:29 smithi070 ceph-mon[39288]: pgmap v81: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:30.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:29 smithi070 ceph-mon[27495]: pgmap v81: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:31.964 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:47:31 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:47:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:47:32.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:31 smithi094 ceph-mon[31053]: pgmap v82: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:32.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:31 smithi070 ceph-mon[27495]: pgmap v82: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:32.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:31 smithi070 ceph-mon[39288]: pgmap v82: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:34.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:33 smithi094 ceph-mon[31053]: pgmap v83: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:34.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:33 smithi070 ceph-mon[27495]: pgmap v83: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:34.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:33 smithi070 ceph-mon[39288]: pgmap v83: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:36.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:35 smithi094 ceph-mon[31053]: pgmap v84: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:36.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:35 smithi070 ceph-mon[39288]: pgmap v84: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:36.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:35 smithi070 ceph-mon[27495]: pgmap v84: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:36.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:36 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:47:36] "GET /metrics HTTP/1.1" 200 236458 "" "Prometheus/2.43.0" 2024-09-14T03:47:37.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:36 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:47:37.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:36 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:47:37.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:36 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:47:38.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:37 smithi094 ceph-mon[31053]: pgmap v85: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:38.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:37 smithi070 ceph-mon[27495]: pgmap v85: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:38.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:37 smithi070 ceph-mon[39288]: pgmap v85: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:40.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:39 smithi094 ceph-mon[31053]: pgmap v86: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:40.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:39 smithi070 ceph-mon[27495]: pgmap v86: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:40.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:39 smithi070 ceph-mon[39288]: pgmap v86: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:41.978 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:47:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:47:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:47:42.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:41 smithi094 ceph-mon[31053]: pgmap v87: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:42.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:41 smithi070 ceph-mon[27495]: pgmap v87: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:42.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:41 smithi070 ceph-mon[39288]: pgmap v87: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:43.566 INFO:tasks.workunit.client.0.smithi070.stderr:Note: switching to '1c941a66c1b420259b8fc5ac19e792dbbf60aded'. 2024-09-14T03:47:43.567 INFO:tasks.workunit.client.0.smithi070.stderr: 2024-09-14T03:47:43.567 INFO:tasks.workunit.client.0.smithi070.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2024-09-14T03:47:43.567 INFO:tasks.workunit.client.0.smithi070.stderr:changes and commit them, and you can discard any commits you make in this 2024-09-14T03:47:43.567 INFO:tasks.workunit.client.0.smithi070.stderr:state without impacting any branches by switching back to a branch. 2024-09-14T03:47:43.567 INFO:tasks.workunit.client.0.smithi070.stderr: 2024-09-14T03:47:43.567 INFO:tasks.workunit.client.0.smithi070.stderr:If you want to create a new branch to retain commits you create, you may 2024-09-14T03:47:43.567 INFO:tasks.workunit.client.0.smithi070.stderr:do so (now or later) by using -c with the switch command. Example: 2024-09-14T03:47:43.567 INFO:tasks.workunit.client.0.smithi070.stderr: 2024-09-14T03:47:43.567 INFO:tasks.workunit.client.0.smithi070.stderr: git switch -c 2024-09-14T03:47:43.567 INFO:tasks.workunit.client.0.smithi070.stderr: 2024-09-14T03:47:43.567 INFO:tasks.workunit.client.0.smithi070.stderr:Or undo this operation with: 2024-09-14T03:47:43.567 INFO:tasks.workunit.client.0.smithi070.stderr: 2024-09-14T03:47:43.568 INFO:tasks.workunit.client.0.smithi070.stderr: git switch - 2024-09-14T03:47:43.568 INFO:tasks.workunit.client.0.smithi070.stderr: 2024-09-14T03:47:43.568 INFO:tasks.workunit.client.0.smithi070.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2024-09-14T03:47:43.568 INFO:tasks.workunit.client.0.smithi070.stderr: 2024-09-14T03:47:43.568 INFO:tasks.workunit.client.0.smithi070.stderr:HEAD is now at 1c941a66c1b Merge pull request #54777 from cbodley/wip-rgw-meta-handlers 2024-09-14T03:47:43.569 DEBUG:teuthology.orchestra.run.smithi070:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-09-14T03:47:43.634 INFO:tasks.workunit.client.0.smithi070.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-14T03:47:43.636 INFO:tasks.workunit.client.0.smithi070.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-14T03:47:43.636 INFO:tasks.workunit.client.0.smithi070.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-14T03:47:44.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:43 smithi094 ceph-mon[31053]: pgmap v88: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:44.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:43 smithi070 ceph-mon[39288]: pgmap v88: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:44.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:43 smithi070 ceph-mon[27495]: pgmap v88: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:44.375 INFO:tasks.workunit.client.0.smithi070.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-14T03:47:44.451 INFO:tasks.workunit.client.0.smithi070.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-14T03:47:44.493 INFO:tasks.workunit.client.0.smithi070.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-14T03:47:44.495 INFO:tasks.workunit.client.0.smithi070.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-14T03:47:44.495 INFO:tasks.workunit.client.0.smithi070.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-14T03:47:44.534 INFO:tasks.workunit.client.0.smithi070.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-14T03:47:44.538 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2024-09-14T03:47:44.538 DEBUG:teuthology.orchestra.run.smithi070:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-09-14T03:47:44.595 INFO:tasks.workunit:Running workunits matching test_telemetry_quincy.sh on client.0... 2024-09-14T03:47:44.596 INFO:tasks.workunit:Running workunit test_telemetry_quincy.sh... 2024-09-14T03:47:44.596 DEBUG:teuthology.orchestra.run.smithi070:workunit test test_telemetry_quincy.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=1c941a66c1b420259b8fc5ac19e792dbbf60aded TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/test_telemetry_quincy.sh 2024-09-14T03:47:44.665 INFO:tasks.workunit.client.0.smithi070.stderr:+ ceph config set mgr mgr/telemetry/channel_ident true 2024-09-14T03:47:45.034 INFO:tasks.workunit.client.0.smithi070.stderr:+ ceph config set mgr mgr/telemetry/organization ceph-qa 2024-09-14T03:47:45.285 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:44 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:47:44] ENGINE Bus STOPPING 2024-09-14T03:47:45.285 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:44 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:47:44] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:47:45.286 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:44 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:47:44] ENGINE Bus STOPPED 2024-09-14T03:47:45.286 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:44 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:47:44] ENGINE Bus STARTING 2024-09-14T03:47:45.286 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:47:45] ENGINE Serving on http://:::9283 2024-09-14T03:47:45.286 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:47:45] ENGINE Bus STARTED 2024-09-14T03:47:45.401 INFO:tasks.workunit.client.0.smithi070.stderr:+ ceph config set mgr mgr/telemetry/description 'upgrade test cluster' 2024-09-14T03:47:45.550 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:47:45] ENGINE Bus STOPPING 2024-09-14T03:47:45.551 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:47:45] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:47:45.760 INFO:tasks.workunit.client.0.smithi070.stderr:+ ceph telemetry preview 2024-09-14T03:47:45.906 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:47:45] ENGINE Bus STOPPED 2024-09-14T03:47:45.906 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:47:45] ENGINE Bus STARTING 2024-09-14T03:47:45.907 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:47:45] ENGINE Serving on http://:::9283 2024-09-14T03:47:45.907 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:47:45] ENGINE Bus STARTED 2024-09-14T03:47:45.907 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:47:45] ENGINE Bus STOPPING 2024-09-14T03:47:46.159 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-mon[39288]: pgmap v89: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:46.160 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/498712575' entity='client.admin' 2024-09-14T03:47:46.160 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:46.160 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/2502820960' entity='client.admin' 2024-09-14T03:47:46.160 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:46.160 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:46.160 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:46.160 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:46.160 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:46.160 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:46.160 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:46.160 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/3890719547' entity='client.admin' 2024-09-14T03:47:46.160 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:46.160 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:46.161 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:46.161 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:46.161 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-mon[27495]: pgmap v89: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:46.161 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/498712575' entity='client.admin' 2024-09-14T03:47:46.161 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:46.161 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/2502820960' entity='client.admin' 2024-09-14T03:47:46.161 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:46.161 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:46.161 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:46.161 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:46.161 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:46.162 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:46.162 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:46.162 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/3890719547' entity='client.admin' 2024-09-14T03:47:46.162 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:46.162 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:46.162 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:46.162 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:45 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:46.205 INFO:tasks.workunit.client.0.smithi070.stdout:{ 2024-09-14T03:47:46.205 INFO:tasks.workunit.client.0.smithi070.stdout: "balancer": { 2024-09-14T03:47:46.205 INFO:tasks.workunit.client.0.smithi070.stdout: "active": true, 2024-09-14T03:47:46.205 INFO:tasks.workunit.client.0.smithi070.stdout: "mode": "" 2024-09-14T03:47:46.206 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.206 INFO:tasks.workunit.client.0.smithi070.stdout: "channels": [ 2024-09-14T03:47:46.206 INFO:tasks.workunit.client.0.smithi070.stdout: "basic", 2024-09-14T03:47:46.206 INFO:tasks.workunit.client.0.smithi070.stdout: "crash", 2024-09-14T03:47:46.206 INFO:tasks.workunit.client.0.smithi070.stdout: "device", 2024-09-14T03:47:46.206 INFO:tasks.workunit.client.0.smithi070.stdout: "ident" 2024-09-14T03:47:46.206 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:46.206 INFO:tasks.workunit.client.0.smithi070.stdout: "channels_available": [ 2024-09-14T03:47:46.206 INFO:tasks.workunit.client.0.smithi070.stdout: "basic", 2024-09-14T03:47:46.206 INFO:tasks.workunit.client.0.smithi070.stdout: "ident", 2024-09-14T03:47:46.206 INFO:tasks.workunit.client.0.smithi070.stdout: "crash", 2024-09-14T03:47:46.206 INFO:tasks.workunit.client.0.smithi070.stdout: "device", 2024-09-14T03:47:46.207 INFO:tasks.workunit.client.0.smithi070.stdout: "perf" 2024-09-14T03:47:46.207 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:46.207 INFO:tasks.workunit.client.0.smithi070.stdout: "collections_available": [ 2024-09-14T03:47:46.207 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_base", 2024-09-14T03:47:46.207 INFO:tasks.workunit.client.0.smithi070.stdout: "device_base", 2024-09-14T03:47:46.207 INFO:tasks.workunit.client.0.smithi070.stdout: "crash_base", 2024-09-14T03:47:46.207 INFO:tasks.workunit.client.0.smithi070.stdout: "ident_base", 2024-09-14T03:47:46.207 INFO:tasks.workunit.client.0.smithi070.stdout: "perf_perf", 2024-09-14T03:47:46.207 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_mds_metadata", 2024-09-14T03:47:46.207 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_pool_usage", 2024-09-14T03:47:46.207 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_usage_by_class", 2024-09-14T03:47:46.207 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_rook_v01", 2024-09-14T03:47:46.207 INFO:tasks.workunit.client.0.smithi070.stdout: "perf_memory_metrics", 2024-09-14T03:47:46.207 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_pool_options_bluestore" 2024-09-14T03:47:46.208 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:46.208 INFO:tasks.workunit.client.0.smithi070.stdout: "collections_opted_in": [ 2024-09-14T03:47:46.208 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_base", 2024-09-14T03:47:46.208 INFO:tasks.workunit.client.0.smithi070.stdout: "device_base", 2024-09-14T03:47:46.208 INFO:tasks.workunit.client.0.smithi070.stdout: "crash_base", 2024-09-14T03:47:46.208 INFO:tasks.workunit.client.0.smithi070.stdout: "ident_base", 2024-09-14T03:47:46.208 INFO:tasks.workunit.client.0.smithi070.stdout: "perf_perf", 2024-09-14T03:47:46.208 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_mds_metadata", 2024-09-14T03:47:46.208 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_pool_usage", 2024-09-14T03:47:46.208 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_usage_by_class", 2024-09-14T03:47:46.208 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_rook_v01", 2024-09-14T03:47:46.208 INFO:tasks.workunit.client.0.smithi070.stdout: "perf_memory_metrics", 2024-09-14T03:47:46.208 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_pool_options_bluestore" 2024-09-14T03:47:46.209 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:46.209 INFO:tasks.workunit.client.0.smithi070.stdout: "config": { 2024-09-14T03:47:46.209 INFO:tasks.workunit.client.0.smithi070.stdout: "active_changed": [ 2024-09-14T03:47:46.209 INFO:tasks.workunit.client.0.smithi070.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-14T03:47:46.209 INFO:tasks.workunit.client.0.smithi070.stdout: "auth_debug", 2024-09-14T03:47:46.209 INFO:tasks.workunit.client.0.smithi070.stdout: "auth_mon_ticket_ttl", 2024-09-14T03:47:46.209 INFO:tasks.workunit.client.0.smithi070.stdout: "auth_service_ticket_ttl", 2024-09-14T03:47:46.209 INFO:tasks.workunit.client.0.smithi070.stdout: "bdev_debug_aio", 2024-09-14T03:47:46.209 INFO:tasks.workunit.client.0.smithi070.stdout: "client_check_pool_perm", 2024-09-14T03:47:46.209 INFO:tasks.workunit.client.0.smithi070.stdout: "client_mount_gid", 2024-09-14T03:47:46.209 INFO:tasks.workunit.client.0.smithi070.stdout: "client_mount_uid", 2024-09-14T03:47:46.210 INFO:tasks.workunit.client.0.smithi070.stdout: "client_quota", 2024-09-14T03:47:46.210 INFO:tasks.workunit.client.0.smithi070.stdout: "container_image", 2024-09-14T03:47:46.210 INFO:tasks.workunit.client.0.smithi070.stdout: "daemonize", 2024-09-14T03:47:46.210 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_asserts_on_shutdown", 2024-09-14T03:47:46.210 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_mgr", 2024-09-14T03:47:46.210 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_mon", 2024-09-14T03:47:46.210 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_ms", 2024-09-14T03:47:46.210 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_osd", 2024-09-14T03:47:46.210 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_paxos", 2024-09-14T03:47:46.210 INFO:tasks.workunit.client.0.smithi070.stdout: "keyring", 2024-09-14T03:47:46.210 INFO:tasks.workunit.client.0.smithi070.stdout: "leveldb_block_size", 2024-09-14T03:47:46.210 INFO:tasks.workunit.client.0.smithi070.stdout: "leveldb_cache_size", 2024-09-14T03:47:46.210 INFO:tasks.workunit.client.0.smithi070.stdout: "leveldb_compression", 2024-09-14T03:47:46.211 INFO:tasks.workunit.client.0.smithi070.stdout: "leveldb_log", 2024-09-14T03:47:46.211 INFO:tasks.workunit.client.0.smithi070.stdout: "leveldb_write_buffer_size", 2024-09-14T03:47:46.211 INFO:tasks.workunit.client.0.smithi070.stdout: "log_to_file", 2024-09-14T03:47:46.211 INFO:tasks.workunit.client.0.smithi070.stdout: "log_to_journald", 2024-09-14T03:47:46.211 INFO:tasks.workunit.client.0.smithi070.stdout: "log_to_stderr", 2024-09-14T03:47:46.211 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_allow_pool_delete", 2024-09-14T03:47:46.211 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_clock_drift_allowed", 2024-09-14T03:47:46.211 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_cluster_log_to_file", 2024-09-14T03:47:46.211 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_cluster_log_to_journald", 2024-09-14T03:47:46.211 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_cluster_log_to_stderr", 2024-09-14T03:47:46.211 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_data_avail_warn", 2024-09-14T03:47:46.211 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_election_default_strategy", 2024-09-14T03:47:46.211 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_host", 2024-09-14T03:47:46.211 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_max_pg_per_osd", 2024-09-14T03:47:46.212 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_mgr_mkfs_grace", 2024-09-14T03:47:46.212 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_osd_prime_pg_temp", 2024-09-14T03:47:46.212 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_osd_reporter_subtree_level", 2024-09-14T03:47:46.212 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_pg_warn_max_object_skew", 2024-09-14T03:47:46.212 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-14T03:47:46.212 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-14T03:47:46.212 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-14T03:47:46.212 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-14T03:47:46.212 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-14T03:47:46.212 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-14T03:47:46.212 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-14T03:47:46.212 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-14T03:47:46.213 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_too_few_osds", 2024-09-14T03:47:46.213 INFO:tasks.workunit.client.0.smithi070.stdout: "ms_die_on_bug", 2024-09-14T03:47:46.213 INFO:tasks.workunit.client.0.smithi070.stdout: "ms_die_on_old_message", 2024-09-14T03:47:46.213 INFO:tasks.workunit.client.0.smithi070.stdout: "no_config_file", 2024-09-14T03:47:46.213 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_class_default_list", 2024-09-14T03:47:46.213 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_class_load_list", 2024-09-14T03:47:46.213 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_misdirected_ops", 2024-09-14T03:47:46.213 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_op_order", 2024-09-14T03:47:46.213 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_pg_log_writeout", 2024-09-14T03:47:46.213 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_shutdown", 2024-09-14T03:47:46.213 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_verify_cached_snaps", 2024-09-14T03:47:46.213 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_verify_missing_on_start", 2024-09-14T03:47:46.213 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_verify_stray_on_activate", 2024-09-14T03:47:46.214 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-14T03:47:46.214 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_profile", 2024-09-14T03:47:46.214 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_background_best_effort_lim", 2024-09-14T03:47:46.214 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_background_best_effort_res", 2024-09-14T03:47:46.214 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_background_best_effort_wgt", 2024-09-14T03:47:46.214 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_background_recovery_lim", 2024-09-14T03:47:46.214 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_background_recovery_res", 2024-09-14T03:47:46.214 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_background_recovery_wgt", 2024-09-14T03:47:46.214 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_client_lim", 2024-09-14T03:47:46.214 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_client_res", 2024-09-14T03:47:46.214 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_client_wgt", 2024-09-14T03:47:46.214 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_memory_target", 2024-09-14T03:47:46.214 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_memory_target_autotune", 2024-09-14T03:47:46.214 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_op_queue", 2024-09-14T03:47:46.215 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_op_queue_cut_off", 2024-09-14T03:47:46.215 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pool_default_erasure_code_profile", 2024-09-14T03:47:46.215 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-14T03:47:46.215 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_recover_clone_overlap", 2024-09-14T03:47:46.215 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_recovery_max_chunk", 2024-09-14T03:47:46.215 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_scrub_load_threshold", 2024-09-14T03:47:46.215 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_scrub_max_interval", 2024-09-14T03:47:46.215 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_shutdown_pgref_assert", 2024-09-14T03:47:46.215 INFO:tasks.workunit.client.0.smithi070.stdout: "public_addr", 2024-09-14T03:47:46.215 INFO:tasks.workunit.client.0.smithi070.stdout: "public_addrv", 2024-09-14T03:47:46.215 INFO:tasks.workunit.client.0.smithi070.stdout: "public_network", 2024-09-14T03:47:46.215 INFO:tasks.workunit.client.0.smithi070.stdout: "rbd_default_features", 2024-09-14T03:47:46.215 INFO:tasks.workunit.client.0.smithi070.stdout: "rbd_qos_exclude_ops", 2024-09-14T03:47:46.216 INFO:tasks.workunit.client.0.smithi070.stdout: "setgroup", 2024-09-14T03:47:46.216 INFO:tasks.workunit.client.0.smithi070.stdout: "setuser" 2024-09-14T03:47:46.216 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:46.216 INFO:tasks.workunit.client.0.smithi070.stdout: "cluster_changed": [ 2024-09-14T03:47:46.216 INFO:tasks.workunit.client.0.smithi070.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-14T03:47:46.216 INFO:tasks.workunit.client.0.smithi070.stdout: "auth_debug", 2024-09-14T03:47:46.216 INFO:tasks.workunit.client.0.smithi070.stdout: "auth_mon_ticket_ttl", 2024-09-14T03:47:46.216 INFO:tasks.workunit.client.0.smithi070.stdout: "auth_service_ticket_ttl", 2024-09-14T03:47:46.216 INFO:tasks.workunit.client.0.smithi070.stdout: "bdev_debug_aio", 2024-09-14T03:47:46.216 INFO:tasks.workunit.client.0.smithi070.stdout: "container_image", 2024-09-14T03:47:46.216 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_asserts_on_shutdown", 2024-09-14T03:47:46.216 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_mgr", 2024-09-14T03:47:46.216 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_mon", 2024-09-14T03:47:46.216 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_ms", 2024-09-14T03:47:46.217 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_osd", 2024-09-14T03:47:46.217 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_paxos", 2024-09-14T03:47:46.217 INFO:tasks.workunit.client.0.smithi070.stdout: "log_to_file", 2024-09-14T03:47:46.217 INFO:tasks.workunit.client.0.smithi070.stdout: "log_to_journald", 2024-09-14T03:47:46.217 INFO:tasks.workunit.client.0.smithi070.stdout: "log_to_stderr", 2024-09-14T03:47:46.217 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_join_fs", 2024-09-14T03:47:46.217 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/cephadm/allow_ptrace", 2024-09-14T03:47:46.217 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/cephadm/container_init", 2024-09-14T03:47:46.217 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/cephadm/migration_current", 2024-09-14T03:47:46.217 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/cephadm/use_repo_digest", 2024-09-14T03:47:46.217 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/dashboard/ALERTMANAGER_API_HOST", 2024-09-14T03:47:46.217 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/dashboard/GRAFANA_API_SSL_VERIFY", 2024-09-14T03:47:46.217 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/dashboard/GRAFANA_API_URL", 2024-09-14T03:47:46.217 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/dashboard/PROMETHEUS_API_HOST", 2024-09-14T03:47:46.218 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/dashboard/ssl_server_port", 2024-09-14T03:47:46.218 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/orchestrator/orchestrator", 2024-09-14T03:47:46.218 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/telemetry/channel_ident", 2024-09-14T03:47:46.218 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/telemetry/description", 2024-09-14T03:47:46.218 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/telemetry/organization", 2024-09-14T03:47:46.218 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_allow_pool_delete", 2024-09-14T03:47:46.218 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_clock_drift_allowed", 2024-09-14T03:47:46.218 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_cluster_log_to_file", 2024-09-14T03:47:46.218 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_data_avail_warn", 2024-09-14T03:47:46.218 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_election_default_strategy", 2024-09-14T03:47:46.218 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_max_pg_per_osd", 2024-09-14T03:47:46.218 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_mgr_mkfs_grace", 2024-09-14T03:47:46.218 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_osd_prime_pg_temp", 2024-09-14T03:47:46.218 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_osd_reporter_subtree_level", 2024-09-14T03:47:46.219 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_pg_warn_max_object_skew", 2024-09-14T03:47:46.219 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-14T03:47:46.219 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-14T03:47:46.219 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-14T03:47:46.219 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-14T03:47:46.219 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-14T03:47:46.219 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-14T03:47:46.219 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-14T03:47:46.219 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-14T03:47:46.219 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_too_few_osds", 2024-09-14T03:47:46.219 INFO:tasks.workunit.client.0.smithi070.stdout: "ms_die_on_bug", 2024-09-14T03:47:46.219 INFO:tasks.workunit.client.0.smithi070.stdout: "ms_die_on_old_message", 2024-09-14T03:47:46.219 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_class_default_list", 2024-09-14T03:47:46.219 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_class_load_list", 2024-09-14T03:47:46.220 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_misdirected_ops", 2024-09-14T03:47:46.220 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_op_order", 2024-09-14T03:47:46.220 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_pg_log_writeout", 2024-09-14T03:47:46.220 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_shutdown", 2024-09-14T03:47:46.220 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_verify_cached_snaps", 2024-09-14T03:47:46.220 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_verify_missing_on_start", 2024-09-14T03:47:46.220 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_verify_stray_on_activate", 2024-09-14T03:47:46.220 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-14T03:47:46.220 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_profile", 2024-09-14T03:47:46.220 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_memory_target", 2024-09-14T03:47:46.220 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_memory_target_autotune", 2024-09-14T03:47:46.220 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_op_queue", 2024-09-14T03:47:46.220 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_op_queue_cut_off", 2024-09-14T03:47:46.220 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pool_default_erasure_code_profile", 2024-09-14T03:47:46.221 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-14T03:47:46.221 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_recover_clone_overlap", 2024-09-14T03:47:46.221 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_recovery_max_chunk", 2024-09-14T03:47:46.221 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_scrub_load_threshold", 2024-09-14T03:47:46.221 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_scrub_max_interval", 2024-09-14T03:47:46.221 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_shutdown_pgref_assert", 2024-09-14T03:47:46.221 INFO:tasks.workunit.client.0.smithi070.stdout: "public_network", 2024-09-14T03:47:46.221 INFO:tasks.workunit.client.0.smithi070.stdout: "rgw_cache_enabled", 2024-09-14T03:47:46.221 INFO:tasks.workunit.client.0.smithi070.stdout: "rgw_enable_ops_log", 2024-09-14T03:47:46.221 INFO:tasks.workunit.client.0.smithi070.stdout: "rgw_enable_usage_log" 2024-09-14T03:47:46.221 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:46.221 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.221 INFO:tasks.workunit.client.0.smithi070.stdout: "contact": null, 2024-09-14T03:47:46.222 INFO:tasks.workunit.client.0.smithi070.stdout: "crashes": [], 2024-09-14T03:47:46.222 INFO:tasks.workunit.client.0.smithi070.stdout: "created": "2024-09-14T03:38:26.272229Z", 2024-09-14T03:47:46.222 INFO:tasks.workunit.client.0.smithi070.stdout: "crush": { 2024-09-14T03:47:46.222 INFO:tasks.workunit.client.0.smithi070.stdout: "bucket_algs": { 2024-09-14T03:47:46.222 INFO:tasks.workunit.client.0.smithi070.stdout: "straw2": 3 2024-09-14T03:47:46.222 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.222 INFO:tasks.workunit.client.0.smithi070.stdout: "bucket_sizes": { 2024-09-14T03:47:46.222 INFO:tasks.workunit.client.0.smithi070.stdout: "2": 1, 2024-09-14T03:47:46.222 INFO:tasks.workunit.client.0.smithi070.stdout: "4": 2 2024-09-14T03:47:46.222 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.222 INFO:tasks.workunit.client.0.smithi070.stdout: "bucket_types": { 2024-09-14T03:47:46.222 INFO:tasks.workunit.client.0.smithi070.stdout: "1": 2, 2024-09-14T03:47:46.222 INFO:tasks.workunit.client.0.smithi070.stdout: "11": 1 2024-09-14T03:47:46.222 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.223 INFO:tasks.workunit.client.0.smithi070.stdout: "compat_weight_set": false, 2024-09-14T03:47:46.223 INFO:tasks.workunit.client.0.smithi070.stdout: "device_classes": [ 2024-09-14T03:47:46.223 INFO:tasks.workunit.client.0.smithi070.stdout: 8 2024-09-14T03:47:46.223 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:46.223 INFO:tasks.workunit.client.0.smithi070.stdout: "num_buckets": 6, 2024-09-14T03:47:46.223 INFO:tasks.workunit.client.0.smithi070.stdout: "num_devices": 8, 2024-09-14T03:47:46.223 INFO:tasks.workunit.client.0.smithi070.stdout: "num_rules": 1, 2024-09-14T03:47:46.223 INFO:tasks.workunit.client.0.smithi070.stdout: "num_types": 12, 2024-09-14T03:47:46.223 INFO:tasks.workunit.client.0.smithi070.stdout: "num_weight_sets": 0, 2024-09-14T03:47:46.223 INFO:tasks.workunit.client.0.smithi070.stdout: "tunables": { 2024-09-14T03:47:46.223 INFO:tasks.workunit.client.0.smithi070.stdout: "allowed_bucket_algs": 54, 2024-09-14T03:47:46.223 INFO:tasks.workunit.client.0.smithi070.stdout: "choose_local_fallback_tries": 0, 2024-09-14T03:47:46.223 INFO:tasks.workunit.client.0.smithi070.stdout: "choose_local_tries": 0, 2024-09-14T03:47:46.223 INFO:tasks.workunit.client.0.smithi070.stdout: "choose_total_tries": 50, 2024-09-14T03:47:46.224 INFO:tasks.workunit.client.0.smithi070.stdout: "chooseleaf_descend_once": 1, 2024-09-14T03:47:46.224 INFO:tasks.workunit.client.0.smithi070.stdout: "chooseleaf_stable": 1, 2024-09-14T03:47:46.224 INFO:tasks.workunit.client.0.smithi070.stdout: "chooseleaf_vary_r": 1, 2024-09-14T03:47:46.224 INFO:tasks.workunit.client.0.smithi070.stdout: "has_v2_rules": 0, 2024-09-14T03:47:46.224 INFO:tasks.workunit.client.0.smithi070.stdout: "has_v3_rules": 0, 2024-09-14T03:47:46.224 INFO:tasks.workunit.client.0.smithi070.stdout: "has_v4_buckets": 1, 2024-09-14T03:47:46.224 INFO:tasks.workunit.client.0.smithi070.stdout: "has_v5_rules": 0, 2024-09-14T03:47:46.224 INFO:tasks.workunit.client.0.smithi070.stdout: "legacy_tunables": 0, 2024-09-14T03:47:46.224 INFO:tasks.workunit.client.0.smithi070.stdout: "minimum_required_version": "jewel", 2024-09-14T03:47:46.224 INFO:tasks.workunit.client.0.smithi070.stdout: "optimal_tunables": 1, 2024-09-14T03:47:46.224 INFO:tasks.workunit.client.0.smithi070.stdout: "profile": "jewel", 2024-09-14T03:47:46.224 INFO:tasks.workunit.client.0.smithi070.stdout: "require_feature_tunables": 1, 2024-09-14T03:47:46.224 INFO:tasks.workunit.client.0.smithi070.stdout: "require_feature_tunables2": 1, 2024-09-14T03:47:46.225 INFO:tasks.workunit.client.0.smithi070.stdout: "require_feature_tunables3": 1, 2024-09-14T03:47:46.225 INFO:tasks.workunit.client.0.smithi070.stdout: "require_feature_tunables5": 1, 2024-09-14T03:47:46.225 INFO:tasks.workunit.client.0.smithi070.stdout: "straw_calc_version": 1 2024-09-14T03:47:46.225 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:46.225 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.225 INFO:tasks.workunit.client.0.smithi070.stdout: "description": "upgrade test cluster", 2024-09-14T03:47:46.225 INFO:tasks.workunit.client.0.smithi070.stdout: "fs": { 2024-09-14T03:47:46.225 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:46.225 INFO:tasks.workunit.client.0.smithi070.stdout: "feature_flags": { 2024-09-14T03:47:46.225 INFO:tasks.workunit.client.0.smithi070.stdout: "enable_multiple": true, 2024-09-14T03:47:46.225 INFO:tasks.workunit.client.0.smithi070.stdout: "ever_enabled_multiple": true 2024-09-14T03:47:46.225 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.225 INFO:tasks.workunit.client.0.smithi070.stdout: "filesystems": [ 2024-09-14T03:47:46.226 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:46.226 INFO:tasks.workunit.client.0.smithi070.stdout: "approx_ctime": "2024-09", 2024-09-14T03:47:46.226 INFO:tasks.workunit.client.0.smithi070.stdout: "balancer_enabled": false, 2024-09-14T03:47:46.226 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:46.226 INFO:tasks.workunit.client.0.smithi070.stdout: "cached_caps": 0, 2024-09-14T03:47:46.226 INFO:tasks.workunit.client.0.smithi070.stdout: "cached_dns": 10, 2024-09-14T03:47:46.226 INFO:tasks.workunit.client.0.smithi070.stdout: "cached_inos": 13, 2024-09-14T03:47:46.226 INFO:tasks.workunit.client.0.smithi070.stdout: "cached_subtrees": 2, 2024-09-14T03:47:46.226 INFO:tasks.workunit.client.0.smithi070.stdout: "ever_allowed_features": 0, 2024-09-14T03:47:46.226 INFO:tasks.workunit.client.0.smithi070.stdout: "explicitly_allowed_features": 0, 2024-09-14T03:47:46.226 INFO:tasks.workunit.client.0.smithi070.stdout: "files": 0, 2024-09-14T03:47:46.226 INFO:tasks.workunit.client.0.smithi070.stdout: "max_mds": 1, 2024-09-14T03:47:46.226 INFO:tasks.workunit.client.0.smithi070.stdout: "num_data_pools": 1, 2024-09-14T03:47:46.226 INFO:tasks.workunit.client.0.smithi070.stdout: "num_in": 1, 2024-09-14T03:47:46.227 INFO:tasks.workunit.client.0.smithi070.stdout: "num_mds": 1, 2024-09-14T03:47:46.227 INFO:tasks.workunit.client.0.smithi070.stdout: "num_sessions": 0, 2024-09-14T03:47:46.227 INFO:tasks.workunit.client.0.smithi070.stdout: "num_standby_replay": 0, 2024-09-14T03:47:46.227 INFO:tasks.workunit.client.0.smithi070.stdout: "num_up": 1, 2024-09-14T03:47:46.227 INFO:tasks.workunit.client.0.smithi070.stdout: "snaps": 0, 2024-09-14T03:47:46.227 INFO:tasks.workunit.client.0.smithi070.stdout: "standby_count_wanted": 0 2024-09-14T03:47:46.227 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:46.227 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:46.227 INFO:tasks.workunit.client.0.smithi070.stdout: "num_standby_mds": 1, 2024-09-14T03:47:46.227 INFO:tasks.workunit.client.0.smithi070.stdout: "total_num_mds": 2 2024-09-14T03:47:46.227 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.227 INFO:tasks.workunit.client.0.smithi070.stdout: "hosts": { 2024-09-14T03:47:46.227 INFO:tasks.workunit.client.0.smithi070.stdout: "num": 2, 2024-09-14T03:47:46.228 INFO:tasks.workunit.client.0.smithi070.stdout: "num_with_mds": 2, 2024-09-14T03:47:46.228 INFO:tasks.workunit.client.0.smithi070.stdout: "num_with_mgr": 2, 2024-09-14T03:47:46.228 INFO:tasks.workunit.client.0.smithi070.stdout: "num_with_mon": 2, 2024-09-14T03:47:46.228 INFO:tasks.workunit.client.0.smithi070.stdout: "num_with_osd": 2 2024-09-14T03:47:46.228 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.228 INFO:tasks.workunit.client.0.smithi070.stdout: "leaderboard": false, 2024-09-14T03:47:46.228 INFO:tasks.workunit.client.0.smithi070.stdout: "license": "sharing-1-0", 2024-09-14T03:47:46.228 INFO:tasks.workunit.client.0.smithi070.stdout: "metadata": { 2024-09-14T03:47:46.228 INFO:tasks.workunit.client.0.smithi070.stdout: "mds": { 2024-09-14T03:47:46.228 INFO:tasks.workunit.client.0.smithi070.stdout: "arch": { 2024-09-14T03:47:46.228 INFO:tasks.workunit.client.0.smithi070.stdout: "x86_64": 2 2024-09-14T03:47:46.228 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.228 INFO:tasks.workunit.client.0.smithi070.stdout: "ceph_version": { 2024-09-14T03:47:46.229 INFO:tasks.workunit.client.0.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:47:46.229 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.229 INFO:tasks.workunit.client.0.smithi070.stdout: "cpu": { 2024-09-14T03:47:46.229 INFO:tasks.workunit.client.0.smithi070.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 2 2024-09-14T03:47:46.229 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.229 INFO:tasks.workunit.client.0.smithi070.stdout: "distro": { 2024-09-14T03:47:46.229 INFO:tasks.workunit.client.0.smithi070.stdout: "centos": 2 2024-09-14T03:47:46.229 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.229 INFO:tasks.workunit.client.0.smithi070.stdout: "distro_description": { 2024-09-14T03:47:46.229 INFO:tasks.workunit.client.0.smithi070.stdout: "CentOS Stream 9": 2 2024-09-14T03:47:46.229 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.229 INFO:tasks.workunit.client.0.smithi070.stdout: "kernel_description": { 2024-09-14T03:47:46.230 INFO:tasks.workunit.client.0.smithi070.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 2 2024-09-14T03:47:46.230 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.230 INFO:tasks.workunit.client.0.smithi070.stdout: "kernel_version": { 2024-09-14T03:47:46.230 INFO:tasks.workunit.client.0.smithi070.stdout: "5.14.0-505.el9.x86_64": 2 2024-09-14T03:47:46.230 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.230 INFO:tasks.workunit.client.0.smithi070.stdout: "os": { 2024-09-14T03:47:46.230 INFO:tasks.workunit.client.0.smithi070.stdout: "Linux": 2 2024-09-14T03:47:46.230 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:46.230 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.230 INFO:tasks.workunit.client.0.smithi070.stdout: "mon": { 2024-09-14T03:47:46.230 INFO:tasks.workunit.client.0.smithi070.stdout: "arch": { 2024-09-14T03:47:46.230 INFO:tasks.workunit.client.0.smithi070.stdout: "x86_64": 3 2024-09-14T03:47:46.230 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.231 INFO:tasks.workunit.client.0.smithi070.stdout: "ceph_version": { 2024-09-14T03:47:46.231 INFO:tasks.workunit.client.0.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 3 2024-09-14T03:47:46.231 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.231 INFO:tasks.workunit.client.0.smithi070.stdout: "cpu": { 2024-09-14T03:47:46.231 INFO:tasks.workunit.client.0.smithi070.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 3 2024-09-14T03:47:46.231 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.231 INFO:tasks.workunit.client.0.smithi070.stdout: "distro": { 2024-09-14T03:47:46.231 INFO:tasks.workunit.client.0.smithi070.stdout: "centos": 3 2024-09-14T03:47:46.231 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.231 INFO:tasks.workunit.client.0.smithi070.stdout: "distro_description": { 2024-09-14T03:47:46.231 INFO:tasks.workunit.client.0.smithi070.stdout: "CentOS Stream 9": 3 2024-09-14T03:47:46.231 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.231 INFO:tasks.workunit.client.0.smithi070.stdout: "kernel_description": { 2024-09-14T03:47:46.232 INFO:tasks.workunit.client.0.smithi070.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 3 2024-09-14T03:47:46.232 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.232 INFO:tasks.workunit.client.0.smithi070.stdout: "kernel_version": { 2024-09-14T03:47:46.232 INFO:tasks.workunit.client.0.smithi070.stdout: "5.14.0-505.el9.x86_64": 3 2024-09-14T03:47:46.232 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.232 INFO:tasks.workunit.client.0.smithi070.stdout: "os": { 2024-09-14T03:47:46.232 INFO:tasks.workunit.client.0.smithi070.stdout: "Linux": 3 2024-09-14T03:47:46.232 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:46.232 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.232 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:46.232 INFO:tasks.workunit.client.0.smithi070.stdout: "arch": { 2024-09-14T03:47:46.232 INFO:tasks.workunit.client.0.smithi070.stdout: "x86_64": 8 2024-09-14T03:47:46.232 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.233 INFO:tasks.workunit.client.0.smithi070.stdout: "ceph_version": { 2024-09-14T03:47:46.233 INFO:tasks.workunit.client.0.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 8 2024-09-14T03:47:46.233 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.233 INFO:tasks.workunit.client.0.smithi070.stdout: "cpu": { 2024-09-14T03:47:46.233 INFO:tasks.workunit.client.0.smithi070.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 8 2024-09-14T03:47:46.233 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.233 INFO:tasks.workunit.client.0.smithi070.stdout: "distro": { 2024-09-14T03:47:46.233 INFO:tasks.workunit.client.0.smithi070.stdout: "centos": 8 2024-09-14T03:47:46.233 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.233 INFO:tasks.workunit.client.0.smithi070.stdout: "distro_description": { 2024-09-14T03:47:46.233 INFO:tasks.workunit.client.0.smithi070.stdout: "CentOS Stream 9": 8 2024-09-14T03:47:46.233 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.233 INFO:tasks.workunit.client.0.smithi070.stdout: "kernel_description": { 2024-09-14T03:47:46.234 INFO:tasks.workunit.client.0.smithi070.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 8 2024-09-14T03:47:46.234 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.234 INFO:tasks.workunit.client.0.smithi070.stdout: "kernel_version": { 2024-09-14T03:47:46.234 INFO:tasks.workunit.client.0.smithi070.stdout: "5.14.0-505.el9.x86_64": 8 2024-09-14T03:47:46.234 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.234 INFO:tasks.workunit.client.0.smithi070.stdout: "os": { 2024-09-14T03:47:46.234 INFO:tasks.workunit.client.0.smithi070.stdout: "Linux": 8 2024-09-14T03:47:46.234 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.234 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_objectstore": { 2024-09-14T03:47:46.234 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore": 8 2024-09-14T03:47:46.234 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.234 INFO:tasks.workunit.client.0.smithi070.stdout: "rotational": { 2024-09-14T03:47:46.234 INFO:tasks.workunit.client.0.smithi070.stdout: "0": 8 2024-09-14T03:47:46.234 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:46.235 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:46.235 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.235 INFO:tasks.workunit.client.0.smithi070.stdout: "mon": { 2024-09-14T03:47:46.235 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 3, 2024-09-14T03:47:46.235 INFO:tasks.workunit.client.0.smithi070.stdout: "features": { 2024-09-14T03:47:46.235 INFO:tasks.workunit.client.0.smithi070.stdout: "optional": [], 2024-09-14T03:47:46.235 INFO:tasks.workunit.client.0.smithi070.stdout: "persistent": [ 2024-09-14T03:47:46.235 INFO:tasks.workunit.client.0.smithi070.stdout: "kraken", 2024-09-14T03:47:46.235 INFO:tasks.workunit.client.0.smithi070.stdout: "luminous", 2024-09-14T03:47:46.235 INFO:tasks.workunit.client.0.smithi070.stdout: "mimic", 2024-09-14T03:47:46.235 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap-prune", 2024-09-14T03:47:46.235 INFO:tasks.workunit.client.0.smithi070.stdout: "nautilus", 2024-09-14T03:47:46.235 INFO:tasks.workunit.client.0.smithi070.stdout: "octopus", 2024-09-14T03:47:46.235 INFO:tasks.workunit.client.0.smithi070.stdout: "pacific", 2024-09-14T03:47:46.236 INFO:tasks.workunit.client.0.smithi070.stdout: "elector-pinging", 2024-09-14T03:47:46.236 INFO:tasks.workunit.client.0.smithi070.stdout: "quincy" 2024-09-14T03:47:46.236 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:46.236 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.236 INFO:tasks.workunit.client.0.smithi070.stdout: "ipv4_addr_mons": 6, 2024-09-14T03:47:46.236 INFO:tasks.workunit.client.0.smithi070.stdout: "ipv6_addr_mons": 0, 2024-09-14T03:47:46.236 INFO:tasks.workunit.client.0.smithi070.stdout: "min_mon_release": 17, 2024-09-14T03:47:46.236 INFO:tasks.workunit.client.0.smithi070.stdout: "v1_addr_mons": 3, 2024-09-14T03:47:46.236 INFO:tasks.workunit.client.0.smithi070.stdout: "v2_addr_mons": 3 2024-09-14T03:47:46.236 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.236 INFO:tasks.workunit.client.0.smithi070.stdout: "organization": "ceph-qa", 2024-09-14T03:47:46.236 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:46.236 INFO:tasks.workunit.client.0.smithi070.stdout: "cluster_network": false, 2024-09-14T03:47:46.237 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 8, 2024-09-14T03:47:46.237 INFO:tasks.workunit.client.0.smithi070.stdout: "require_min_compat_client": "luminous", 2024-09-14T03:47:46.237 INFO:tasks.workunit.client.0.smithi070.stdout: "require_osd_release": "quincy" 2024-09-14T03:47:46.237 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.237 INFO:tasks.workunit.client.0.smithi070.stdout: "pools": [ 2024-09-14T03:47:46.237 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:46.237 INFO:tasks.workunit.client.0.smithi070.stdout: "application": [ 2024-09-14T03:47:46.237 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr" 2024-09-14T03:47:46.237 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:46.237 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_mode": "none", 2024-09-14T03:47:46.237 INFO:tasks.workunit.client.0.smithi070.stdout: "erasure_code_profile": {}, 2024-09-14T03:47:46.237 INFO:tasks.workunit.client.0.smithi070.stdout: "min_size": 2, 2024-09-14T03:47:46.237 INFO:tasks.workunit.client.0.smithi070.stdout: "options": {}, 2024-09-14T03:47:46.237 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_autoscale_mode": "off", 2024-09-14T03:47:46.238 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_num": 1, 2024-09-14T03:47:46.238 INFO:tasks.workunit.client.0.smithi070.stdout: "pgp_num": 1, 2024-09-14T03:47:46.238 INFO:tasks.workunit.client.0.smithi070.stdout: "pool": 1, 2024-09-14T03:47:46.238 INFO:tasks.workunit.client.0.smithi070.stdout: "size": 3, 2024-09-14T03:47:46.238 INFO:tasks.workunit.client.0.smithi070.stdout: "stats": { 2024-09-14T03:47:46.238 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_raw": 727115901194, 2024-09-14T03:47:46.238 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_used": 1781760, 2024-09-14T03:47:46.238 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_bytes_used": 0, 2024-09-14T03:47:46.238 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_under_bytes": 0, 2024-09-14T03:47:46.238 INFO:tasks.workunit.client.0.smithi070.stdout: "data_bytes_used": 1781760, 2024-09-14T03:47:46.238 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty": 0, 2024-09-14T03:47:46.238 INFO:tasks.workunit.client.0.smithi070.stdout: "max_avail": 242371969024, 2024-09-14T03:47:46.238 INFO:tasks.workunit.client.0.smithi070.stdout: "objects": 2, 2024-09-14T03:47:46.239 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_bytes_used": 0, 2024-09-14T03:47:46.239 INFO:tasks.workunit.client.0.smithi070.stdout: "percent_used": 2.450442480039783e-06, 2024-09-14T03:47:46.239 INFO:tasks.workunit.client.0.smithi070.stdout: "quota_bytes": 0, 2024-09-14T03:47:46.239 INFO:tasks.workunit.client.0.smithi070.stdout: "quota_objects": 0, 2024-09-14T03:47:46.239 INFO:tasks.workunit.client.0.smithi070.stdout: "rd": 126, 2024-09-14T03:47:46.239 INFO:tasks.workunit.client.0.smithi070.stdout: "rd_bytes": 236544, 2024-09-14T03:47:46.239 INFO:tasks.workunit.client.0.smithi070.stdout: "stored": 590368, 2024-09-14T03:47:46.239 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_data": 590368, 2024-09-14T03:47:46.239 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_omap": 0, 2024-09-14T03:47:46.239 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_raw": 1771104, 2024-09-14T03:47:46.239 INFO:tasks.workunit.client.0.smithi070.stdout: "wr": 113, 2024-09-14T03:47:46.239 INFO:tasks.workunit.client.0.smithi070.stdout: "wr_bytes": 1667072 2024-09-14T03:47:46.239 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.240 INFO:tasks.workunit.client.0.smithi070.stdout: "target_max_bytes": 0, 2024-09-14T03:47:46.240 INFO:tasks.workunit.client.0.smithi070.stdout: "target_max_objects": 0, 2024-09-14T03:47:46.240 INFO:tasks.workunit.client.0.smithi070.stdout: "type": "replicated" 2024-09-14T03:47:46.240 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.240 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:46.240 INFO:tasks.workunit.client.0.smithi070.stdout: "application": [ 2024-09-14T03:47:46.240 INFO:tasks.workunit.client.0.smithi070.stdout: "cephfs" 2024-09-14T03:47:46.240 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:46.240 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_mode": "none", 2024-09-14T03:47:46.240 INFO:tasks.workunit.client.0.smithi070.stdout: "erasure_code_profile": {}, 2024-09-14T03:47:46.240 INFO:tasks.workunit.client.0.smithi070.stdout: "min_size": 2, 2024-09-14T03:47:46.240 INFO:tasks.workunit.client.0.smithi070.stdout: "options": {}, 2024-09-14T03:47:46.240 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_autoscale_mode": "off", 2024-09-14T03:47:46.241 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_num": 32, 2024-09-14T03:47:46.241 INFO:tasks.workunit.client.0.smithi070.stdout: "pgp_num": 32, 2024-09-14T03:47:46.241 INFO:tasks.workunit.client.0.smithi070.stdout: "pool": 2, 2024-09-14T03:47:46.241 INFO:tasks.workunit.client.0.smithi070.stdout: "size": 3, 2024-09-14T03:47:46.241 INFO:tasks.workunit.client.0.smithi070.stdout: "stats": { 2024-09-14T03:47:46.241 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_raw": 727115901194, 2024-09-14T03:47:46.241 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_used": 98304, 2024-09-14T03:47:46.241 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_bytes_used": 0, 2024-09-14T03:47:46.241 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_under_bytes": 0, 2024-09-14T03:47:46.241 INFO:tasks.workunit.client.0.smithi070.stdout: "data_bytes_used": 98304, 2024-09-14T03:47:46.241 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty": 0, 2024-09-14T03:47:46.241 INFO:tasks.workunit.client.0.smithi070.stdout: "max_avail": 242371969024, 2024-09-14T03:47:46.241 INFO:tasks.workunit.client.0.smithi070.stdout: "objects": 22, 2024-09-14T03:47:46.241 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_bytes_used": 0, 2024-09-14T03:47:46.242 INFO:tasks.workunit.client.0.smithi070.stdout: "percent_used": 1.3519712638299097e-07, 2024-09-14T03:47:46.242 INFO:tasks.workunit.client.0.smithi070.stdout: "quota_bytes": 0, 2024-09-14T03:47:46.242 INFO:tasks.workunit.client.0.smithi070.stdout: "quota_objects": 0, 2024-09-14T03:47:46.242 INFO:tasks.workunit.client.0.smithi070.stdout: "rd": 0, 2024-09-14T03:47:46.242 INFO:tasks.workunit.client.0.smithi070.stdout: "rd_bytes": 0, 2024-09-14T03:47:46.242 INFO:tasks.workunit.client.0.smithi070.stdout: "stored": 3410, 2024-09-14T03:47:46.242 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_data": 3410, 2024-09-14T03:47:46.242 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_omap": 0, 2024-09-14T03:47:46.242 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_raw": 10230, 2024-09-14T03:47:46.242 INFO:tasks.workunit.client.0.smithi070.stdout: "wr": 51, 2024-09-14T03:47:46.242 INFO:tasks.workunit.client.0.smithi070.stdout: "wr_bytes": 19456 2024-09-14T03:47:46.242 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.242 INFO:tasks.workunit.client.0.smithi070.stdout: "target_max_bytes": 0, 2024-09-14T03:47:46.242 INFO:tasks.workunit.client.0.smithi070.stdout: "target_max_objects": 0, 2024-09-14T03:47:46.242 INFO:tasks.workunit.client.0.smithi070.stdout: "type": "replicated" 2024-09-14T03:47:46.243 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.243 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:46.243 INFO:tasks.workunit.client.0.smithi070.stdout: "application": [ 2024-09-14T03:47:46.243 INFO:tasks.workunit.client.0.smithi070.stdout: "cephfs" 2024-09-14T03:47:46.243 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:46.243 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_mode": "none", 2024-09-14T03:47:46.243 INFO:tasks.workunit.client.0.smithi070.stdout: "erasure_code_profile": {}, 2024-09-14T03:47:46.243 INFO:tasks.workunit.client.0.smithi070.stdout: "min_size": 2, 2024-09-14T03:47:46.243 INFO:tasks.workunit.client.0.smithi070.stdout: "options": {}, 2024-09-14T03:47:46.243 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_autoscale_mode": "off", 2024-09-14T03:47:46.243 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_num": 64, 2024-09-14T03:47:46.243 INFO:tasks.workunit.client.0.smithi070.stdout: "pgp_num": 64, 2024-09-14T03:47:46.243 INFO:tasks.workunit.client.0.smithi070.stdout: "pool": 3, 2024-09-14T03:47:46.243 INFO:tasks.workunit.client.0.smithi070.stdout: "size": 3, 2024-09-14T03:47:46.243 INFO:tasks.workunit.client.0.smithi070.stdout: "stats": { 2024-09-14T03:47:46.243 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_raw": 727115901194, 2024-09-14T03:47:46.244 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_used": 0, 2024-09-14T03:47:46.244 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_bytes_used": 0, 2024-09-14T03:47:46.244 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_under_bytes": 0, 2024-09-14T03:47:46.244 INFO:tasks.workunit.client.0.smithi070.stdout: "data_bytes_used": 0, 2024-09-14T03:47:46.244 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty": 0, 2024-09-14T03:47:46.244 INFO:tasks.workunit.client.0.smithi070.stdout: "max_avail": 242371969024, 2024-09-14T03:47:46.244 INFO:tasks.workunit.client.0.smithi070.stdout: "objects": 0, 2024-09-14T03:47:46.244 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_bytes_used": 0, 2024-09-14T03:47:46.244 INFO:tasks.workunit.client.0.smithi070.stdout: "percent_used": 0.0, 2024-09-14T03:47:46.244 INFO:tasks.workunit.client.0.smithi070.stdout: "quota_bytes": 0, 2024-09-14T03:47:46.244 INFO:tasks.workunit.client.0.smithi070.stdout: "quota_objects": 0, 2024-09-14T03:47:46.244 INFO:tasks.workunit.client.0.smithi070.stdout: "rd": 0, 2024-09-14T03:47:46.245 INFO:tasks.workunit.client.0.smithi070.stdout: "rd_bytes": 0, 2024-09-14T03:47:46.245 INFO:tasks.workunit.client.0.smithi070.stdout: "stored": 0, 2024-09-14T03:47:46.245 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_data": 0, 2024-09-14T03:47:46.245 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_omap": 0, 2024-09-14T03:47:46.245 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_raw": 0, 2024-09-14T03:47:46.245 INFO:tasks.workunit.client.0.smithi070.stdout: "wr": 0, 2024-09-14T03:47:46.245 INFO:tasks.workunit.client.0.smithi070.stdout: "wr_bytes": 0 2024-09-14T03:47:46.245 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.245 INFO:tasks.workunit.client.0.smithi070.stdout: "target_max_bytes": 0, 2024-09-14T03:47:46.245 INFO:tasks.workunit.client.0.smithi070.stdout: "target_max_objects": 0, 2024-09-14T03:47:46.245 INFO:tasks.workunit.client.0.smithi070.stdout: "type": "replicated" 2024-09-14T03:47:46.245 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.245 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:46.245 INFO:tasks.workunit.client.0.smithi070.stdout: "application": [ 2024-09-14T03:47:46.246 INFO:tasks.workunit.client.0.smithi070.stdout: "rbd" 2024-09-14T03:47:46.246 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:46.246 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_mode": "none", 2024-09-14T03:47:46.246 INFO:tasks.workunit.client.0.smithi070.stdout: "erasure_code_profile": {}, 2024-09-14T03:47:46.246 INFO:tasks.workunit.client.0.smithi070.stdout: "min_size": 2, 2024-09-14T03:47:46.246 INFO:tasks.workunit.client.0.smithi070.stdout: "options": {}, 2024-09-14T03:47:46.246 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_autoscale_mode": "off", 2024-09-14T03:47:46.246 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_num": 8, 2024-09-14T03:47:46.246 INFO:tasks.workunit.client.0.smithi070.stdout: "pgp_num": 8, 2024-09-14T03:47:46.246 INFO:tasks.workunit.client.0.smithi070.stdout: "pool": 4, 2024-09-14T03:47:46.246 INFO:tasks.workunit.client.0.smithi070.stdout: "size": 3, 2024-09-14T03:47:46.246 INFO:tasks.workunit.client.0.smithi070.stdout: "stats": { 2024-09-14T03:47:46.246 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_raw": 727115901194, 2024-09-14T03:47:46.247 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_used": 12288, 2024-09-14T03:47:46.247 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_bytes_used": 0, 2024-09-14T03:47:46.247 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_under_bytes": 0, 2024-09-14T03:47:46.247 INFO:tasks.workunit.client.0.smithi070.stdout: "data_bytes_used": 12288, 2024-09-14T03:47:46.247 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty": 0, 2024-09-14T03:47:46.247 INFO:tasks.workunit.client.0.smithi070.stdout: "max_avail": 242371969024, 2024-09-14T03:47:46.247 INFO:tasks.workunit.client.0.smithi070.stdout: "objects": 2, 2024-09-14T03:47:46.247 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_bytes_used": 0, 2024-09-14T03:47:46.247 INFO:tasks.workunit.client.0.smithi070.stdout: "percent_used": 1.689964435058755e-08, 2024-09-14T03:47:46.247 INFO:tasks.workunit.client.0.smithi070.stdout: "quota_bytes": 0, 2024-09-14T03:47:46.247 INFO:tasks.workunit.client.0.smithi070.stdout: "quota_objects": 0, 2024-09-14T03:47:46.247 INFO:tasks.workunit.client.0.smithi070.stdout: "rd": 0, 2024-09-14T03:47:46.247 INFO:tasks.workunit.client.0.smithi070.stdout: "rd_bytes": 0, 2024-09-14T03:47:46.247 INFO:tasks.workunit.client.0.smithi070.stdout: "stored": 19, 2024-09-14T03:47:46.248 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_data": 19, 2024-09-14T03:47:46.248 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_omap": 0, 2024-09-14T03:47:46.248 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_raw": 57, 2024-09-14T03:47:46.248 INFO:tasks.workunit.client.0.smithi070.stdout: "wr": 2, 2024-09-14T03:47:46.248 INFO:tasks.workunit.client.0.smithi070.stdout: "wr_bytes": 2048 2024-09-14T03:47:46.248 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.248 INFO:tasks.workunit.client.0.smithi070.stdout: "target_max_bytes": 0, 2024-09-14T03:47:46.248 INFO:tasks.workunit.client.0.smithi070.stdout: "target_max_objects": 0, 2024-09-14T03:47:46.248 INFO:tasks.workunit.client.0.smithi070.stdout: "type": "replicated" 2024-09-14T03:47:46.248 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:46.248 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:46.248 INFO:tasks.workunit.client.0.smithi070.stdout: "rbd": { 2024-09-14T03:47:46.248 INFO:tasks.workunit.client.0.smithi070.stdout: "mirroring_by_pool": [ 2024-09-14T03:47:46.249 INFO:tasks.workunit.client.0.smithi070.stdout: false 2024-09-14T03:47:46.249 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:46.249 INFO:tasks.workunit.client.0.smithi070.stdout: "num_images_by_pool": [ 2024-09-14T03:47:46.249 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:46.249 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:46.249 INFO:tasks.workunit.client.0.smithi070.stdout: "num_pools": 1 2024-09-14T03:47:46.249 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.249 INFO:tasks.workunit.client.0.smithi070.stdout: "report_id": "04c26ace-6e52-41dd-9ad2-e917daf8e93d", 2024-09-14T03:47:46.249 INFO:tasks.workunit.client.0.smithi070.stdout: "report_timestamp": "2024-09-14T03:47:46.199215", 2024-09-14T03:47:46.249 INFO:tasks.workunit.client.0.smithi070.stdout: "report_version": 1, 2024-09-14T03:47:46.249 INFO:tasks.workunit.client.0.smithi070.stdout: "rook": { 2024-09-14T03:47:46.249 INFO:tasks.workunit.client.0.smithi070.stdout: "cluster": { 2024-09-14T03:47:46.249 INFO:tasks.workunit.client.0.smithi070.stdout: "external-mode": null, 2024-09-14T03:47:46.250 INFO:tasks.workunit.client.0.smithi070.stdout: "mon": { 2024-09-14T03:47:46.250 INFO:tasks.workunit.client.0.smithi070.stdout: "allow-multiple-per-node": null, 2024-09-14T03:47:46.250 INFO:tasks.workunit.client.0.smithi070.stdout: "count": null, 2024-09-14T03:47:46.250 INFO:tasks.workunit.client.0.smithi070.stdout: "max-id": null, 2024-09-14T03:47:46.250 INFO:tasks.workunit.client.0.smithi070.stdout: "pvc": { 2024-09-14T03:47:46.250 INFO:tasks.workunit.client.0.smithi070.stdout: "enabled": null 2024-09-14T03:47:46.250 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.250 INFO:tasks.workunit.client.0.smithi070.stdout: "stretch": { 2024-09-14T03:47:46.250 INFO:tasks.workunit.client.0.smithi070.stdout: "enabled": null 2024-09-14T03:47:46.250 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:46.250 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.250 INFO:tasks.workunit.client.0.smithi070.stdout: "network": { 2024-09-14T03:47:46.250 INFO:tasks.workunit.client.0.smithi070.stdout: "provider": null 2024-09-14T03:47:46.250 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.251 INFO:tasks.workunit.client.0.smithi070.stdout: "storage": { 2024-09-14T03:47:46.251 INFO:tasks.workunit.client.0.smithi070.stdout: "device-set": { 2024-09-14T03:47:46.251 INFO:tasks.workunit.client.0.smithi070.stdout: "count": { 2024-09-14T03:47:46.251 INFO:tasks.workunit.client.0.smithi070.stdout: "non-portable": null, 2024-09-14T03:47:46.251 INFO:tasks.workunit.client.0.smithi070.stdout: "portable": null, 2024-09-14T03:47:46.251 INFO:tasks.workunit.client.0.smithi070.stdout: "total": null 2024-09-14T03:47:46.251 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:46.251 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:46.251 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:46.251 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.251 INFO:tasks.workunit.client.0.smithi070.stdout: "csi": { 2024-09-14T03:47:46.251 INFO:tasks.workunit.client.0.smithi070.stdout: "version": null 2024-09-14T03:47:46.251 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.252 INFO:tasks.workunit.client.0.smithi070.stdout: "kubernetes": { 2024-09-14T03:47:46.252 INFO:tasks.workunit.client.0.smithi070.stdout: "version": null 2024-09-14T03:47:46.252 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.252 INFO:tasks.workunit.client.0.smithi070.stdout: "node": { 2024-09-14T03:47:46.252 INFO:tasks.workunit.client.0.smithi070.stdout: "count": { 2024-09-14T03:47:46.252 INFO:tasks.workunit.client.0.smithi070.stdout: "kubernetes-total": null, 2024-09-14T03:47:46.252 INFO:tasks.workunit.client.0.smithi070.stdout: "with-ceph-daemons": null, 2024-09-14T03:47:46.252 INFO:tasks.workunit.client.0.smithi070.stdout: "with-csi-cephfs-plugin": null, 2024-09-14T03:47:46.252 INFO:tasks.workunit.client.0.smithi070.stdout: "with-csi-nfs-plugin": null, 2024-09-14T03:47:46.252 INFO:tasks.workunit.client.0.smithi070.stdout: "with-csi-rbd-plugin": null 2024-09-14T03:47:46.252 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:46.252 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.252 INFO:tasks.workunit.client.0.smithi070.stdout: "usage": { 2024-09-14T03:47:46.253 INFO:tasks.workunit.client.0.smithi070.stdout: "storage-class": { 2024-09-14T03:47:46.253 INFO:tasks.workunit.client.0.smithi070.stdout: "count": { 2024-09-14T03:47:46.253 INFO:tasks.workunit.client.0.smithi070.stdout: "bucket": null, 2024-09-14T03:47:46.253 INFO:tasks.workunit.client.0.smithi070.stdout: "cephfs": null, 2024-09-14T03:47:46.253 INFO:tasks.workunit.client.0.smithi070.stdout: "nfs": null, 2024-09-14T03:47:46.253 INFO:tasks.workunit.client.0.smithi070.stdout: "rbd": null, 2024-09-14T03:47:46.253 INFO:tasks.workunit.client.0.smithi070.stdout: "total": null 2024-09-14T03:47:46.253 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:46.253 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:46.253 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.253 INFO:tasks.workunit.client.0.smithi070.stdout: "version": null 2024-09-14T03:47:46.253 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.253 INFO:tasks.workunit.client.0.smithi070.stdout: "services": {}, 2024-09-14T03:47:46.253 INFO:tasks.workunit.client.0.smithi070.stdout: "usage": { 2024-09-14T03:47:46.254 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_num": 105, 2024-09-14T03:47:46.254 INFO:tasks.workunit.client.0.smithi070.stdout: "pools": 4, 2024-09-14T03:47:46.254 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_by_class": { 2024-09-14T03:47:46.254 INFO:tasks.workunit.client.0.smithi070.stdout: "ssd": { 2024-09-14T03:47:46.254 INFO:tasks.workunit.client.0.smithi070.stdout: "total_avail_bytes": 765516926976, 2024-09-14T03:47:46.254 INFO:tasks.workunit.client.0.smithi070.stdout: "total_bytes": 767960285184, 2024-09-14T03:47:46.254 INFO:tasks.workunit.client.0.smithi070.stdout: "total_used_bytes": 2443358208, 2024-09-14T03:47:46.254 INFO:tasks.workunit.client.0.smithi070.stdout: "total_used_raw_bytes": 2443358208, 2024-09-14T03:47:46.254 INFO:tasks.workunit.client.0.smithi070.stdout: "total_used_raw_ratio": 0.0031816205009818077 2024-09-14T03:47:46.254 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:46.254 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.254 INFO:tasks.workunit.client.0.smithi070.stdout: "total_avail_bytes": 765516926976, 2024-09-14T03:47:46.254 INFO:tasks.workunit.client.0.smithi070.stdout: "total_bytes": 767960285184, 2024-09-14T03:47:46.254 INFO:tasks.workunit.client.0.smithi070.stdout: "total_used_bytes": 2443358208 2024-09-14T03:47:46.255 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:46.255 INFO:tasks.workunit.client.0.smithi070.stdout:} 2024-09-14T03:47:46.255 INFO:tasks.workunit.client.0.smithi070.stdout:Device report is generated separately. To see it run 'ceph telemetry preview-device'. 2024-09-14T03:47:46.255 INFO:tasks.workunit.client.0.smithi070.stderr:+ ceph telemetry preview-device 2024-09-14T03:47:46.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:45 smithi094 ceph-mon[31053]: pgmap v89: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:46.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:45 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/498712575' entity='client.admin' 2024-09-14T03:47:46.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:45 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:46.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:45 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/2502820960' entity='client.admin' 2024-09-14T03:47:46.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:45 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:46.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:45 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:46.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:45 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:46.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:45 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:46.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:45 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:46.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:45 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:46.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:45 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:46.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:45 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/3890719547' entity='client.admin' 2024-09-14T03:47:46.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:45 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:46.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:45 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:46.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:45 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:46.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:45 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:46.428 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:46 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:47:46] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:47:46.428 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:46 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:47:46] ENGINE Bus STOPPED 2024-09-14T03:47:46.428 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:46 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:47:46] ENGINE Bus STARTING 2024-09-14T03:47:46.428 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:46 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:47:46] ENGINE Serving on http://:::9283 2024-09-14T03:47:46.428 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:46 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:47:46] ENGINE Bus STARTED 2024-09-14T03:47:46.428 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:46 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:47:46] "GET /metrics HTTP/1.1" 200 236436 "" "Prometheus/2.43.0" 2024-09-14T03:47:46.676 INFO:tasks.workunit.client.0.smithi070.stdout:{ 2024-09-14T03:47:46.676 INFO:tasks.workunit.client.0.smithi070.stdout: "1ad5ed40-724c-11ef-9ddf-0cc47a8fd3e7": { 2024-09-14T03:47:46.676 INFO:tasks.workunit.client.0.smithi070.stdout: "INTEL_SSDPED1D480GA_1ad66d74-724c-11ef-9ddf-0cc47a8fd3e7": { 2024-09-14T03:47:46.676 INFO:tasks.workunit.client.0.smithi070.stdout: "20240914-034143": { 2024-09-14T03:47:46.676 INFO:tasks.workunit.client.0.smithi070.stdout: "device": { 2024-09-14T03:47:46.677 INFO:tasks.workunit.client.0.smithi070.stdout: "info_name": "/dev/nvme0n1", 2024-09-14T03:47:46.677 INFO:tasks.workunit.client.0.smithi070.stdout: "name": "/dev/nvme0n1", 2024-09-14T03:47:46.677 INFO:tasks.workunit.client.0.smithi070.stdout: "protocol": "NVMe", 2024-09-14T03:47:46.677 INFO:tasks.workunit.client.0.smithi070.stdout: "type": "nvme" 2024-09-14T03:47:46.677 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.677 INFO:tasks.workunit.client.0.smithi070.stdout: "firmware_version": "E2010325", 2024-09-14T03:47:46.677 INFO:tasks.workunit.client.0.smithi070.stdout: "host_id": "1ad5ed40-724c-11ef-9ddf-0cc47a8fd3e7", 2024-09-14T03:47:46.677 INFO:tasks.workunit.client.0.smithi070.stdout: "json_format_version": [ 2024-09-14T03:47:46.677 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:46.677 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:46.677 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:46.677 INFO:tasks.workunit.client.0.smithi070.stdout: "local_time": { 2024-09-14T03:47:46.677 INFO:tasks.workunit.client.0.smithi070.stdout: "asctime": "Sat Sep 14 03:41:43 2024 UTC", 2024-09-14T03:47:46.678 INFO:tasks.workunit.client.0.smithi070.stdout: "time_t": 1726285303 2024-09-14T03:47:46.678 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.678 INFO:tasks.workunit.client.0.smithi070.stdout: "logical_block_size": 512, 2024-09-14T03:47:46.678 INFO:tasks.workunit.client.0.smithi070.stdout: "model_name": "INTEL SSDPED1D480GA", 2024-09-14T03:47:46.678 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_controller_id": 0, 2024-09-14T03:47:46.678 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-14T03:47:46.678 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_namespaces": [ 2024-09-14T03:47:46.678 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:46.678 INFO:tasks.workunit.client.0.smithi070.stdout: "capacity": { 2024-09-14T03:47:46.678 INFO:tasks.workunit.client.0.smithi070.stdout: "blocks": 937703088, 2024-09-14T03:47:46.678 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 480103981056 2024-09-14T03:47:46.678 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.678 INFO:tasks.workunit.client.0.smithi070.stdout: "formatted_lba_size": 512, 2024-09-14T03:47:46.678 INFO:tasks.workunit.client.0.smithi070.stdout: "id": 1, 2024-09-14T03:47:46.679 INFO:tasks.workunit.client.0.smithi070.stdout: "size": { 2024-09-14T03:47:46.679 INFO:tasks.workunit.client.0.smithi070.stdout: "blocks": 937703088, 2024-09-14T03:47:46.679 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 480103981056 2024-09-14T03:47:46.679 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.679 INFO:tasks.workunit.client.0.smithi070.stdout: "utilization": { 2024-09-14T03:47:46.679 INFO:tasks.workunit.client.0.smithi070.stdout: "blocks": 937703088, 2024-09-14T03:47:46.679 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 480103981056 2024-09-14T03:47:46.679 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:46.679 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:46.679 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:46.679 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_number_of_namespaces": 1, 2024-09-14T03:47:46.679 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_pci_vendor": { 2024-09-14T03:47:46.679 INFO:tasks.workunit.client.0.smithi070.stdout: "id": 32902, 2024-09-14T03:47:46.680 INFO:tasks.workunit.client.0.smithi070.stdout: "subsystem_id": 32902 2024-09-14T03:47:46.680 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.680 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_smart_health_information_add_log": { 2024-09-14T03:47:46.680 INFO:tasks.workunit.client.0.smithi070.stdout: "Device stats": { 2024-09-14T03:47:46.680 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_fw_downgrades": { 2024-09-14T03:47:46.680 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:46.680 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:46.680 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.680 INFO:tasks.workunit.client.0.smithi070.stdout: "crc_error_count": { 2024-09-14T03:47:46.680 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:46.680 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:46.680 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.680 INFO:tasks.workunit.client.0.smithi070.stdout: "end_to_end_error_detection_count": { 2024-09-14T03:47:46.680 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:46.681 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:46.681 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.681 INFO:tasks.workunit.client.0.smithi070.stdout: "erase_fail_count": { 2024-09-14T03:47:46.681 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:46.681 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:46.681 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.681 INFO:tasks.workunit.client.0.smithi070.stdout: "host_bytes_written": { 2024-09-14T03:47:46.681 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:46.681 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 13063793 2024-09-14T03:47:46.681 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.681 INFO:tasks.workunit.client.0.smithi070.stdout: "host_ctx_wear_used": { 2024-09-14T03:47:46.681 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:46.681 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:46.682 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.682 INFO:tasks.workunit.client.0.smithi070.stdout: "media_bytes_read": { 2024-09-14T03:47:46.682 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:46.682 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:46.682 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.682 INFO:tasks.workunit.client.0.smithi070.stdout: "nand_bytes_written": { 2024-09-14T03:47:46.682 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:46.682 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:46.682 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.682 INFO:tasks.workunit.client.0.smithi070.stdout: "perf_stat_indicator": { 2024-09-14T03:47:46.682 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:46.682 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:46.682 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.682 INFO:tasks.workunit.client.0.smithi070.stdout: "pll_lock_loss_count": { 2024-09-14T03:47:46.683 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:46.683 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:46.683 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.683 INFO:tasks.workunit.client.0.smithi070.stdout: "program_fail_count": { 2024-09-14T03:47:46.683 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:46.683 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:46.683 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.683 INFO:tasks.workunit.client.0.smithi070.stdout: "re_alloc_sectr_cnt": { 2024-09-14T03:47:46.683 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:46.683 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:46.683 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.683 INFO:tasks.workunit.client.0.smithi070.stdout: "retry_buffer_overflow_count": { 2024-09-14T03:47:46.683 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:46.684 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:46.684 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.684 INFO:tasks.workunit.client.0.smithi070.stdout: "soft_ecc_err_rate": { 2024-09-14T03:47:46.684 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:46.684 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:46.684 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.684 INFO:tasks.workunit.client.0.smithi070.stdout: "thermal_throttle_status": { 2024-09-14T03:47:46.684 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:46.684 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": { 2024-09-14T03:47:46.684 INFO:tasks.workunit.client.0.smithi070.stdout: "cnt": 0, 2024-09-14T03:47:46.684 INFO:tasks.workunit.client.0.smithi070.stdout: "pct": 0 2024-09-14T03:47:46.684 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:46.684 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.684 INFO:tasks.workunit.client.0.smithi070.stdout: "timed_workload_host_reads": { 2024-09-14T03:47:46.685 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:46.685 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 14 2024-09-14T03:47:46.685 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.685 INFO:tasks.workunit.client.0.smithi070.stdout: "timed_workload_media_wear": { 2024-09-14T03:47:46.685 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:46.685 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": "9.029297" 2024-09-14T03:47:46.685 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.685 INFO:tasks.workunit.client.0.smithi070.stdout: "timed_workload_timer": { 2024-09-14T03:47:46.685 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:46.685 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 1544484 2024-09-14T03:47:46.685 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.685 INFO:tasks.workunit.client.0.smithi070.stdout: "unexp_power_loss": { 2024-09-14T03:47:46.685 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:46.686 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:46.686 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.686 INFO:tasks.workunit.client.0.smithi070.stdout: "wear_leveling": { 2024-09-14T03:47:46.686 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:46.686 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": { 2024-09-14T03:47:46.686 INFO:tasks.workunit.client.0.smithi070.stdout: "avg": 0, 2024-09-14T03:47:46.686 INFO:tasks.workunit.client.0.smithi070.stdout: "max": 0, 2024-09-14T03:47:46.686 INFO:tasks.workunit.client.0.smithi070.stdout: "min": 0 2024-09-14T03:47:46.686 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:46.686 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:46.686 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.686 INFO:tasks.workunit.client.0.smithi070.stdout: "Intel Smart log": "nvme0n1" 2024-09-14T03:47:46.686 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.686 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_smart_health_information_log": { 2024-09-14T03:47:46.687 INFO:tasks.workunit.client.0.smithi070.stdout: "available_spare": 100, 2024-09-14T03:47:46.687 INFO:tasks.workunit.client.0.smithi070.stdout: "available_spare_threshold": 0, 2024-09-14T03:47:46.687 INFO:tasks.workunit.client.0.smithi070.stdout: "controller_busy_time": 5166, 2024-09-14T03:47:46.687 INFO:tasks.workunit.client.0.smithi070.stdout: "critical_warning": 0, 2024-09-14T03:47:46.687 INFO:tasks.workunit.client.0.smithi070.stdout: "data_units_read": 177890366, 2024-09-14T03:47:46.687 INFO:tasks.workunit.client.0.smithi070.stdout: "data_units_written": 856148757, 2024-09-14T03:47:46.687 INFO:tasks.workunit.client.0.smithi070.stdout: "host_reads": 3130935174, 2024-09-14T03:47:46.687 INFO:tasks.workunit.client.0.smithi070.stdout: "host_writes": 18536697060, 2024-09-14T03:47:46.687 INFO:tasks.workunit.client.0.smithi070.stdout: "media_errors": 0, 2024-09-14T03:47:46.687 INFO:tasks.workunit.client.0.smithi070.stdout: "num_err_log_entries": 0, 2024-09-14T03:47:46.687 INFO:tasks.workunit.client.0.smithi070.stdout: "percentage_used": 14, 2024-09-14T03:47:46.687 INFO:tasks.workunit.client.0.smithi070.stdout: "power_cycles": 35890, 2024-09-14T03:47:46.687 INFO:tasks.workunit.client.0.smithi070.stdout: "power_on_hours": 40704, 2024-09-14T03:47:46.687 INFO:tasks.workunit.client.0.smithi070.stdout: "temperature": 39, 2024-09-14T03:47:46.688 INFO:tasks.workunit.client.0.smithi070.stdout: "unsafe_shutdowns": 10931 2024-09-14T03:47:46.688 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.688 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_vendor": "intel", 2024-09-14T03:47:46.688 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_version": { 2024-09-14T03:47:46.688 INFO:tasks.workunit.client.0.smithi070.stdout: "string": "<1.2", 2024-09-14T03:47:46.688 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:46.688 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.688 INFO:tasks.workunit.client.0.smithi070.stdout: "power_cycle_count": 35890, 2024-09-14T03:47:46.688 INFO:tasks.workunit.client.0.smithi070.stdout: "power_on_time": { 2024-09-14T03:47:46.688 INFO:tasks.workunit.client.0.smithi070.stdout: "hours": 40704 2024-09-14T03:47:46.688 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.688 INFO:tasks.workunit.client.0.smithi070.stdout: "serial_number": "deleted", 2024-09-14T03:47:46.688 INFO:tasks.workunit.client.0.smithi070.stdout: "smart_status": { 2024-09-14T03:47:46.688 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme": { 2024-09-14T03:47:46.689 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:46.689 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.689 INFO:tasks.workunit.client.0.smithi070.stdout: "passed": true 2024-09-14T03:47:46.689 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.689 INFO:tasks.workunit.client.0.smithi070.stdout: "smartctl": { 2024-09-14T03:47:46.689 INFO:tasks.workunit.client.0.smithi070.stdout: "argv": [ 2024-09-14T03:47:46.689 INFO:tasks.workunit.client.0.smithi070.stdout: "smartctl", 2024-09-14T03:47:46.689 INFO:tasks.workunit.client.0.smithi070.stdout: "-x", 2024-09-14T03:47:46.689 INFO:tasks.workunit.client.0.smithi070.stdout: "--json=o", 2024-09-14T03:47:46.689 INFO:tasks.workunit.client.0.smithi070.stdout: "/dev/nvme0n1" 2024-09-14T03:47:46.689 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:46.689 INFO:tasks.workunit.client.0.smithi070.stdout: "build_info": "(local build)", 2024-09-14T03:47:46.689 INFO:tasks.workunit.client.0.smithi070.stdout: "exit_status": 0, 2024-09-14T03:47:46.689 INFO:tasks.workunit.client.0.smithi070.stdout: "output": [ 2024-09-14T03:47:46.690 INFO:tasks.workunit.client.0.smithi070.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-505.el9.x86_64] (local build)", 2024-09-14T03:47:46.690 INFO:tasks.workunit.client.0.smithi070.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-14T03:47:46.690 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:46.690 INFO:tasks.workunit.client.0.smithi070.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-14T03:47:46.690 INFO:tasks.workunit.client.0.smithi070.stdout: "Model Number: INTEL SSDPED1D480GA", 2024-09-14T03:47:46.690 INFO:tasks.workunit.client.0.smithi070.stdout: "Serial Number: deleted", 2024-09-14T03:47:46.690 INFO:tasks.workunit.client.0.smithi070.stdout: "Firmware Version: E2010325", 2024-09-14T03:47:46.690 INFO:tasks.workunit.client.0.smithi070.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-14T03:47:46.690 INFO:tasks.workunit.client.0.smithi070.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-14T03:47:46.690 INFO:tasks.workunit.client.0.smithi070.stdout: "Controller ID: 0", 2024-09-14T03:47:46.690 INFO:tasks.workunit.client.0.smithi070.stdout: "NVMe Version: <1.2", 2024-09-14T03:47:46.690 INFO:tasks.workunit.client.0.smithi070.stdout: "Number of Namespaces: 1", 2024-09-14T03:47:46.690 INFO:tasks.workunit.client.0.smithi070.stdout: "Namespace 1 Size/Capacity: 480,103,981,056 [480 GB]", 2024-09-14T03:47:46.691 INFO:tasks.workunit.client.0.smithi070.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-14T03:47:46.691 INFO:tasks.workunit.client.0.smithi070.stdout: "Local Time is: Sat Sep 14 03:41:43 2024 UTC", 2024-09-14T03:47:46.691 INFO:tasks.workunit.client.0.smithi070.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-14T03:47:46.691 INFO:tasks.workunit.client.0.smithi070.stdout: "Optional Admin Commands (0x0007): Security Format Frmw_DL", 2024-09-14T03:47:46.691 INFO:tasks.workunit.client.0.smithi070.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-14T03:47:46.691 INFO:tasks.workunit.client.0.smithi070.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-14T03:47:46.691 INFO:tasks.workunit.client.0.smithi070.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-14T03:47:46.691 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:46.691 INFO:tasks.workunit.client.0.smithi070.stdout: "Supported Power States", 2024-09-14T03:47:46.691 INFO:tasks.workunit.client.0.smithi070.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-14T03:47:46.691 INFO:tasks.workunit.client.0.smithi070.stdout: " 0 + 18.00W - - 0 0 0 0 0 0", 2024-09-14T03:47:46.691 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:46.691 INFO:tasks.workunit.client.0.smithi070.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-14T03:47:46.691 INFO:tasks.workunit.client.0.smithi070.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-14T03:47:46.692 INFO:tasks.workunit.client.0.smithi070.stdout: " 0 + 512 0 2", 2024-09-14T03:47:46.692 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:46.692 INFO:tasks.workunit.client.0.smithi070.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-14T03:47:46.692 INFO:tasks.workunit.client.0.smithi070.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-14T03:47:46.692 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:46.692 INFO:tasks.workunit.client.0.smithi070.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-14T03:47:46.692 INFO:tasks.workunit.client.0.smithi070.stdout: "Critical Warning: 0x00", 2024-09-14T03:47:46.692 INFO:tasks.workunit.client.0.smithi070.stdout: "Temperature: 39 Celsius", 2024-09-14T03:47:46.692 INFO:tasks.workunit.client.0.smithi070.stdout: "Available Spare: 100%", 2024-09-14T03:47:46.692 INFO:tasks.workunit.client.0.smithi070.stdout: "Available Spare Threshold: 0%", 2024-09-14T03:47:46.692 INFO:tasks.workunit.client.0.smithi070.stdout: "Percentage Used: 14%", 2024-09-14T03:47:46.692 INFO:tasks.workunit.client.0.smithi070.stdout: "Data Units Read: 177,890,366 [91.0 TB]", 2024-09-14T03:47:46.692 INFO:tasks.workunit.client.0.smithi070.stdout: "Data Units Written: 856,148,757 [438 TB]", 2024-09-14T03:47:46.692 INFO:tasks.workunit.client.0.smithi070.stdout: "Host Read Commands: 3,130,935,174", 2024-09-14T03:47:46.693 INFO:tasks.workunit.client.0.smithi070.stdout: "Host Write Commands: 18,536,697,060", 2024-09-14T03:47:46.693 INFO:tasks.workunit.client.0.smithi070.stdout: "Controller Busy Time: 5,166", 2024-09-14T03:47:46.693 INFO:tasks.workunit.client.0.smithi070.stdout: "Power Cycles: 35,890", 2024-09-14T03:47:46.693 INFO:tasks.workunit.client.0.smithi070.stdout: "Power On Hours: 40,704", 2024-09-14T03:47:46.693 INFO:tasks.workunit.client.0.smithi070.stdout: "Unsafe Shutdowns: 10,931", 2024-09-14T03:47:46.693 INFO:tasks.workunit.client.0.smithi070.stdout: "Media and Data Integrity Errors: 0", 2024-09-14T03:47:46.693 INFO:tasks.workunit.client.0.smithi070.stdout: "Error Information Log Entries: 0", 2024-09-14T03:47:46.693 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:46.693 INFO:tasks.workunit.client.0.smithi070.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-14T03:47:46.693 INFO:tasks.workunit.client.0.smithi070.stdout: "No Errors Logged", 2024-09-14T03:47:46.693 INFO:tasks.workunit.client.0.smithi070.stdout: "" 2024-09-14T03:47:46.693 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:46.693 INFO:tasks.workunit.client.0.smithi070.stdout: "platform_info": "x86_64-linux-5.14.0-505.el9.x86_64", 2024-09-14T03:47:46.693 INFO:tasks.workunit.client.0.smithi070.stdout: "svn_revision": "5155", 2024-09-14T03:47:46.694 INFO:tasks.workunit.client.0.smithi070.stdout: "version": [ 2024-09-14T03:47:46.694 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:46.694 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:46.694 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:46.694 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.694 INFO:tasks.workunit.client.0.smithi070.stdout: "temperature": { 2024-09-14T03:47:46.694 INFO:tasks.workunit.client.0.smithi070.stdout: "current": 39 2024-09-14T03:47:46.694 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.694 INFO:tasks.workunit.client.0.smithi070.stdout: "user_capacity": { 2024-09-14T03:47:46.694 INFO:tasks.workunit.client.0.smithi070.stdout: "blocks": 937703088, 2024-09-14T03:47:46.694 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 480103981056 2024-09-14T03:47:46.694 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:46.694 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:46.694 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:46.695 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.695 INFO:tasks.workunit.client.0.smithi070.stdout: "1ad766de-724c-11ef-9ddf-0cc47a8fd3e7": { 2024-09-14T03:47:46.695 INFO:tasks.workunit.client.0.smithi070.stdout: "INTEL_SSDPED1D480GA_1ad7d2f4-724c-11ef-9ddf-0cc47a8fd3e7": { 2024-09-14T03:47:46.695 INFO:tasks.workunit.client.0.smithi070.stdout: "20240914-034141": { 2024-09-14T03:47:46.695 INFO:tasks.workunit.client.0.smithi070.stdout: "device": { 2024-09-14T03:47:46.695 INFO:tasks.workunit.client.0.smithi070.stdout: "info_name": "/dev/nvme0n1", 2024-09-14T03:47:46.695 INFO:tasks.workunit.client.0.smithi070.stdout: "name": "/dev/nvme0n1", 2024-09-14T03:47:46.695 INFO:tasks.workunit.client.0.smithi070.stdout: "protocol": "NVMe", 2024-09-14T03:47:46.695 INFO:tasks.workunit.client.0.smithi070.stdout: "type": "nvme" 2024-09-14T03:47:46.695 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.695 INFO:tasks.workunit.client.0.smithi070.stdout: "firmware_version": "E2010325", 2024-09-14T03:47:46.695 INFO:tasks.workunit.client.0.smithi070.stdout: "host_id": "1ad766de-724c-11ef-9ddf-0cc47a8fd3e7", 2024-09-14T03:47:46.695 INFO:tasks.workunit.client.0.smithi070.stdout: "json_format_version": [ 2024-09-14T03:47:46.695 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:46.696 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:46.696 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:46.696 INFO:tasks.workunit.client.0.smithi070.stdout: "local_time": { 2024-09-14T03:47:46.696 INFO:tasks.workunit.client.0.smithi070.stdout: "asctime": "Sat Sep 14 03:41:40 2024 UTC", 2024-09-14T03:47:46.696 INFO:tasks.workunit.client.0.smithi070.stdout: "time_t": 1726285300 2024-09-14T03:47:46.696 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.696 INFO:tasks.workunit.client.0.smithi070.stdout: "logical_block_size": 512, 2024-09-14T03:47:46.696 INFO:tasks.workunit.client.0.smithi070.stdout: "model_name": "INTEL SSDPED1D480GA", 2024-09-14T03:47:46.696 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_controller_id": 0, 2024-09-14T03:47:46.696 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-14T03:47:46.696 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_namespaces": [ 2024-09-14T03:47:46.696 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:46.696 INFO:tasks.workunit.client.0.smithi070.stdout: "capacity": { 2024-09-14T03:47:46.697 INFO:tasks.workunit.client.0.smithi070.stdout: "blocks": 937703088, 2024-09-14T03:47:46.697 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 480103981056 2024-09-14T03:47:46.697 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.697 INFO:tasks.workunit.client.0.smithi070.stdout: "formatted_lba_size": 512, 2024-09-14T03:47:46.697 INFO:tasks.workunit.client.0.smithi070.stdout: "id": 1, 2024-09-14T03:47:46.697 INFO:tasks.workunit.client.0.smithi070.stdout: "size": { 2024-09-14T03:47:46.697 INFO:tasks.workunit.client.0.smithi070.stdout: "blocks": 937703088, 2024-09-14T03:47:46.697 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 480103981056 2024-09-14T03:47:46.697 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.697 INFO:tasks.workunit.client.0.smithi070.stdout: "utilization": { 2024-09-14T03:47:46.697 INFO:tasks.workunit.client.0.smithi070.stdout: "blocks": 937703088, 2024-09-14T03:47:46.697 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 480103981056 2024-09-14T03:47:46.697 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:46.697 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:46.698 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:46.698 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_number_of_namespaces": 1, 2024-09-14T03:47:46.698 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_pci_vendor": { 2024-09-14T03:47:46.698 INFO:tasks.workunit.client.0.smithi070.stdout: "id": 32902, 2024-09-14T03:47:46.698 INFO:tasks.workunit.client.0.smithi070.stdout: "subsystem_id": 32902 2024-09-14T03:47:46.698 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.698 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_smart_health_information_add_log": { 2024-09-14T03:47:46.698 INFO:tasks.workunit.client.0.smithi070.stdout: "Device stats": { 2024-09-14T03:47:46.698 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_fw_downgrades": { 2024-09-14T03:47:46.698 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:46.698 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:46.698 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.698 INFO:tasks.workunit.client.0.smithi070.stdout: "crc_error_count": { 2024-09-14T03:47:46.698 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:46.699 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:46.699 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.699 INFO:tasks.workunit.client.0.smithi070.stdout: "end_to_end_error_detection_count": { 2024-09-14T03:47:46.699 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:46.699 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:46.699 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.699 INFO:tasks.workunit.client.0.smithi070.stdout: "erase_fail_count": { 2024-09-14T03:47:46.699 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:46.699 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:46.699 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.699 INFO:tasks.workunit.client.0.smithi070.stdout: "host_bytes_written": { 2024-09-14T03:47:46.699 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:46.699 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 11224882 2024-09-14T03:47:46.699 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.700 INFO:tasks.workunit.client.0.smithi070.stdout: "host_ctx_wear_used": { 2024-09-14T03:47:46.700 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:46.700 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:46.700 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.700 INFO:tasks.workunit.client.0.smithi070.stdout: "media_bytes_read": { 2024-09-14T03:47:46.700 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:46.700 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:46.700 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.700 INFO:tasks.workunit.client.0.smithi070.stdout: "nand_bytes_written": { 2024-09-14T03:47:46.700 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:46.700 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:46.700 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.700 INFO:tasks.workunit.client.0.smithi070.stdout: "perf_stat_indicator": { 2024-09-14T03:47:46.700 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:46.701 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:46.701 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.701 INFO:tasks.workunit.client.0.smithi070.stdout: "pll_lock_loss_count": { 2024-09-14T03:47:46.701 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:46.701 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:46.701 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.701 INFO:tasks.workunit.client.0.smithi070.stdout: "program_fail_count": { 2024-09-14T03:47:46.701 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:46.701 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:46.701 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.701 INFO:tasks.workunit.client.0.smithi070.stdout: "re_alloc_sectr_cnt": { 2024-09-14T03:47:46.701 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:46.701 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:46.701 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.702 INFO:tasks.workunit.client.0.smithi070.stdout: "retry_buffer_overflow_count": { 2024-09-14T03:47:46.702 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:46.702 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:46.702 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.702 INFO:tasks.workunit.client.0.smithi070.stdout: "soft_ecc_err_rate": { 2024-09-14T03:47:46.702 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:46.702 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:46.702 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.702 INFO:tasks.workunit.client.0.smithi070.stdout: "thermal_throttle_status": { 2024-09-14T03:47:46.702 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:46.702 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": { 2024-09-14T03:47:46.702 INFO:tasks.workunit.client.0.smithi070.stdout: "cnt": 0, 2024-09-14T03:47:46.702 INFO:tasks.workunit.client.0.smithi070.stdout: "pct": 0 2024-09-14T03:47:46.702 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:46.703 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.703 INFO:tasks.workunit.client.0.smithi070.stdout: "timed_workload_host_reads": { 2024-09-14T03:47:46.703 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:46.703 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 14 2024-09-14T03:47:46.703 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.703 INFO:tasks.workunit.client.0.smithi070.stdout: "timed_workload_media_wear": { 2024-09-14T03:47:46.703 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:46.703 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": "0.000000" 2024-09-14T03:47:46.703 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.703 INFO:tasks.workunit.client.0.smithi070.stdout: "timed_workload_timer": { 2024-09-14T03:47:46.703 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:46.703 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 1493348 2024-09-14T03:47:46.703 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.703 INFO:tasks.workunit.client.0.smithi070.stdout: "unexp_power_loss": { 2024-09-14T03:47:46.704 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:46.704 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:46.704 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.704 INFO:tasks.workunit.client.0.smithi070.stdout: "wear_leveling": { 2024-09-14T03:47:46.704 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:46.704 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": { 2024-09-14T03:47:46.704 INFO:tasks.workunit.client.0.smithi070.stdout: "avg": 0, 2024-09-14T03:47:46.704 INFO:tasks.workunit.client.0.smithi070.stdout: "max": 0, 2024-09-14T03:47:46.704 INFO:tasks.workunit.client.0.smithi070.stdout: "min": 0 2024-09-14T03:47:46.704 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:46.704 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:46.704 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.704 INFO:tasks.workunit.client.0.smithi070.stdout: "Intel Smart log": "nvme0n1" 2024-09-14T03:47:46.704 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.705 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_smart_health_information_log": { 2024-09-14T03:47:46.705 INFO:tasks.workunit.client.0.smithi070.stdout: "available_spare": 100, 2024-09-14T03:47:46.705 INFO:tasks.workunit.client.0.smithi070.stdout: "available_spare_threshold": 0, 2024-09-14T03:47:46.705 INFO:tasks.workunit.client.0.smithi070.stdout: "controller_busy_time": 4368, 2024-09-14T03:47:46.705 INFO:tasks.workunit.client.0.smithi070.stdout: "critical_warning": 0, 2024-09-14T03:47:46.705 INFO:tasks.workunit.client.0.smithi070.stdout: "data_units_read": 155028165, 2024-09-14T03:47:46.705 INFO:tasks.workunit.client.0.smithi070.stdout: "data_units_written": 735633918, 2024-09-14T03:47:46.705 INFO:tasks.workunit.client.0.smithi070.stdout: "host_reads": 2607915323, 2024-09-14T03:47:46.705 INFO:tasks.workunit.client.0.smithi070.stdout: "host_writes": 16464089664, 2024-09-14T03:47:46.705 INFO:tasks.workunit.client.0.smithi070.stdout: "media_errors": 0, 2024-09-14T03:47:46.705 INFO:tasks.workunit.client.0.smithi070.stdout: "num_err_log_entries": 0, 2024-09-14T03:47:46.705 INFO:tasks.workunit.client.0.smithi070.stdout: "percentage_used": 0, 2024-09-14T03:47:46.705 INFO:tasks.workunit.client.0.smithi070.stdout: "power_cycles": 30550, 2024-09-14T03:47:46.706 INFO:tasks.workunit.client.0.smithi070.stdout: "power_on_hours": 37550, 2024-09-14T03:47:46.706 INFO:tasks.workunit.client.0.smithi070.stdout: "temperature": 34, 2024-09-14T03:47:46.706 INFO:tasks.workunit.client.0.smithi070.stdout: "unsafe_shutdowns": 9447 2024-09-14T03:47:46.706 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.706 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_vendor": "intel", 2024-09-14T03:47:46.706 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_version": { 2024-09-14T03:47:46.706 INFO:tasks.workunit.client.0.smithi070.stdout: "string": "<1.2", 2024-09-14T03:47:46.706 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:46.706 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.706 INFO:tasks.workunit.client.0.smithi070.stdout: "power_cycle_count": 30550, 2024-09-14T03:47:46.706 INFO:tasks.workunit.client.0.smithi070.stdout: "power_on_time": { 2024-09-14T03:47:46.706 INFO:tasks.workunit.client.0.smithi070.stdout: "hours": 37550 2024-09-14T03:47:46.706 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.706 INFO:tasks.workunit.client.0.smithi070.stdout: "serial_number": "deleted", 2024-09-14T03:47:46.706 INFO:tasks.workunit.client.0.smithi070.stdout: "smart_status": { 2024-09-14T03:47:46.707 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme": { 2024-09-14T03:47:46.707 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:46.707 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.707 INFO:tasks.workunit.client.0.smithi070.stdout: "passed": true 2024-09-14T03:47:46.707 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.707 INFO:tasks.workunit.client.0.smithi070.stdout: "smartctl": { 2024-09-14T03:47:46.707 INFO:tasks.workunit.client.0.smithi070.stdout: "argv": [ 2024-09-14T03:47:46.707 INFO:tasks.workunit.client.0.smithi070.stdout: "smartctl", 2024-09-14T03:47:46.707 INFO:tasks.workunit.client.0.smithi070.stdout: "-x", 2024-09-14T03:47:46.707 INFO:tasks.workunit.client.0.smithi070.stdout: "--json=o", 2024-09-14T03:47:46.707 INFO:tasks.workunit.client.0.smithi070.stdout: "/dev/nvme0n1" 2024-09-14T03:47:46.707 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:46.707 INFO:tasks.workunit.client.0.smithi070.stdout: "build_info": "(local build)", 2024-09-14T03:47:46.707 INFO:tasks.workunit.client.0.smithi070.stdout: "exit_status": 0, 2024-09-14T03:47:46.708 INFO:tasks.workunit.client.0.smithi070.stdout: "output": [ 2024-09-14T03:47:46.708 INFO:tasks.workunit.client.0.smithi070.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-505.el9.x86_64] (local build)", 2024-09-14T03:47:46.708 INFO:tasks.workunit.client.0.smithi070.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-14T03:47:46.708 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:46.708 INFO:tasks.workunit.client.0.smithi070.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-14T03:47:46.708 INFO:tasks.workunit.client.0.smithi070.stdout: "Model Number: INTEL SSDPED1D480GA", 2024-09-14T03:47:46.708 INFO:tasks.workunit.client.0.smithi070.stdout: "Serial Number: deleted", 2024-09-14T03:47:46.708 INFO:tasks.workunit.client.0.smithi070.stdout: "Firmware Version: E2010325", 2024-09-14T03:47:46.708 INFO:tasks.workunit.client.0.smithi070.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-14T03:47:46.708 INFO:tasks.workunit.client.0.smithi070.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-14T03:47:46.708 INFO:tasks.workunit.client.0.smithi070.stdout: "Controller ID: 0", 2024-09-14T03:47:46.708 INFO:tasks.workunit.client.0.smithi070.stdout: "NVMe Version: <1.2", 2024-09-14T03:47:46.709 INFO:tasks.workunit.client.0.smithi070.stdout: "Number of Namespaces: 1", 2024-09-14T03:47:46.709 INFO:tasks.workunit.client.0.smithi070.stdout: "Namespace 1 Size/Capacity: 480,103,981,056 [480 GB]", 2024-09-14T03:47:46.709 INFO:tasks.workunit.client.0.smithi070.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-14T03:47:46.709 INFO:tasks.workunit.client.0.smithi070.stdout: "Local Time is: Sat Sep 14 03:41:40 2024 UTC", 2024-09-14T03:47:46.709 INFO:tasks.workunit.client.0.smithi070.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-14T03:47:46.709 INFO:tasks.workunit.client.0.smithi070.stdout: "Optional Admin Commands (0x0007): Security Format Frmw_DL", 2024-09-14T03:47:46.709 INFO:tasks.workunit.client.0.smithi070.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-14T03:47:46.709 INFO:tasks.workunit.client.0.smithi070.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-14T03:47:46.709 INFO:tasks.workunit.client.0.smithi070.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-14T03:47:46.709 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:46.709 INFO:tasks.workunit.client.0.smithi070.stdout: "Supported Power States", 2024-09-14T03:47:46.709 INFO:tasks.workunit.client.0.smithi070.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-14T03:47:46.709 INFO:tasks.workunit.client.0.smithi070.stdout: " 0 + 18.00W - - 0 0 0 0 0 0", 2024-09-14T03:47:46.709 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:46.710 INFO:tasks.workunit.client.0.smithi070.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-14T03:47:46.710 INFO:tasks.workunit.client.0.smithi070.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-14T03:47:46.710 INFO:tasks.workunit.client.0.smithi070.stdout: " 0 + 512 0 2", 2024-09-14T03:47:46.710 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:46.710 INFO:tasks.workunit.client.0.smithi070.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-14T03:47:46.710 INFO:tasks.workunit.client.0.smithi070.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-14T03:47:46.710 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:46.710 INFO:tasks.workunit.client.0.smithi070.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-14T03:47:46.710 INFO:tasks.workunit.client.0.smithi070.stdout: "Critical Warning: 0x00", 2024-09-14T03:47:46.710 INFO:tasks.workunit.client.0.smithi070.stdout: "Temperature: 34 Celsius", 2024-09-14T03:47:46.710 INFO:tasks.workunit.client.0.smithi070.stdout: "Available Spare: 100%", 2024-09-14T03:47:46.710 INFO:tasks.workunit.client.0.smithi070.stdout: "Available Spare Threshold: 0%", 2024-09-14T03:47:46.710 INFO:tasks.workunit.client.0.smithi070.stdout: "Percentage Used: 0%", 2024-09-14T03:47:46.710 INFO:tasks.workunit.client.0.smithi070.stdout: "Data Units Read: 155,028,165 [79.3 TB]", 2024-09-14T03:47:46.711 INFO:tasks.workunit.client.0.smithi070.stdout: "Data Units Written: 735,633,918 [376 TB]", 2024-09-14T03:47:46.711 INFO:tasks.workunit.client.0.smithi070.stdout: "Host Read Commands: 2,607,915,323", 2024-09-14T03:47:46.711 INFO:tasks.workunit.client.0.smithi070.stdout: "Host Write Commands: 16,464,089,664", 2024-09-14T03:47:46.711 INFO:tasks.workunit.client.0.smithi070.stdout: "Controller Busy Time: 4,368", 2024-09-14T03:47:46.711 INFO:tasks.workunit.client.0.smithi070.stdout: "Power Cycles: 30,550", 2024-09-14T03:47:46.711 INFO:tasks.workunit.client.0.smithi070.stdout: "Power On Hours: 37,550", 2024-09-14T03:47:46.711 INFO:tasks.workunit.client.0.smithi070.stdout: "Unsafe Shutdowns: 9,447", 2024-09-14T03:47:46.711 INFO:tasks.workunit.client.0.smithi070.stdout: "Media and Data Integrity Errors: 0", 2024-09-14T03:47:46.711 INFO:tasks.workunit.client.0.smithi070.stdout: "Error Information Log Entries: 0", 2024-09-14T03:47:46.711 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:46.711 INFO:tasks.workunit.client.0.smithi070.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-14T03:47:46.711 INFO:tasks.workunit.client.0.smithi070.stdout: "No Errors Logged", 2024-09-14T03:47:46.711 INFO:tasks.workunit.client.0.smithi070.stdout: "" 2024-09-14T03:47:46.711 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:46.712 INFO:tasks.workunit.client.0.smithi070.stdout: "platform_info": "x86_64-linux-5.14.0-505.el9.x86_64", 2024-09-14T03:47:46.712 INFO:tasks.workunit.client.0.smithi070.stdout: "svn_revision": "5155", 2024-09-14T03:47:46.712 INFO:tasks.workunit.client.0.smithi070.stdout: "version": [ 2024-09-14T03:47:46.712 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:46.712 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:46.712 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:46.712 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.712 INFO:tasks.workunit.client.0.smithi070.stdout: "temperature": { 2024-09-14T03:47:46.712 INFO:tasks.workunit.client.0.smithi070.stdout: "current": 34 2024-09-14T03:47:46.712 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:46.712 INFO:tasks.workunit.client.0.smithi070.stdout: "user_capacity": { 2024-09-14T03:47:46.712 INFO:tasks.workunit.client.0.smithi070.stdout: "blocks": 937703088, 2024-09-14T03:47:46.712 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 480103981056 2024-09-14T03:47:46.713 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:46.713 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:46.713 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:46.713 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:46.713 INFO:tasks.workunit.client.0.smithi070.stdout:} 2024-09-14T03:47:46.713 INFO:tasks.workunit.client.0.smithi070.stderr:+ ceph telemetry preview-all 2024-09-14T03:47:47.136 INFO:tasks.workunit.client.0.smithi070.stdout:{ 2024-09-14T03:47:47.136 INFO:tasks.workunit.client.0.smithi070.stdout: "device_report": { 2024-09-14T03:47:47.136 INFO:tasks.workunit.client.0.smithi070.stdout: "1ad5ed40-724c-11ef-9ddf-0cc47a8fd3e7": { 2024-09-14T03:47:47.136 INFO:tasks.workunit.client.0.smithi070.stdout: "INTEL_SSDPED1D480GA_1ad66d74-724c-11ef-9ddf-0cc47a8fd3e7": { 2024-09-14T03:47:47.136 INFO:tasks.workunit.client.0.smithi070.stdout: "20240914-034143": { 2024-09-14T03:47:47.136 INFO:tasks.workunit.client.0.smithi070.stdout: "device": { 2024-09-14T03:47:47.136 INFO:tasks.workunit.client.0.smithi070.stdout: "info_name": "/dev/nvme0n1", 2024-09-14T03:47:47.136 INFO:tasks.workunit.client.0.smithi070.stdout: "name": "/dev/nvme0n1", 2024-09-14T03:47:47.136 INFO:tasks.workunit.client.0.smithi070.stdout: "protocol": "NVMe", 2024-09-14T03:47:47.136 INFO:tasks.workunit.client.0.smithi070.stdout: "type": "nvme" 2024-09-14T03:47:47.137 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.137 INFO:tasks.workunit.client.0.smithi070.stdout: "firmware_version": "E2010325", 2024-09-14T03:47:47.137 INFO:tasks.workunit.client.0.smithi070.stdout: "host_id": "1ad5ed40-724c-11ef-9ddf-0cc47a8fd3e7", 2024-09-14T03:47:47.137 INFO:tasks.workunit.client.0.smithi070.stdout: "json_format_version": [ 2024-09-14T03:47:47.137 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:47.137 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:47.137 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:47.137 INFO:tasks.workunit.client.0.smithi070.stdout: "local_time": { 2024-09-14T03:47:47.137 INFO:tasks.workunit.client.0.smithi070.stdout: "asctime": "Sat Sep 14 03:41:43 2024 UTC", 2024-09-14T03:47:47.137 INFO:tasks.workunit.client.0.smithi070.stdout: "time_t": 1726285303 2024-09-14T03:47:47.137 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.137 INFO:tasks.workunit.client.0.smithi070.stdout: "logical_block_size": 512, 2024-09-14T03:47:47.137 INFO:tasks.workunit.client.0.smithi070.stdout: "model_name": "INTEL SSDPED1D480GA", 2024-09-14T03:47:47.137 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_controller_id": 0, 2024-09-14T03:47:47.138 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-14T03:47:47.138 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_namespaces": [ 2024-09-14T03:47:47.138 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:47.138 INFO:tasks.workunit.client.0.smithi070.stdout: "capacity": { 2024-09-14T03:47:47.138 INFO:tasks.workunit.client.0.smithi070.stdout: "blocks": 937703088, 2024-09-14T03:47:47.138 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 480103981056 2024-09-14T03:47:47.138 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.138 INFO:tasks.workunit.client.0.smithi070.stdout: "formatted_lba_size": 512, 2024-09-14T03:47:47.138 INFO:tasks.workunit.client.0.smithi070.stdout: "id": 1, 2024-09-14T03:47:47.138 INFO:tasks.workunit.client.0.smithi070.stdout: "size": { 2024-09-14T03:47:47.138 INFO:tasks.workunit.client.0.smithi070.stdout: "blocks": 937703088, 2024-09-14T03:47:47.138 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 480103981056 2024-09-14T03:47:47.138 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.139 INFO:tasks.workunit.client.0.smithi070.stdout: "utilization": { 2024-09-14T03:47:47.139 INFO:tasks.workunit.client.0.smithi070.stdout: "blocks": 937703088, 2024-09-14T03:47:47.139 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 480103981056 2024-09-14T03:47:47.139 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:47.139 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:47.139 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:47.139 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_number_of_namespaces": 1, 2024-09-14T03:47:47.139 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_pci_vendor": { 2024-09-14T03:47:47.139 INFO:tasks.workunit.client.0.smithi070.stdout: "id": 32902, 2024-09-14T03:47:47.139 INFO:tasks.workunit.client.0.smithi070.stdout: "subsystem_id": 32902 2024-09-14T03:47:47.139 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.139 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_smart_health_information_add_log": { 2024-09-14T03:47:47.139 INFO:tasks.workunit.client.0.smithi070.stdout: "Device stats": { 2024-09-14T03:47:47.139 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_fw_downgrades": { 2024-09-14T03:47:47.140 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:47.140 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:47.140 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.140 INFO:tasks.workunit.client.0.smithi070.stdout: "crc_error_count": { 2024-09-14T03:47:47.140 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:47.140 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:47.140 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.140 INFO:tasks.workunit.client.0.smithi070.stdout: "end_to_end_error_detection_count": { 2024-09-14T03:47:47.140 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:47.140 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:47.140 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.140 INFO:tasks.workunit.client.0.smithi070.stdout: "erase_fail_count": { 2024-09-14T03:47:47.140 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:47.140 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:47.141 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.141 INFO:tasks.workunit.client.0.smithi070.stdout: "host_bytes_written": { 2024-09-14T03:47:47.141 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:47.141 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 13063793 2024-09-14T03:47:47.141 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.141 INFO:tasks.workunit.client.0.smithi070.stdout: "host_ctx_wear_used": { 2024-09-14T03:47:47.141 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:47.141 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:47.141 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.141 INFO:tasks.workunit.client.0.smithi070.stdout: "media_bytes_read": { 2024-09-14T03:47:47.141 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:47.141 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:47.141 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.141 INFO:tasks.workunit.client.0.smithi070.stdout: "nand_bytes_written": { 2024-09-14T03:47:47.142 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:47.142 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:47.142 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.142 INFO:tasks.workunit.client.0.smithi070.stdout: "perf_stat_indicator": { 2024-09-14T03:47:47.142 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:47.142 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:47.142 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.142 INFO:tasks.workunit.client.0.smithi070.stdout: "pll_lock_loss_count": { 2024-09-14T03:47:47.142 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:47.142 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:47.142 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.142 INFO:tasks.workunit.client.0.smithi070.stdout: "program_fail_count": { 2024-09-14T03:47:47.142 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:47.142 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:47.143 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.143 INFO:tasks.workunit.client.0.smithi070.stdout: "re_alloc_sectr_cnt": { 2024-09-14T03:47:47.143 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:47.143 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:47.143 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.143 INFO:tasks.workunit.client.0.smithi070.stdout: "retry_buffer_overflow_count": { 2024-09-14T03:47:47.143 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:47.143 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:47.143 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.143 INFO:tasks.workunit.client.0.smithi070.stdout: "soft_ecc_err_rate": { 2024-09-14T03:47:47.143 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:47.143 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:47.143 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.143 INFO:tasks.workunit.client.0.smithi070.stdout: "thermal_throttle_status": { 2024-09-14T03:47:47.144 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:47.144 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": { 2024-09-14T03:47:47.144 INFO:tasks.workunit.client.0.smithi070.stdout: "cnt": 0, 2024-09-14T03:47:47.144 INFO:tasks.workunit.client.0.smithi070.stdout: "pct": 0 2024-09-14T03:47:47.144 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:47.144 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.144 INFO:tasks.workunit.client.0.smithi070.stdout: "timed_workload_host_reads": { 2024-09-14T03:47:47.144 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:47.144 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 14 2024-09-14T03:47:47.144 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.144 INFO:tasks.workunit.client.0.smithi070.stdout: "timed_workload_media_wear": { 2024-09-14T03:47:47.144 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:47.144 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": "9.029297" 2024-09-14T03:47:47.145 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.145 INFO:tasks.workunit.client.0.smithi070.stdout: "timed_workload_timer": { 2024-09-14T03:47:47.145 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:47.145 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 1544484 2024-09-14T03:47:47.145 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.145 INFO:tasks.workunit.client.0.smithi070.stdout: "unexp_power_loss": { 2024-09-14T03:47:47.145 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:47.145 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:47.145 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.145 INFO:tasks.workunit.client.0.smithi070.stdout: "wear_leveling": { 2024-09-14T03:47:47.145 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:47.145 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": { 2024-09-14T03:47:47.145 INFO:tasks.workunit.client.0.smithi070.stdout: "avg": 0, 2024-09-14T03:47:47.145 INFO:tasks.workunit.client.0.smithi070.stdout: "max": 0, 2024-09-14T03:47:47.146 INFO:tasks.workunit.client.0.smithi070.stdout: "min": 0 2024-09-14T03:47:47.146 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:47.146 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:47.146 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.146 INFO:tasks.workunit.client.0.smithi070.stdout: "Intel Smart log": "nvme0n1" 2024-09-14T03:47:47.146 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.146 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_smart_health_information_log": { 2024-09-14T03:47:47.146 INFO:tasks.workunit.client.0.smithi070.stdout: "available_spare": 100, 2024-09-14T03:47:47.146 INFO:tasks.workunit.client.0.smithi070.stdout: "available_spare_threshold": 0, 2024-09-14T03:47:47.146 INFO:tasks.workunit.client.0.smithi070.stdout: "controller_busy_time": 5166, 2024-09-14T03:47:47.146 INFO:tasks.workunit.client.0.smithi070.stdout: "critical_warning": 0, 2024-09-14T03:47:47.146 INFO:tasks.workunit.client.0.smithi070.stdout: "data_units_read": 177890366, 2024-09-14T03:47:47.146 INFO:tasks.workunit.client.0.smithi070.stdout: "data_units_written": 856148757, 2024-09-14T03:47:47.147 INFO:tasks.workunit.client.0.smithi070.stdout: "host_reads": 3130935174, 2024-09-14T03:47:47.147 INFO:tasks.workunit.client.0.smithi070.stdout: "host_writes": 18536697060, 2024-09-14T03:47:47.147 INFO:tasks.workunit.client.0.smithi070.stdout: "media_errors": 0, 2024-09-14T03:47:47.147 INFO:tasks.workunit.client.0.smithi070.stdout: "num_err_log_entries": 0, 2024-09-14T03:47:47.147 INFO:tasks.workunit.client.0.smithi070.stdout: "percentage_used": 14, 2024-09-14T03:47:47.147 INFO:tasks.workunit.client.0.smithi070.stdout: "power_cycles": 35890, 2024-09-14T03:47:47.147 INFO:tasks.workunit.client.0.smithi070.stdout: "power_on_hours": 40704, 2024-09-14T03:47:47.147 INFO:tasks.workunit.client.0.smithi070.stdout: "temperature": 39, 2024-09-14T03:47:47.147 INFO:tasks.workunit.client.0.smithi070.stdout: "unsafe_shutdowns": 10931 2024-09-14T03:47:47.147 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.147 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_vendor": "intel", 2024-09-14T03:47:47.147 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_version": { 2024-09-14T03:47:47.147 INFO:tasks.workunit.client.0.smithi070.stdout: "string": "<1.2", 2024-09-14T03:47:47.148 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:47.148 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.148 INFO:tasks.workunit.client.0.smithi070.stdout: "power_cycle_count": 35890, 2024-09-14T03:47:47.148 INFO:tasks.workunit.client.0.smithi070.stdout: "power_on_time": { 2024-09-14T03:47:47.148 INFO:tasks.workunit.client.0.smithi070.stdout: "hours": 40704 2024-09-14T03:47:47.148 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.148 INFO:tasks.workunit.client.0.smithi070.stdout: "serial_number": "deleted", 2024-09-14T03:47:47.148 INFO:tasks.workunit.client.0.smithi070.stdout: "smart_status": { 2024-09-14T03:47:47.148 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme": { 2024-09-14T03:47:47.148 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:47.148 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.148 INFO:tasks.workunit.client.0.smithi070.stdout: "passed": true 2024-09-14T03:47:47.148 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.149 INFO:tasks.workunit.client.0.smithi070.stdout: "smartctl": { 2024-09-14T03:47:47.149 INFO:tasks.workunit.client.0.smithi070.stdout: "argv": [ 2024-09-14T03:47:47.149 INFO:tasks.workunit.client.0.smithi070.stdout: "smartctl", 2024-09-14T03:47:47.149 INFO:tasks.workunit.client.0.smithi070.stdout: "-x", 2024-09-14T03:47:47.149 INFO:tasks.workunit.client.0.smithi070.stdout: "--json=o", 2024-09-14T03:47:47.149 INFO:tasks.workunit.client.0.smithi070.stdout: "/dev/nvme0n1" 2024-09-14T03:47:47.149 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:47.149 INFO:tasks.workunit.client.0.smithi070.stdout: "build_info": "(local build)", 2024-09-14T03:47:47.149 INFO:tasks.workunit.client.0.smithi070.stdout: "exit_status": 0, 2024-09-14T03:47:47.149 INFO:tasks.workunit.client.0.smithi070.stdout: "output": [ 2024-09-14T03:47:47.149 INFO:tasks.workunit.client.0.smithi070.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-505.el9.x86_64] (local build)", 2024-09-14T03:47:47.149 INFO:tasks.workunit.client.0.smithi070.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-14T03:47:47.149 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:47.150 INFO:tasks.workunit.client.0.smithi070.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-14T03:47:47.150 INFO:tasks.workunit.client.0.smithi070.stdout: "Model Number: INTEL SSDPED1D480GA", 2024-09-14T03:47:47.150 INFO:tasks.workunit.client.0.smithi070.stdout: "Serial Number: deleted", 2024-09-14T03:47:47.150 INFO:tasks.workunit.client.0.smithi070.stdout: "Firmware Version: E2010325", 2024-09-14T03:47:47.150 INFO:tasks.workunit.client.0.smithi070.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-14T03:47:47.150 INFO:tasks.workunit.client.0.smithi070.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-14T03:47:47.150 INFO:tasks.workunit.client.0.smithi070.stdout: "Controller ID: 0", 2024-09-14T03:47:47.150 INFO:tasks.workunit.client.0.smithi070.stdout: "NVMe Version: <1.2", 2024-09-14T03:47:47.150 INFO:tasks.workunit.client.0.smithi070.stdout: "Number of Namespaces: 1", 2024-09-14T03:47:47.150 INFO:tasks.workunit.client.0.smithi070.stdout: "Namespace 1 Size/Capacity: 480,103,981,056 [480 GB]", 2024-09-14T03:47:47.150 INFO:tasks.workunit.client.0.smithi070.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-14T03:47:47.150 INFO:tasks.workunit.client.0.smithi070.stdout: "Local Time is: Sat Sep 14 03:41:43 2024 UTC", 2024-09-14T03:47:47.150 INFO:tasks.workunit.client.0.smithi070.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-14T03:47:47.150 INFO:tasks.workunit.client.0.smithi070.stdout: "Optional Admin Commands (0x0007): Security Format Frmw_DL", 2024-09-14T03:47:47.151 INFO:tasks.workunit.client.0.smithi070.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-14T03:47:47.151 INFO:tasks.workunit.client.0.smithi070.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-14T03:47:47.151 INFO:tasks.workunit.client.0.smithi070.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-14T03:47:47.151 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:47.151 INFO:tasks.workunit.client.0.smithi070.stdout: "Supported Power States", 2024-09-14T03:47:47.151 INFO:tasks.workunit.client.0.smithi070.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-14T03:47:47.151 INFO:tasks.workunit.client.0.smithi070.stdout: " 0 + 18.00W - - 0 0 0 0 0 0", 2024-09-14T03:47:47.151 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:47.151 INFO:tasks.workunit.client.0.smithi070.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-14T03:47:47.151 INFO:tasks.workunit.client.0.smithi070.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-14T03:47:47.151 INFO:tasks.workunit.client.0.smithi070.stdout: " 0 + 512 0 2", 2024-09-14T03:47:47.151 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:47.151 INFO:tasks.workunit.client.0.smithi070.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-14T03:47:47.152 INFO:tasks.workunit.client.0.smithi070.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-14T03:47:47.152 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:47.152 INFO:tasks.workunit.client.0.smithi070.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-14T03:47:47.152 INFO:tasks.workunit.client.0.smithi070.stdout: "Critical Warning: 0x00", 2024-09-14T03:47:47.152 INFO:tasks.workunit.client.0.smithi070.stdout: "Temperature: 39 Celsius", 2024-09-14T03:47:47.152 INFO:tasks.workunit.client.0.smithi070.stdout: "Available Spare: 100%", 2024-09-14T03:47:47.152 INFO:tasks.workunit.client.0.smithi070.stdout: "Available Spare Threshold: 0%", 2024-09-14T03:47:47.152 INFO:tasks.workunit.client.0.smithi070.stdout: "Percentage Used: 14%", 2024-09-14T03:47:47.152 INFO:tasks.workunit.client.0.smithi070.stdout: "Data Units Read: 177,890,366 [91.0 TB]", 2024-09-14T03:47:47.152 INFO:tasks.workunit.client.0.smithi070.stdout: "Data Units Written: 856,148,757 [438 TB]", 2024-09-14T03:47:47.152 INFO:tasks.workunit.client.0.smithi070.stdout: "Host Read Commands: 3,130,935,174", 2024-09-14T03:47:47.152 INFO:tasks.workunit.client.0.smithi070.stdout: "Host Write Commands: 18,536,697,060", 2024-09-14T03:47:47.152 INFO:tasks.workunit.client.0.smithi070.stdout: "Controller Busy Time: 5,166", 2024-09-14T03:47:47.152 INFO:tasks.workunit.client.0.smithi070.stdout: "Power Cycles: 35,890", 2024-09-14T03:47:47.153 INFO:tasks.workunit.client.0.smithi070.stdout: "Power On Hours: 40,704", 2024-09-14T03:47:47.153 INFO:tasks.workunit.client.0.smithi070.stdout: "Unsafe Shutdowns: 10,931", 2024-09-14T03:47:47.153 INFO:tasks.workunit.client.0.smithi070.stdout: "Media and Data Integrity Errors: 0", 2024-09-14T03:47:47.153 INFO:tasks.workunit.client.0.smithi070.stdout: "Error Information Log Entries: 0", 2024-09-14T03:47:47.153 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:47.153 INFO:tasks.workunit.client.0.smithi070.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-14T03:47:47.153 INFO:tasks.workunit.client.0.smithi070.stdout: "No Errors Logged", 2024-09-14T03:47:47.153 INFO:tasks.workunit.client.0.smithi070.stdout: "" 2024-09-14T03:47:47.153 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:47.153 INFO:tasks.workunit.client.0.smithi070.stdout: "platform_info": "x86_64-linux-5.14.0-505.el9.x86_64", 2024-09-14T03:47:47.153 INFO:tasks.workunit.client.0.smithi070.stdout: "svn_revision": "5155", 2024-09-14T03:47:47.153 INFO:tasks.workunit.client.0.smithi070.stdout: "version": [ 2024-09-14T03:47:47.153 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:47.153 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:47.154 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:47.154 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.154 INFO:tasks.workunit.client.0.smithi070.stdout: "temperature": { 2024-09-14T03:47:47.154 INFO:tasks.workunit.client.0.smithi070.stdout: "current": 39 2024-09-14T03:47:47.154 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.154 INFO:tasks.workunit.client.0.smithi070.stdout: "user_capacity": { 2024-09-14T03:47:47.154 INFO:tasks.workunit.client.0.smithi070.stdout: "blocks": 937703088, 2024-09-14T03:47:47.154 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 480103981056 2024-09-14T03:47:47.154 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:47.154 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:47.154 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:47.154 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.154 INFO:tasks.workunit.client.0.smithi070.stdout: "1ad766de-724c-11ef-9ddf-0cc47a8fd3e7": { 2024-09-14T03:47:47.154 INFO:tasks.workunit.client.0.smithi070.stdout: "INTEL_SSDPED1D480GA_1ad7d2f4-724c-11ef-9ddf-0cc47a8fd3e7": { 2024-09-14T03:47:47.155 INFO:tasks.workunit.client.0.smithi070.stdout: "20240914-034141": { 2024-09-14T03:47:47.155 INFO:tasks.workunit.client.0.smithi070.stdout: "device": { 2024-09-14T03:47:47.155 INFO:tasks.workunit.client.0.smithi070.stdout: "info_name": "/dev/nvme0n1", 2024-09-14T03:47:47.155 INFO:tasks.workunit.client.0.smithi070.stdout: "name": "/dev/nvme0n1", 2024-09-14T03:47:47.155 INFO:tasks.workunit.client.0.smithi070.stdout: "protocol": "NVMe", 2024-09-14T03:47:47.155 INFO:tasks.workunit.client.0.smithi070.stdout: "type": "nvme" 2024-09-14T03:47:47.155 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.155 INFO:tasks.workunit.client.0.smithi070.stdout: "firmware_version": "E2010325", 2024-09-14T03:47:47.155 INFO:tasks.workunit.client.0.smithi070.stdout: "host_id": "1ad766de-724c-11ef-9ddf-0cc47a8fd3e7", 2024-09-14T03:47:47.155 INFO:tasks.workunit.client.0.smithi070.stdout: "json_format_version": [ 2024-09-14T03:47:47.155 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:47.155 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:47.155 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:47.156 INFO:tasks.workunit.client.0.smithi070.stdout: "local_time": { 2024-09-14T03:47:47.156 INFO:tasks.workunit.client.0.smithi070.stdout: "asctime": "Sat Sep 14 03:41:40 2024 UTC", 2024-09-14T03:47:47.156 INFO:tasks.workunit.client.0.smithi070.stdout: "time_t": 1726285300 2024-09-14T03:47:47.156 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.156 INFO:tasks.workunit.client.0.smithi070.stdout: "logical_block_size": 512, 2024-09-14T03:47:47.156 INFO:tasks.workunit.client.0.smithi070.stdout: "model_name": "INTEL SSDPED1D480GA", 2024-09-14T03:47:47.156 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_controller_id": 0, 2024-09-14T03:47:47.156 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-14T03:47:47.156 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_namespaces": [ 2024-09-14T03:47:47.156 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:47.156 INFO:tasks.workunit.client.0.smithi070.stdout: "capacity": { 2024-09-14T03:47:47.156 INFO:tasks.workunit.client.0.smithi070.stdout: "blocks": 937703088, 2024-09-14T03:47:47.156 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 480103981056 2024-09-14T03:47:47.156 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.157 INFO:tasks.workunit.client.0.smithi070.stdout: "formatted_lba_size": 512, 2024-09-14T03:47:47.157 INFO:tasks.workunit.client.0.smithi070.stdout: "id": 1, 2024-09-14T03:47:47.157 INFO:tasks.workunit.client.0.smithi070.stdout: "size": { 2024-09-14T03:47:47.157 INFO:tasks.workunit.client.0.smithi070.stdout: "blocks": 937703088, 2024-09-14T03:47:47.157 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 480103981056 2024-09-14T03:47:47.157 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.157 INFO:tasks.workunit.client.0.smithi070.stdout: "utilization": { 2024-09-14T03:47:47.157 INFO:tasks.workunit.client.0.smithi070.stdout: "blocks": 937703088, 2024-09-14T03:47:47.157 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 480103981056 2024-09-14T03:47:47.157 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:47.157 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:47.157 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:47.157 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_number_of_namespaces": 1, 2024-09-14T03:47:47.157 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_pci_vendor": { 2024-09-14T03:47:47.158 INFO:tasks.workunit.client.0.smithi070.stdout: "id": 32902, 2024-09-14T03:47:47.158 INFO:tasks.workunit.client.0.smithi070.stdout: "subsystem_id": 32902 2024-09-14T03:47:47.158 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.158 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_smart_health_information_add_log": { 2024-09-14T03:47:47.158 INFO:tasks.workunit.client.0.smithi070.stdout: "Device stats": { 2024-09-14T03:47:47.158 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_fw_downgrades": { 2024-09-14T03:47:47.158 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:47.158 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:47.158 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.158 INFO:tasks.workunit.client.0.smithi070.stdout: "crc_error_count": { 2024-09-14T03:47:47.158 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:47.158 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:47.159 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.159 INFO:tasks.workunit.client.0.smithi070.stdout: "end_to_end_error_detection_count": { 2024-09-14T03:47:47.159 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:47.159 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:47.159 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.159 INFO:tasks.workunit.client.0.smithi070.stdout: "erase_fail_count": { 2024-09-14T03:47:47.159 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:47.159 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:47.159 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.159 INFO:tasks.workunit.client.0.smithi070.stdout: "host_bytes_written": { 2024-09-14T03:47:47.159 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:47.159 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 11224882 2024-09-14T03:47:47.159 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.159 INFO:tasks.workunit.client.0.smithi070.stdout: "host_ctx_wear_used": { 2024-09-14T03:47:47.160 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:47.160 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:47.160 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.160 INFO:tasks.workunit.client.0.smithi070.stdout: "media_bytes_read": { 2024-09-14T03:47:47.160 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:47.160 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:47.160 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.160 INFO:tasks.workunit.client.0.smithi070.stdout: "nand_bytes_written": { 2024-09-14T03:47:47.160 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:47.160 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:47.160 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.160 INFO:tasks.workunit.client.0.smithi070.stdout: "perf_stat_indicator": { 2024-09-14T03:47:47.160 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:47.160 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:47.161 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.161 INFO:tasks.workunit.client.0.smithi070.stdout: "pll_lock_loss_count": { 2024-09-14T03:47:47.161 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:47.161 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:47.161 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.161 INFO:tasks.workunit.client.0.smithi070.stdout: "program_fail_count": { 2024-09-14T03:47:47.161 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:47.161 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:47.161 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.161 INFO:tasks.workunit.client.0.smithi070.stdout: "re_alloc_sectr_cnt": { 2024-09-14T03:47:47.161 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:47.161 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:47.161 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.161 INFO:tasks.workunit.client.0.smithi070.stdout: "retry_buffer_overflow_count": { 2024-09-14T03:47:47.162 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:47.162 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:47.162 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.162 INFO:tasks.workunit.client.0.smithi070.stdout: "soft_ecc_err_rate": { 2024-09-14T03:47:47.162 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:47.162 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:47.162 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.162 INFO:tasks.workunit.client.0.smithi070.stdout: "thermal_throttle_status": { 2024-09-14T03:47:47.162 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:47.162 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": { 2024-09-14T03:47:47.162 INFO:tasks.workunit.client.0.smithi070.stdout: "cnt": 0, 2024-09-14T03:47:47.162 INFO:tasks.workunit.client.0.smithi070.stdout: "pct": 0 2024-09-14T03:47:47.163 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:47.163 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.163 INFO:tasks.workunit.client.0.smithi070.stdout: "timed_workload_host_reads": { 2024-09-14T03:47:47.163 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:47.163 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 14 2024-09-14T03:47:47.163 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.163 INFO:tasks.workunit.client.0.smithi070.stdout: "timed_workload_media_wear": { 2024-09-14T03:47:47.163 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:47.163 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": "0.000000" 2024-09-14T03:47:47.163 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.163 INFO:tasks.workunit.client.0.smithi070.stdout: "timed_workload_timer": { 2024-09-14T03:47:47.163 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:47.163 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 1493348 2024-09-14T03:47:47.163 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.164 INFO:tasks.workunit.client.0.smithi070.stdout: "unexp_power_loss": { 2024-09-14T03:47:47.164 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:47.164 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:47.164 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.164 INFO:tasks.workunit.client.0.smithi070.stdout: "wear_leveling": { 2024-09-14T03:47:47.164 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:47.164 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": { 2024-09-14T03:47:47.164 INFO:tasks.workunit.client.0.smithi070.stdout: "avg": 0, 2024-09-14T03:47:47.164 INFO:tasks.workunit.client.0.smithi070.stdout: "max": 0, 2024-09-14T03:47:47.164 INFO:tasks.workunit.client.0.smithi070.stdout: "min": 0 2024-09-14T03:47:47.164 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:47.164 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:47.164 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.165 INFO:tasks.workunit.client.0.smithi070.stdout: "Intel Smart log": "nvme0n1" 2024-09-14T03:47:47.165 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.165 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_smart_health_information_log": { 2024-09-14T03:47:47.165 INFO:tasks.workunit.client.0.smithi070.stdout: "available_spare": 100, 2024-09-14T03:47:47.165 INFO:tasks.workunit.client.0.smithi070.stdout: "available_spare_threshold": 0, 2024-09-14T03:47:47.165 INFO:tasks.workunit.client.0.smithi070.stdout: "controller_busy_time": 4368, 2024-09-14T03:47:47.165 INFO:tasks.workunit.client.0.smithi070.stdout: "critical_warning": 0, 2024-09-14T03:47:47.165 INFO:tasks.workunit.client.0.smithi070.stdout: "data_units_read": 155028165, 2024-09-14T03:47:47.165 INFO:tasks.workunit.client.0.smithi070.stdout: "data_units_written": 735633918, 2024-09-14T03:47:47.165 INFO:tasks.workunit.client.0.smithi070.stdout: "host_reads": 2607915323, 2024-09-14T03:47:47.165 INFO:tasks.workunit.client.0.smithi070.stdout: "host_writes": 16464089664, 2024-09-14T03:47:47.165 INFO:tasks.workunit.client.0.smithi070.stdout: "media_errors": 0, 2024-09-14T03:47:47.165 INFO:tasks.workunit.client.0.smithi070.stdout: "num_err_log_entries": 0, 2024-09-14T03:47:47.165 INFO:tasks.workunit.client.0.smithi070.stdout: "percentage_used": 0, 2024-09-14T03:47:47.166 INFO:tasks.workunit.client.0.smithi070.stdout: "power_cycles": 30550, 2024-09-14T03:47:47.166 INFO:tasks.workunit.client.0.smithi070.stdout: "power_on_hours": 37550, 2024-09-14T03:47:47.166 INFO:tasks.workunit.client.0.smithi070.stdout: "temperature": 34, 2024-09-14T03:47:47.166 INFO:tasks.workunit.client.0.smithi070.stdout: "unsafe_shutdowns": 9447 2024-09-14T03:47:47.166 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.166 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_vendor": "intel", 2024-09-14T03:47:47.166 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_version": { 2024-09-14T03:47:47.166 INFO:tasks.workunit.client.0.smithi070.stdout: "string": "<1.2", 2024-09-14T03:47:47.166 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:47.166 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.166 INFO:tasks.workunit.client.0.smithi070.stdout: "power_cycle_count": 30550, 2024-09-14T03:47:47.166 INFO:tasks.workunit.client.0.smithi070.stdout: "power_on_time": { 2024-09-14T03:47:47.166 INFO:tasks.workunit.client.0.smithi070.stdout: "hours": 37550 2024-09-14T03:47:47.166 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.167 INFO:tasks.workunit.client.0.smithi070.stdout: "serial_number": "deleted", 2024-09-14T03:47:47.167 INFO:tasks.workunit.client.0.smithi070.stdout: "smart_status": { 2024-09-14T03:47:47.167 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme": { 2024-09-14T03:47:47.167 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:47.167 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.167 INFO:tasks.workunit.client.0.smithi070.stdout: "passed": true 2024-09-14T03:47:47.167 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.167 INFO:tasks.workunit.client.0.smithi070.stdout: "smartctl": { 2024-09-14T03:47:47.167 INFO:tasks.workunit.client.0.smithi070.stdout: "argv": [ 2024-09-14T03:47:47.167 INFO:tasks.workunit.client.0.smithi070.stdout: "smartctl", 2024-09-14T03:47:47.167 INFO:tasks.workunit.client.0.smithi070.stdout: "-x", 2024-09-14T03:47:47.167 INFO:tasks.workunit.client.0.smithi070.stdout: "--json=o", 2024-09-14T03:47:47.167 INFO:tasks.workunit.client.0.smithi070.stdout: "/dev/nvme0n1" 2024-09-14T03:47:47.167 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:47.168 INFO:tasks.workunit.client.0.smithi070.stdout: "build_info": "(local build)", 2024-09-14T03:47:47.168 INFO:tasks.workunit.client.0.smithi070.stdout: "exit_status": 0, 2024-09-14T03:47:47.168 INFO:tasks.workunit.client.0.smithi070.stdout: "output": [ 2024-09-14T03:47:47.168 INFO:tasks.workunit.client.0.smithi070.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-505.el9.x86_64] (local build)", 2024-09-14T03:47:47.168 INFO:tasks.workunit.client.0.smithi070.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-14T03:47:47.168 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:47.168 INFO:tasks.workunit.client.0.smithi070.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-14T03:47:47.168 INFO:tasks.workunit.client.0.smithi070.stdout: "Model Number: INTEL SSDPED1D480GA", 2024-09-14T03:47:47.168 INFO:tasks.workunit.client.0.smithi070.stdout: "Serial Number: deleted", 2024-09-14T03:47:47.168 INFO:tasks.workunit.client.0.smithi070.stdout: "Firmware Version: E2010325", 2024-09-14T03:47:47.168 INFO:tasks.workunit.client.0.smithi070.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-14T03:47:47.168 INFO:tasks.workunit.client.0.smithi070.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-14T03:47:47.168 INFO:tasks.workunit.client.0.smithi070.stdout: "Controller ID: 0", 2024-09-14T03:47:47.168 INFO:tasks.workunit.client.0.smithi070.stdout: "NVMe Version: <1.2", 2024-09-14T03:47:47.169 INFO:tasks.workunit.client.0.smithi070.stdout: "Number of Namespaces: 1", 2024-09-14T03:47:47.169 INFO:tasks.workunit.client.0.smithi070.stdout: "Namespace 1 Size/Capacity: 480,103,981,056 [480 GB]", 2024-09-14T03:47:47.169 INFO:tasks.workunit.client.0.smithi070.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-14T03:47:47.169 INFO:tasks.workunit.client.0.smithi070.stdout: "Local Time is: Sat Sep 14 03:41:40 2024 UTC", 2024-09-14T03:47:47.169 INFO:tasks.workunit.client.0.smithi070.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-14T03:47:47.169 INFO:tasks.workunit.client.0.smithi070.stdout: "Optional Admin Commands (0x0007): Security Format Frmw_DL", 2024-09-14T03:47:47.169 INFO:tasks.workunit.client.0.smithi070.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-14T03:47:47.169 INFO:tasks.workunit.client.0.smithi070.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-14T03:47:47.169 INFO:tasks.workunit.client.0.smithi070.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-14T03:47:47.169 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:47.169 INFO:tasks.workunit.client.0.smithi070.stdout: "Supported Power States", 2024-09-14T03:47:47.169 INFO:tasks.workunit.client.0.smithi070.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-14T03:47:47.169 INFO:tasks.workunit.client.0.smithi070.stdout: " 0 + 18.00W - - 0 0 0 0 0 0", 2024-09-14T03:47:47.170 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:47.170 INFO:tasks.workunit.client.0.smithi070.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-14T03:47:47.170 INFO:tasks.workunit.client.0.smithi070.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-14T03:47:47.170 INFO:tasks.workunit.client.0.smithi070.stdout: " 0 + 512 0 2", 2024-09-14T03:47:47.170 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:47.170 INFO:tasks.workunit.client.0.smithi070.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-14T03:47:47.170 INFO:tasks.workunit.client.0.smithi070.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-14T03:47:47.170 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:47.170 INFO:tasks.workunit.client.0.smithi070.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-14T03:47:47.170 INFO:tasks.workunit.client.0.smithi070.stdout: "Critical Warning: 0x00", 2024-09-14T03:47:47.170 INFO:tasks.workunit.client.0.smithi070.stdout: "Temperature: 34 Celsius", 2024-09-14T03:47:47.170 INFO:tasks.workunit.client.0.smithi070.stdout: "Available Spare: 100%", 2024-09-14T03:47:47.170 INFO:tasks.workunit.client.0.smithi070.stdout: "Available Spare Threshold: 0%", 2024-09-14T03:47:47.170 INFO:tasks.workunit.client.0.smithi070.stdout: "Percentage Used: 0%", 2024-09-14T03:47:47.170 INFO:tasks.workunit.client.0.smithi070.stdout: "Data Units Read: 155,028,165 [79.3 TB]", 2024-09-14T03:47:47.171 INFO:tasks.workunit.client.0.smithi070.stdout: "Data Units Written: 735,633,918 [376 TB]", 2024-09-14T03:47:47.171 INFO:tasks.workunit.client.0.smithi070.stdout: "Host Read Commands: 2,607,915,323", 2024-09-14T03:47:47.171 INFO:tasks.workunit.client.0.smithi070.stdout: "Host Write Commands: 16,464,089,664", 2024-09-14T03:47:47.171 INFO:tasks.workunit.client.0.smithi070.stdout: "Controller Busy Time: 4,368", 2024-09-14T03:47:47.171 INFO:tasks.workunit.client.0.smithi070.stdout: "Power Cycles: 30,550", 2024-09-14T03:47:47.171 INFO:tasks.workunit.client.0.smithi070.stdout: "Power On Hours: 37,550", 2024-09-14T03:47:47.171 INFO:tasks.workunit.client.0.smithi070.stdout: "Unsafe Shutdowns: 9,447", 2024-09-14T03:47:47.171 INFO:tasks.workunit.client.0.smithi070.stdout: "Media and Data Integrity Errors: 0", 2024-09-14T03:47:47.171 INFO:tasks.workunit.client.0.smithi070.stdout: "Error Information Log Entries: 0", 2024-09-14T03:47:47.171 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:47.171 INFO:tasks.workunit.client.0.smithi070.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-14T03:47:47.171 INFO:tasks.workunit.client.0.smithi070.stdout: "No Errors Logged", 2024-09-14T03:47:47.171 INFO:tasks.workunit.client.0.smithi070.stdout: "" 2024-09-14T03:47:47.171 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:47.172 INFO:tasks.workunit.client.0.smithi070.stdout: "platform_info": "x86_64-linux-5.14.0-505.el9.x86_64", 2024-09-14T03:47:47.172 INFO:tasks.workunit.client.0.smithi070.stdout: "svn_revision": "5155", 2024-09-14T03:47:47.172 INFO:tasks.workunit.client.0.smithi070.stdout: "version": [ 2024-09-14T03:47:47.172 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:47.172 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:47.172 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:47.172 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.172 INFO:tasks.workunit.client.0.smithi070.stdout: "temperature": { 2024-09-14T03:47:47.172 INFO:tasks.workunit.client.0.smithi070.stdout: "current": 34 2024-09-14T03:47:47.172 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.172 INFO:tasks.workunit.client.0.smithi070.stdout: "user_capacity": { 2024-09-14T03:47:47.172 INFO:tasks.workunit.client.0.smithi070.stdout: "blocks": 937703088, 2024-09-14T03:47:47.172 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 480103981056 2024-09-14T03:47:47.172 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:47.173 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:47.173 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:47.173 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:47.173 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.173 INFO:tasks.workunit.client.0.smithi070.stdout: "report": { 2024-09-14T03:47:47.173 INFO:tasks.workunit.client.0.smithi070.stdout: "balancer": { 2024-09-14T03:47:47.173 INFO:tasks.workunit.client.0.smithi070.stdout: "active": true, 2024-09-14T03:47:47.173 INFO:tasks.workunit.client.0.smithi070.stdout: "mode": "" 2024-09-14T03:47:47.173 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.173 INFO:tasks.workunit.client.0.smithi070.stdout: "channels": [ 2024-09-14T03:47:47.173 INFO:tasks.workunit.client.0.smithi070.stdout: "basic", 2024-09-14T03:47:47.173 INFO:tasks.workunit.client.0.smithi070.stdout: "crash", 2024-09-14T03:47:47.173 INFO:tasks.workunit.client.0.smithi070.stdout: "device", 2024-09-14T03:47:47.174 INFO:tasks.workunit.client.0.smithi070.stdout: "ident" 2024-09-14T03:47:47.174 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:47.174 INFO:tasks.workunit.client.0.smithi070.stdout: "channels_available": [ 2024-09-14T03:47:47.174 INFO:tasks.workunit.client.0.smithi070.stdout: "basic", 2024-09-14T03:47:47.174 INFO:tasks.workunit.client.0.smithi070.stdout: "ident", 2024-09-14T03:47:47.174 INFO:tasks.workunit.client.0.smithi070.stdout: "crash", 2024-09-14T03:47:47.174 INFO:tasks.workunit.client.0.smithi070.stdout: "device", 2024-09-14T03:47:47.174 INFO:tasks.workunit.client.0.smithi070.stdout: "perf" 2024-09-14T03:47:47.174 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:47.174 INFO:tasks.workunit.client.0.smithi070.stdout: "collections_available": [ 2024-09-14T03:47:47.174 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_base", 2024-09-14T03:47:47.174 INFO:tasks.workunit.client.0.smithi070.stdout: "device_base", 2024-09-14T03:47:47.174 INFO:tasks.workunit.client.0.smithi070.stdout: "crash_base", 2024-09-14T03:47:47.175 INFO:tasks.workunit.client.0.smithi070.stdout: "ident_base", 2024-09-14T03:47:47.175 INFO:tasks.workunit.client.0.smithi070.stdout: "perf_perf", 2024-09-14T03:47:47.175 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_mds_metadata", 2024-09-14T03:47:47.175 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_pool_usage", 2024-09-14T03:47:47.175 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_usage_by_class", 2024-09-14T03:47:47.175 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_rook_v01", 2024-09-14T03:47:47.175 INFO:tasks.workunit.client.0.smithi070.stdout: "perf_memory_metrics", 2024-09-14T03:47:47.175 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_pool_options_bluestore" 2024-09-14T03:47:47.175 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:47.175 INFO:tasks.workunit.client.0.smithi070.stdout: "collections_opted_in": [ 2024-09-14T03:47:47.175 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_base", 2024-09-14T03:47:47.175 INFO:tasks.workunit.client.0.smithi070.stdout: "device_base", 2024-09-14T03:47:47.175 INFO:tasks.workunit.client.0.smithi070.stdout: "crash_base", 2024-09-14T03:47:47.175 INFO:tasks.workunit.client.0.smithi070.stdout: "ident_base", 2024-09-14T03:47:47.176 INFO:tasks.workunit.client.0.smithi070.stdout: "perf_perf", 2024-09-14T03:47:47.176 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_mds_metadata", 2024-09-14T03:47:47.176 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_pool_usage", 2024-09-14T03:47:47.176 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_usage_by_class", 2024-09-14T03:47:47.176 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_rook_v01", 2024-09-14T03:47:47.176 INFO:tasks.workunit.client.0.smithi070.stdout: "perf_memory_metrics", 2024-09-14T03:47:47.176 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_pool_options_bluestore" 2024-09-14T03:47:47.176 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:47.176 INFO:tasks.workunit.client.0.smithi070.stdout: "config": { 2024-09-14T03:47:47.176 INFO:tasks.workunit.client.0.smithi070.stdout: "active_changed": [ 2024-09-14T03:47:47.176 INFO:tasks.workunit.client.0.smithi070.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-14T03:47:47.176 INFO:tasks.workunit.client.0.smithi070.stdout: "auth_debug", 2024-09-14T03:47:47.176 INFO:tasks.workunit.client.0.smithi070.stdout: "auth_mon_ticket_ttl", 2024-09-14T03:47:47.177 INFO:tasks.workunit.client.0.smithi070.stdout: "auth_service_ticket_ttl", 2024-09-14T03:47:47.177 INFO:tasks.workunit.client.0.smithi070.stdout: "bdev_debug_aio", 2024-09-14T03:47:47.177 INFO:tasks.workunit.client.0.smithi070.stdout: "client_check_pool_perm", 2024-09-14T03:47:47.177 INFO:tasks.workunit.client.0.smithi070.stdout: "client_mount_gid", 2024-09-14T03:47:47.177 INFO:tasks.workunit.client.0.smithi070.stdout: "client_mount_uid", 2024-09-14T03:47:47.177 INFO:tasks.workunit.client.0.smithi070.stdout: "client_quota", 2024-09-14T03:47:47.177 INFO:tasks.workunit.client.0.smithi070.stdout: "container_image", 2024-09-14T03:47:47.177 INFO:tasks.workunit.client.0.smithi070.stdout: "daemonize", 2024-09-14T03:47:47.177 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_asserts_on_shutdown", 2024-09-14T03:47:47.177 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_mgr", 2024-09-14T03:47:47.177 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_mon", 2024-09-14T03:47:47.177 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_ms", 2024-09-14T03:47:47.177 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_osd", 2024-09-14T03:47:47.177 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_paxos", 2024-09-14T03:47:47.178 INFO:tasks.workunit.client.0.smithi070.stdout: "keyring", 2024-09-14T03:47:47.178 INFO:tasks.workunit.client.0.smithi070.stdout: "leveldb_block_size", 2024-09-14T03:47:47.178 INFO:tasks.workunit.client.0.smithi070.stdout: "leveldb_cache_size", 2024-09-14T03:47:47.178 INFO:tasks.workunit.client.0.smithi070.stdout: "leveldb_compression", 2024-09-14T03:47:47.178 INFO:tasks.workunit.client.0.smithi070.stdout: "leveldb_log", 2024-09-14T03:47:47.178 INFO:tasks.workunit.client.0.smithi070.stdout: "leveldb_write_buffer_size", 2024-09-14T03:47:47.178 INFO:tasks.workunit.client.0.smithi070.stdout: "log_to_file", 2024-09-14T03:47:47.178 INFO:tasks.workunit.client.0.smithi070.stdout: "log_to_journald", 2024-09-14T03:47:47.178 INFO:tasks.workunit.client.0.smithi070.stdout: "log_to_stderr", 2024-09-14T03:47:47.178 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_allow_pool_delete", 2024-09-14T03:47:47.178 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_clock_drift_allowed", 2024-09-14T03:47:47.178 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_cluster_log_to_file", 2024-09-14T03:47:47.178 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_cluster_log_to_journald", 2024-09-14T03:47:47.178 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_cluster_log_to_stderr", 2024-09-14T03:47:47.179 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_data_avail_warn", 2024-09-14T03:47:47.179 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_election_default_strategy", 2024-09-14T03:47:47.179 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_host", 2024-09-14T03:47:47.179 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_max_pg_per_osd", 2024-09-14T03:47:47.179 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_mgr_mkfs_grace", 2024-09-14T03:47:47.179 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_osd_prime_pg_temp", 2024-09-14T03:47:47.179 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_osd_reporter_subtree_level", 2024-09-14T03:47:47.179 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_pg_warn_max_object_skew", 2024-09-14T03:47:47.179 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-14T03:47:47.179 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-14T03:47:47.179 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-14T03:47:47.179 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-14T03:47:47.179 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-14T03:47:47.179 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-14T03:47:47.180 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-14T03:47:47.180 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-14T03:47:47.180 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_too_few_osds", 2024-09-14T03:47:47.180 INFO:tasks.workunit.client.0.smithi070.stdout: "ms_die_on_bug", 2024-09-14T03:47:47.180 INFO:tasks.workunit.client.0.smithi070.stdout: "ms_die_on_old_message", 2024-09-14T03:47:47.180 INFO:tasks.workunit.client.0.smithi070.stdout: "no_config_file", 2024-09-14T03:47:47.180 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_class_default_list", 2024-09-14T03:47:47.180 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_class_load_list", 2024-09-14T03:47:47.180 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_misdirected_ops", 2024-09-14T03:47:47.180 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_op_order", 2024-09-14T03:47:47.180 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_pg_log_writeout", 2024-09-14T03:47:47.180 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_shutdown", 2024-09-14T03:47:47.180 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_verify_cached_snaps", 2024-09-14T03:47:47.181 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_verify_missing_on_start", 2024-09-14T03:47:47.181 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_verify_stray_on_activate", 2024-09-14T03:47:47.181 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-14T03:47:47.181 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_profile", 2024-09-14T03:47:47.181 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_background_best_effort_lim", 2024-09-14T03:47:47.181 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_background_best_effort_res", 2024-09-14T03:47:47.181 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_background_best_effort_wgt", 2024-09-14T03:47:47.181 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_background_recovery_lim", 2024-09-14T03:47:47.181 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_background_recovery_res", 2024-09-14T03:47:47.181 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_background_recovery_wgt", 2024-09-14T03:47:47.181 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_client_lim", 2024-09-14T03:47:47.181 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_client_res", 2024-09-14T03:47:47.181 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_client_wgt", 2024-09-14T03:47:47.182 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_memory_target", 2024-09-14T03:47:47.182 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_memory_target_autotune", 2024-09-14T03:47:47.182 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_op_queue", 2024-09-14T03:47:47.182 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_op_queue_cut_off", 2024-09-14T03:47:47.182 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pool_default_erasure_code_profile", 2024-09-14T03:47:47.182 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-14T03:47:47.182 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_recover_clone_overlap", 2024-09-14T03:47:47.182 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_recovery_max_chunk", 2024-09-14T03:47:47.182 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_scrub_load_threshold", 2024-09-14T03:47:47.182 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_scrub_max_interval", 2024-09-14T03:47:47.182 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_shutdown_pgref_assert", 2024-09-14T03:47:47.182 INFO:tasks.workunit.client.0.smithi070.stdout: "public_addr", 2024-09-14T03:47:47.182 INFO:tasks.workunit.client.0.smithi070.stdout: "public_addrv", 2024-09-14T03:47:47.182 INFO:tasks.workunit.client.0.smithi070.stdout: "public_network", 2024-09-14T03:47:47.183 INFO:tasks.workunit.client.0.smithi070.stdout: "rbd_default_features", 2024-09-14T03:47:47.183 INFO:tasks.workunit.client.0.smithi070.stdout: "rbd_qos_exclude_ops", 2024-09-14T03:47:47.183 INFO:tasks.workunit.client.0.smithi070.stdout: "setgroup", 2024-09-14T03:47:47.183 INFO:tasks.workunit.client.0.smithi070.stdout: "setuser" 2024-09-14T03:47:47.183 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:47.183 INFO:tasks.workunit.client.0.smithi070.stdout: "cluster_changed": [ 2024-09-14T03:47:47.183 INFO:tasks.workunit.client.0.smithi070.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-14T03:47:47.183 INFO:tasks.workunit.client.0.smithi070.stdout: "auth_debug", 2024-09-14T03:47:47.183 INFO:tasks.workunit.client.0.smithi070.stdout: "auth_mon_ticket_ttl", 2024-09-14T03:47:47.183 INFO:tasks.workunit.client.0.smithi070.stdout: "auth_service_ticket_ttl", 2024-09-14T03:47:47.183 INFO:tasks.workunit.client.0.smithi070.stdout: "bdev_debug_aio", 2024-09-14T03:47:47.183 INFO:tasks.workunit.client.0.smithi070.stdout: "container_image", 2024-09-14T03:47:47.183 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_asserts_on_shutdown", 2024-09-14T03:47:47.183 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_mgr", 2024-09-14T03:47:47.184 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_mon", 2024-09-14T03:47:47.184 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_ms", 2024-09-14T03:47:47.184 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_osd", 2024-09-14T03:47:47.184 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_paxos", 2024-09-14T03:47:47.184 INFO:tasks.workunit.client.0.smithi070.stdout: "log_to_file", 2024-09-14T03:47:47.184 INFO:tasks.workunit.client.0.smithi070.stdout: "log_to_journald", 2024-09-14T03:47:47.184 INFO:tasks.workunit.client.0.smithi070.stdout: "log_to_stderr", 2024-09-14T03:47:47.184 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_join_fs", 2024-09-14T03:47:47.184 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/cephadm/allow_ptrace", 2024-09-14T03:47:47.184 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/cephadm/container_init", 2024-09-14T03:47:47.184 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/cephadm/migration_current", 2024-09-14T03:47:47.184 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/cephadm/use_repo_digest", 2024-09-14T03:47:47.184 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/dashboard/ALERTMANAGER_API_HOST", 2024-09-14T03:47:47.185 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/dashboard/GRAFANA_API_SSL_VERIFY", 2024-09-14T03:47:47.185 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/dashboard/GRAFANA_API_URL", 2024-09-14T03:47:47.185 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/dashboard/PROMETHEUS_API_HOST", 2024-09-14T03:47:47.185 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/dashboard/ssl_server_port", 2024-09-14T03:47:47.185 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/orchestrator/orchestrator", 2024-09-14T03:47:47.185 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/telemetry/channel_ident", 2024-09-14T03:47:47.185 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/telemetry/description", 2024-09-14T03:47:47.185 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/telemetry/organization", 2024-09-14T03:47:47.185 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_allow_pool_delete", 2024-09-14T03:47:47.185 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_clock_drift_allowed", 2024-09-14T03:47:47.185 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_cluster_log_to_file", 2024-09-14T03:47:47.185 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_data_avail_warn", 2024-09-14T03:47:47.185 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_election_default_strategy", 2024-09-14T03:47:47.185 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_max_pg_per_osd", 2024-09-14T03:47:47.185 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_mgr_mkfs_grace", 2024-09-14T03:47:47.186 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_osd_prime_pg_temp", 2024-09-14T03:47:47.186 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_osd_reporter_subtree_level", 2024-09-14T03:47:47.186 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_pg_warn_max_object_skew", 2024-09-14T03:47:47.186 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-14T03:47:47.186 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-14T03:47:47.186 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-14T03:47:47.186 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-14T03:47:47.186 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-14T03:47:47.186 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-14T03:47:47.186 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-14T03:47:47.186 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-14T03:47:47.186 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_too_few_osds", 2024-09-14T03:47:47.186 INFO:tasks.workunit.client.0.smithi070.stdout: "ms_die_on_bug", 2024-09-14T03:47:47.186 INFO:tasks.workunit.client.0.smithi070.stdout: "ms_die_on_old_message", 2024-09-14T03:47:47.187 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_class_default_list", 2024-09-14T03:47:47.187 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_class_load_list", 2024-09-14T03:47:47.187 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_misdirected_ops", 2024-09-14T03:47:47.187 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_op_order", 2024-09-14T03:47:47.187 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_pg_log_writeout", 2024-09-14T03:47:47.187 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_shutdown", 2024-09-14T03:47:47.187 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_verify_cached_snaps", 2024-09-14T03:47:47.187 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_verify_missing_on_start", 2024-09-14T03:47:47.187 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_verify_stray_on_activate", 2024-09-14T03:47:47.187 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-14T03:47:47.187 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_profile", 2024-09-14T03:47:47.187 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_memory_target", 2024-09-14T03:47:47.187 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_memory_target_autotune", 2024-09-14T03:47:47.187 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_op_queue", 2024-09-14T03:47:47.188 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_op_queue_cut_off", 2024-09-14T03:47:47.188 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pool_default_erasure_code_profile", 2024-09-14T03:47:47.188 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-14T03:47:47.188 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_recover_clone_overlap", 2024-09-14T03:47:47.188 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_recovery_max_chunk", 2024-09-14T03:47:47.188 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_scrub_load_threshold", 2024-09-14T03:47:47.188 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_scrub_max_interval", 2024-09-14T03:47:47.188 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_shutdown_pgref_assert", 2024-09-14T03:47:47.188 INFO:tasks.workunit.client.0.smithi070.stdout: "public_network", 2024-09-14T03:47:47.188 INFO:tasks.workunit.client.0.smithi070.stdout: "rgw_cache_enabled", 2024-09-14T03:47:47.188 INFO:tasks.workunit.client.0.smithi070.stdout: "rgw_enable_ops_log", 2024-09-14T03:47:47.188 INFO:tasks.workunit.client.0.smithi070.stdout: "rgw_enable_usage_log" 2024-09-14T03:47:47.188 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:47.188 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.189 INFO:tasks.workunit.client.0.smithi070.stdout: "contact": null, 2024-09-14T03:47:47.189 INFO:tasks.workunit.client.0.smithi070.stdout: "crashes": [], 2024-09-14T03:47:47.189 INFO:tasks.workunit.client.0.smithi070.stdout: "created": "2024-09-14T03:38:26.272229Z", 2024-09-14T03:47:47.189 INFO:tasks.workunit.client.0.smithi070.stdout: "crush": { 2024-09-14T03:47:47.189 INFO:tasks.workunit.client.0.smithi070.stdout: "bucket_algs": { 2024-09-14T03:47:47.189 INFO:tasks.workunit.client.0.smithi070.stdout: "straw2": 3 2024-09-14T03:47:47.189 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.189 INFO:tasks.workunit.client.0.smithi070.stdout: "bucket_sizes": { 2024-09-14T03:47:47.189 INFO:tasks.workunit.client.0.smithi070.stdout: "2": 1, 2024-09-14T03:47:47.189 INFO:tasks.workunit.client.0.smithi070.stdout: "4": 2 2024-09-14T03:47:47.189 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.189 INFO:tasks.workunit.client.0.smithi070.stdout: "bucket_types": { 2024-09-14T03:47:47.189 INFO:tasks.workunit.client.0.smithi070.stdout: "1": 2, 2024-09-14T03:47:47.190 INFO:tasks.workunit.client.0.smithi070.stdout: "11": 1 2024-09-14T03:47:47.190 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.190 INFO:tasks.workunit.client.0.smithi070.stdout: "compat_weight_set": false, 2024-09-14T03:47:47.190 INFO:tasks.workunit.client.0.smithi070.stdout: "device_classes": [ 2024-09-14T03:47:47.190 INFO:tasks.workunit.client.0.smithi070.stdout: 8 2024-09-14T03:47:47.190 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:47.190 INFO:tasks.workunit.client.0.smithi070.stdout: "num_buckets": 6, 2024-09-14T03:47:47.190 INFO:tasks.workunit.client.0.smithi070.stdout: "num_devices": 8, 2024-09-14T03:47:47.190 INFO:tasks.workunit.client.0.smithi070.stdout: "num_rules": 1, 2024-09-14T03:47:47.190 INFO:tasks.workunit.client.0.smithi070.stdout: "num_types": 12, 2024-09-14T03:47:47.190 INFO:tasks.workunit.client.0.smithi070.stdout: "num_weight_sets": 0, 2024-09-14T03:47:47.190 INFO:tasks.workunit.client.0.smithi070.stdout: "tunables": { 2024-09-14T03:47:47.190 INFO:tasks.workunit.client.0.smithi070.stdout: "allowed_bucket_algs": 54, 2024-09-14T03:47:47.191 INFO:tasks.workunit.client.0.smithi070.stdout: "choose_local_fallback_tries": 0, 2024-09-14T03:47:47.191 INFO:tasks.workunit.client.0.smithi070.stdout: "choose_local_tries": 0, 2024-09-14T03:47:47.191 INFO:tasks.workunit.client.0.smithi070.stdout: "choose_total_tries": 50, 2024-09-14T03:47:47.191 INFO:tasks.workunit.client.0.smithi070.stdout: "chooseleaf_descend_once": 1, 2024-09-14T03:47:47.191 INFO:tasks.workunit.client.0.smithi070.stdout: "chooseleaf_stable": 1, 2024-09-14T03:47:47.191 INFO:tasks.workunit.client.0.smithi070.stdout: "chooseleaf_vary_r": 1, 2024-09-14T03:47:47.191 INFO:tasks.workunit.client.0.smithi070.stdout: "has_v2_rules": 0, 2024-09-14T03:47:47.191 INFO:tasks.workunit.client.0.smithi070.stdout: "has_v3_rules": 0, 2024-09-14T03:47:47.191 INFO:tasks.workunit.client.0.smithi070.stdout: "has_v4_buckets": 1, 2024-09-14T03:47:47.191 INFO:tasks.workunit.client.0.smithi070.stdout: "has_v5_rules": 0, 2024-09-14T03:47:47.191 INFO:tasks.workunit.client.0.smithi070.stdout: "legacy_tunables": 0, 2024-09-14T03:47:47.191 INFO:tasks.workunit.client.0.smithi070.stdout: "minimum_required_version": "jewel", 2024-09-14T03:47:47.191 INFO:tasks.workunit.client.0.smithi070.stdout: "optimal_tunables": 1, 2024-09-14T03:47:47.191 INFO:tasks.workunit.client.0.smithi070.stdout: "profile": "jewel", 2024-09-14T03:47:47.191 INFO:tasks.workunit.client.0.smithi070.stdout: "require_feature_tunables": 1, 2024-09-14T03:47:47.192 INFO:tasks.workunit.client.0.smithi070.stdout: "require_feature_tunables2": 1, 2024-09-14T03:47:47.192 INFO:tasks.workunit.client.0.smithi070.stdout: "require_feature_tunables3": 1, 2024-09-14T03:47:47.192 INFO:tasks.workunit.client.0.smithi070.stdout: "require_feature_tunables5": 1, 2024-09-14T03:47:47.192 INFO:tasks.workunit.client.0.smithi070.stdout: "straw_calc_version": 1 2024-09-14T03:47:47.192 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:47.192 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.192 INFO:tasks.workunit.client.0.smithi070.stdout: "description": "upgrade test cluster", 2024-09-14T03:47:47.192 INFO:tasks.workunit.client.0.smithi070.stdout: "fs": { 2024-09-14T03:47:47.192 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:47.192 INFO:tasks.workunit.client.0.smithi070.stdout: "feature_flags": { 2024-09-14T03:47:47.192 INFO:tasks.workunit.client.0.smithi070.stdout: "enable_multiple": true, 2024-09-14T03:47:47.192 INFO:tasks.workunit.client.0.smithi070.stdout: "ever_enabled_multiple": true 2024-09-14T03:47:47.192 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.192 INFO:tasks.workunit.client.0.smithi070.stdout: "filesystems": [ 2024-09-14T03:47:47.193 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:47.193 INFO:tasks.workunit.client.0.smithi070.stdout: "approx_ctime": "2024-09", 2024-09-14T03:47:47.193 INFO:tasks.workunit.client.0.smithi070.stdout: "balancer_enabled": false, 2024-09-14T03:47:47.193 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:47.193 INFO:tasks.workunit.client.0.smithi070.stdout: "cached_caps": 0, 2024-09-14T03:47:47.193 INFO:tasks.workunit.client.0.smithi070.stdout: "cached_dns": 10, 2024-09-14T03:47:47.193 INFO:tasks.workunit.client.0.smithi070.stdout: "cached_inos": 13, 2024-09-14T03:47:47.193 INFO:tasks.workunit.client.0.smithi070.stdout: "cached_subtrees": 2, 2024-09-14T03:47:47.193 INFO:tasks.workunit.client.0.smithi070.stdout: "ever_allowed_features": 0, 2024-09-14T03:47:47.193 INFO:tasks.workunit.client.0.smithi070.stdout: "explicitly_allowed_features": 0, 2024-09-14T03:47:47.193 INFO:tasks.workunit.client.0.smithi070.stdout: "files": 0, 2024-09-14T03:47:47.193 INFO:tasks.workunit.client.0.smithi070.stdout: "max_mds": 1, 2024-09-14T03:47:47.193 INFO:tasks.workunit.client.0.smithi070.stdout: "num_data_pools": 1, 2024-09-14T03:47:47.193 INFO:tasks.workunit.client.0.smithi070.stdout: "num_in": 1, 2024-09-14T03:47:47.194 INFO:tasks.workunit.client.0.smithi070.stdout: "num_mds": 1, 2024-09-14T03:47:47.194 INFO:tasks.workunit.client.0.smithi070.stdout: "num_sessions": 0, 2024-09-14T03:47:47.194 INFO:tasks.workunit.client.0.smithi070.stdout: "num_standby_replay": 0, 2024-09-14T03:47:47.194 INFO:tasks.workunit.client.0.smithi070.stdout: "num_up": 1, 2024-09-14T03:47:47.194 INFO:tasks.workunit.client.0.smithi070.stdout: "snaps": 0, 2024-09-14T03:47:47.194 INFO:tasks.workunit.client.0.smithi070.stdout: "standby_count_wanted": 0 2024-09-14T03:47:47.194 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:47.194 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:47.194 INFO:tasks.workunit.client.0.smithi070.stdout: "num_standby_mds": 1, 2024-09-14T03:47:47.194 INFO:tasks.workunit.client.0.smithi070.stdout: "total_num_mds": 2 2024-09-14T03:47:47.194 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.194 INFO:tasks.workunit.client.0.smithi070.stdout: "hosts": { 2024-09-14T03:47:47.194 INFO:tasks.workunit.client.0.smithi070.stdout: "num": 2, 2024-09-14T03:47:47.194 INFO:tasks.workunit.client.0.smithi070.stdout: "num_with_mds": 2, 2024-09-14T03:47:47.195 INFO:tasks.workunit.client.0.smithi070.stdout: "num_with_mgr": 2, 2024-09-14T03:47:47.195 INFO:tasks.workunit.client.0.smithi070.stdout: "num_with_mon": 2, 2024-09-14T03:47:47.195 INFO:tasks.workunit.client.0.smithi070.stdout: "num_with_osd": 2 2024-09-14T03:47:47.195 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.195 INFO:tasks.workunit.client.0.smithi070.stdout: "leaderboard": false, 2024-09-14T03:47:47.195 INFO:tasks.workunit.client.0.smithi070.stdout: "license": "sharing-1-0", 2024-09-14T03:47:47.195 INFO:tasks.workunit.client.0.smithi070.stdout: "metadata": { 2024-09-14T03:47:47.195 INFO:tasks.workunit.client.0.smithi070.stdout: "mds": { 2024-09-14T03:47:47.195 INFO:tasks.workunit.client.0.smithi070.stdout: "arch": { 2024-09-14T03:47:47.195 INFO:tasks.workunit.client.0.smithi070.stdout: "x86_64": 2 2024-09-14T03:47:47.195 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.195 INFO:tasks.workunit.client.0.smithi070.stdout: "ceph_version": { 2024-09-14T03:47:47.195 INFO:tasks.workunit.client.0.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:47:47.196 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.196 INFO:tasks.workunit.client.0.smithi070.stdout: "cpu": { 2024-09-14T03:47:47.196 INFO:tasks.workunit.client.0.smithi070.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 2 2024-09-14T03:47:47.196 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.196 INFO:tasks.workunit.client.0.smithi070.stdout: "distro": { 2024-09-14T03:47:47.196 INFO:tasks.workunit.client.0.smithi070.stdout: "centos": 2 2024-09-14T03:47:47.196 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.196 INFO:tasks.workunit.client.0.smithi070.stdout: "distro_description": { 2024-09-14T03:47:47.196 INFO:tasks.workunit.client.0.smithi070.stdout: "CentOS Stream 9": 2 2024-09-14T03:47:47.196 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.196 INFO:tasks.workunit.client.0.smithi070.stdout: "kernel_description": { 2024-09-14T03:47:47.196 INFO:tasks.workunit.client.0.smithi070.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 2 2024-09-14T03:47:47.196 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.196 INFO:tasks.workunit.client.0.smithi070.stdout: "kernel_version": { 2024-09-14T03:47:47.197 INFO:tasks.workunit.client.0.smithi070.stdout: "5.14.0-505.el9.x86_64": 2 2024-09-14T03:47:47.197 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.197 INFO:tasks.workunit.client.0.smithi070.stdout: "os": { 2024-09-14T03:47:47.197 INFO:tasks.workunit.client.0.smithi070.stdout: "Linux": 2 2024-09-14T03:47:47.197 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:47.197 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.197 INFO:tasks.workunit.client.0.smithi070.stdout: "mon": { 2024-09-14T03:47:47.197 INFO:tasks.workunit.client.0.smithi070.stdout: "arch": { 2024-09-14T03:47:47.197 INFO:tasks.workunit.client.0.smithi070.stdout: "x86_64": 3 2024-09-14T03:47:47.197 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.197 INFO:tasks.workunit.client.0.smithi070.stdout: "ceph_version": { 2024-09-14T03:47:47.197 INFO:tasks.workunit.client.0.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 3 2024-09-14T03:47:47.197 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.197 INFO:tasks.workunit.client.0.smithi070.stdout: "cpu": { 2024-09-14T03:47:47.198 INFO:tasks.workunit.client.0.smithi070.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 3 2024-09-14T03:47:47.198 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.198 INFO:tasks.workunit.client.0.smithi070.stdout: "distro": { 2024-09-14T03:47:47.198 INFO:tasks.workunit.client.0.smithi070.stdout: "centos": 3 2024-09-14T03:47:47.198 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.198 INFO:tasks.workunit.client.0.smithi070.stdout: "distro_description": { 2024-09-14T03:47:47.198 INFO:tasks.workunit.client.0.smithi070.stdout: "CentOS Stream 9": 3 2024-09-14T03:47:47.198 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.198 INFO:tasks.workunit.client.0.smithi070.stdout: "kernel_description": { 2024-09-14T03:47:47.198 INFO:tasks.workunit.client.0.smithi070.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 3 2024-09-14T03:47:47.198 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.198 INFO:tasks.workunit.client.0.smithi070.stdout: "kernel_version": { 2024-09-14T03:47:47.198 INFO:tasks.workunit.client.0.smithi070.stdout: "5.14.0-505.el9.x86_64": 3 2024-09-14T03:47:47.198 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.199 INFO:tasks.workunit.client.0.smithi070.stdout: "os": { 2024-09-14T03:47:47.199 INFO:tasks.workunit.client.0.smithi070.stdout: "Linux": 3 2024-09-14T03:47:47.199 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:47.199 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.199 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:47.199 INFO:tasks.workunit.client.0.smithi070.stdout: "arch": { 2024-09-14T03:47:47.199 INFO:tasks.workunit.client.0.smithi070.stdout: "x86_64": 8 2024-09-14T03:47:47.199 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.199 INFO:tasks.workunit.client.0.smithi070.stdout: "ceph_version": { 2024-09-14T03:47:47.199 INFO:tasks.workunit.client.0.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 8 2024-09-14T03:47:47.199 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.199 INFO:tasks.workunit.client.0.smithi070.stdout: "cpu": { 2024-09-14T03:47:47.199 INFO:tasks.workunit.client.0.smithi070.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 8 2024-09-14T03:47:47.200 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.200 INFO:tasks.workunit.client.0.smithi070.stdout: "distro": { 2024-09-14T03:47:47.200 INFO:tasks.workunit.client.0.smithi070.stdout: "centos": 8 2024-09-14T03:47:47.200 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.200 INFO:tasks.workunit.client.0.smithi070.stdout: "distro_description": { 2024-09-14T03:47:47.200 INFO:tasks.workunit.client.0.smithi070.stdout: "CentOS Stream 9": 8 2024-09-14T03:47:47.200 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.200 INFO:tasks.workunit.client.0.smithi070.stdout: "kernel_description": { 2024-09-14T03:47:47.200 INFO:tasks.workunit.client.0.smithi070.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 8 2024-09-14T03:47:47.200 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.200 INFO:tasks.workunit.client.0.smithi070.stdout: "kernel_version": { 2024-09-14T03:47:47.200 INFO:tasks.workunit.client.0.smithi070.stdout: "5.14.0-505.el9.x86_64": 8 2024-09-14T03:47:47.200 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.200 INFO:tasks.workunit.client.0.smithi070.stdout: "os": { 2024-09-14T03:47:47.201 INFO:tasks.workunit.client.0.smithi070.stdout: "Linux": 8 2024-09-14T03:47:47.201 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.201 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_objectstore": { 2024-09-14T03:47:47.201 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore": 8 2024-09-14T03:47:47.201 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.201 INFO:tasks.workunit.client.0.smithi070.stdout: "rotational": { 2024-09-14T03:47:47.201 INFO:tasks.workunit.client.0.smithi070.stdout: "0": 8 2024-09-14T03:47:47.201 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:47.201 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:47.201 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.201 INFO:tasks.workunit.client.0.smithi070.stdout: "mon": { 2024-09-14T03:47:47.201 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 3, 2024-09-14T03:47:47.201 INFO:tasks.workunit.client.0.smithi070.stdout: "features": { 2024-09-14T03:47:47.201 INFO:tasks.workunit.client.0.smithi070.stdout: "optional": [], 2024-09-14T03:47:47.202 INFO:tasks.workunit.client.0.smithi070.stdout: "persistent": [ 2024-09-14T03:47:47.202 INFO:tasks.workunit.client.0.smithi070.stdout: "kraken", 2024-09-14T03:47:47.202 INFO:tasks.workunit.client.0.smithi070.stdout: "luminous", 2024-09-14T03:47:47.202 INFO:tasks.workunit.client.0.smithi070.stdout: "mimic", 2024-09-14T03:47:47.202 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap-prune", 2024-09-14T03:47:47.202 INFO:tasks.workunit.client.0.smithi070.stdout: "nautilus", 2024-09-14T03:47:47.202 INFO:tasks.workunit.client.0.smithi070.stdout: "octopus", 2024-09-14T03:47:47.202 INFO:tasks.workunit.client.0.smithi070.stdout: "pacific", 2024-09-14T03:47:47.202 INFO:tasks.workunit.client.0.smithi070.stdout: "elector-pinging", 2024-09-14T03:47:47.202 INFO:tasks.workunit.client.0.smithi070.stdout: "quincy" 2024-09-14T03:47:47.202 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:47.202 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.202 INFO:tasks.workunit.client.0.smithi070.stdout: "ipv4_addr_mons": 6, 2024-09-14T03:47:47.202 INFO:tasks.workunit.client.0.smithi070.stdout: "ipv6_addr_mons": 0, 2024-09-14T03:47:47.203 INFO:tasks.workunit.client.0.smithi070.stdout: "min_mon_release": 17, 2024-09-14T03:47:47.203 INFO:tasks.workunit.client.0.smithi070.stdout: "v1_addr_mons": 3, 2024-09-14T03:47:47.203 INFO:tasks.workunit.client.0.smithi070.stdout: "v2_addr_mons": 3 2024-09-14T03:47:47.203 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.203 INFO:tasks.workunit.client.0.smithi070.stdout: "organization": "ceph-qa", 2024-09-14T03:47:47.203 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:47.203 INFO:tasks.workunit.client.0.smithi070.stdout: "cluster_network": false, 2024-09-14T03:47:47.203 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 8, 2024-09-14T03:47:47.203 INFO:tasks.workunit.client.0.smithi070.stdout: "require_min_compat_client": "luminous", 2024-09-14T03:47:47.203 INFO:tasks.workunit.client.0.smithi070.stdout: "require_osd_release": "quincy" 2024-09-14T03:47:47.203 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.203 INFO:tasks.workunit.client.0.smithi070.stdout: "pools": [ 2024-09-14T03:47:47.203 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:47.203 INFO:tasks.workunit.client.0.smithi070.stdout: "application": [ 2024-09-14T03:47:47.204 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr" 2024-09-14T03:47:47.204 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:47.204 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_mode": "none", 2024-09-14T03:47:47.204 INFO:tasks.workunit.client.0.smithi070.stdout: "erasure_code_profile": {}, 2024-09-14T03:47:47.204 INFO:tasks.workunit.client.0.smithi070.stdout: "min_size": 2, 2024-09-14T03:47:47.204 INFO:tasks.workunit.client.0.smithi070.stdout: "options": {}, 2024-09-14T03:47:47.204 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_autoscale_mode": "off", 2024-09-14T03:47:47.204 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_num": 1, 2024-09-14T03:47:47.204 INFO:tasks.workunit.client.0.smithi070.stdout: "pgp_num": 1, 2024-09-14T03:47:47.204 INFO:tasks.workunit.client.0.smithi070.stdout: "pool": 1, 2024-09-14T03:47:47.204 INFO:tasks.workunit.client.0.smithi070.stdout: "size": 3, 2024-09-14T03:47:47.204 INFO:tasks.workunit.client.0.smithi070.stdout: "stats": { 2024-09-14T03:47:47.204 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_raw": 727115901194, 2024-09-14T03:47:47.204 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_used": 1781760, 2024-09-14T03:47:47.205 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_bytes_used": 0, 2024-09-14T03:47:47.205 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_under_bytes": 0, 2024-09-14T03:47:47.205 INFO:tasks.workunit.client.0.smithi070.stdout: "data_bytes_used": 1781760, 2024-09-14T03:47:47.205 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty": 0, 2024-09-14T03:47:47.205 INFO:tasks.workunit.client.0.smithi070.stdout: "max_avail": 242371969024, 2024-09-14T03:47:47.205 INFO:tasks.workunit.client.0.smithi070.stdout: "objects": 2, 2024-09-14T03:47:47.205 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_bytes_used": 0, 2024-09-14T03:47:47.205 INFO:tasks.workunit.client.0.smithi070.stdout: "percent_used": 2.450442480039783e-06, 2024-09-14T03:47:47.205 INFO:tasks.workunit.client.0.smithi070.stdout: "quota_bytes": 0, 2024-09-14T03:47:47.205 INFO:tasks.workunit.client.0.smithi070.stdout: "quota_objects": 0, 2024-09-14T03:47:47.205 INFO:tasks.workunit.client.0.smithi070.stdout: "rd": 126, 2024-09-14T03:47:47.205 INFO:tasks.workunit.client.0.smithi070.stdout: "rd_bytes": 236544, 2024-09-14T03:47:47.205 INFO:tasks.workunit.client.0.smithi070.stdout: "stored": 590368, 2024-09-14T03:47:47.205 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_data": 590368, 2024-09-14T03:47:47.206 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_omap": 0, 2024-09-14T03:47:47.206 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_raw": 1771104, 2024-09-14T03:47:47.206 INFO:tasks.workunit.client.0.smithi070.stdout: "wr": 113, 2024-09-14T03:47:47.206 INFO:tasks.workunit.client.0.smithi070.stdout: "wr_bytes": 1667072 2024-09-14T03:47:47.206 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.206 INFO:tasks.workunit.client.0.smithi070.stdout: "target_max_bytes": 0, 2024-09-14T03:47:47.206 INFO:tasks.workunit.client.0.smithi070.stdout: "target_max_objects": 0, 2024-09-14T03:47:47.206 INFO:tasks.workunit.client.0.smithi070.stdout: "type": "replicated" 2024-09-14T03:47:47.206 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.206 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:47.206 INFO:tasks.workunit.client.0.smithi070.stdout: "application": [ 2024-09-14T03:47:47.206 INFO:tasks.workunit.client.0.smithi070.stdout: "cephfs" 2024-09-14T03:47:47.206 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:47.206 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_mode": "none", 2024-09-14T03:47:47.206 INFO:tasks.workunit.client.0.smithi070.stdout: "erasure_code_profile": {}, 2024-09-14T03:47:47.207 INFO:tasks.workunit.client.0.smithi070.stdout: "min_size": 2, 2024-09-14T03:47:47.207 INFO:tasks.workunit.client.0.smithi070.stdout: "options": {}, 2024-09-14T03:47:47.207 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_autoscale_mode": "off", 2024-09-14T03:47:47.207 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_num": 32, 2024-09-14T03:47:47.207 INFO:tasks.workunit.client.0.smithi070.stdout: "pgp_num": 32, 2024-09-14T03:47:47.207 INFO:tasks.workunit.client.0.smithi070.stdout: "pool": 2, 2024-09-14T03:47:47.207 INFO:tasks.workunit.client.0.smithi070.stdout: "size": 3, 2024-09-14T03:47:47.207 INFO:tasks.workunit.client.0.smithi070.stdout: "stats": { 2024-09-14T03:47:47.207 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_raw": 727115901194, 2024-09-14T03:47:47.207 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_used": 98304, 2024-09-14T03:47:47.207 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_bytes_used": 0, 2024-09-14T03:47:47.207 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_under_bytes": 0, 2024-09-14T03:47:47.208 INFO:tasks.workunit.client.0.smithi070.stdout: "data_bytes_used": 98304, 2024-09-14T03:47:47.208 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty": 0, 2024-09-14T03:47:47.208 INFO:tasks.workunit.client.0.smithi070.stdout: "max_avail": 242371969024, 2024-09-14T03:47:47.208 INFO:tasks.workunit.client.0.smithi070.stdout: "objects": 22, 2024-09-14T03:47:47.208 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_bytes_used": 0, 2024-09-14T03:47:47.208 INFO:tasks.workunit.client.0.smithi070.stdout: "percent_used": 1.3519712638299097e-07, 2024-09-14T03:47:47.208 INFO:tasks.workunit.client.0.smithi070.stdout: "quota_bytes": 0, 2024-09-14T03:47:47.208 INFO:tasks.workunit.client.0.smithi070.stdout: "quota_objects": 0, 2024-09-14T03:47:47.208 INFO:tasks.workunit.client.0.smithi070.stdout: "rd": 0, 2024-09-14T03:47:47.208 INFO:tasks.workunit.client.0.smithi070.stdout: "rd_bytes": 0, 2024-09-14T03:47:47.208 INFO:tasks.workunit.client.0.smithi070.stdout: "stored": 3410, 2024-09-14T03:47:47.208 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_data": 3410, 2024-09-14T03:47:47.208 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_omap": 0, 2024-09-14T03:47:47.208 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_raw": 10230, 2024-09-14T03:47:47.209 INFO:tasks.workunit.client.0.smithi070.stdout: "wr": 51, 2024-09-14T03:47:47.209 INFO:tasks.workunit.client.0.smithi070.stdout: "wr_bytes": 19456 2024-09-14T03:47:47.209 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.209 INFO:tasks.workunit.client.0.smithi070.stdout: "target_max_bytes": 0, 2024-09-14T03:47:47.209 INFO:tasks.workunit.client.0.smithi070.stdout: "target_max_objects": 0, 2024-09-14T03:47:47.209 INFO:tasks.workunit.client.0.smithi070.stdout: "type": "replicated" 2024-09-14T03:47:47.209 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.209 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:47.209 INFO:tasks.workunit.client.0.smithi070.stdout: "application": [ 2024-09-14T03:47:47.209 INFO:tasks.workunit.client.0.smithi070.stdout: "cephfs" 2024-09-14T03:47:47.209 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:47.209 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_mode": "none", 2024-09-14T03:47:47.209 INFO:tasks.workunit.client.0.smithi070.stdout: "erasure_code_profile": {}, 2024-09-14T03:47:47.210 INFO:tasks.workunit.client.0.smithi070.stdout: "min_size": 2, 2024-09-14T03:47:47.210 INFO:tasks.workunit.client.0.smithi070.stdout: "options": {}, 2024-09-14T03:47:47.210 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_autoscale_mode": "off", 2024-09-14T03:47:47.210 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_num": 64, 2024-09-14T03:47:47.210 INFO:tasks.workunit.client.0.smithi070.stdout: "pgp_num": 64, 2024-09-14T03:47:47.210 INFO:tasks.workunit.client.0.smithi070.stdout: "pool": 3, 2024-09-14T03:47:47.210 INFO:tasks.workunit.client.0.smithi070.stdout: "size": 3, 2024-09-14T03:47:47.210 INFO:tasks.workunit.client.0.smithi070.stdout: "stats": { 2024-09-14T03:47:47.210 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_raw": 727115901194, 2024-09-14T03:47:47.210 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_used": 0, 2024-09-14T03:47:47.210 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_bytes_used": 0, 2024-09-14T03:47:47.210 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_under_bytes": 0, 2024-09-14T03:47:47.210 INFO:tasks.workunit.client.0.smithi070.stdout: "data_bytes_used": 0, 2024-09-14T03:47:47.210 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty": 0, 2024-09-14T03:47:47.210 INFO:tasks.workunit.client.0.smithi070.stdout: "max_avail": 242371969024, 2024-09-14T03:47:47.211 INFO:tasks.workunit.client.0.smithi070.stdout: "objects": 0, 2024-09-14T03:47:47.211 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_bytes_used": 0, 2024-09-14T03:47:47.211 INFO:tasks.workunit.client.0.smithi070.stdout: "percent_used": 0.0, 2024-09-14T03:47:47.211 INFO:tasks.workunit.client.0.smithi070.stdout: "quota_bytes": 0, 2024-09-14T03:47:47.211 INFO:tasks.workunit.client.0.smithi070.stdout: "quota_objects": 0, 2024-09-14T03:47:47.211 INFO:tasks.workunit.client.0.smithi070.stdout: "rd": 0, 2024-09-14T03:47:47.211 INFO:tasks.workunit.client.0.smithi070.stdout: "rd_bytes": 0, 2024-09-14T03:47:47.211 INFO:tasks.workunit.client.0.smithi070.stdout: "stored": 0, 2024-09-14T03:47:47.211 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_data": 0, 2024-09-14T03:47:47.211 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_omap": 0, 2024-09-14T03:47:47.211 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_raw": 0, 2024-09-14T03:47:47.211 INFO:tasks.workunit.client.0.smithi070.stdout: "wr": 0, 2024-09-14T03:47:47.211 INFO:tasks.workunit.client.0.smithi070.stdout: "wr_bytes": 0 2024-09-14T03:47:47.211 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.212 INFO:tasks.workunit.client.0.smithi070.stdout: "target_max_bytes": 0, 2024-09-14T03:47:47.212 INFO:tasks.workunit.client.0.smithi070.stdout: "target_max_objects": 0, 2024-09-14T03:47:47.212 INFO:tasks.workunit.client.0.smithi070.stdout: "type": "replicated" 2024-09-14T03:47:47.212 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.212 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:47.212 INFO:tasks.workunit.client.0.smithi070.stdout: "application": [ 2024-09-14T03:47:47.212 INFO:tasks.workunit.client.0.smithi070.stdout: "rbd" 2024-09-14T03:47:47.212 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:47.212 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_mode": "none", 2024-09-14T03:47:47.212 INFO:tasks.workunit.client.0.smithi070.stdout: "erasure_code_profile": {}, 2024-09-14T03:47:47.212 INFO:tasks.workunit.client.0.smithi070.stdout: "min_size": 2, 2024-09-14T03:47:47.212 INFO:tasks.workunit.client.0.smithi070.stdout: "options": {}, 2024-09-14T03:47:47.212 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_autoscale_mode": "off", 2024-09-14T03:47:47.213 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_num": 8, 2024-09-14T03:47:47.213 INFO:tasks.workunit.client.0.smithi070.stdout: "pgp_num": 8, 2024-09-14T03:47:47.213 INFO:tasks.workunit.client.0.smithi070.stdout: "pool": 4, 2024-09-14T03:47:47.213 INFO:tasks.workunit.client.0.smithi070.stdout: "size": 3, 2024-09-14T03:47:47.213 INFO:tasks.workunit.client.0.smithi070.stdout: "stats": { 2024-09-14T03:47:47.213 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_raw": 727115901194, 2024-09-14T03:47:47.213 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_used": 12288, 2024-09-14T03:47:47.213 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_bytes_used": 0, 2024-09-14T03:47:47.213 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_under_bytes": 0, 2024-09-14T03:47:47.213 INFO:tasks.workunit.client.0.smithi070.stdout: "data_bytes_used": 12288, 2024-09-14T03:47:47.213 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty": 0, 2024-09-14T03:47:47.213 INFO:tasks.workunit.client.0.smithi070.stdout: "max_avail": 242371969024, 2024-09-14T03:47:47.213 INFO:tasks.workunit.client.0.smithi070.stdout: "objects": 2, 2024-09-14T03:47:47.213 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_bytes_used": 0, 2024-09-14T03:47:47.214 INFO:tasks.workunit.client.0.smithi070.stdout: "percent_used": 1.689964435058755e-08, 2024-09-14T03:47:47.214 INFO:tasks.workunit.client.0.smithi070.stdout: "quota_bytes": 0, 2024-09-14T03:47:47.214 INFO:tasks.workunit.client.0.smithi070.stdout: "quota_objects": 0, 2024-09-14T03:47:47.214 INFO:tasks.workunit.client.0.smithi070.stdout: "rd": 0, 2024-09-14T03:47:47.214 INFO:tasks.workunit.client.0.smithi070.stdout: "rd_bytes": 0, 2024-09-14T03:47:47.214 INFO:tasks.workunit.client.0.smithi070.stdout: "stored": 19, 2024-09-14T03:47:47.214 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_data": 19, 2024-09-14T03:47:47.214 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_omap": 0, 2024-09-14T03:47:47.214 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_raw": 57, 2024-09-14T03:47:47.214 INFO:tasks.workunit.client.0.smithi070.stdout: "wr": 2, 2024-09-14T03:47:47.214 INFO:tasks.workunit.client.0.smithi070.stdout: "wr_bytes": 2048 2024-09-14T03:47:47.214 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.214 INFO:tasks.workunit.client.0.smithi070.stdout: "target_max_bytes": 0, 2024-09-14T03:47:47.214 INFO:tasks.workunit.client.0.smithi070.stdout: "target_max_objects": 0, 2024-09-14T03:47:47.214 INFO:tasks.workunit.client.0.smithi070.stdout: "type": "replicated" 2024-09-14T03:47:47.215 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:47.215 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:47.215 INFO:tasks.workunit.client.0.smithi070.stdout: "rbd": { 2024-09-14T03:47:47.215 INFO:tasks.workunit.client.0.smithi070.stdout: "mirroring_by_pool": [ 2024-09-14T03:47:47.215 INFO:tasks.workunit.client.0.smithi070.stdout: false 2024-09-14T03:47:47.215 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:47.215 INFO:tasks.workunit.client.0.smithi070.stdout: "num_images_by_pool": [ 2024-09-14T03:47:47.215 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:47.215 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:47.215 INFO:tasks.workunit.client.0.smithi070.stdout: "num_pools": 1 2024-09-14T03:47:47.215 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.215 INFO:tasks.workunit.client.0.smithi070.stdout: "report_id": "04c26ace-6e52-41dd-9ad2-e917daf8e93d", 2024-09-14T03:47:47.215 INFO:tasks.workunit.client.0.smithi070.stdout: "report_timestamp": "2024-09-14T03:47:47.120039", 2024-09-14T03:47:47.215 INFO:tasks.workunit.client.0.smithi070.stdout: "report_version": 1, 2024-09-14T03:47:47.216 INFO:tasks.workunit.client.0.smithi070.stdout: "rook": { 2024-09-14T03:47:47.216 INFO:tasks.workunit.client.0.smithi070.stdout: "cluster": { 2024-09-14T03:47:47.216 INFO:tasks.workunit.client.0.smithi070.stdout: "external-mode": null, 2024-09-14T03:47:47.216 INFO:tasks.workunit.client.0.smithi070.stdout: "mon": { 2024-09-14T03:47:47.216 INFO:tasks.workunit.client.0.smithi070.stdout: "allow-multiple-per-node": null, 2024-09-14T03:47:47.216 INFO:tasks.workunit.client.0.smithi070.stdout: "count": null, 2024-09-14T03:47:47.216 INFO:tasks.workunit.client.0.smithi070.stdout: "max-id": null, 2024-09-14T03:47:47.216 INFO:tasks.workunit.client.0.smithi070.stdout: "pvc": { 2024-09-14T03:47:47.216 INFO:tasks.workunit.client.0.smithi070.stdout: "enabled": null 2024-09-14T03:47:47.216 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.216 INFO:tasks.workunit.client.0.smithi070.stdout: "stretch": { 2024-09-14T03:47:47.216 INFO:tasks.workunit.client.0.smithi070.stdout: "enabled": null 2024-09-14T03:47:47.216 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:47.217 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.217 INFO:tasks.workunit.client.0.smithi070.stdout: "network": { 2024-09-14T03:47:47.217 INFO:tasks.workunit.client.0.smithi070.stdout: "provider": null 2024-09-14T03:47:47.217 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.217 INFO:tasks.workunit.client.0.smithi070.stdout: "storage": { 2024-09-14T03:47:47.217 INFO:tasks.workunit.client.0.smithi070.stdout: "device-set": { 2024-09-14T03:47:47.217 INFO:tasks.workunit.client.0.smithi070.stdout: "count": { 2024-09-14T03:47:47.217 INFO:tasks.workunit.client.0.smithi070.stdout: "non-portable": null, 2024-09-14T03:47:47.217 INFO:tasks.workunit.client.0.smithi070.stdout: "portable": null, 2024-09-14T03:47:47.217 INFO:tasks.workunit.client.0.smithi070.stdout: "total": null 2024-09-14T03:47:47.217 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:47.217 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:47.217 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:47.217 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.217 INFO:tasks.workunit.client.0.smithi070.stdout: "csi": { 2024-09-14T03:47:47.218 INFO:tasks.workunit.client.0.smithi070.stdout: "version": null 2024-09-14T03:47:47.218 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.218 INFO:tasks.workunit.client.0.smithi070.stdout: "kubernetes": { 2024-09-14T03:47:47.218 INFO:tasks.workunit.client.0.smithi070.stdout: "version": null 2024-09-14T03:47:47.218 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.218 INFO:tasks.workunit.client.0.smithi070.stdout: "node": { 2024-09-14T03:47:47.218 INFO:tasks.workunit.client.0.smithi070.stdout: "count": { 2024-09-14T03:47:47.218 INFO:tasks.workunit.client.0.smithi070.stdout: "kubernetes-total": null, 2024-09-14T03:47:47.218 INFO:tasks.workunit.client.0.smithi070.stdout: "with-ceph-daemons": null, 2024-09-14T03:47:47.218 INFO:tasks.workunit.client.0.smithi070.stdout: "with-csi-cephfs-plugin": null, 2024-09-14T03:47:47.218 INFO:tasks.workunit.client.0.smithi070.stdout: "with-csi-nfs-plugin": null, 2024-09-14T03:47:47.218 INFO:tasks.workunit.client.0.smithi070.stdout: "with-csi-rbd-plugin": null 2024-09-14T03:47:47.218 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:47.218 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.219 INFO:tasks.workunit.client.0.smithi070.stdout: "usage": { 2024-09-14T03:47:47.219 INFO:tasks.workunit.client.0.smithi070.stdout: "storage-class": { 2024-09-14T03:47:47.219 INFO:tasks.workunit.client.0.smithi070.stdout: "count": { 2024-09-14T03:47:47.219 INFO:tasks.workunit.client.0.smithi070.stdout: "bucket": null, 2024-09-14T03:47:47.219 INFO:tasks.workunit.client.0.smithi070.stdout: "cephfs": null, 2024-09-14T03:47:47.219 INFO:tasks.workunit.client.0.smithi070.stdout: "nfs": null, 2024-09-14T03:47:47.219 INFO:tasks.workunit.client.0.smithi070.stdout: "rbd": null, 2024-09-14T03:47:47.219 INFO:tasks.workunit.client.0.smithi070.stdout: "total": null 2024-09-14T03:47:47.219 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:47.219 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:47.219 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.219 INFO:tasks.workunit.client.0.smithi070.stdout: "version": null 2024-09-14T03:47:47.219 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.219 INFO:tasks.workunit.client.0.smithi070.stdout: "services": {}, 2024-09-14T03:47:47.220 INFO:tasks.workunit.client.0.smithi070.stdout: "usage": { 2024-09-14T03:47:47.220 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_num": 105, 2024-09-14T03:47:47.220 INFO:tasks.workunit.client.0.smithi070.stdout: "pools": 4, 2024-09-14T03:47:47.220 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_by_class": { 2024-09-14T03:47:47.220 INFO:tasks.workunit.client.0.smithi070.stdout: "ssd": { 2024-09-14T03:47:47.220 INFO:tasks.workunit.client.0.smithi070.stdout: "total_avail_bytes": 765516926976, 2024-09-14T03:47:47.220 INFO:tasks.workunit.client.0.smithi070.stdout: "total_bytes": 767960285184, 2024-09-14T03:47:47.220 INFO:tasks.workunit.client.0.smithi070.stdout: "total_used_bytes": 2443358208, 2024-09-14T03:47:47.220 INFO:tasks.workunit.client.0.smithi070.stdout: "total_used_raw_bytes": 2443358208, 2024-09-14T03:47:47.220 INFO:tasks.workunit.client.0.smithi070.stdout: "total_used_raw_ratio": 0.0031816205009818077 2024-09-14T03:47:47.220 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:47.220 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:47.220 INFO:tasks.workunit.client.0.smithi070.stdout: "total_avail_bytes": 765516926976, 2024-09-14T03:47:47.220 INFO:tasks.workunit.client.0.smithi070.stdout: "total_bytes": 767960285184, 2024-09-14T03:47:47.221 INFO:tasks.workunit.client.0.smithi070.stdout: "total_used_bytes": 2443358208 2024-09-14T03:47:47.221 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:47.221 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:47.221 INFO:tasks.workunit.client.0.smithi070.stdout:} 2024-09-14T03:47:47.221 INFO:tasks.workunit.client.0.smithi070.stderr:++ ceph telemetry collection ls 2024-09-14T03:47:47.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:46 smithi094 ceph-mon[31053]: from='client.24716 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:47.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:46 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:47.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:46 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:47.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:46 smithi094 ceph-mon[31053]: from='client.24716 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:47.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:46 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:47.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:46 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:47.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:46 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:47.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:46 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:47.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:46 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:47.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:46 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:47.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:46 smithi070 ceph-mon[39288]: from='client.24716 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:47.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:46 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:47.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:46 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:47.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:46 smithi070 ceph-mon[39288]: from='client.24716 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:47.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:46 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:47.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:46 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:47.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:46 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:47.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:46 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:47.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:46 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:47.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:46 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:47.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:46 smithi070 ceph-mon[27495]: from='client.24716 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:47.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:46 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:47.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:46 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:47.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:46 smithi070 ceph-mon[27495]: from='client.24716 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:47.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:46 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:47.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:46 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:47.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:46 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:47.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:46 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:47.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:46 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:47.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:46 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:47.583 INFO:tasks.workunit.client.0.smithi070.stderr:+ COLLECTIONS='New collections are available: 2024-09-14T03:47:47.583 INFO:tasks.workunit.client.0.smithi070.stderr:['\''basic_base'\'', '\''basic_mds_metadata'\'', '\''basic_pool_options_bluestore'\'', '\''basic_pool_usage'\'', '\''basic_rook_v01'\'', '\''basic_usage_by_class'\'', '\''crash_base'\'', '\''device_base'\'', '\''ident_base'\'', '\''perf_memory_metrics'\'', '\''perf_perf'\''] 2024-09-14T03:47:47.583 INFO:tasks.workunit.client.0.smithi070.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-14T03:47:47.583 INFO:tasks.workunit.client.0.smithi070.stderr: 2024-09-14T03:47:47.583 INFO:tasks.workunit.client.0.smithi070.stderr:NAME STATUS DESC 2024-09-14T03:47:47.584 INFO:tasks.workunit.client.0.smithi070.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-14T03:47:47.584 INFO:tasks.workunit.client.0.smithi070.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-14T03:47:47.584 INFO:tasks.workunit.client.0.smithi070.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-14T03:47:47.584 INFO:tasks.workunit.client.0.smithi070.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-14T03:47:47.584 INFO:tasks.workunit.client.0.smithi070.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-14T03:47:47.584 INFO:tasks.workunit.client.0.smithi070.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-14T03:47:47.584 INFO:tasks.workunit.client.0.smithi070.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-14T03:47:47.584 INFO:tasks.workunit.client.0.smithi070.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-14T03:47:47.584 INFO:tasks.workunit.client.0.smithi070.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-14T03:47:47.584 INFO:tasks.workunit.client.0.smithi070.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-14T03:47:47.584 INFO:tasks.workunit.client.0.smithi070.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster ' 2024-09-14T03:47:47.584 INFO:tasks.workunit.client.0.smithi070.stderr:+ NEW_COLLECTIONS=("perf_perf" "basic_mds_metadata" "basic_pool_usage" "basic_rook_v01" "perf_memory_metrics" "basic_pool_options_bluestore") 2024-09-14T03:47:47.585 INFO:tasks.workunit.client.0.smithi070.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-14T03:47:47.585 INFO:tasks.workunit.client.0.smithi070.stderr:+ [[ New collections are available: 2024-09-14T03:47:47.585 INFO:tasks.workunit.client.0.smithi070.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-14T03:47:47.585 INFO:tasks.workunit.client.0.smithi070.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-14T03:47:47.585 INFO:tasks.workunit.client.0.smithi070.stderr: 2024-09-14T03:47:47.585 INFO:tasks.workunit.client.0.smithi070.stderr:NAME STATUS DESC 2024-09-14T03:47:47.585 INFO:tasks.workunit.client.0.smithi070.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-14T03:47:47.585 INFO:tasks.workunit.client.0.smithi070.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-14T03:47:47.585 INFO:tasks.workunit.client.0.smithi070.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-14T03:47:47.586 INFO:tasks.workunit.client.0.smithi070.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-14T03:47:47.586 INFO:tasks.workunit.client.0.smithi070.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-14T03:47:47.586 INFO:tasks.workunit.client.0.smithi070.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-14T03:47:47.586 INFO:tasks.workunit.client.0.smithi070.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-14T03:47:47.586 INFO:tasks.workunit.client.0.smithi070.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-14T03:47:47.586 INFO:tasks.workunit.client.0.smithi070.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-14T03:47:47.586 INFO:tasks.workunit.client.0.smithi070.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-14T03:47:47.586 INFO:tasks.workunit.client.0.smithi070.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *perf_perf* ]] 2024-09-14T03:47:47.586 INFO:tasks.workunit.client.0.smithi070.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-14T03:47:47.586 INFO:tasks.workunit.client.0.smithi070.stderr:+ [[ New collections are available: 2024-09-14T03:47:47.586 INFO:tasks.workunit.client.0.smithi070.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-14T03:47:47.586 INFO:tasks.workunit.client.0.smithi070.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-14T03:47:47.587 INFO:tasks.workunit.client.0.smithi070.stderr: 2024-09-14T03:47:47.587 INFO:tasks.workunit.client.0.smithi070.stderr:NAME STATUS DESC 2024-09-14T03:47:47.587 INFO:tasks.workunit.client.0.smithi070.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-14T03:47:47.587 INFO:tasks.workunit.client.0.smithi070.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-14T03:47:47.587 INFO:tasks.workunit.client.0.smithi070.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-14T03:47:47.587 INFO:tasks.workunit.client.0.smithi070.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-14T03:47:47.587 INFO:tasks.workunit.client.0.smithi070.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-14T03:47:47.587 INFO:tasks.workunit.client.0.smithi070.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-14T03:47:47.587 INFO:tasks.workunit.client.0.smithi070.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-14T03:47:47.587 INFO:tasks.workunit.client.0.smithi070.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-14T03:47:47.587 INFO:tasks.workunit.client.0.smithi070.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-14T03:47:47.587 INFO:tasks.workunit.client.0.smithi070.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-14T03:47:47.587 INFO:tasks.workunit.client.0.smithi070.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *basic_mds_metadata* ]] 2024-09-14T03:47:47.588 INFO:tasks.workunit.client.0.smithi070.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-14T03:47:47.588 INFO:tasks.workunit.client.0.smithi070.stderr:+ [[ New collections are available: 2024-09-14T03:47:47.588 INFO:tasks.workunit.client.0.smithi070.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-14T03:47:47.588 INFO:tasks.workunit.client.0.smithi070.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-14T03:47:47.588 INFO:tasks.workunit.client.0.smithi070.stderr: 2024-09-14T03:47:47.588 INFO:tasks.workunit.client.0.smithi070.stderr:NAME STATUS DESC 2024-09-14T03:47:47.588 INFO:tasks.workunit.client.0.smithi070.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-14T03:47:47.588 INFO:tasks.workunit.client.0.smithi070.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-14T03:47:47.588 INFO:tasks.workunit.client.0.smithi070.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-14T03:47:47.588 INFO:tasks.workunit.client.0.smithi070.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-14T03:47:47.588 INFO:tasks.workunit.client.0.smithi070.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-14T03:47:47.589 INFO:tasks.workunit.client.0.smithi070.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-14T03:47:47.589 INFO:tasks.workunit.client.0.smithi070.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-14T03:47:47.589 INFO:tasks.workunit.client.0.smithi070.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-14T03:47:47.589 INFO:tasks.workunit.client.0.smithi070.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-14T03:47:47.589 INFO:tasks.workunit.client.0.smithi070.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-14T03:47:47.589 INFO:tasks.workunit.client.0.smithi070.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *basic_pool_usage* ]] 2024-09-14T03:47:47.589 INFO:tasks.workunit.client.0.smithi070.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-14T03:47:47.589 INFO:tasks.workunit.client.0.smithi070.stderr:+ [[ New collections are available: 2024-09-14T03:47:47.589 INFO:tasks.workunit.client.0.smithi070.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-14T03:47:47.589 INFO:tasks.workunit.client.0.smithi070.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-14T03:47:47.589 INFO:tasks.workunit.client.0.smithi070.stderr: 2024-09-14T03:47:47.589 INFO:tasks.workunit.client.0.smithi070.stderr:NAME STATUS DESC 2024-09-14T03:47:47.589 INFO:tasks.workunit.client.0.smithi070.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-14T03:47:47.590 INFO:tasks.workunit.client.0.smithi070.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-14T03:47:47.590 INFO:tasks.workunit.client.0.smithi070.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-14T03:47:47.590 INFO:tasks.workunit.client.0.smithi070.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-14T03:47:47.590 INFO:tasks.workunit.client.0.smithi070.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-14T03:47:47.590 INFO:tasks.workunit.client.0.smithi070.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-14T03:47:47.590 INFO:tasks.workunit.client.0.smithi070.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-14T03:47:47.590 INFO:tasks.workunit.client.0.smithi070.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-14T03:47:47.590 INFO:tasks.workunit.client.0.smithi070.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-14T03:47:47.590 INFO:tasks.workunit.client.0.smithi070.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-14T03:47:47.590 INFO:tasks.workunit.client.0.smithi070.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *basic_rook_v01* ]] 2024-09-14T03:47:47.590 INFO:tasks.workunit.client.0.smithi070.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-14T03:47:47.590 INFO:tasks.workunit.client.0.smithi070.stderr:+ [[ New collections are available: 2024-09-14T03:47:47.590 INFO:tasks.workunit.client.0.smithi070.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-14T03:47:47.591 INFO:tasks.workunit.client.0.smithi070.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-14T03:47:47.591 INFO:tasks.workunit.client.0.smithi070.stderr: 2024-09-14T03:47:47.591 INFO:tasks.workunit.client.0.smithi070.stderr:NAME STATUS DESC 2024-09-14T03:47:47.591 INFO:tasks.workunit.client.0.smithi070.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-14T03:47:47.591 INFO:tasks.workunit.client.0.smithi070.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-14T03:47:47.591 INFO:tasks.workunit.client.0.smithi070.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-14T03:47:47.591 INFO:tasks.workunit.client.0.smithi070.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-14T03:47:47.591 INFO:tasks.workunit.client.0.smithi070.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-14T03:47:47.591 INFO:tasks.workunit.client.0.smithi070.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-14T03:47:47.591 INFO:tasks.workunit.client.0.smithi070.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-14T03:47:47.591 INFO:tasks.workunit.client.0.smithi070.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-14T03:47:47.591 INFO:tasks.workunit.client.0.smithi070.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-14T03:47:47.591 INFO:tasks.workunit.client.0.smithi070.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-14T03:47:47.592 INFO:tasks.workunit.client.0.smithi070.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *perf_memory_metrics* ]] 2024-09-14T03:47:47.592 INFO:tasks.workunit.client.0.smithi070.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-14T03:47:47.592 INFO:tasks.workunit.client.0.smithi070.stderr:+ [[ New collections are available: 2024-09-14T03:47:47.592 INFO:tasks.workunit.client.0.smithi070.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-14T03:47:47.592 INFO:tasks.workunit.client.0.smithi070.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-14T03:47:47.592 INFO:tasks.workunit.client.0.smithi070.stderr: 2024-09-14T03:47:47.592 INFO:tasks.workunit.client.0.smithi070.stderr:NAME STATUS DESC 2024-09-14T03:47:47.592 INFO:tasks.workunit.client.0.smithi070.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-14T03:47:47.592 INFO:tasks.workunit.client.0.smithi070.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-14T03:47:47.592 INFO:tasks.workunit.client.0.smithi070.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-14T03:47:47.592 INFO:tasks.workunit.client.0.smithi070.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-14T03:47:47.592 INFO:tasks.workunit.client.0.smithi070.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-14T03:47:47.593 INFO:tasks.workunit.client.0.smithi070.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-14T03:47:47.593 INFO:tasks.workunit.client.0.smithi070.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-14T03:47:47.593 INFO:tasks.workunit.client.0.smithi070.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-14T03:47:47.593 INFO:tasks.workunit.client.0.smithi070.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-14T03:47:47.593 INFO:tasks.workunit.client.0.smithi070.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-14T03:47:47.593 INFO:tasks.workunit.client.0.smithi070.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *basic_pool_options_bluestore* ]] 2024-09-14T03:47:47.593 INFO:tasks.workunit.client.0.smithi070.stderr:+ ceph telemetry on --license sharing-1-0 2024-09-14T03:47:48.035 INFO:tasks.workunit.client.0.smithi070.stdout:Telemetry is on. 2024-09-14T03:47:48.035 INFO:tasks.workunit.client.0.smithi070.stdout:Some channels are disabled, please enable with: 2024-09-14T03:47:48.035 INFO:tasks.workunit.client.0.smithi070.stdout:`ceph telemetry enable channel perf` 2024-09-14T03:47:48.046 INFO:tasks.workunit.client.0.smithi070.stderr:+ ceph telemetry enable channel perf 2024-09-14T03:47:48.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:47 smithi094 ceph-mon[31053]: from='client.24728 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:48.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:47 smithi094 ceph-mon[31053]: pgmap v90: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:48.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:47 smithi094 ceph-mon[31053]: from='client.24728 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:48.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:47 smithi094 ceph-mon[31053]: from='client.14826 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:48.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:47 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:48.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:47 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:48.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:47 smithi094 ceph-mon[31053]: from='client.14826 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:48.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:47 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:48.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:47 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:48.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:47 smithi094 ceph-mon[31053]: mgrmap e22: y(active, since 2m), standbys: x 2024-09-14T03:47:48.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:47 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:48.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:47 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:48.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:47 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:48.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:47 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:48.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:47 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:48.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:47 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:48.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:47 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:48.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:47 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:48.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:47 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:48.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[27495]: from='client.24728 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:48.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[27495]: pgmap v90: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:48.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[27495]: from='client.24728 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:48.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[27495]: from='client.14826 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:48.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:48.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:48.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[27495]: from='client.14826 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:48.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:48.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:48.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[27495]: mgrmap e22: y(active, since 2m), standbys: x 2024-09-14T03:47:48.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:48.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:48.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:48.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:48.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:48.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:48.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:48.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:48.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:48.287 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:47:47] ENGINE Bus STOPPING 2024-09-14T03:47:48.287 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:47:47] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:47:48.287 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:47:47] ENGINE Bus STOPPED 2024-09-14T03:47:48.287 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:47:47] ENGINE Bus STARTING 2024-09-14T03:47:48.287 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:47:48] ENGINE Serving on http://:::9283 2024-09-14T03:47:48.287 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:47:48] ENGINE Bus STARTED 2024-09-14T03:47:48.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[39288]: from='client.24728 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:48.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[39288]: pgmap v90: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:47:48.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[39288]: from='client.24728 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:48.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[39288]: from='client.14826 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:48.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:48.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:48.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[39288]: from='client.14826 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:48.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:48.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:48.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[39288]: mgrmap e22: y(active, since 2m), standbys: x 2024-09-14T03:47:48.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:48.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:48.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:48.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:48.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:48.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:48.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:48.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:48.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:47 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:48.491 INFO:tasks.workunit.client.0.smithi070.stdout:channel_perf is enabled 2024-09-14T03:47:48.502 INFO:tasks.workunit.client.0.smithi070.stderr:++ ceph config get mgr mgr/telemetry/last_opt_revision 2024-09-14T03:47:48.638 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:47:48] ENGINE Bus STOPPING 2024-09-14T03:47:48.639 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:47:48] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:47:48.639 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:47:48] ENGINE Bus STOPPED 2024-09-14T03:47:48.639 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:47:48] ENGINE Bus STARTING 2024-09-14T03:47:48.639 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:47:48] ENGINE Serving on http://:::9283 2024-09-14T03:47:48.845 INFO:tasks.workunit.client.0.smithi070.stderr:+ LAST_OPT_REVISION=1 2024-09-14T03:47:48.845 INFO:tasks.workunit.client.0.smithi070.stderr:+ '[' 1 -ne 1 ']' 2024-09-14T03:47:48.846 INFO:tasks.workunit.client.0.smithi070.stderr:+ ceph telemetry show 2024-09-14T03:47:48.910 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:47:48] ENGINE Bus STARTED 2024-09-14T03:47:49.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='client.24740 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:49.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='client.24740 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:49.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='client.24746 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:49.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:49.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:49.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='client.24746 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:49.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:49.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:49.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:49.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:49.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:49.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:49.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:49.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:49.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:49.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:49.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:49.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:49.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:49.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:49.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:49.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:49.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:49.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:49.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:49.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:49.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:49.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:49.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:49.280 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:49.280 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:49.280 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.280 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:49.280 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:49.280 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:49.280 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:49.280 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.280 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/3195802717' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-14T03:47:49.280 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:48 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/3195802717' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-14T03:47:49.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='client.24740 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:49.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='client.24740 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:49.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='client.24746 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:49.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:49.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:49.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='client.24746 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:49.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:49.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:49.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:49.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:49.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:49.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:49.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:49.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:49.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:49.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:49.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:49.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:49.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:49.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:49.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:49.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:49.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:49.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:49.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='client.24740 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:49.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='client.24740 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:49.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='client.24746 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:49.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:49.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:49.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='client.24746 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:49.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:49.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:49.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:49.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:49.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:49.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:49.290 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:49.290 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:49.290 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:49.290 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:49.290 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:47:49.290 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:49.290 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:49.290 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.290 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:49.290 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:49.290 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.290 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:49.290 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:49.291 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.291 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:49.291 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:49.291 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:49.291 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:49.291 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.291 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:49.291 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:49.291 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:49.292 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:49.292 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.292 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:49.292 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:49.292 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:49.292 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:49.292 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.292 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/3195802717' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-14T03:47:49.292 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/3195802717' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-14T03:47:49.293 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:49.293 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:49.293 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:49.293 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.293 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:49.293 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:49.293 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:49.293 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:49.293 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.293 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:49.294 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:49.294 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:49.294 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:49.294 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.294 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/3195802717' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-14T03:47:49.294 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:48 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/3195802717' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-14T03:47:49.481 INFO:tasks.workunit.client.0.smithi070.stdout:{ 2024-09-14T03:47:49.481 INFO:tasks.workunit.client.0.smithi070.stdout: "balancer": { 2024-09-14T03:47:49.481 INFO:tasks.workunit.client.0.smithi070.stdout: "active": true, 2024-09-14T03:47:49.482 INFO:tasks.workunit.client.0.smithi070.stdout: "mode": "" 2024-09-14T03:47:49.482 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.482 INFO:tasks.workunit.client.0.smithi070.stdout: "channels": [ 2024-09-14T03:47:49.482 INFO:tasks.workunit.client.0.smithi070.stdout: "basic", 2024-09-14T03:47:49.482 INFO:tasks.workunit.client.0.smithi070.stdout: "crash", 2024-09-14T03:47:49.482 INFO:tasks.workunit.client.0.smithi070.stdout: "device", 2024-09-14T03:47:49.482 INFO:tasks.workunit.client.0.smithi070.stdout: "ident", 2024-09-14T03:47:49.482 INFO:tasks.workunit.client.0.smithi070.stdout: "perf" 2024-09-14T03:47:49.482 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:49.482 INFO:tasks.workunit.client.0.smithi070.stdout: "channels_available": [ 2024-09-14T03:47:49.482 INFO:tasks.workunit.client.0.smithi070.stdout: "basic", 2024-09-14T03:47:49.482 INFO:tasks.workunit.client.0.smithi070.stdout: "ident", 2024-09-14T03:47:49.483 INFO:tasks.workunit.client.0.smithi070.stdout: "crash", 2024-09-14T03:47:49.483 INFO:tasks.workunit.client.0.smithi070.stdout: "device", 2024-09-14T03:47:49.483 INFO:tasks.workunit.client.0.smithi070.stdout: "perf" 2024-09-14T03:47:49.483 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:49.483 INFO:tasks.workunit.client.0.smithi070.stdout: "collections_available": [ 2024-09-14T03:47:49.483 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_base", 2024-09-14T03:47:49.483 INFO:tasks.workunit.client.0.smithi070.stdout: "device_base", 2024-09-14T03:47:49.483 INFO:tasks.workunit.client.0.smithi070.stdout: "crash_base", 2024-09-14T03:47:49.483 INFO:tasks.workunit.client.0.smithi070.stdout: "ident_base", 2024-09-14T03:47:49.483 INFO:tasks.workunit.client.0.smithi070.stdout: "perf_perf", 2024-09-14T03:47:49.483 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_mds_metadata", 2024-09-14T03:47:49.483 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_pool_usage", 2024-09-14T03:47:49.483 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_usage_by_class", 2024-09-14T03:47:49.483 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_rook_v01", 2024-09-14T03:47:49.484 INFO:tasks.workunit.client.0.smithi070.stdout: "perf_memory_metrics", 2024-09-14T03:47:49.484 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_pool_options_bluestore" 2024-09-14T03:47:49.484 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:49.484 INFO:tasks.workunit.client.0.smithi070.stdout: "collections_opted_in": [ 2024-09-14T03:47:49.484 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_base", 2024-09-14T03:47:49.484 INFO:tasks.workunit.client.0.smithi070.stdout: "device_base", 2024-09-14T03:47:49.484 INFO:tasks.workunit.client.0.smithi070.stdout: "crash_base", 2024-09-14T03:47:49.484 INFO:tasks.workunit.client.0.smithi070.stdout: "ident_base", 2024-09-14T03:47:49.484 INFO:tasks.workunit.client.0.smithi070.stdout: "perf_perf", 2024-09-14T03:47:49.484 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_mds_metadata", 2024-09-14T03:47:49.484 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_pool_usage", 2024-09-14T03:47:49.484 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_usage_by_class", 2024-09-14T03:47:49.484 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_rook_v01", 2024-09-14T03:47:49.485 INFO:tasks.workunit.client.0.smithi070.stdout: "perf_memory_metrics", 2024-09-14T03:47:49.485 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_pool_options_bluestore" 2024-09-14T03:47:49.485 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:49.485 INFO:tasks.workunit.client.0.smithi070.stdout: "config": { 2024-09-14T03:47:49.485 INFO:tasks.workunit.client.0.smithi070.stdout: "active_changed": [ 2024-09-14T03:47:49.485 INFO:tasks.workunit.client.0.smithi070.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-14T03:47:49.485 INFO:tasks.workunit.client.0.smithi070.stdout: "auth_debug", 2024-09-14T03:47:49.485 INFO:tasks.workunit.client.0.smithi070.stdout: "auth_mon_ticket_ttl", 2024-09-14T03:47:49.485 INFO:tasks.workunit.client.0.smithi070.stdout: "auth_service_ticket_ttl", 2024-09-14T03:47:49.485 INFO:tasks.workunit.client.0.smithi070.stdout: "bdev_debug_aio", 2024-09-14T03:47:49.485 INFO:tasks.workunit.client.0.smithi070.stdout: "client_check_pool_perm", 2024-09-14T03:47:49.485 INFO:tasks.workunit.client.0.smithi070.stdout: "client_mount_gid", 2024-09-14T03:47:49.486 INFO:tasks.workunit.client.0.smithi070.stdout: "client_mount_uid", 2024-09-14T03:47:49.486 INFO:tasks.workunit.client.0.smithi070.stdout: "client_quota", 2024-09-14T03:47:49.486 INFO:tasks.workunit.client.0.smithi070.stdout: "container_image", 2024-09-14T03:47:49.486 INFO:tasks.workunit.client.0.smithi070.stdout: "daemonize", 2024-09-14T03:47:49.486 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_asserts_on_shutdown", 2024-09-14T03:47:49.486 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_mgr", 2024-09-14T03:47:49.486 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_mon", 2024-09-14T03:47:49.486 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_ms", 2024-09-14T03:47:49.486 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_osd", 2024-09-14T03:47:49.486 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_paxos", 2024-09-14T03:47:49.486 INFO:tasks.workunit.client.0.smithi070.stdout: "keyring", 2024-09-14T03:47:49.486 INFO:tasks.workunit.client.0.smithi070.stdout: "leveldb_block_size", 2024-09-14T03:47:49.487 INFO:tasks.workunit.client.0.smithi070.stdout: "leveldb_cache_size", 2024-09-14T03:47:49.487 INFO:tasks.workunit.client.0.smithi070.stdout: "leveldb_compression", 2024-09-14T03:47:49.487 INFO:tasks.workunit.client.0.smithi070.stdout: "leveldb_log", 2024-09-14T03:47:49.487 INFO:tasks.workunit.client.0.smithi070.stdout: "leveldb_write_buffer_size", 2024-09-14T03:47:49.487 INFO:tasks.workunit.client.0.smithi070.stdout: "log_to_file", 2024-09-14T03:47:49.487 INFO:tasks.workunit.client.0.smithi070.stdout: "log_to_journald", 2024-09-14T03:47:49.487 INFO:tasks.workunit.client.0.smithi070.stdout: "log_to_stderr", 2024-09-14T03:47:49.487 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_allow_pool_delete", 2024-09-14T03:47:49.487 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_clock_drift_allowed", 2024-09-14T03:47:49.487 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_cluster_log_to_file", 2024-09-14T03:47:49.487 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_cluster_log_to_journald", 2024-09-14T03:47:49.487 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_cluster_log_to_stderr", 2024-09-14T03:47:49.488 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_data_avail_warn", 2024-09-14T03:47:49.488 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_election_default_strategy", 2024-09-14T03:47:49.488 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_host", 2024-09-14T03:47:49.488 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_max_pg_per_osd", 2024-09-14T03:47:49.488 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_mgr_mkfs_grace", 2024-09-14T03:47:49.488 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_osd_prime_pg_temp", 2024-09-14T03:47:49.488 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_osd_reporter_subtree_level", 2024-09-14T03:47:49.488 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_pg_warn_max_object_skew", 2024-09-14T03:47:49.488 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-14T03:47:49.488 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-14T03:47:49.488 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-14T03:47:49.488 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-14T03:47:49.489 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-14T03:47:49.489 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-14T03:47:49.489 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-14T03:47:49.489 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-14T03:47:49.489 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_too_few_osds", 2024-09-14T03:47:49.489 INFO:tasks.workunit.client.0.smithi070.stdout: "ms_die_on_bug", 2024-09-14T03:47:49.489 INFO:tasks.workunit.client.0.smithi070.stdout: "ms_die_on_old_message", 2024-09-14T03:47:49.489 INFO:tasks.workunit.client.0.smithi070.stdout: "no_config_file", 2024-09-14T03:47:49.489 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_class_default_list", 2024-09-14T03:47:49.489 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_class_load_list", 2024-09-14T03:47:49.489 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_misdirected_ops", 2024-09-14T03:47:49.489 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_op_order", 2024-09-14T03:47:49.489 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_pg_log_writeout", 2024-09-14T03:47:49.490 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_shutdown", 2024-09-14T03:47:49.490 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_verify_cached_snaps", 2024-09-14T03:47:49.490 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_verify_missing_on_start", 2024-09-14T03:47:49.490 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_verify_stray_on_activate", 2024-09-14T03:47:49.490 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-14T03:47:49.490 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_profile", 2024-09-14T03:47:49.490 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_background_best_effort_lim", 2024-09-14T03:47:49.490 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_background_best_effort_res", 2024-09-14T03:47:49.490 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_background_best_effort_wgt", 2024-09-14T03:47:49.490 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_background_recovery_lim", 2024-09-14T03:47:49.491 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_background_recovery_res", 2024-09-14T03:47:49.491 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_background_recovery_wgt", 2024-09-14T03:47:49.491 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_client_lim", 2024-09-14T03:47:49.491 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_client_res", 2024-09-14T03:47:49.491 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_client_wgt", 2024-09-14T03:47:49.491 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_memory_target", 2024-09-14T03:47:49.491 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_memory_target_autotune", 2024-09-14T03:47:49.491 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_op_queue", 2024-09-14T03:47:49.491 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_op_queue_cut_off", 2024-09-14T03:47:49.491 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pool_default_erasure_code_profile", 2024-09-14T03:47:49.491 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-14T03:47:49.491 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_recover_clone_overlap", 2024-09-14T03:47:49.491 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_recovery_max_chunk", 2024-09-14T03:47:49.492 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_scrub_load_threshold", 2024-09-14T03:47:49.492 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_scrub_max_interval", 2024-09-14T03:47:49.492 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_shutdown_pgref_assert", 2024-09-14T03:47:49.492 INFO:tasks.workunit.client.0.smithi070.stdout: "public_addr", 2024-09-14T03:47:49.492 INFO:tasks.workunit.client.0.smithi070.stdout: "public_addrv", 2024-09-14T03:47:49.492 INFO:tasks.workunit.client.0.smithi070.stdout: "public_network", 2024-09-14T03:47:49.492 INFO:tasks.workunit.client.0.smithi070.stdout: "rbd_default_features", 2024-09-14T03:47:49.492 INFO:tasks.workunit.client.0.smithi070.stdout: "rbd_qos_exclude_ops", 2024-09-14T03:47:49.492 INFO:tasks.workunit.client.0.smithi070.stdout: "setgroup", 2024-09-14T03:47:49.492 INFO:tasks.workunit.client.0.smithi070.stdout: "setuser" 2024-09-14T03:47:49.492 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:49.492 INFO:tasks.workunit.client.0.smithi070.stdout: "cluster_changed": [ 2024-09-14T03:47:49.492 INFO:tasks.workunit.client.0.smithi070.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-14T03:47:49.492 INFO:tasks.workunit.client.0.smithi070.stdout: "auth_debug", 2024-09-14T03:47:49.493 INFO:tasks.workunit.client.0.smithi070.stdout: "auth_mon_ticket_ttl", 2024-09-14T03:47:49.493 INFO:tasks.workunit.client.0.smithi070.stdout: "auth_service_ticket_ttl", 2024-09-14T03:47:49.493 INFO:tasks.workunit.client.0.smithi070.stdout: "bdev_debug_aio", 2024-09-14T03:47:49.493 INFO:tasks.workunit.client.0.smithi070.stdout: "container_image", 2024-09-14T03:47:49.493 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_asserts_on_shutdown", 2024-09-14T03:47:49.493 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_mgr", 2024-09-14T03:47:49.493 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_mon", 2024-09-14T03:47:49.493 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_ms", 2024-09-14T03:47:49.493 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_osd", 2024-09-14T03:47:49.493 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_paxos", 2024-09-14T03:47:49.493 INFO:tasks.workunit.client.0.smithi070.stdout: "log_to_file", 2024-09-14T03:47:49.493 INFO:tasks.workunit.client.0.smithi070.stdout: "log_to_journald", 2024-09-14T03:47:49.494 INFO:tasks.workunit.client.0.smithi070.stdout: "log_to_stderr", 2024-09-14T03:47:49.494 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_join_fs", 2024-09-14T03:47:49.494 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/cephadm/allow_ptrace", 2024-09-14T03:47:49.494 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/cephadm/container_init", 2024-09-14T03:47:49.494 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/cephadm/migration_current", 2024-09-14T03:47:49.494 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/cephadm/use_repo_digest", 2024-09-14T03:47:49.494 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/dashboard/ALERTMANAGER_API_HOST", 2024-09-14T03:47:49.494 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/dashboard/GRAFANA_API_SSL_VERIFY", 2024-09-14T03:47:49.494 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/dashboard/GRAFANA_API_URL", 2024-09-14T03:47:49.494 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/dashboard/PROMETHEUS_API_HOST", 2024-09-14T03:47:49.495 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/dashboard/ssl_server_port", 2024-09-14T03:47:49.495 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/orchestrator/orchestrator", 2024-09-14T03:47:49.495 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/telemetry/channel_ident", 2024-09-14T03:47:49.495 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/telemetry/channel_perf", 2024-09-14T03:47:49.495 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/telemetry/description", 2024-09-14T03:47:49.495 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/telemetry/enabled", 2024-09-14T03:47:49.495 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/telemetry/organization", 2024-09-14T03:47:49.495 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_allow_pool_delete", 2024-09-14T03:47:49.495 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_clock_drift_allowed", 2024-09-14T03:47:49.495 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_cluster_log_to_file", 2024-09-14T03:47:49.495 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_data_avail_warn", 2024-09-14T03:47:49.496 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_election_default_strategy", 2024-09-14T03:47:49.496 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_max_pg_per_osd", 2024-09-14T03:47:49.496 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_mgr_mkfs_grace", 2024-09-14T03:47:49.496 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_osd_prime_pg_temp", 2024-09-14T03:47:49.496 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_osd_reporter_subtree_level", 2024-09-14T03:47:49.496 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_pg_warn_max_object_skew", 2024-09-14T03:47:49.496 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-14T03:47:49.496 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-14T03:47:49.496 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-14T03:47:49.496 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-14T03:47:49.497 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-14T03:47:49.497 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-14T03:47:49.497 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-14T03:47:49.497 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-14T03:47:49.497 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_too_few_osds", 2024-09-14T03:47:49.497 INFO:tasks.workunit.client.0.smithi070.stdout: "ms_die_on_bug", 2024-09-14T03:47:49.497 INFO:tasks.workunit.client.0.smithi070.stdout: "ms_die_on_old_message", 2024-09-14T03:47:49.497 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_class_default_list", 2024-09-14T03:47:49.497 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_class_load_list", 2024-09-14T03:47:49.497 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_misdirected_ops", 2024-09-14T03:47:49.497 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_op_order", 2024-09-14T03:47:49.497 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_pg_log_writeout", 2024-09-14T03:47:49.497 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_shutdown", 2024-09-14T03:47:49.498 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_verify_cached_snaps", 2024-09-14T03:47:49.498 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_verify_missing_on_start", 2024-09-14T03:47:49.498 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_verify_stray_on_activate", 2024-09-14T03:47:49.498 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-14T03:47:49.498 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_profile", 2024-09-14T03:47:49.498 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_memory_target", 2024-09-14T03:47:49.498 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_memory_target_autotune", 2024-09-14T03:47:49.498 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_op_queue", 2024-09-14T03:47:49.498 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_op_queue_cut_off", 2024-09-14T03:47:49.498 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pool_default_erasure_code_profile", 2024-09-14T03:47:49.498 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-14T03:47:49.498 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_recover_clone_overlap", 2024-09-14T03:47:49.498 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_recovery_max_chunk", 2024-09-14T03:47:49.499 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_scrub_load_threshold", 2024-09-14T03:47:49.499 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_scrub_max_interval", 2024-09-14T03:47:49.499 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_shutdown_pgref_assert", 2024-09-14T03:47:49.499 INFO:tasks.workunit.client.0.smithi070.stdout: "public_network", 2024-09-14T03:47:49.499 INFO:tasks.workunit.client.0.smithi070.stdout: "rgw_cache_enabled", 2024-09-14T03:47:49.499 INFO:tasks.workunit.client.0.smithi070.stdout: "rgw_enable_ops_log", 2024-09-14T03:47:49.499 INFO:tasks.workunit.client.0.smithi070.stdout: "rgw_enable_usage_log" 2024-09-14T03:47:49.499 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.499 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.499 INFO:tasks.workunit.client.0.smithi070.stdout: "contact": null, 2024-09-14T03:47:49.499 INFO:tasks.workunit.client.0.smithi070.stdout: "crashes": [], 2024-09-14T03:47:49.499 INFO:tasks.workunit.client.0.smithi070.stdout: "created": "2024-09-14T03:38:26.272229Z", 2024-09-14T03:47:49.499 INFO:tasks.workunit.client.0.smithi070.stdout: "crush": { 2024-09-14T03:47:49.500 INFO:tasks.workunit.client.0.smithi070.stdout: "bucket_algs": { 2024-09-14T03:47:49.500 INFO:tasks.workunit.client.0.smithi070.stdout: "straw2": 3 2024-09-14T03:47:49.500 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.500 INFO:tasks.workunit.client.0.smithi070.stdout: "bucket_sizes": { 2024-09-14T03:47:49.500 INFO:tasks.workunit.client.0.smithi070.stdout: "2": 1, 2024-09-14T03:47:49.500 INFO:tasks.workunit.client.0.smithi070.stdout: "4": 2 2024-09-14T03:47:49.500 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.500 INFO:tasks.workunit.client.0.smithi070.stdout: "bucket_types": { 2024-09-14T03:47:49.500 INFO:tasks.workunit.client.0.smithi070.stdout: "1": 2, 2024-09-14T03:47:49.500 INFO:tasks.workunit.client.0.smithi070.stdout: "11": 1 2024-09-14T03:47:49.500 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.500 INFO:tasks.workunit.client.0.smithi070.stdout: "compat_weight_set": false, 2024-09-14T03:47:49.501 INFO:tasks.workunit.client.0.smithi070.stdout: "device_classes": [ 2024-09-14T03:47:49.501 INFO:tasks.workunit.client.0.smithi070.stdout: 8 2024-09-14T03:47:49.501 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:49.501 INFO:tasks.workunit.client.0.smithi070.stdout: "num_buckets": 6, 2024-09-14T03:47:49.501 INFO:tasks.workunit.client.0.smithi070.stdout: "num_devices": 8, 2024-09-14T03:47:49.501 INFO:tasks.workunit.client.0.smithi070.stdout: "num_rules": 1, 2024-09-14T03:47:49.501 INFO:tasks.workunit.client.0.smithi070.stdout: "num_types": 12, 2024-09-14T03:47:49.501 INFO:tasks.workunit.client.0.smithi070.stdout: "num_weight_sets": 0, 2024-09-14T03:47:49.501 INFO:tasks.workunit.client.0.smithi070.stdout: "tunables": { 2024-09-14T03:47:49.501 INFO:tasks.workunit.client.0.smithi070.stdout: "allowed_bucket_algs": 54, 2024-09-14T03:47:49.501 INFO:tasks.workunit.client.0.smithi070.stdout: "choose_local_fallback_tries": 0, 2024-09-14T03:47:49.502 INFO:tasks.workunit.client.0.smithi070.stdout: "choose_local_tries": 0, 2024-09-14T03:47:49.502 INFO:tasks.workunit.client.0.smithi070.stdout: "choose_total_tries": 50, 2024-09-14T03:47:49.502 INFO:tasks.workunit.client.0.smithi070.stdout: "chooseleaf_descend_once": 1, 2024-09-14T03:47:49.502 INFO:tasks.workunit.client.0.smithi070.stdout: "chooseleaf_stable": 1, 2024-09-14T03:47:49.502 INFO:tasks.workunit.client.0.smithi070.stdout: "chooseleaf_vary_r": 1, 2024-09-14T03:47:49.502 INFO:tasks.workunit.client.0.smithi070.stdout: "has_v2_rules": 0, 2024-09-14T03:47:49.502 INFO:tasks.workunit.client.0.smithi070.stdout: "has_v3_rules": 0, 2024-09-14T03:47:49.502 INFO:tasks.workunit.client.0.smithi070.stdout: "has_v4_buckets": 1, 2024-09-14T03:47:49.502 INFO:tasks.workunit.client.0.smithi070.stdout: "has_v5_rules": 0, 2024-09-14T03:47:49.502 INFO:tasks.workunit.client.0.smithi070.stdout: "legacy_tunables": 0, 2024-09-14T03:47:49.502 INFO:tasks.workunit.client.0.smithi070.stdout: "minimum_required_version": "jewel", 2024-09-14T03:47:49.502 INFO:tasks.workunit.client.0.smithi070.stdout: "optimal_tunables": 1, 2024-09-14T03:47:49.502 INFO:tasks.workunit.client.0.smithi070.stdout: "profile": "jewel", 2024-09-14T03:47:49.503 INFO:tasks.workunit.client.0.smithi070.stdout: "require_feature_tunables": 1, 2024-09-14T03:47:49.503 INFO:tasks.workunit.client.0.smithi070.stdout: "require_feature_tunables2": 1, 2024-09-14T03:47:49.503 INFO:tasks.workunit.client.0.smithi070.stdout: "require_feature_tunables3": 1, 2024-09-14T03:47:49.503 INFO:tasks.workunit.client.0.smithi070.stdout: "require_feature_tunables5": 1, 2024-09-14T03:47:49.503 INFO:tasks.workunit.client.0.smithi070.stdout: "straw_calc_version": 1 2024-09-14T03:47:49.503 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.503 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.503 INFO:tasks.workunit.client.0.smithi070.stdout: "description": "upgrade test cluster", 2024-09-14T03:47:49.503 INFO:tasks.workunit.client.0.smithi070.stdout: "fs": { 2024-09-14T03:47:49.503 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:49.504 INFO:tasks.workunit.client.0.smithi070.stdout: "feature_flags": { 2024-09-14T03:47:49.504 INFO:tasks.workunit.client.0.smithi070.stdout: "enable_multiple": true, 2024-09-14T03:47:49.504 INFO:tasks.workunit.client.0.smithi070.stdout: "ever_enabled_multiple": true 2024-09-14T03:47:49.504 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.504 INFO:tasks.workunit.client.0.smithi070.stdout: "filesystems": [ 2024-09-14T03:47:49.504 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.504 INFO:tasks.workunit.client.0.smithi070.stdout: "approx_ctime": "2024-09", 2024-09-14T03:47:49.504 INFO:tasks.workunit.client.0.smithi070.stdout: "balancer_enabled": false, 2024-09-14T03:47:49.504 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.504 INFO:tasks.workunit.client.0.smithi070.stdout: "cached_caps": 0, 2024-09-14T03:47:49.504 INFO:tasks.workunit.client.0.smithi070.stdout: "cached_dns": 10, 2024-09-14T03:47:49.504 INFO:tasks.workunit.client.0.smithi070.stdout: "cached_inos": 13, 2024-09-14T03:47:49.504 INFO:tasks.workunit.client.0.smithi070.stdout: "cached_subtrees": 2, 2024-09-14T03:47:49.505 INFO:tasks.workunit.client.0.smithi070.stdout: "ever_allowed_features": 0, 2024-09-14T03:47:49.505 INFO:tasks.workunit.client.0.smithi070.stdout: "explicitly_allowed_features": 0, 2024-09-14T03:47:49.505 INFO:tasks.workunit.client.0.smithi070.stdout: "files": 0, 2024-09-14T03:47:49.505 INFO:tasks.workunit.client.0.smithi070.stdout: "max_mds": 1, 2024-09-14T03:47:49.505 INFO:tasks.workunit.client.0.smithi070.stdout: "num_data_pools": 1, 2024-09-14T03:47:49.505 INFO:tasks.workunit.client.0.smithi070.stdout: "num_in": 1, 2024-09-14T03:47:49.505 INFO:tasks.workunit.client.0.smithi070.stdout: "num_mds": 1, 2024-09-14T03:47:49.505 INFO:tasks.workunit.client.0.smithi070.stdout: "num_sessions": 0, 2024-09-14T03:47:49.505 INFO:tasks.workunit.client.0.smithi070.stdout: "num_standby_replay": 0, 2024-09-14T03:47:49.505 INFO:tasks.workunit.client.0.smithi070.stdout: "num_up": 1, 2024-09-14T03:47:49.505 INFO:tasks.workunit.client.0.smithi070.stdout: "snaps": 0, 2024-09-14T03:47:49.505 INFO:tasks.workunit.client.0.smithi070.stdout: "standby_count_wanted": 0 2024-09-14T03:47:49.505 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.506 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:49.506 INFO:tasks.workunit.client.0.smithi070.stdout: "num_standby_mds": 1, 2024-09-14T03:47:49.506 INFO:tasks.workunit.client.0.smithi070.stdout: "total_num_mds": 2 2024-09-14T03:47:49.506 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.506 INFO:tasks.workunit.client.0.smithi070.stdout: "heap_stats": { 2024-09-14T03:47:49.506 INFO:tasks.workunit.client.0.smithi070.stdout: "mds": { 2024-09-14T03:47:49.506 INFO:tasks.workunit.client.0.smithi070.stdout: "mds.a8e17c6c2bb7d35df7944161574528c366e6a388": { 2024-09-14T03:47:49.506 INFO:tasks.workunit.client.0.smithi070.stdout: "actual_memory_used": 20578304, 2024-09-14T03:47:49.506 INFO:tasks.workunit.client.0.smithi070.stdout: "central_cache_freelist": 389384, 2024-09-14T03:47:49.506 INFO:tasks.workunit.client.0.smithi070.stdout: "malloc_metadata": 2621440, 2024-09-14T03:47:49.506 INFO:tasks.workunit.client.0.smithi070.stdout: "page_heap_freelist": 0, 2024-09-14T03:47:49.506 INFO:tasks.workunit.client.0.smithi070.stdout: "released_to_os": 606208, 2024-09-14T03:47:49.506 INFO:tasks.workunit.client.0.smithi070.stdout: "spans_in_use": 432, 2024-09-14T03:47:49.507 INFO:tasks.workunit.client.0.smithi070.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:47:49.507 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_cache_freelists": 1291776, 2024-09-14T03:47:49.507 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_heaps_in_use": 20, 2024-09-14T03:47:49.507 INFO:tasks.workunit.client.0.smithi070.stdout: "transfer_cache_freelist": 1171968, 2024-09-14T03:47:49.507 INFO:tasks.workunit.client.0.smithi070.stdout: "use_by_application": 15103736, 2024-09-14T03:47:49.507 INFO:tasks.workunit.client.0.smithi070.stdout: "virtual_address_space_used": 21184512 2024-09-14T03:47:49.507 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.507 INFO:tasks.workunit.client.0.smithi070.stdout: "mds.b1d2edfb86a8f225d9283baf06ff7497eda48d2d": { 2024-09-14T03:47:49.507 INFO:tasks.workunit.client.0.smithi070.stdout: "actual_memory_used": 20135936, 2024-09-14T03:47:49.507 INFO:tasks.workunit.client.0.smithi070.stdout: "central_cache_freelist": 325496, 2024-09-14T03:47:49.507 INFO:tasks.workunit.client.0.smithi070.stdout: "malloc_metadata": 2621440, 2024-09-14T03:47:49.508 INFO:tasks.workunit.client.0.smithi070.stdout: "page_heap_freelist": 778240, 2024-09-14T03:47:49.508 INFO:tasks.workunit.client.0.smithi070.stdout: "released_to_os": 0, 2024-09-14T03:47:49.508 INFO:tasks.workunit.client.0.smithi070.stdout: "spans_in_use": 353, 2024-09-14T03:47:49.508 INFO:tasks.workunit.client.0.smithi070.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:47:49.508 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_cache_freelists": 815576, 2024-09-14T03:47:49.508 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_heaps_in_use": 12, 2024-09-14T03:47:49.508 INFO:tasks.workunit.client.0.smithi070.stdout: "transfer_cache_freelist": 910848, 2024-09-14T03:47:49.508 INFO:tasks.workunit.client.0.smithi070.stdout: "use_by_application": 14684336, 2024-09-14T03:47:49.508 INFO:tasks.workunit.client.0.smithi070.stdout: "virtual_address_space_used": 20135936 2024-09-14T03:47:49.509 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.509 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.509 INFO:tasks.workunit.client.0.smithi070.stdout: "mon": { 2024-09-14T03:47:49.509 INFO:tasks.workunit.client.0.smithi070.stdout: "mon.a8e17c6c2bb7d35df7944161574528c366e6a388": { 2024-09-14T03:47:49.509 INFO:tasks.workunit.client.0.smithi070.stdout: "actual_memory_used": 46120960, 2024-09-14T03:47:49.509 INFO:tasks.workunit.client.0.smithi070.stdout: "central_cache_freelist": 1385880, 2024-09-14T03:47:49.509 INFO:tasks.workunit.client.0.smithi070.stdout: "malloc_metadata": 2621440, 2024-09-14T03:47:49.509 INFO:tasks.workunit.client.0.smithi070.stdout: "page_heap_freelist": 630784, 2024-09-14T03:47:49.509 INFO:tasks.workunit.client.0.smithi070.stdout: "released_to_os": 6537216, 2024-09-14T03:47:49.509 INFO:tasks.workunit.client.0.smithi070.stdout: "spans_in_use": 1884, 2024-09-14T03:47:49.509 INFO:tasks.workunit.client.0.smithi070.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:47:49.509 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_cache_freelists": 11019696, 2024-09-14T03:47:49.510 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_heaps_in_use": 23, 2024-09-14T03:47:49.510 INFO:tasks.workunit.client.0.smithi070.stdout: "transfer_cache_freelist": 2778112, 2024-09-14T03:47:49.510 INFO:tasks.workunit.client.0.smithi070.stdout: "use_by_application": 27685048, 2024-09-14T03:47:49.510 INFO:tasks.workunit.client.0.smithi070.stdout: "virtual_address_space_used": 52658176 2024-09-14T03:47:49.510 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.510 INFO:tasks.workunit.client.0.smithi070.stdout: "mon.b1d2edfb86a8f225d9283baf06ff7497eda48d2d": { 2024-09-14T03:47:49.510 INFO:tasks.workunit.client.0.smithi070.stdout: "actual_memory_used": 36216832, 2024-09-14T03:47:49.510 INFO:tasks.workunit.client.0.smithi070.stdout: "central_cache_freelist": 1019712, 2024-09-14T03:47:49.510 INFO:tasks.workunit.client.0.smithi070.stdout: "malloc_metadata": 2621440, 2024-09-14T03:47:49.510 INFO:tasks.workunit.client.0.smithi070.stdout: "page_heap_freelist": 16384, 2024-09-14T03:47:49.510 INFO:tasks.workunit.client.0.smithi070.stdout: "released_to_os": 15482880, 2024-09-14T03:47:49.510 INFO:tasks.workunit.client.0.smithi070.stdout: "spans_in_use": 1139, 2024-09-14T03:47:49.510 INFO:tasks.workunit.client.0.smithi070.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:47:49.511 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_cache_freelists": 7025256, 2024-09-14T03:47:49.511 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_heaps_in_use": 21, 2024-09-14T03:47:49.511 INFO:tasks.workunit.client.0.smithi070.stdout: "transfer_cache_freelist": 1793024, 2024-09-14T03:47:49.511 INFO:tasks.workunit.client.0.smithi070.stdout: "use_by_application": 23741016, 2024-09-14T03:47:49.511 INFO:tasks.workunit.client.0.smithi070.stdout: "virtual_address_space_used": 51699712 2024-09-14T03:47:49.511 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.511 INFO:tasks.workunit.client.0.smithi070.stdout: "mon.bb10d63b3f8e26c4a0c5da991c12ff03df4a20b0": { 2024-09-14T03:47:49.511 INFO:tasks.workunit.client.0.smithi070.stdout: "actual_memory_used": 37101568, 2024-09-14T03:47:49.511 INFO:tasks.workunit.client.0.smithi070.stdout: "central_cache_freelist": 1256280, 2024-09-14T03:47:49.511 INFO:tasks.workunit.client.0.smithi070.stdout: "malloc_metadata": 2621440, 2024-09-14T03:47:49.511 INFO:tasks.workunit.client.0.smithi070.stdout: "page_heap_freelist": 540672, 2024-09-14T03:47:49.511 INFO:tasks.workunit.client.0.smithi070.stdout: "released_to_os": 13549568, 2024-09-14T03:47:49.512 INFO:tasks.workunit.client.0.smithi070.stdout: "spans_in_use": 1047, 2024-09-14T03:47:49.512 INFO:tasks.workunit.client.0.smithi070.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:47:49.512 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_cache_freelists": 7127792, 2024-09-14T03:47:49.512 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_heaps_in_use": 21, 2024-09-14T03:47:49.512 INFO:tasks.workunit.client.0.smithi070.stdout: "transfer_cache_freelist": 1832704, 2024-09-14T03:47:49.512 INFO:tasks.workunit.client.0.smithi070.stdout: "use_by_application": 23722680, 2024-09-14T03:47:49.512 INFO:tasks.workunit.client.0.smithi070.stdout: "virtual_address_space_used": 50651136 2024-09-14T03:47:49.512 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.512 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.512 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:49.512 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.0": { 2024-09-14T03:47:49.512 INFO:tasks.workunit.client.0.smithi070.stdout: "actual_memory_used": 273850368, 2024-09-14T03:47:49.512 INFO:tasks.workunit.client.0.smithi070.stdout: "central_cache_freelist": 795032, 2024-09-14T03:47:49.513 INFO:tasks.workunit.client.0.smithi070.stdout: "malloc_metadata": 2752512, 2024-09-14T03:47:49.513 INFO:tasks.workunit.client.0.smithi070.stdout: "page_heap_freelist": 0, 2024-09-14T03:47:49.513 INFO:tasks.workunit.client.0.smithi070.stdout: "released_to_os": 925696, 2024-09-14T03:47:49.513 INFO:tasks.workunit.client.0.smithi070.stdout: "spans_in_use": 1378, 2024-09-14T03:47:49.513 INFO:tasks.workunit.client.0.smithi070.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:47:49.513 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_cache_freelists": 5256720, 2024-09-14T03:47:49.513 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_heaps_in_use": 45, 2024-09-14T03:47:49.513 INFO:tasks.workunit.client.0.smithi070.stdout: "transfer_cache_freelist": 2561024, 2024-09-14T03:47:49.513 INFO:tasks.workunit.client.0.smithi070.stdout: "use_by_application": 262485080, 2024-09-14T03:47:49.513 INFO:tasks.workunit.client.0.smithi070.stdout: "virtual_address_space_used": 274776064 2024-09-14T03:47:49.513 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.513 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.1": { 2024-09-14T03:47:49.513 INFO:tasks.workunit.client.0.smithi070.stdout: "actual_memory_used": 273268736, 2024-09-14T03:47:49.514 INFO:tasks.workunit.client.0.smithi070.stdout: "central_cache_freelist": 748944, 2024-09-14T03:47:49.514 INFO:tasks.workunit.client.0.smithi070.stdout: "malloc_metadata": 2752512, 2024-09-14T03:47:49.514 INFO:tasks.workunit.client.0.smithi070.stdout: "page_heap_freelist": 0, 2024-09-14T03:47:49.514 INFO:tasks.workunit.client.0.smithi070.stdout: "released_to_os": 393216, 2024-09-14T03:47:49.514 INFO:tasks.workunit.client.0.smithi070.stdout: "spans_in_use": 1343, 2024-09-14T03:47:49.514 INFO:tasks.workunit.client.0.smithi070.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:47:49.514 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_cache_freelists": 5430904, 2024-09-14T03:47:49.514 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_heaps_in_use": 46, 2024-09-14T03:47:49.514 INFO:tasks.workunit.client.0.smithi070.stdout: "transfer_cache_freelist": 2209280, 2024-09-14T03:47:49.514 INFO:tasks.workunit.client.0.smithi070.stdout: "use_by_application": 262127096, 2024-09-14T03:47:49.514 INFO:tasks.workunit.client.0.smithi070.stdout: "virtual_address_space_used": 273661952 2024-09-14T03:47:49.514 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.514 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.2": { 2024-09-14T03:47:49.514 INFO:tasks.workunit.client.0.smithi070.stdout: "actual_memory_used": 272015360, 2024-09-14T03:47:49.515 INFO:tasks.workunit.client.0.smithi070.stdout: "central_cache_freelist": 682600, 2024-09-14T03:47:49.515 INFO:tasks.workunit.client.0.smithi070.stdout: "malloc_metadata": 2752512, 2024-09-14T03:47:49.515 INFO:tasks.workunit.client.0.smithi070.stdout: "page_heap_freelist": 0, 2024-09-14T03:47:49.515 INFO:tasks.workunit.client.0.smithi070.stdout: "released_to_os": 598016, 2024-09-14T03:47:49.515 INFO:tasks.workunit.client.0.smithi070.stdout: "spans_in_use": 1272, 2024-09-14T03:47:49.515 INFO:tasks.workunit.client.0.smithi070.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:47:49.515 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_cache_freelists": 5093712, 2024-09-14T03:47:49.515 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_heaps_in_use": 47, 2024-09-14T03:47:49.515 INFO:tasks.workunit.client.0.smithi070.stdout: "transfer_cache_freelist": 2334976, 2024-09-14T03:47:49.515 INFO:tasks.workunit.client.0.smithi070.stdout: "use_by_application": 261151560, 2024-09-14T03:47:49.515 INFO:tasks.workunit.client.0.smithi070.stdout: "virtual_address_space_used": 272613376 2024-09-14T03:47:49.515 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.515 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.3": { 2024-09-14T03:47:49.516 INFO:tasks.workunit.client.0.smithi070.stdout: "actual_memory_used": 273817600, 2024-09-14T03:47:49.516 INFO:tasks.workunit.client.0.smithi070.stdout: "central_cache_freelist": 627232, 2024-09-14T03:47:49.516 INFO:tasks.workunit.client.0.smithi070.stdout: "malloc_metadata": 4849664, 2024-09-14T03:47:49.516 INFO:tasks.workunit.client.0.smithi070.stdout: "page_heap_freelist": 0, 2024-09-14T03:47:49.516 INFO:tasks.workunit.client.0.smithi070.stdout: "released_to_os": 958464, 2024-09-14T03:47:49.516 INFO:tasks.workunit.client.0.smithi070.stdout: "spans_in_use": 1247, 2024-09-14T03:47:49.516 INFO:tasks.workunit.client.0.smithi070.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:47:49.516 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_cache_freelists": 4677336, 2024-09-14T03:47:49.516 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_heaps_in_use": 46, 2024-09-14T03:47:49.516 INFO:tasks.workunit.client.0.smithi070.stdout: "transfer_cache_freelist": 1850624, 2024-09-14T03:47:49.516 INFO:tasks.workunit.client.0.smithi070.stdout: "use_by_application": 261812744, 2024-09-14T03:47:49.516 INFO:tasks.workunit.client.0.smithi070.stdout: "virtual_address_space_used": 274776064 2024-09-14T03:47:49.516 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.517 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.4": { 2024-09-14T03:47:49.517 INFO:tasks.workunit.client.0.smithi070.stdout: "actual_memory_used": 272695296, 2024-09-14T03:47:49.517 INFO:tasks.workunit.client.0.smithi070.stdout: "central_cache_freelist": 642432, 2024-09-14T03:47:49.517 INFO:tasks.workunit.client.0.smithi070.stdout: "malloc_metadata": 2752512, 2024-09-14T03:47:49.517 INFO:tasks.workunit.client.0.smithi070.stdout: "page_heap_freelist": 24576, 2024-09-14T03:47:49.517 INFO:tasks.workunit.client.0.smithi070.stdout: "released_to_os": 1032192, 2024-09-14T03:47:49.517 INFO:tasks.workunit.client.0.smithi070.stdout: "spans_in_use": 1260, 2024-09-14T03:47:49.517 INFO:tasks.workunit.client.0.smithi070.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:47:49.517 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_cache_freelists": 4793200, 2024-09-14T03:47:49.517 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_heaps_in_use": 47, 2024-09-14T03:47:49.517 INFO:tasks.workunit.client.0.smithi070.stdout: "transfer_cache_freelist": 2141952, 2024-09-14T03:47:49.517 INFO:tasks.workunit.client.0.smithi070.stdout: "use_by_application": 262340624, 2024-09-14T03:47:49.517 INFO:tasks.workunit.client.0.smithi070.stdout: "virtual_address_space_used": 273727488 2024-09-14T03:47:49.517 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.518 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.5": { 2024-09-14T03:47:49.518 INFO:tasks.workunit.client.0.smithi070.stdout: "actual_memory_used": 272465920, 2024-09-14T03:47:49.518 INFO:tasks.workunit.client.0.smithi070.stdout: "central_cache_freelist": 664424, 2024-09-14T03:47:49.518 INFO:tasks.workunit.client.0.smithi070.stdout: "malloc_metadata": 2752512, 2024-09-14T03:47:49.518 INFO:tasks.workunit.client.0.smithi070.stdout: "page_heap_freelist": 24576, 2024-09-14T03:47:49.518 INFO:tasks.workunit.client.0.smithi070.stdout: "released_to_os": 212992, 2024-09-14T03:47:49.518 INFO:tasks.workunit.client.0.smithi070.stdout: "spans_in_use": 1257, 2024-09-14T03:47:49.518 INFO:tasks.workunit.client.0.smithi070.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:47:49.518 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_cache_freelists": 4617408, 2024-09-14T03:47:49.518 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_heaps_in_use": 46, 2024-09-14T03:47:49.518 INFO:tasks.workunit.client.0.smithi070.stdout: "transfer_cache_freelist": 2065152, 2024-09-14T03:47:49.518 INFO:tasks.workunit.client.0.smithi070.stdout: "use_by_application": 262341848, 2024-09-14T03:47:49.518 INFO:tasks.workunit.client.0.smithi070.stdout: "virtual_address_space_used": 272678912 2024-09-14T03:47:49.519 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.519 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.6": { 2024-09-14T03:47:49.519 INFO:tasks.workunit.client.0.smithi070.stdout: "actual_memory_used": 272310272, 2024-09-14T03:47:49.519 INFO:tasks.workunit.client.0.smithi070.stdout: "central_cache_freelist": 1186184, 2024-09-14T03:47:49.519 INFO:tasks.workunit.client.0.smithi070.stdout: "malloc_metadata": 2752512, 2024-09-14T03:47:49.519 INFO:tasks.workunit.client.0.smithi070.stdout: "page_heap_freelist": 24576, 2024-09-14T03:47:49.519 INFO:tasks.workunit.client.0.smithi070.stdout: "released_to_os": 1417216, 2024-09-14T03:47:49.519 INFO:tasks.workunit.client.0.smithi070.stdout: "spans_in_use": 1267, 2024-09-14T03:47:49.519 INFO:tasks.workunit.client.0.smithi070.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:47:49.519 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_cache_freelists": 4638304, 2024-09-14T03:47:49.519 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_heaps_in_use": 45, 2024-09-14T03:47:49.519 INFO:tasks.workunit.client.0.smithi070.stdout: "transfer_cache_freelist": 2081536, 2024-09-14T03:47:49.519 INFO:tasks.workunit.client.0.smithi070.stdout: "use_by_application": 261627160, 2024-09-14T03:47:49.520 INFO:tasks.workunit.client.0.smithi070.stdout: "virtual_address_space_used": 273727488 2024-09-14T03:47:49.520 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.520 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.7": { 2024-09-14T03:47:49.520 INFO:tasks.workunit.client.0.smithi070.stdout: "actual_memory_used": 273072128, 2024-09-14T03:47:49.520 INFO:tasks.workunit.client.0.smithi070.stdout: "central_cache_freelist": 1241784, 2024-09-14T03:47:49.520 INFO:tasks.workunit.client.0.smithi070.stdout: "malloc_metadata": 2752512, 2024-09-14T03:47:49.520 INFO:tasks.workunit.client.0.smithi070.stdout: "page_heap_freelist": 0, 2024-09-14T03:47:49.520 INFO:tasks.workunit.client.0.smithi070.stdout: "released_to_os": 1638400, 2024-09-14T03:47:49.520 INFO:tasks.workunit.client.0.smithi070.stdout: "spans_in_use": 1334, 2024-09-14T03:47:49.520 INFO:tasks.workunit.client.0.smithi070.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:47:49.520 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_cache_freelists": 4845096, 2024-09-14T03:47:49.520 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_heaps_in_use": 45, 2024-09-14T03:47:49.520 INFO:tasks.workunit.client.0.smithi070.stdout: "transfer_cache_freelist": 2115328, 2024-09-14T03:47:49.521 INFO:tasks.workunit.client.0.smithi070.stdout: "use_by_application": 262117408, 2024-09-14T03:47:49.521 INFO:tasks.workunit.client.0.smithi070.stdout: "virtual_address_space_used": 274710528 2024-09-14T03:47:49.521 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.521 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.521 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.521 INFO:tasks.workunit.client.0.smithi070.stdout: "hosts": { 2024-09-14T03:47:49.521 INFO:tasks.workunit.client.0.smithi070.stdout: "num": 2, 2024-09-14T03:47:49.521 INFO:tasks.workunit.client.0.smithi070.stdout: "num_with_mds": 2, 2024-09-14T03:47:49.521 INFO:tasks.workunit.client.0.smithi070.stdout: "num_with_mgr": 2, 2024-09-14T03:47:49.521 INFO:tasks.workunit.client.0.smithi070.stdout: "num_with_mon": 2, 2024-09-14T03:47:49.521 INFO:tasks.workunit.client.0.smithi070.stdout: "num_with_osd": 2 2024-09-14T03:47:49.521 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.521 INFO:tasks.workunit.client.0.smithi070.stdout: "io_rate": { 2024-09-14T03:47:49.522 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_stats_delta": { 2024-09-14T03:47:49.522 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": 0, 2024-09-14T03:47:49.522 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:49.522 INFO:tasks.workunit.client.0.smithi070.stdout: "num_store_stats": 0, 2024-09-14T03:47:49.522 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:49.522 INFO:tasks.workunit.client.0.smithi070.stdout: "stamp_delta": "12.002341", 2024-09-14T03:47:49.522 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:49.522 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:49.522 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:49.522 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:49.522 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:49.522 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:49.522 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:49.523 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:49.523 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:49.523 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:49.523 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:49.523 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:49.523 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:49.523 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:49.523 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:49.523 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:49.523 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:49.523 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:49.523 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:49.523 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:49.523 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:49.523 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:49.524 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:49.524 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:49.524 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:49.524 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:49.524 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:49.524 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:49.524 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:49.524 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:49.524 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:49.524 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:49.524 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:49.524 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:49.524 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 146, 2024-09-14T03:47:49.525 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 248, 2024-09-14T03:47:49.525 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:49.525 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:49.525 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:49.525 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 32, 2024-09-14T03:47:49.525 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:49.525 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.525 INFO:tasks.workunit.client.0.smithi070.stdout: "store_stats": { 2024-09-14T03:47:49.525 INFO:tasks.workunit.client.0.smithi070.stdout: "allocated": 0, 2024-09-14T03:47:49.525 INFO:tasks.workunit.client.0.smithi070.stdout: "available": 0, 2024-09-14T03:47:49.525 INFO:tasks.workunit.client.0.smithi070.stdout: "data_compressed": 0, 2024-09-14T03:47:49.525 INFO:tasks.workunit.client.0.smithi070.stdout: "data_compressed_allocated": 0, 2024-09-14T03:47:49.525 INFO:tasks.workunit.client.0.smithi070.stdout: "data_compressed_original": 0, 2024-09-14T03:47:49.526 INFO:tasks.workunit.client.0.smithi070.stdout: "data_stored": 0, 2024-09-14T03:47:49.526 INFO:tasks.workunit.client.0.smithi070.stdout: "internal_metadata": 0, 2024-09-14T03:47:49.526 INFO:tasks.workunit.client.0.smithi070.stdout: "internally_reserved": 0, 2024-09-14T03:47:49.526 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_allocated": 0, 2024-09-14T03:47:49.526 INFO:tasks.workunit.client.0.smithi070.stdout: "total": 0 2024-09-14T03:47:49.526 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.526 INFO:tasks.workunit.client.0.smithi070.stdout: "up": 0 2024-09-14T03:47:49.526 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.526 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.526 INFO:tasks.workunit.client.0.smithi070.stdout: "leaderboard": false, 2024-09-14T03:47:49.526 INFO:tasks.workunit.client.0.smithi070.stdout: "license": "sharing-1-0", 2024-09-14T03:47:49.526 INFO:tasks.workunit.client.0.smithi070.stdout: "mempool": { 2024-09-14T03:47:49.526 INFO:tasks.workunit.client.0.smithi070.stdout: "mds": { 2024-09-14T03:47:49.526 INFO:tasks.workunit.client.0.smithi070.stdout: "mds.a8e17c6c2bb7d35df7944161574528c366e6a388": { 2024-09-14T03:47:49.527 INFO:tasks.workunit.client.0.smithi070.stdout: "bloom_filter": { 2024-09-14T03:47:49.527 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.527 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.527 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.527 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:49.527 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.527 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.527 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.527 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_reader": { 2024-09-14T03:47:49.527 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.527 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.527 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.527 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_writer": { 2024-09-14T03:47:49.528 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.528 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.528 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.528 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Blob": { 2024-09-14T03:47:49.528 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.528 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.528 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.528 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Buffer": { 2024-09-14T03:47:49.528 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.528 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.528 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.528 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Extent": { 2024-09-14T03:47:49.528 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.529 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.529 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.529 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_SharedBlob": { 2024-09-14T03:47:49.529 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.529 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.529 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.529 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_alloc": { 2024-09-14T03:47:49.529 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.529 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.529 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.529 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_data": { 2024-09-14T03:47:49.529 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.529 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.529 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.530 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_meta": { 2024-09-14T03:47:49.530 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.530 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.530 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.530 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_onode": { 2024-09-14T03:47:49.530 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.530 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.530 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.530 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_other": { 2024-09-14T03:47:49.530 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.530 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.530 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.530 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_fsck": { 2024-09-14T03:47:49.531 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.531 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.531 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.531 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_inline_bl": { 2024-09-14T03:47:49.531 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.531 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.531 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.531 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_txc": { 2024-09-14T03:47:49.531 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.531 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.531 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.532 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing": { 2024-09-14T03:47:49.532 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.532 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.532 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.532 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing_deferred": { 2024-09-14T03:47:49.532 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.532 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.532 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.532 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_anon": { 2024-09-14T03:47:49.532 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 222471, 2024-09-14T03:47:49.532 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 72 2024-09-14T03:47:49.532 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.532 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_meta": { 2024-09-14T03:47:49.533 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.533 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.533 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.533 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_co": { 2024-09-14T03:47:49.533 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 50488, 2024-09-14T03:47:49.533 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 208 2024-09-14T03:47:49.533 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.533 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:49.533 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.533 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.533 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.533 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mapbl": { 2024-09-14T03:47:49.533 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.534 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.534 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.534 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pglog": { 2024-09-14T03:47:49.534 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.534 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.534 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.534 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap": { 2024-09-14T03:47:49.534 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 8048, 2024-09-14T03:47:49.534 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 138 2024-09-14T03:47:49.534 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.534 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap_mapping": { 2024-09-14T03:47:49.534 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.534 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.534 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.535 INFO:tasks.workunit.client.0.smithi070.stdout: "pgmap": { 2024-09-14T03:47:49.535 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.535 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.535 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.535 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_1": { 2024-09-14T03:47:49.535 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.535 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.535 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.535 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_2": { 2024-09-14T03:47:49.535 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.535 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.535 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.535 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.536 INFO:tasks.workunit.client.0.smithi070.stdout: "mds.b1d2edfb86a8f225d9283baf06ff7497eda48d2d": { 2024-09-14T03:47:49.536 INFO:tasks.workunit.client.0.smithi070.stdout: "bloom_filter": { 2024-09-14T03:47:49.536 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.536 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.536 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.536 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:49.536 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.536 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.536 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.536 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_reader": { 2024-09-14T03:47:49.536 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.536 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.536 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.537 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_writer": { 2024-09-14T03:47:49.537 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.537 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.537 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.537 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Blob": { 2024-09-14T03:47:49.537 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.537 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.537 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.537 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Buffer": { 2024-09-14T03:47:49.537 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.537 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.538 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.538 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Extent": { 2024-09-14T03:47:49.538 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.538 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.538 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.538 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_SharedBlob": { 2024-09-14T03:47:49.538 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.538 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.538 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.538 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_alloc": { 2024-09-14T03:47:49.538 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.538 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.538 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.538 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_data": { 2024-09-14T03:47:49.539 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.539 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.539 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.539 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_meta": { 2024-09-14T03:47:49.539 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.539 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.539 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.539 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_onode": { 2024-09-14T03:47:49.539 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.539 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.539 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.539 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_other": { 2024-09-14T03:47:49.539 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.540 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.540 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.540 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_fsck": { 2024-09-14T03:47:49.540 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.540 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.540 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.540 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_inline_bl": { 2024-09-14T03:47:49.540 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.540 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.540 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.540 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_txc": { 2024-09-14T03:47:49.540 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.540 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.541 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.541 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing": { 2024-09-14T03:47:49.541 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.541 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.541 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.541 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing_deferred": { 2024-09-14T03:47:49.541 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.541 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.541 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.541 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_anon": { 2024-09-14T03:47:49.541 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 149057, 2024-09-14T03:47:49.541 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 32 2024-09-14T03:47:49.541 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.541 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_meta": { 2024-09-14T03:47:49.542 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.542 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.542 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.542 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_co": { 2024-09-14T03:47:49.542 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 856, 2024-09-14T03:47:49.542 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 2 2024-09-14T03:47:49.542 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.542 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:49.542 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.542 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.542 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.542 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mapbl": { 2024-09-14T03:47:49.542 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.542 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.543 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.543 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pglog": { 2024-09-14T03:47:49.543 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.543 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.543 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.543 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap": { 2024-09-14T03:47:49.543 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.543 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.543 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.543 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap_mapping": { 2024-09-14T03:47:49.543 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.543 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.543 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.543 INFO:tasks.workunit.client.0.smithi070.stdout: "pgmap": { 2024-09-14T03:47:49.544 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.544 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.544 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.544 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_1": { 2024-09-14T03:47:49.544 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.544 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.544 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.544 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_2": { 2024-09-14T03:47:49.544 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.544 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.544 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.544 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.544 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.544 INFO:tasks.workunit.client.0.smithi070.stdout: "mon": { 2024-09-14T03:47:49.545 INFO:tasks.workunit.client.0.smithi070.stdout: "mon.a8e17c6c2bb7d35df7944161574528c366e6a388": { 2024-09-14T03:47:49.545 INFO:tasks.workunit.client.0.smithi070.stdout: "bloom_filter": { 2024-09-14T03:47:49.545 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.545 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.545 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.545 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:49.545 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.545 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.545 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.545 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_reader": { 2024-09-14T03:47:49.545 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.545 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.545 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.546 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_writer": { 2024-09-14T03:47:49.546 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.546 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.546 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.546 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Blob": { 2024-09-14T03:47:49.546 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.546 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.546 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.546 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Buffer": { 2024-09-14T03:47:49.546 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.546 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.546 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.546 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Extent": { 2024-09-14T03:47:49.546 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.547 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.547 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.547 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_SharedBlob": { 2024-09-14T03:47:49.547 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.547 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.547 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.547 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_alloc": { 2024-09-14T03:47:49.547 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.547 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.547 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.547 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_data": { 2024-09-14T03:47:49.547 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.547 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.548 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.548 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_meta": { 2024-09-14T03:47:49.548 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.548 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.548 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.548 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_onode": { 2024-09-14T03:47:49.548 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.548 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.548 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.548 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_other": { 2024-09-14T03:47:49.548 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.548 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.548 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.548 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_fsck": { 2024-09-14T03:47:49.549 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.549 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.549 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.549 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_inline_bl": { 2024-09-14T03:47:49.549 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.549 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.549 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.549 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_txc": { 2024-09-14T03:47:49.549 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.549 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.549 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.550 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing": { 2024-09-14T03:47:49.550 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.550 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.550 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.550 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing_deferred": { 2024-09-14T03:47:49.550 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.550 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.550 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.550 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_anon": { 2024-09-14T03:47:49.550 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 1129302, 2024-09-14T03:47:49.550 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 330 2024-09-14T03:47:49.550 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.550 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_meta": { 2024-09-14T03:47:49.550 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 440, 2024-09-14T03:47:49.551 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 5 2024-09-14T03:47:49.551 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.551 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_co": { 2024-09-14T03:47:49.551 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.551 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.551 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.551 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:49.551 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.551 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.551 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.551 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mapbl": { 2024-09-14T03:47:49.551 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.552 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.552 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.552 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pglog": { 2024-09-14T03:47:49.552 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.552 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.552 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.552 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap": { 2024-09-14T03:47:49.552 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 6696, 2024-09-14T03:47:49.552 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 137 2024-09-14T03:47:49.552 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.552 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap_mapping": { 2024-09-14T03:47:49.552 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 12744, 2024-09-14T03:47:49.552 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 1546 2024-09-14T03:47:49.552 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.553 INFO:tasks.workunit.client.0.smithi070.stdout: "pgmap": { 2024-09-14T03:47:49.553 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 10624, 2024-09-14T03:47:49.553 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 206 2024-09-14T03:47:49.553 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.553 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_1": { 2024-09-14T03:47:49.553 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.553 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.553 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.553 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_2": { 2024-09-14T03:47:49.553 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.553 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.553 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.553 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.554 INFO:tasks.workunit.client.0.smithi070.stdout: "mon.b1d2edfb86a8f225d9283baf06ff7497eda48d2d": { 2024-09-14T03:47:49.554 INFO:tasks.workunit.client.0.smithi070.stdout: "bloom_filter": { 2024-09-14T03:47:49.554 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.554 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.554 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.554 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:49.554 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.554 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.554 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.554 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_reader": { 2024-09-14T03:47:49.554 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.554 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.554 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.555 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_writer": { 2024-09-14T03:47:49.555 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.555 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.555 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.555 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Blob": { 2024-09-14T03:47:49.555 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.555 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.555 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.555 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Buffer": { 2024-09-14T03:47:49.555 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.555 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.555 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.555 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Extent": { 2024-09-14T03:47:49.555 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.556 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.556 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.556 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_SharedBlob": { 2024-09-14T03:47:49.556 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.556 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.556 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.556 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_alloc": { 2024-09-14T03:47:49.556 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.556 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.556 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.556 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_data": { 2024-09-14T03:47:49.556 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.557 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.557 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.557 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_meta": { 2024-09-14T03:47:49.557 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.557 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.557 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.557 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_onode": { 2024-09-14T03:47:49.557 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.557 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.557 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.557 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_other": { 2024-09-14T03:47:49.557 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.557 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.558 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.558 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_fsck": { 2024-09-14T03:47:49.558 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.558 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.558 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.558 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_inline_bl": { 2024-09-14T03:47:49.558 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.558 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.558 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.558 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_txc": { 2024-09-14T03:47:49.558 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.558 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.559 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.559 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing": { 2024-09-14T03:47:49.559 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.559 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.559 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.559 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing_deferred": { 2024-09-14T03:47:49.559 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.559 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.559 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.560 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_anon": { 2024-09-14T03:47:49.560 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 1099847, 2024-09-14T03:47:49.560 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 348 2024-09-14T03:47:49.560 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.560 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_meta": { 2024-09-14T03:47:49.560 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 88, 2024-09-14T03:47:49.560 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 1 2024-09-14T03:47:49.560 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.560 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_co": { 2024-09-14T03:47:49.560 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.560 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.561 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.561 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:49.561 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.561 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.561 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.561 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mapbl": { 2024-09-14T03:47:49.561 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.561 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.561 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.561 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pglog": { 2024-09-14T03:47:49.561 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.561 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.561 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.562 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap": { 2024-09-14T03:47:49.562 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 6696, 2024-09-14T03:47:49.562 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 137 2024-09-14T03:47:49.562 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.562 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap_mapping": { 2024-09-14T03:47:49.562 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 12744, 2024-09-14T03:47:49.562 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 1546 2024-09-14T03:47:49.562 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.562 INFO:tasks.workunit.client.0.smithi070.stdout: "pgmap": { 2024-09-14T03:47:49.562 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 5312, 2024-09-14T03:47:49.562 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 103 2024-09-14T03:47:49.563 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.563 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_1": { 2024-09-14T03:47:49.563 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.563 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.563 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.563 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_2": { 2024-09-14T03:47:49.563 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.563 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.563 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.563 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.563 INFO:tasks.workunit.client.0.smithi070.stdout: "mon.bb10d63b3f8e26c4a0c5da991c12ff03df4a20b0": { 2024-09-14T03:47:49.563 INFO:tasks.workunit.client.0.smithi070.stdout: "bloom_filter": { 2024-09-14T03:47:49.564 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.564 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.564 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.564 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:49.564 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.564 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.564 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.564 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_reader": { 2024-09-14T03:47:49.564 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.564 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.564 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.564 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_writer": { 2024-09-14T03:47:49.565 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.565 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.565 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.565 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Blob": { 2024-09-14T03:47:49.565 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.565 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.565 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.565 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Buffer": { 2024-09-14T03:47:49.565 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.565 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.566 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.566 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Extent": { 2024-09-14T03:47:49.566 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.566 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.566 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.566 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_SharedBlob": { 2024-09-14T03:47:49.566 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.566 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.566 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.566 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_alloc": { 2024-09-14T03:47:49.566 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.566 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.566 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.566 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_data": { 2024-09-14T03:47:49.567 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.567 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.567 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.567 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_meta": { 2024-09-14T03:47:49.567 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.567 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.567 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.567 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_onode": { 2024-09-14T03:47:49.567 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.567 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.567 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.567 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_other": { 2024-09-14T03:47:49.567 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.567 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.568 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.568 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_fsck": { 2024-09-14T03:47:49.568 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.568 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.568 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.568 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_inline_bl": { 2024-09-14T03:47:49.568 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.568 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.568 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.568 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_txc": { 2024-09-14T03:47:49.568 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.568 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.568 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.569 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing": { 2024-09-14T03:47:49.569 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.569 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.569 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.569 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing_deferred": { 2024-09-14T03:47:49.569 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.569 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.569 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.569 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_anon": { 2024-09-14T03:47:49.569 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 1100086, 2024-09-14T03:47:49.570 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 356 2024-09-14T03:47:49.570 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.570 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_meta": { 2024-09-14T03:47:49.570 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.570 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.570 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.570 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_co": { 2024-09-14T03:47:49.570 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.570 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.570 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.570 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:49.570 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.570 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.571 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.571 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mapbl": { 2024-09-14T03:47:49.571 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.571 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.571 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.571 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pglog": { 2024-09-14T03:47:49.571 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.571 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.571 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.571 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap": { 2024-09-14T03:47:49.571 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 6696, 2024-09-14T03:47:49.571 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 137 2024-09-14T03:47:49.571 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.571 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap_mapping": { 2024-09-14T03:47:49.571 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 12744, 2024-09-14T03:47:49.572 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 1546 2024-09-14T03:47:49.572 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.572 INFO:tasks.workunit.client.0.smithi070.stdout: "pgmap": { 2024-09-14T03:47:49.572 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 5312, 2024-09-14T03:47:49.572 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 103 2024-09-14T03:47:49.572 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.572 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_1": { 2024-09-14T03:47:49.572 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.572 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.572 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.572 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_2": { 2024-09-14T03:47:49.572 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.572 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.573 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.573 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.573 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.573 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:49.573 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.0": { 2024-09-14T03:47:49.573 INFO:tasks.workunit.client.0.smithi070.stdout: "bloom_filter": { 2024-09-14T03:47:49.573 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.573 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.573 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.574 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:49.574 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 6304, 2024-09-14T03:47:49.574 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 115 2024-09-14T03:47:49.574 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.574 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_reader": { 2024-09-14T03:47:49.576 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 16896, 2024-09-14T03:47:49.576 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 8 2024-09-14T03:47:49.577 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.577 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_writer": { 2024-09-14T03:47:49.577 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 576, 2024-09-14T03:47:49.577 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3 2024-09-14T03:47:49.577 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.577 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Blob": { 2024-09-14T03:47:49.577 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 20496, 2024-09-14T03:47:49.577 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 183 2024-09-14T03:47:49.577 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.577 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Buffer": { 2024-09-14T03:47:49.578 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 224, 2024-09-14T03:47:49.578 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 2 2024-09-14T03:47:49.578 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.578 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Extent": { 2024-09-14T03:47:49.578 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 8784, 2024-09-14T03:47:49.578 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 183 2024-09-14T03:47:49.578 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.578 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_SharedBlob": { 2024-09-14T03:47:49.578 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 20496, 2024-09-14T03:47:49.578 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 183 2024-09-14T03:47:49.578 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.578 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_alloc": { 2024-09-14T03:47:49.578 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 5280, 2024-09-14T03:47:49.578 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 66 2024-09-14T03:47:49.579 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.579 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_data": { 2024-09-14T03:47:49.579 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 8192, 2024-09-14T03:47:49.579 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 2 2024-09-14T03:47:49.579 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.579 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_meta": { 2024-09-14T03:47:49.579 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 54475, 2024-09-14T03:47:49.579 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 9882 2024-09-14T03:47:49.579 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.579 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_onode": { 2024-09-14T03:47:49.580 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 135320, 2024-09-14T03:47:49.580 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 199 2024-09-14T03:47:49.580 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.580 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_other": { 2024-09-14T03:47:49.580 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 11104, 2024-09-14T03:47:49.580 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 711 2024-09-14T03:47:49.580 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.580 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_fsck": { 2024-09-14T03:47:49.580 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.580 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.580 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.580 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_inline_bl": { 2024-09-14T03:47:49.580 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 31588, 2024-09-14T03:47:49.580 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 199 2024-09-14T03:47:49.581 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.581 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_txc": { 2024-09-14T03:47:49.581 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.581 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.581 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.581 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing": { 2024-09-14T03:47:49.581 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 399331, 2024-09-14T03:47:49.581 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 87 2024-09-14T03:47:49.581 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.581 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing_deferred": { 2024-09-14T03:47:49.581 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.581 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.581 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.581 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_anon": { 2024-09-14T03:47:49.582 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 2433372, 2024-09-14T03:47:49.582 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 193 2024-09-14T03:47:49.582 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.582 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_meta": { 2024-09-14T03:47:49.582 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 5632, 2024-09-14T03:47:49.582 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 64 2024-09-14T03:47:49.582 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.582 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_co": { 2024-09-14T03:47:49.582 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.582 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.582 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.582 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:49.582 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 521280, 2024-09-14T03:47:49.582 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 45 2024-09-14T03:47:49.583 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.583 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mapbl": { 2024-09-14T03:47:49.583 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.583 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.583 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.583 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pglog": { 2024-09-14T03:47:49.583 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 70304, 2024-09-14T03:47:49.583 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 140 2024-09-14T03:47:49.583 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.583 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap": { 2024-09-14T03:47:49.583 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 235720, 2024-09-14T03:47:49.583 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3529 2024-09-14T03:47:49.583 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.584 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap_mapping": { 2024-09-14T03:47:49.584 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.584 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.584 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.584 INFO:tasks.workunit.client.0.smithi070.stdout: "pgmap": { 2024-09-14T03:47:49.584 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.584 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.584 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.584 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_1": { 2024-09-14T03:47:49.584 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.584 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.584 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.584 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_2": { 2024-09-14T03:47:49.585 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.585 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.585 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.585 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.585 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.1": { 2024-09-14T03:47:49.585 INFO:tasks.workunit.client.0.smithi070.stdout: "bloom_filter": { 2024-09-14T03:47:49.585 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.585 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.585 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.585 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:49.585 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 6304, 2024-09-14T03:47:49.585 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 115 2024-09-14T03:47:49.585 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.586 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_reader": { 2024-09-14T03:47:49.586 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 16896, 2024-09-14T03:47:49.586 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 8 2024-09-14T03:47:49.586 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.586 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_writer": { 2024-09-14T03:47:49.586 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 576, 2024-09-14T03:47:49.586 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3 2024-09-14T03:47:49.586 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.586 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Blob": { 2024-09-14T03:47:49.586 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 19040, 2024-09-14T03:47:49.586 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 170 2024-09-14T03:47:49.586 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.586 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Buffer": { 2024-09-14T03:47:49.586 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 896, 2024-09-14T03:47:49.587 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 8 2024-09-14T03:47:49.587 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.587 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Extent": { 2024-09-14T03:47:49.587 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 8160, 2024-09-14T03:47:49.587 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 170 2024-09-14T03:47:49.587 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.587 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_SharedBlob": { 2024-09-14T03:47:49.587 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 19040, 2024-09-14T03:47:49.587 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 170 2024-09-14T03:47:49.587 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.587 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_alloc": { 2024-09-14T03:47:49.587 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 4560, 2024-09-14T03:47:49.587 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 57 2024-09-14T03:47:49.587 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.588 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_data": { 2024-09-14T03:47:49.588 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 12288, 2024-09-14T03:47:49.588 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3 2024-09-14T03:47:49.588 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.588 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_meta": { 2024-09-14T03:47:49.588 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 48337, 2024-09-14T03:47:49.588 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 8825 2024-09-14T03:47:49.588 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.588 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_onode": { 2024-09-14T03:47:49.588 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 125120, 2024-09-14T03:47:49.588 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 184 2024-09-14T03:47:49.588 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.588 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_other": { 2024-09-14T03:47:49.589 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 9152, 2024-09-14T03:47:49.589 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 535 2024-09-14T03:47:49.589 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.589 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_fsck": { 2024-09-14T03:47:49.589 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.589 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.589 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.589 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_inline_bl": { 2024-09-14T03:47:49.589 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 27312, 2024-09-14T03:47:49.589 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 184 2024-09-14T03:47:49.589 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.589 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_txc": { 2024-09-14T03:47:49.589 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 736, 2024-09-14T03:47:49.589 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 1 2024-09-14T03:47:49.590 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.590 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing": { 2024-09-14T03:47:49.590 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 427120, 2024-09-14T03:47:49.590 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 94 2024-09-14T03:47:49.590 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.590 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing_deferred": { 2024-09-14T03:47:49.590 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 12096, 2024-09-14T03:47:49.590 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3 2024-09-14T03:47:49.590 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.590 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_anon": { 2024-09-14T03:47:49.590 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 2411092, 2024-09-14T03:47:49.590 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 181 2024-09-14T03:47:49.590 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.590 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_meta": { 2024-09-14T03:47:49.591 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 4928, 2024-09-14T03:47:49.591 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 56 2024-09-14T03:47:49.591 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.591 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_co": { 2024-09-14T03:47:49.591 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.591 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.591 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.591 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:49.591 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 440192, 2024-09-14T03:47:49.591 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 38 2024-09-14T03:47:49.591 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.591 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mapbl": { 2024-09-14T03:47:49.591 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.591 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.592 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.592 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pglog": { 2024-09-14T03:47:49.592 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 8528, 2024-09-14T03:47:49.592 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 22 2024-09-14T03:47:49.592 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.592 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap": { 2024-09-14T03:47:49.592 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 235208, 2024-09-14T03:47:49.592 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3505 2024-09-14T03:47:49.592 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.592 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap_mapping": { 2024-09-14T03:47:49.592 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.592 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.592 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.593 INFO:tasks.workunit.client.0.smithi070.stdout: "pgmap": { 2024-09-14T03:47:49.593 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.593 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.593 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.593 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_1": { 2024-09-14T03:47:49.593 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.593 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.593 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.593 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_2": { 2024-09-14T03:47:49.593 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.593 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.593 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.593 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.594 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.2": { 2024-09-14T03:47:49.594 INFO:tasks.workunit.client.0.smithi070.stdout: "bloom_filter": { 2024-09-14T03:47:49.594 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.594 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.594 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.594 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:49.594 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 6304, 2024-09-14T03:47:49.594 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 115 2024-09-14T03:47:49.594 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.594 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_reader": { 2024-09-14T03:47:49.594 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 16896, 2024-09-14T03:47:49.594 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 8 2024-09-14T03:47:49.594 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.594 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_writer": { 2024-09-14T03:47:49.595 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 576, 2024-09-14T03:47:49.595 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3 2024-09-14T03:47:49.595 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.595 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Blob": { 2024-09-14T03:47:49.595 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 18928, 2024-09-14T03:47:49.595 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 169 2024-09-14T03:47:49.595 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.595 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Buffer": { 2024-09-14T03:47:49.595 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 784, 2024-09-14T03:47:49.595 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 7 2024-09-14T03:47:49.595 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.595 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Extent": { 2024-09-14T03:47:49.595 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 8112, 2024-09-14T03:47:49.595 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 169 2024-09-14T03:47:49.596 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.596 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_SharedBlob": { 2024-09-14T03:47:49.596 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 18928, 2024-09-14T03:47:49.596 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 169 2024-09-14T03:47:49.596 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.596 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_alloc": { 2024-09-14T03:47:49.596 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 4160, 2024-09-14T03:47:49.597 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 52 2024-09-14T03:47:49.597 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.597 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_data": { 2024-09-14T03:47:49.597 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 8192, 2024-09-14T03:47:49.597 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 2 2024-09-14T03:47:49.597 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.597 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_meta": { 2024-09-14T03:47:49.597 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 45727, 2024-09-14T03:47:49.597 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 8489 2024-09-14T03:47:49.597 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.597 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_onode": { 2024-09-14T03:47:49.598 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 120360, 2024-09-14T03:47:49.598 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 177 2024-09-14T03:47:49.598 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.598 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_other": { 2024-09-14T03:47:49.598 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 8348, 2024-09-14T03:47:49.598 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 526 2024-09-14T03:47:49.598 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.598 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_fsck": { 2024-09-14T03:47:49.598 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.598 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.598 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.598 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_inline_bl": { 2024-09-14T03:47:49.598 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 25688, 2024-09-14T03:47:49.599 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 177 2024-09-14T03:47:49.599 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.599 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_txc": { 2024-09-14T03:47:49.599 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 736, 2024-09-14T03:47:49.599 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 1 2024-09-14T03:47:49.599 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.599 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing": { 2024-09-14T03:47:49.599 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 329388, 2024-09-14T03:47:49.599 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 70 2024-09-14T03:47:49.599 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.599 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing_deferred": { 2024-09-14T03:47:49.600 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 12096, 2024-09-14T03:47:49.600 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3 2024-09-14T03:47:49.600 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.600 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_anon": { 2024-09-14T03:47:49.600 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 2422832, 2024-09-14T03:47:49.600 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 191 2024-09-14T03:47:49.600 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.600 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_meta": { 2024-09-14T03:47:49.600 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 4312, 2024-09-14T03:47:49.600 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 49 2024-09-14T03:47:49.600 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.600 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_co": { 2024-09-14T03:47:49.600 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.601 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.601 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.601 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:49.601 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 359104, 2024-09-14T03:47:49.601 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 31 2024-09-14T03:47:49.601 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.601 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mapbl": { 2024-09-14T03:47:49.601 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.601 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.601 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.601 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pglog": { 2024-09-14T03:47:49.601 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 8320, 2024-09-14T03:47:49.601 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 24 2024-09-14T03:47:49.602 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.602 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap": { 2024-09-14T03:47:49.602 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 235720, 2024-09-14T03:47:49.602 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3529 2024-09-14T03:47:49.602 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.602 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap_mapping": { 2024-09-14T03:47:49.602 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.602 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.602 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.602 INFO:tasks.workunit.client.0.smithi070.stdout: "pgmap": { 2024-09-14T03:47:49.602 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.602 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.602 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.602 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_1": { 2024-09-14T03:47:49.602 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.602 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.603 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.603 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_2": { 2024-09-14T03:47:49.603 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.603 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.603 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.603 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.603 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.3": { 2024-09-14T03:47:49.603 INFO:tasks.workunit.client.0.smithi070.stdout: "bloom_filter": { 2024-09-14T03:47:49.603 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.603 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.603 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.603 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:49.603 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 6304, 2024-09-14T03:47:49.603 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 115 2024-09-14T03:47:49.604 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.604 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_reader": { 2024-09-14T03:47:49.604 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 16896, 2024-09-14T03:47:49.604 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 8 2024-09-14T03:47:49.604 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.604 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_writer": { 2024-09-14T03:47:49.604 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 576, 2024-09-14T03:47:49.604 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3 2024-09-14T03:47:49.604 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.604 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Blob": { 2024-09-14T03:47:49.604 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 19488, 2024-09-14T03:47:49.604 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 174 2024-09-14T03:47:49.605 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.605 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Buffer": { 2024-09-14T03:47:49.605 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 224, 2024-09-14T03:47:49.605 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 2 2024-09-14T03:47:49.605 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.605 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Extent": { 2024-09-14T03:47:49.605 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 8352, 2024-09-14T03:47:49.605 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 174 2024-09-14T03:47:49.605 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.605 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_SharedBlob": { 2024-09-14T03:47:49.605 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 19488, 2024-09-14T03:47:49.605 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 174 2024-09-14T03:47:49.606 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.606 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_alloc": { 2024-09-14T03:47:49.606 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 3760, 2024-09-14T03:47:49.606 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 47 2024-09-14T03:47:49.606 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.606 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_data": { 2024-09-14T03:47:49.606 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.606 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.606 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.606 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_meta": { 2024-09-14T03:47:49.606 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 47391, 2024-09-14T03:47:49.606 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 8817 2024-09-14T03:47:49.606 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.606 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_onode": { 2024-09-14T03:47:49.607 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 125120, 2024-09-14T03:47:49.607 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 184 2024-09-14T03:47:49.607 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.607 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_other": { 2024-09-14T03:47:49.607 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 9152, 2024-09-14T03:47:49.607 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 544 2024-09-14T03:47:49.607 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.607 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_fsck": { 2024-09-14T03:47:49.607 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.607 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.607 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.608 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_inline_bl": { 2024-09-14T03:47:49.608 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 26640, 2024-09-14T03:47:49.608 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 184 2024-09-14T03:47:49.608 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.608 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_txc": { 2024-09-14T03:47:49.608 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 2208, 2024-09-14T03:47:49.608 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3 2024-09-14T03:47:49.608 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.608 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing": { 2024-09-14T03:47:49.608 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 276833, 2024-09-14T03:47:49.608 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 57 2024-09-14T03:47:49.608 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.608 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing_deferred": { 2024-09-14T03:47:49.608 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 30637, 2024-09-14T03:47:49.609 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 9 2024-09-14T03:47:49.609 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.609 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_anon": { 2024-09-14T03:47:49.609 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 2413964, 2024-09-14T03:47:49.609 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 185 2024-09-14T03:47:49.609 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.609 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_meta": { 2024-09-14T03:47:49.609 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 4752, 2024-09-14T03:47:49.609 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 54 2024-09-14T03:47:49.609 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.609 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_co": { 2024-09-14T03:47:49.609 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.609 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.609 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.610 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:49.610 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 428608, 2024-09-14T03:47:49.610 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 37 2024-09-14T03:47:49.610 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.610 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mapbl": { 2024-09-14T03:47:49.610 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.610 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.610 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.610 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pglog": { 2024-09-14T03:47:49.610 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 10400, 2024-09-14T03:47:49.610 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 20 2024-09-14T03:47:49.610 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.610 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap": { 2024-09-14T03:47:49.611 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 235720, 2024-09-14T03:47:49.611 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3529 2024-09-14T03:47:49.611 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.611 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap_mapping": { 2024-09-14T03:47:49.611 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.611 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.611 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.611 INFO:tasks.workunit.client.0.smithi070.stdout: "pgmap": { 2024-09-14T03:47:49.611 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.611 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.611 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.611 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_1": { 2024-09-14T03:47:49.612 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.612 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.612 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.612 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_2": { 2024-09-14T03:47:49.612 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.612 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.612 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.612 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.612 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.4": { 2024-09-14T03:47:49.612 INFO:tasks.workunit.client.0.smithi070.stdout: "bloom_filter": { 2024-09-14T03:47:49.612 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.612 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.613 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.613 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:49.613 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 6304, 2024-09-14T03:47:49.613 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 115 2024-09-14T03:47:49.613 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.613 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_reader": { 2024-09-14T03:47:49.613 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 16896, 2024-09-14T03:47:49.613 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 8 2024-09-14T03:47:49.613 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.613 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_writer": { 2024-09-14T03:47:49.614 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 576, 2024-09-14T03:47:49.614 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3 2024-09-14T03:47:49.614 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.614 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Blob": { 2024-09-14T03:47:49.614 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 19600, 2024-09-14T03:47:49.614 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 175 2024-09-14T03:47:49.614 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.614 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Buffer": { 2024-09-14T03:47:49.614 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 560, 2024-09-14T03:47:49.614 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 5 2024-09-14T03:47:49.614 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.614 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Extent": { 2024-09-14T03:47:49.614 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 8400, 2024-09-14T03:47:49.614 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 175 2024-09-14T03:47:49.614 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.615 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_SharedBlob": { 2024-09-14T03:47:49.615 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 19600, 2024-09-14T03:47:49.615 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 175 2024-09-14T03:47:49.615 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.615 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_alloc": { 2024-09-14T03:47:49.615 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 3280, 2024-09-14T03:47:49.615 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 41 2024-09-14T03:47:49.615 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.615 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_data": { 2024-09-14T03:47:49.615 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.615 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.615 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.615 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_meta": { 2024-09-14T03:47:49.615 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 48760, 2024-09-14T03:47:49.616 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 9014 2024-09-14T03:47:49.616 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.616 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_onode": { 2024-09-14T03:47:49.616 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 127840, 2024-09-14T03:47:49.616 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 188 2024-09-14T03:47:49.616 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.616 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_other": { 2024-09-14T03:47:49.616 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 9364, 2024-09-14T03:47:49.616 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 546 2024-09-14T03:47:49.616 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.616 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_fsck": { 2024-09-14T03:47:49.616 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.616 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.617 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.617 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_inline_bl": { 2024-09-14T03:47:49.617 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 27496, 2024-09-14T03:47:49.617 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 188 2024-09-14T03:47:49.617 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.617 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_txc": { 2024-09-14T03:47:49.617 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 736, 2024-09-14T03:47:49.617 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 1 2024-09-14T03:47:49.617 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.617 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing": { 2024-09-14T03:47:49.617 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 901365, 2024-09-14T03:47:49.617 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 67 2024-09-14T03:47:49.617 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.617 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing_deferred": { 2024-09-14T03:47:49.618 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 12096, 2024-09-14T03:47:49.618 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3 2024-09-14T03:47:49.618 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.618 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_anon": { 2024-09-14T03:47:49.618 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 2405394, 2024-09-14T03:47:49.618 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 170 2024-09-14T03:47:49.618 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.618 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_meta": { 2024-09-14T03:47:49.618 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 4752, 2024-09-14T03:47:49.618 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 54 2024-09-14T03:47:49.618 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.618 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_co": { 2024-09-14T03:47:49.618 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.619 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.619 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.619 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:49.619 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 451776, 2024-09-14T03:47:49.619 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 39 2024-09-14T03:47:49.619 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.619 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mapbl": { 2024-09-14T03:47:49.619 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.619 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.619 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.619 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pglog": { 2024-09-14T03:47:49.620 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 8944, 2024-09-14T03:47:49.620 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 18 2024-09-14T03:47:49.620 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.620 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap": { 2024-09-14T03:47:49.620 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 235464, 2024-09-14T03:47:49.620 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3517 2024-09-14T03:47:49.620 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.620 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap_mapping": { 2024-09-14T03:47:49.620 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.620 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.620 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.620 INFO:tasks.workunit.client.0.smithi070.stdout: "pgmap": { 2024-09-14T03:47:49.620 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.620 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.621 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.621 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_1": { 2024-09-14T03:47:49.621 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.621 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.621 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.621 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_2": { 2024-09-14T03:47:49.621 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.621 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.621 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.621 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.621 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.5": { 2024-09-14T03:47:49.621 INFO:tasks.workunit.client.0.smithi070.stdout: "bloom_filter": { 2024-09-14T03:47:49.621 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.621 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.622 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.622 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:49.622 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 6304, 2024-09-14T03:47:49.622 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 115 2024-09-14T03:47:49.622 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.622 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_reader": { 2024-09-14T03:47:49.622 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 16896, 2024-09-14T03:47:49.622 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 8 2024-09-14T03:47:49.622 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.622 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_writer": { 2024-09-14T03:47:49.622 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 576, 2024-09-14T03:47:49.622 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3 2024-09-14T03:47:49.622 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.622 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Blob": { 2024-09-14T03:47:49.623 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 19488, 2024-09-14T03:47:49.623 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 174 2024-09-14T03:47:49.623 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.623 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Buffer": { 2024-09-14T03:47:49.623 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 224, 2024-09-14T03:47:49.623 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 2 2024-09-14T03:47:49.623 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.623 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Extent": { 2024-09-14T03:47:49.623 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 8352, 2024-09-14T03:47:49.623 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 174 2024-09-14T03:47:49.623 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.623 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_SharedBlob": { 2024-09-14T03:47:49.623 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 19488, 2024-09-14T03:47:49.623 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 174 2024-09-14T03:47:49.624 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.624 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_alloc": { 2024-09-14T03:47:49.624 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 3040, 2024-09-14T03:47:49.624 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 38 2024-09-14T03:47:49.624 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.624 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_data": { 2024-09-14T03:47:49.624 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.624 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.624 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.624 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_meta": { 2024-09-14T03:47:49.624 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 49727, 2024-09-14T03:47:49.624 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 9103 2024-09-14T03:47:49.625 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.625 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_onode": { 2024-09-14T03:47:49.625 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 129200, 2024-09-14T03:47:49.625 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 190 2024-09-14T03:47:49.625 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.625 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_other": { 2024-09-14T03:47:49.625 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 9600, 2024-09-14T03:47:49.625 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 548 2024-09-14T03:47:49.625 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.625 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_fsck": { 2024-09-14T03:47:49.625 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.625 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.625 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.626 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_inline_bl": { 2024-09-14T03:47:49.626 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 28176, 2024-09-14T03:47:49.626 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 190 2024-09-14T03:47:49.626 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.626 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_txc": { 2024-09-14T03:47:49.626 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 2208, 2024-09-14T03:47:49.626 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3 2024-09-14T03:47:49.626 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.626 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing": { 2024-09-14T03:47:49.626 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 846007, 2024-09-14T03:47:49.626 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 53 2024-09-14T03:47:49.626 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.626 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing_deferred": { 2024-09-14T03:47:49.626 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 30637, 2024-09-14T03:47:49.627 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 9 2024-09-14T03:47:49.627 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.627 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_anon": { 2024-09-14T03:47:49.627 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 2403941, 2024-09-14T03:47:49.627 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 169 2024-09-14T03:47:49.627 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.627 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_meta": { 2024-09-14T03:47:49.627 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 5456, 2024-09-14T03:47:49.627 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 62 2024-09-14T03:47:49.627 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.627 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_co": { 2024-09-14T03:47:49.627 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.628 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.628 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.628 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:49.628 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 474944, 2024-09-14T03:47:49.628 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 41 2024-09-14T03:47:49.628 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.628 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mapbl": { 2024-09-14T03:47:49.628 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.628 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.628 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.628 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pglog": { 2024-09-14T03:47:49.628 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 13000, 2024-09-14T03:47:49.629 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 25 2024-09-14T03:47:49.629 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.629 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap": { 2024-09-14T03:47:49.629 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 235720, 2024-09-14T03:47:49.629 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3529 2024-09-14T03:47:49.629 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.629 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap_mapping": { 2024-09-14T03:47:49.629 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.629 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.629 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.629 INFO:tasks.workunit.client.0.smithi070.stdout: "pgmap": { 2024-09-14T03:47:49.629 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.629 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.630 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.630 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_1": { 2024-09-14T03:47:49.630 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.630 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.630 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.630 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_2": { 2024-09-14T03:47:49.630 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.630 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.630 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.630 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.630 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.6": { 2024-09-14T03:47:49.630 INFO:tasks.workunit.client.0.smithi070.stdout: "bloom_filter": { 2024-09-14T03:47:49.630 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.630 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.631 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.631 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:49.631 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 6304, 2024-09-14T03:47:49.631 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 115 2024-09-14T03:47:49.631 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.631 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_reader": { 2024-09-14T03:47:49.631 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 16896, 2024-09-14T03:47:49.631 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 8 2024-09-14T03:47:49.631 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.631 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_writer": { 2024-09-14T03:47:49.631 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 576, 2024-09-14T03:47:49.632 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3 2024-09-14T03:47:49.632 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.632 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Blob": { 2024-09-14T03:47:49.632 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 20384, 2024-09-14T03:47:49.632 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 182 2024-09-14T03:47:49.632 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.632 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Buffer": { 2024-09-14T03:47:49.632 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 224, 2024-09-14T03:47:49.632 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 2 2024-09-14T03:47:49.632 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.633 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Extent": { 2024-09-14T03:47:49.633 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 8736, 2024-09-14T03:47:49.633 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 182 2024-09-14T03:47:49.633 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.633 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_SharedBlob": { 2024-09-14T03:47:49.633 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 20384, 2024-09-14T03:47:49.633 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 182 2024-09-14T03:47:49.633 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.633 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_alloc": { 2024-09-14T03:47:49.633 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 2560, 2024-09-14T03:47:49.633 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 32 2024-09-14T03:47:49.633 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.633 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_data": { 2024-09-14T03:47:49.634 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 14723, 2024-09-14T03:47:49.634 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3 2024-09-14T03:47:49.634 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.634 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_meta": { 2024-09-14T03:47:49.634 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 48913, 2024-09-14T03:47:49.634 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 9250 2024-09-14T03:47:49.634 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.634 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_onode": { 2024-09-14T03:47:49.634 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 126480, 2024-09-14T03:47:49.634 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 186 2024-09-14T03:47:49.634 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.634 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_other": { 2024-09-14T03:47:49.634 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 10492, 2024-09-14T03:47:49.634 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 702 2024-09-14T03:47:49.634 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.635 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_fsck": { 2024-09-14T03:47:49.635 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.635 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.635 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.635 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_inline_bl": { 2024-09-14T03:47:49.635 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 28348, 2024-09-14T03:47:49.635 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 186 2024-09-14T03:47:49.635 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.635 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_txc": { 2024-09-14T03:47:49.635 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.636 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.636 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.636 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing": { 2024-09-14T03:47:49.636 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 210427, 2024-09-14T03:47:49.636 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 42 2024-09-14T03:47:49.636 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.636 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing_deferred": { 2024-09-14T03:47:49.636 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.636 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.636 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.636 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_anon": { 2024-09-14T03:47:49.636 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 2410728, 2024-09-14T03:47:49.636 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 177 2024-09-14T03:47:49.637 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.637 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_meta": { 2024-09-14T03:47:49.637 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 4576, 2024-09-14T03:47:49.637 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 52 2024-09-14T03:47:49.637 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.637 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_co": { 2024-09-14T03:47:49.637 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.637 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.637 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.637 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:49.637 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 440192, 2024-09-14T03:47:49.637 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 38 2024-09-14T03:47:49.637 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.638 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mapbl": { 2024-09-14T03:47:49.638 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.638 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.638 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.638 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pglog": { 2024-09-14T03:47:49.638 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 69472, 2024-09-14T03:47:49.638 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 144 2024-09-14T03:47:49.638 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.638 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap": { 2024-09-14T03:47:49.638 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 235208, 2024-09-14T03:47:49.638 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3505 2024-09-14T03:47:49.639 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.639 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap_mapping": { 2024-09-14T03:47:49.639 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.639 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.639 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.639 INFO:tasks.workunit.client.0.smithi070.stdout: "pgmap": { 2024-09-14T03:47:49.639 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.639 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.639 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.639 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_1": { 2024-09-14T03:47:49.639 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.639 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.639 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.639 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_2": { 2024-09-14T03:47:49.640 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.640 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.640 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.640 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.640 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.7": { 2024-09-14T03:47:49.640 INFO:tasks.workunit.client.0.smithi070.stdout: "bloom_filter": { 2024-09-14T03:47:49.640 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.640 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.640 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.640 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:49.640 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 6304, 2024-09-14T03:47:49.640 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 115 2024-09-14T03:47:49.640 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.640 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_reader": { 2024-09-14T03:47:49.641 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 16896, 2024-09-14T03:47:49.641 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 8 2024-09-14T03:47:49.641 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.641 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_writer": { 2024-09-14T03:47:49.641 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 576, 2024-09-14T03:47:49.641 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3 2024-09-14T03:47:49.641 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.641 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Blob": { 2024-09-14T03:47:49.641 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 20384, 2024-09-14T03:47:49.641 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 182 2024-09-14T03:47:49.641 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.641 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Buffer": { 2024-09-14T03:47:49.641 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 1232, 2024-09-14T03:47:49.642 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 11 2024-09-14T03:47:49.642 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.642 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Extent": { 2024-09-14T03:47:49.642 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 8736, 2024-09-14T03:47:49.642 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 182 2024-09-14T03:47:49.642 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.642 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_SharedBlob": { 2024-09-14T03:47:49.642 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 20384, 2024-09-14T03:47:49.642 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 182 2024-09-14T03:47:49.642 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.642 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_alloc": { 2024-09-14T03:47:49.642 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 2240, 2024-09-14T03:47:49.642 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 28 2024-09-14T03:47:49.642 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.643 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_data": { 2024-09-14T03:47:49.643 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 293251, 2024-09-14T03:47:49.643 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 12 2024-09-14T03:47:49.643 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.643 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_meta": { 2024-09-14T03:47:49.643 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 53184, 2024-09-14T03:47:49.643 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 9789 2024-09-14T03:47:49.643 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.643 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_onode": { 2024-09-14T03:47:49.643 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 133960, 2024-09-14T03:47:49.643 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 197 2024-09-14T03:47:49.643 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.643 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_other": { 2024-09-14T03:47:49.644 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 11388, 2024-09-14T03:47:49.644 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 710 2024-09-14T03:47:49.644 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.644 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_fsck": { 2024-09-14T03:47:49.644 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.644 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.644 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.644 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_inline_bl": { 2024-09-14T03:47:49.644 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 31164, 2024-09-14T03:47:49.644 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 197 2024-09-14T03:47:49.644 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.644 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_txc": { 2024-09-14T03:47:49.644 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.645 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.645 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.645 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing": { 2024-09-14T03:47:49.645 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 181403, 2024-09-14T03:47:49.645 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 35 2024-09-14T03:47:49.645 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.645 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing_deferred": { 2024-09-14T03:47:49.645 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.645 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.645 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.645 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_anon": { 2024-09-14T03:47:49.645 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 2430622, 2024-09-14T03:47:49.645 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 217 2024-09-14T03:47:49.646 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.646 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_meta": { 2024-09-14T03:47:49.646 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 6336, 2024-09-14T03:47:49.646 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 72 2024-09-14T03:47:49.646 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.646 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_co": { 2024-09-14T03:47:49.646 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.646 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.646 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.646 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:49.646 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 532864, 2024-09-14T03:47:49.647 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 46 2024-09-14T03:47:49.647 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.647 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mapbl": { 2024-09-14T03:47:49.647 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.647 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.647 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.647 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pglog": { 2024-09-14T03:47:49.647 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 72072, 2024-09-14T03:47:49.647 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 173 2024-09-14T03:47:49.647 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.647 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap": { 2024-09-14T03:47:49.647 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 235208, 2024-09-14T03:47:49.647 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3505 2024-09-14T03:47:49.647 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.648 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap_mapping": { 2024-09-14T03:47:49.648 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.648 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.648 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.648 INFO:tasks.workunit.client.0.smithi070.stdout: "pgmap": { 2024-09-14T03:47:49.648 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.648 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.648 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.648 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_1": { 2024-09-14T03:47:49.648 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.648 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.648 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.648 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_2": { 2024-09-14T03:47:49.649 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:49.649 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:49.649 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.649 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.649 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.649 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.649 INFO:tasks.workunit.client.0.smithi070.stdout: "metadata": { 2024-09-14T03:47:49.649 INFO:tasks.workunit.client.0.smithi070.stdout: "mds": { 2024-09-14T03:47:49.649 INFO:tasks.workunit.client.0.smithi070.stdout: "arch": { 2024-09-14T03:47:49.649 INFO:tasks.workunit.client.0.smithi070.stdout: "x86_64": 2 2024-09-14T03:47:49.649 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.649 INFO:tasks.workunit.client.0.smithi070.stdout: "ceph_version": { 2024-09-14T03:47:49.649 INFO:tasks.workunit.client.0.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:47:49.650 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.650 INFO:tasks.workunit.client.0.smithi070.stdout: "cpu": { 2024-09-14T03:47:49.650 INFO:tasks.workunit.client.0.smithi070.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 2 2024-09-14T03:47:49.650 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.650 INFO:tasks.workunit.client.0.smithi070.stdout: "distro": { 2024-09-14T03:47:49.650 INFO:tasks.workunit.client.0.smithi070.stdout: "centos": 2 2024-09-14T03:47:49.650 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.650 INFO:tasks.workunit.client.0.smithi070.stdout: "distro_description": { 2024-09-14T03:47:49.650 INFO:tasks.workunit.client.0.smithi070.stdout: "CentOS Stream 9": 2 2024-09-14T03:47:49.650 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.650 INFO:tasks.workunit.client.0.smithi070.stdout: "kernel_description": { 2024-09-14T03:47:49.651 INFO:tasks.workunit.client.0.smithi070.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 2 2024-09-14T03:47:49.651 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.651 INFO:tasks.workunit.client.0.smithi070.stdout: "kernel_version": { 2024-09-14T03:47:49.651 INFO:tasks.workunit.client.0.smithi070.stdout: "5.14.0-505.el9.x86_64": 2 2024-09-14T03:47:49.651 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.651 INFO:tasks.workunit.client.0.smithi070.stdout: "os": { 2024-09-14T03:47:49.651 INFO:tasks.workunit.client.0.smithi070.stdout: "Linux": 2 2024-09-14T03:47:49.652 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.652 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.652 INFO:tasks.workunit.client.0.smithi070.stdout: "mon": { 2024-09-14T03:47:49.652 INFO:tasks.workunit.client.0.smithi070.stdout: "arch": { 2024-09-14T03:47:49.652 INFO:tasks.workunit.client.0.smithi070.stdout: "x86_64": 3 2024-09-14T03:47:49.652 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.652 INFO:tasks.workunit.client.0.smithi070.stdout: "ceph_version": { 2024-09-14T03:47:49.652 INFO:tasks.workunit.client.0.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 3 2024-09-14T03:47:49.652 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.652 INFO:tasks.workunit.client.0.smithi070.stdout: "cpu": { 2024-09-14T03:47:49.653 INFO:tasks.workunit.client.0.smithi070.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 3 2024-09-14T03:47:49.653 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.653 INFO:tasks.workunit.client.0.smithi070.stdout: "distro": { 2024-09-14T03:47:49.653 INFO:tasks.workunit.client.0.smithi070.stdout: "centos": 3 2024-09-14T03:47:49.653 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.653 INFO:tasks.workunit.client.0.smithi070.stdout: "distro_description": { 2024-09-14T03:47:49.653 INFO:tasks.workunit.client.0.smithi070.stdout: "CentOS Stream 9": 3 2024-09-14T03:47:49.653 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.653 INFO:tasks.workunit.client.0.smithi070.stdout: "kernel_description": { 2024-09-14T03:47:49.653 INFO:tasks.workunit.client.0.smithi070.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 3 2024-09-14T03:47:49.654 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.654 INFO:tasks.workunit.client.0.smithi070.stdout: "kernel_version": { 2024-09-14T03:47:49.654 INFO:tasks.workunit.client.0.smithi070.stdout: "5.14.0-505.el9.x86_64": 3 2024-09-14T03:47:49.654 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.654 INFO:tasks.workunit.client.0.smithi070.stdout: "os": { 2024-09-14T03:47:49.654 INFO:tasks.workunit.client.0.smithi070.stdout: "Linux": 3 2024-09-14T03:47:49.655 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.655 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.655 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:49.655 INFO:tasks.workunit.client.0.smithi070.stdout: "arch": { 2024-09-14T03:47:49.655 INFO:tasks.workunit.client.0.smithi070.stdout: "x86_64": 8 2024-09-14T03:47:49.656 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.656 INFO:tasks.workunit.client.0.smithi070.stdout: "ceph_version": { 2024-09-14T03:47:49.656 INFO:tasks.workunit.client.0.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 8 2024-09-14T03:47:49.656 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.656 INFO:tasks.workunit.client.0.smithi070.stdout: "cpu": { 2024-09-14T03:47:49.656 INFO:tasks.workunit.client.0.smithi070.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 8 2024-09-14T03:47:49.656 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.656 INFO:tasks.workunit.client.0.smithi070.stdout: "distro": { 2024-09-14T03:47:49.657 INFO:tasks.workunit.client.0.smithi070.stdout: "centos": 8 2024-09-14T03:47:49.657 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.657 INFO:tasks.workunit.client.0.smithi070.stdout: "distro_description": { 2024-09-14T03:47:49.657 INFO:tasks.workunit.client.0.smithi070.stdout: "CentOS Stream 9": 8 2024-09-14T03:47:49.657 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.657 INFO:tasks.workunit.client.0.smithi070.stdout: "kernel_description": { 2024-09-14T03:47:49.657 INFO:tasks.workunit.client.0.smithi070.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 8 2024-09-14T03:47:49.657 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.657 INFO:tasks.workunit.client.0.smithi070.stdout: "kernel_version": { 2024-09-14T03:47:49.657 INFO:tasks.workunit.client.0.smithi070.stdout: "5.14.0-505.el9.x86_64": 8 2024-09-14T03:47:49.658 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.658 INFO:tasks.workunit.client.0.smithi070.stdout: "os": { 2024-09-14T03:47:49.658 INFO:tasks.workunit.client.0.smithi070.stdout: "Linux": 8 2024-09-14T03:47:49.658 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.658 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_objectstore": { 2024-09-14T03:47:49.658 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore": 8 2024-09-14T03:47:49.658 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.659 INFO:tasks.workunit.client.0.smithi070.stdout: "rotational": { 2024-09-14T03:47:49.659 INFO:tasks.workunit.client.0.smithi070.stdout: "0": 8 2024-09-14T03:47:49.659 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.659 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.659 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.659 INFO:tasks.workunit.client.0.smithi070.stdout: "mon": { 2024-09-14T03:47:49.659 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 3, 2024-09-14T03:47:49.659 INFO:tasks.workunit.client.0.smithi070.stdout: "features": { 2024-09-14T03:47:49.659 INFO:tasks.workunit.client.0.smithi070.stdout: "optional": [], 2024-09-14T03:47:49.659 INFO:tasks.workunit.client.0.smithi070.stdout: "persistent": [ 2024-09-14T03:47:49.659 INFO:tasks.workunit.client.0.smithi070.stdout: "kraken", 2024-09-14T03:47:49.659 INFO:tasks.workunit.client.0.smithi070.stdout: "luminous", 2024-09-14T03:47:49.659 INFO:tasks.workunit.client.0.smithi070.stdout: "mimic", 2024-09-14T03:47:49.660 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap-prune", 2024-09-14T03:47:49.660 INFO:tasks.workunit.client.0.smithi070.stdout: "nautilus", 2024-09-14T03:47:49.660 INFO:tasks.workunit.client.0.smithi070.stdout: "octopus", 2024-09-14T03:47:49.660 INFO:tasks.workunit.client.0.smithi070.stdout: "pacific", 2024-09-14T03:47:49.660 INFO:tasks.workunit.client.0.smithi070.stdout: "elector-pinging", 2024-09-14T03:47:49.660 INFO:tasks.workunit.client.0.smithi070.stdout: "quincy" 2024-09-14T03:47:49.660 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.660 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.660 INFO:tasks.workunit.client.0.smithi070.stdout: "ipv4_addr_mons": 6, 2024-09-14T03:47:49.660 INFO:tasks.workunit.client.0.smithi070.stdout: "ipv6_addr_mons": 0, 2024-09-14T03:47:49.660 INFO:tasks.workunit.client.0.smithi070.stdout: "min_mon_release": 17, 2024-09-14T03:47:49.660 INFO:tasks.workunit.client.0.smithi070.stdout: "v1_addr_mons": 3, 2024-09-14T03:47:49.660 INFO:tasks.workunit.client.0.smithi070.stdout: "v2_addr_mons": 3 2024-09-14T03:47:49.661 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.661 INFO:tasks.workunit.client.0.smithi070.stdout: "organization": "ceph-qa", 2024-09-14T03:47:49.661 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:49.661 INFO:tasks.workunit.client.0.smithi070.stdout: "cluster_network": false, 2024-09-14T03:47:49.661 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 8, 2024-09-14T03:47:49.661 INFO:tasks.workunit.client.0.smithi070.stdout: "require_min_compat_client": "luminous", 2024-09-14T03:47:49.661 INFO:tasks.workunit.client.0.smithi070.stdout: "require_osd_release": "quincy" 2024-09-14T03:47:49.661 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.661 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_perf_histograms": [ 2024-09-14T03:47:49.661 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.661 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:47:49.661 INFO:tasks.workunit.client.0.smithi070.stdout: "axes": [ 2024-09-14T03:47:49.661 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.662 INFO:tasks.workunit.client.0.smithi070.stdout: "buckets": 32, 2024-09-14T03:47:49.662 INFO:tasks.workunit.client.0.smithi070.stdout: "min": 0, 2024-09-14T03:47:49.662 INFO:tasks.workunit.client.0.smithi070.stdout: "name": "Latency (usec)", 2024-09-14T03:47:49.662 INFO:tasks.workunit.client.0.smithi070.stdout: "quant_size": 100000, 2024-09-14T03:47:49.662 INFO:tasks.workunit.client.0.smithi070.stdout: "ranges": [ 2024-09-14T03:47:49.662 INFO:tasks.workunit.client.0.smithi070.stdout: "[None, -1]", 2024-09-14T03:47:49.662 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 99999]", 2024-09-14T03:47:49.662 INFO:tasks.workunit.client.0.smithi070.stdout: "[100000, 199999]", 2024-09-14T03:47:49.662 INFO:tasks.workunit.client.0.smithi070.stdout: "[200000, 399999]", 2024-09-14T03:47:49.662 INFO:tasks.workunit.client.0.smithi070.stdout: "[400000, 799999]", 2024-09-14T03:47:49.662 INFO:tasks.workunit.client.0.smithi070.stdout: "[800000, 1599999]", 2024-09-14T03:47:49.662 INFO:tasks.workunit.client.0.smithi070.stdout: "[1600000, 3199999]", 2024-09-14T03:47:49.662 INFO:tasks.workunit.client.0.smithi070.stdout: "[3200000, 6399999]", 2024-09-14T03:47:49.662 INFO:tasks.workunit.client.0.smithi070.stdout: "[6400000, 12799999]", 2024-09-14T03:47:49.663 INFO:tasks.workunit.client.0.smithi070.stdout: "[12800000, 25599999]", 2024-09-14T03:47:49.663 INFO:tasks.workunit.client.0.smithi070.stdout: "[25600000, 51199999]", 2024-09-14T03:47:49.663 INFO:tasks.workunit.client.0.smithi070.stdout: "[51200000, 102399999]", 2024-09-14T03:47:49.663 INFO:tasks.workunit.client.0.smithi070.stdout: "[102400000, 204799999]", 2024-09-14T03:47:49.663 INFO:tasks.workunit.client.0.smithi070.stdout: "[204800000, 409599999]", 2024-09-14T03:47:49.663 INFO:tasks.workunit.client.0.smithi070.stdout: "[409600000, 819199999]", 2024-09-14T03:47:49.663 INFO:tasks.workunit.client.0.smithi070.stdout: "[819200000, 1638399999]", 2024-09-14T03:47:49.663 INFO:tasks.workunit.client.0.smithi070.stdout: "[1638400000, 3276799999]", 2024-09-14T03:47:49.663 INFO:tasks.workunit.client.0.smithi070.stdout: "[3276800000, 6553599999]", 2024-09-14T03:47:49.663 INFO:tasks.workunit.client.0.smithi070.stdout: "[6553600000, 13107199999]", 2024-09-14T03:47:49.663 INFO:tasks.workunit.client.0.smithi070.stdout: "[13107200000, 26214399999]", 2024-09-14T03:47:49.663 INFO:tasks.workunit.client.0.smithi070.stdout: "[26214400000, 52428799999]", 2024-09-14T03:47:49.663 INFO:tasks.workunit.client.0.smithi070.stdout: "[52428800000, 104857599999]", 2024-09-14T03:47:49.664 INFO:tasks.workunit.client.0.smithi070.stdout: "[104857600000, 209715199999]", 2024-09-14T03:47:49.664 INFO:tasks.workunit.client.0.smithi070.stdout: "[209715200000, 419430399999]", 2024-09-14T03:47:49.664 INFO:tasks.workunit.client.0.smithi070.stdout: "[419430400000, 838860799999]", 2024-09-14T03:47:49.664 INFO:tasks.workunit.client.0.smithi070.stdout: "[838860800000, 1677721599999]", 2024-09-14T03:47:49.664 INFO:tasks.workunit.client.0.smithi070.stdout: "[1677721600000, 3355443199999]", 2024-09-14T03:47:49.664 INFO:tasks.workunit.client.0.smithi070.stdout: "[3355443200000, 6710886399999]", 2024-09-14T03:47:49.664 INFO:tasks.workunit.client.0.smithi070.stdout: "[6710886400000, 13421772799999]", 2024-09-14T03:47:49.664 INFO:tasks.workunit.client.0.smithi070.stdout: "[13421772800000, 26843545599999]", 2024-09-14T03:47:49.664 INFO:tasks.workunit.client.0.smithi070.stdout: "[26843545600000, 53687091199999]", 2024-09-14T03:47:49.664 INFO:tasks.workunit.client.0.smithi070.stdout: "[53687091200000, None]" 2024-09-14T03:47:49.664 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:49.664 INFO:tasks.workunit.client.0.smithi070.stdout: "scale_type": "log2" 2024-09-14T03:47:49.665 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.665 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.665 INFO:tasks.workunit.client.0.smithi070.stdout: "buckets": 32, 2024-09-14T03:47:49.665 INFO:tasks.workunit.client.0.smithi070.stdout: "min": 0, 2024-09-14T03:47:49.665 INFO:tasks.workunit.client.0.smithi070.stdout: "name": "Request size (bytes)", 2024-09-14T03:47:49.665 INFO:tasks.workunit.client.0.smithi070.stdout: "quant_size": 512, 2024-09-14T03:47:49.665 INFO:tasks.workunit.client.0.smithi070.stdout: "ranges": [ 2024-09-14T03:47:49.665 INFO:tasks.workunit.client.0.smithi070.stdout: "[None, -1]", 2024-09-14T03:47:49.665 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 511]", 2024-09-14T03:47:49.665 INFO:tasks.workunit.client.0.smithi070.stdout: "[512, 1023]", 2024-09-14T03:47:49.666 INFO:tasks.workunit.client.0.smithi070.stdout: "[1024, 2047]", 2024-09-14T03:47:49.666 INFO:tasks.workunit.client.0.smithi070.stdout: "[2048, 4095]", 2024-09-14T03:47:49.666 INFO:tasks.workunit.client.0.smithi070.stdout: "[4096, 8191]", 2024-09-14T03:47:49.666 INFO:tasks.workunit.client.0.smithi070.stdout: "[8192, 16383]", 2024-09-14T03:47:49.666 INFO:tasks.workunit.client.0.smithi070.stdout: "[16384, 32767]", 2024-09-14T03:47:49.666 INFO:tasks.workunit.client.0.smithi070.stdout: "[32768, 65535]", 2024-09-14T03:47:49.666 INFO:tasks.workunit.client.0.smithi070.stdout: "[65536, 131071]", 2024-09-14T03:47:49.666 INFO:tasks.workunit.client.0.smithi070.stdout: "[131072, 262143]", 2024-09-14T03:47:49.666 INFO:tasks.workunit.client.0.smithi070.stdout: "[262144, 524287]", 2024-09-14T03:47:49.666 INFO:tasks.workunit.client.0.smithi070.stdout: "[524288, 1048575]", 2024-09-14T03:47:49.666 INFO:tasks.workunit.client.0.smithi070.stdout: "[1048576, 2097151]", 2024-09-14T03:47:49.666 INFO:tasks.workunit.client.0.smithi070.stdout: "[2097152, 4194303]", 2024-09-14T03:47:49.666 INFO:tasks.workunit.client.0.smithi070.stdout: "[4194304, 8388607]", 2024-09-14T03:47:49.667 INFO:tasks.workunit.client.0.smithi070.stdout: "[8388608, 16777215]", 2024-09-14T03:47:49.667 INFO:tasks.workunit.client.0.smithi070.stdout: "[16777216, 33554431]", 2024-09-14T03:47:49.667 INFO:tasks.workunit.client.0.smithi070.stdout: "[33554432, 67108863]", 2024-09-14T03:47:49.667 INFO:tasks.workunit.client.0.smithi070.stdout: "[67108864, 134217727]", 2024-09-14T03:47:49.667 INFO:tasks.workunit.client.0.smithi070.stdout: "[134217728, 268435455]", 2024-09-14T03:47:49.667 INFO:tasks.workunit.client.0.smithi070.stdout: "[268435456, 536870911]", 2024-09-14T03:47:49.667 INFO:tasks.workunit.client.0.smithi070.stdout: "[536870912, 1073741823]", 2024-09-14T03:47:49.667 INFO:tasks.workunit.client.0.smithi070.stdout: "[1073741824, 2147483647]", 2024-09-14T03:47:49.667 INFO:tasks.workunit.client.0.smithi070.stdout: "[2147483648, 4294967295]", 2024-09-14T03:47:49.667 INFO:tasks.workunit.client.0.smithi070.stdout: "[4294967296, 8589934591]", 2024-09-14T03:47:49.667 INFO:tasks.workunit.client.0.smithi070.stdout: "[8589934592, 17179869183]", 2024-09-14T03:47:49.668 INFO:tasks.workunit.client.0.smithi070.stdout: "[17179869184, 34359738367]", 2024-09-14T03:47:49.668 INFO:tasks.workunit.client.0.smithi070.stdout: "[34359738368, 68719476735]", 2024-09-14T03:47:49.668 INFO:tasks.workunit.client.0.smithi070.stdout: "[68719476736, 137438953471]", 2024-09-14T03:47:49.668 INFO:tasks.workunit.client.0.smithi070.stdout: "[137438953472, 274877906943]", 2024-09-14T03:47:49.668 INFO:tasks.workunit.client.0.smithi070.stdout: "[274877906944, None]" 2024-09-14T03:47:49.668 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:49.668 INFO:tasks.workunit.client.0.smithi070.stdout: "scale_type": "log2" 2024-09-14T03:47:49.668 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.668 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:49.668 INFO:tasks.workunit.client.0.smithi070.stdout: "osds": [ 2024-09-14T03:47:49.668 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.669 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 0, 2024-09-14T03:47:49.669 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:49.669 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.669 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.669 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.669 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.669 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.669 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.669 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.669 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.670 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.670 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.670 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.670 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.670 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.670 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.670 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.670 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.670 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.670 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.670 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.670 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.670 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.670 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.671 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.671 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.671 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.671 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.671 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.671 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.671 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.671 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.671 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.671 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:49.671 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.671 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.672 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.672 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 1, 2024-09-14T03:47:49.672 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:49.672 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.672 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.672 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.672 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.672 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.672 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.672 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.672 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.672 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.672 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.672 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.673 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.673 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.673 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.673 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.673 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.673 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.673 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.673 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.673 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.673 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.673 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.674 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.674 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.674 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.674 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.674 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.674 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.674 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.674 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.674 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.674 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:49.674 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.674 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.675 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.675 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 2, 2024-09-14T03:47:49.675 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:49.675 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.675 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.675 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.675 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.675 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.675 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.675 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.675 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.675 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.675 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.675 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.676 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.676 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.676 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.676 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.676 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.676 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.676 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.676 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.676 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.676 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.676 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.676 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.676 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.677 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.677 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.677 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.677 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.677 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.677 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.677 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.677 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:49.677 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.678 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.678 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.678 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 3, 2024-09-14T03:47:49.678 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:49.678 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.678 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.678 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.678 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.678 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.678 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.678 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.678 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.678 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.678 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.679 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.679 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.679 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.679 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.679 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.679 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.679 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.679 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.679 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.679 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.679 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.679 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.679 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.679 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.679 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.680 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.680 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.680 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.680 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.680 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.680 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.680 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:49.680 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.680 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.680 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.680 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 4, 2024-09-14T03:47:49.680 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:49.680 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.680 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.681 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.681 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.681 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.681 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.681 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.681 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.681 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.681 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.681 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.681 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.681 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.681 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.681 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.682 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.682 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.682 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.682 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.682 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.682 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.682 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.682 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.682 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.682 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.682 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.682 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.682 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.683 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.683 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.683 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.683 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:49.683 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.683 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.683 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.683 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 5, 2024-09-14T03:47:49.683 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:49.683 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.683 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.683 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.683 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.683 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.684 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.684 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.684 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.684 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.684 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.684 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.684 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.684 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.684 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.684 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.684 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.684 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.685 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.685 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.685 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.685 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.685 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.685 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.685 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.685 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.685 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.685 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.686 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.686 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.686 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.686 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.686 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:49.686 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.686 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.686 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.686 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 6, 2024-09-14T03:47:49.686 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:49.686 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.686 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.687 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.687 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.687 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.687 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.687 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.687 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.687 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.687 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.687 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.687 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.687 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.687 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.687 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.687 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.688 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.688 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.688 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.688 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.688 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.688 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.688 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.688 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.688 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.688 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.688 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.688 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.688 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.688 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.689 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.689 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:49.689 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.689 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.689 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.689 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 7, 2024-09-14T03:47:49.689 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:49.689 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.689 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.689 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 74, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.689 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.689 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.689 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.690 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.690 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.690 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.690 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.690 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.690 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.690 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.690 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.690 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.690 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.690 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.690 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.690 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.690 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.691 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.691 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.691 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.691 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.691 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.691 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.691 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.691 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.691 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.691 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.691 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.692 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:49.692 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.692 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.692 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.692 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.692 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:47:49.692 INFO:tasks.workunit.client.0.smithi070.stdout: "axes": [ 2024-09-14T03:47:49.692 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.692 INFO:tasks.workunit.client.0.smithi070.stdout: "buckets": 32, 2024-09-14T03:47:49.692 INFO:tasks.workunit.client.0.smithi070.stdout: "min": 0, 2024-09-14T03:47:49.692 INFO:tasks.workunit.client.0.smithi070.stdout: "name": "Latency (usec)", 2024-09-14T03:47:49.692 INFO:tasks.workunit.client.0.smithi070.stdout: "quant_size": 100000, 2024-09-14T03:47:49.692 INFO:tasks.workunit.client.0.smithi070.stdout: "ranges": [ 2024-09-14T03:47:49.693 INFO:tasks.workunit.client.0.smithi070.stdout: "[None, -1]", 2024-09-14T03:47:49.693 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 99999]", 2024-09-14T03:47:49.693 INFO:tasks.workunit.client.0.smithi070.stdout: "[100000, 199999]", 2024-09-14T03:47:49.693 INFO:tasks.workunit.client.0.smithi070.stdout: "[200000, 399999]", 2024-09-14T03:47:49.693 INFO:tasks.workunit.client.0.smithi070.stdout: "[400000, 799999]", 2024-09-14T03:47:49.693 INFO:tasks.workunit.client.0.smithi070.stdout: "[800000, 1599999]", 2024-09-14T03:47:49.693 INFO:tasks.workunit.client.0.smithi070.stdout: "[1600000, 3199999]", 2024-09-14T03:47:49.693 INFO:tasks.workunit.client.0.smithi070.stdout: "[3200000, 6399999]", 2024-09-14T03:47:49.693 INFO:tasks.workunit.client.0.smithi070.stdout: "[6400000, 12799999]", 2024-09-14T03:47:49.693 INFO:tasks.workunit.client.0.smithi070.stdout: "[12800000, 25599999]", 2024-09-14T03:47:49.693 INFO:tasks.workunit.client.0.smithi070.stdout: "[25600000, 51199999]", 2024-09-14T03:47:49.693 INFO:tasks.workunit.client.0.smithi070.stdout: "[51200000, 102399999]", 2024-09-14T03:47:49.694 INFO:tasks.workunit.client.0.smithi070.stdout: "[102400000, 204799999]", 2024-09-14T03:47:49.694 INFO:tasks.workunit.client.0.smithi070.stdout: "[204800000, 409599999]", 2024-09-14T03:47:49.694 INFO:tasks.workunit.client.0.smithi070.stdout: "[409600000, 819199999]", 2024-09-14T03:47:49.694 INFO:tasks.workunit.client.0.smithi070.stdout: "[819200000, 1638399999]", 2024-09-14T03:47:49.694 INFO:tasks.workunit.client.0.smithi070.stdout: "[1638400000, 3276799999]", 2024-09-14T03:47:49.694 INFO:tasks.workunit.client.0.smithi070.stdout: "[3276800000, 6553599999]", 2024-09-14T03:47:49.694 INFO:tasks.workunit.client.0.smithi070.stdout: "[6553600000, 13107199999]", 2024-09-14T03:47:49.694 INFO:tasks.workunit.client.0.smithi070.stdout: "[13107200000, 26214399999]", 2024-09-14T03:47:49.694 INFO:tasks.workunit.client.0.smithi070.stdout: "[26214400000, 52428799999]", 2024-09-14T03:47:49.694 INFO:tasks.workunit.client.0.smithi070.stdout: "[52428800000, 104857599999]", 2024-09-14T03:47:49.694 INFO:tasks.workunit.client.0.smithi070.stdout: "[104857600000, 209715199999]", 2024-09-14T03:47:49.694 INFO:tasks.workunit.client.0.smithi070.stdout: "[209715200000, 419430399999]", 2024-09-14T03:47:49.694 INFO:tasks.workunit.client.0.smithi070.stdout: "[419430400000, 838860799999]", 2024-09-14T03:47:49.695 INFO:tasks.workunit.client.0.smithi070.stdout: "[838860800000, 1677721599999]", 2024-09-14T03:47:49.695 INFO:tasks.workunit.client.0.smithi070.stdout: "[1677721600000, 3355443199999]", 2024-09-14T03:47:49.695 INFO:tasks.workunit.client.0.smithi070.stdout: "[3355443200000, 6710886399999]", 2024-09-14T03:47:49.695 INFO:tasks.workunit.client.0.smithi070.stdout: "[6710886400000, 13421772799999]", 2024-09-14T03:47:49.695 INFO:tasks.workunit.client.0.smithi070.stdout: "[13421772800000, 26843545599999]", 2024-09-14T03:47:49.695 INFO:tasks.workunit.client.0.smithi070.stdout: "[26843545600000, 53687091199999]", 2024-09-14T03:47:49.695 INFO:tasks.workunit.client.0.smithi070.stdout: "[53687091200000, None]" 2024-09-14T03:47:49.695 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:49.695 INFO:tasks.workunit.client.0.smithi070.stdout: "scale_type": "log2" 2024-09-14T03:47:49.695 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.695 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.695 INFO:tasks.workunit.client.0.smithi070.stdout: "buckets": 32, 2024-09-14T03:47:49.695 INFO:tasks.workunit.client.0.smithi070.stdout: "min": 0, 2024-09-14T03:47:49.695 INFO:tasks.workunit.client.0.smithi070.stdout: "name": "Request size (bytes)", 2024-09-14T03:47:49.696 INFO:tasks.workunit.client.0.smithi070.stdout: "quant_size": 512, 2024-09-14T03:47:49.696 INFO:tasks.workunit.client.0.smithi070.stdout: "ranges": [ 2024-09-14T03:47:49.696 INFO:tasks.workunit.client.0.smithi070.stdout: "[None, -1]", 2024-09-14T03:47:49.696 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 511]", 2024-09-14T03:47:49.696 INFO:tasks.workunit.client.0.smithi070.stdout: "[512, 1023]", 2024-09-14T03:47:49.696 INFO:tasks.workunit.client.0.smithi070.stdout: "[1024, 2047]", 2024-09-14T03:47:49.696 INFO:tasks.workunit.client.0.smithi070.stdout: "[2048, 4095]", 2024-09-14T03:47:49.696 INFO:tasks.workunit.client.0.smithi070.stdout: "[4096, 8191]", 2024-09-14T03:47:49.696 INFO:tasks.workunit.client.0.smithi070.stdout: "[8192, 16383]", 2024-09-14T03:47:49.696 INFO:tasks.workunit.client.0.smithi070.stdout: "[16384, 32767]", 2024-09-14T03:47:49.696 INFO:tasks.workunit.client.0.smithi070.stdout: "[32768, 65535]", 2024-09-14T03:47:49.696 INFO:tasks.workunit.client.0.smithi070.stdout: "[65536, 131071]", 2024-09-14T03:47:49.696 INFO:tasks.workunit.client.0.smithi070.stdout: "[131072, 262143]", 2024-09-14T03:47:49.697 INFO:tasks.workunit.client.0.smithi070.stdout: "[262144, 524287]", 2024-09-14T03:47:49.697 INFO:tasks.workunit.client.0.smithi070.stdout: "[524288, 1048575]", 2024-09-14T03:47:49.697 INFO:tasks.workunit.client.0.smithi070.stdout: "[1048576, 2097151]", 2024-09-14T03:47:49.697 INFO:tasks.workunit.client.0.smithi070.stdout: "[2097152, 4194303]", 2024-09-14T03:47:49.697 INFO:tasks.workunit.client.0.smithi070.stdout: "[4194304, 8388607]", 2024-09-14T03:47:49.697 INFO:tasks.workunit.client.0.smithi070.stdout: "[8388608, 16777215]", 2024-09-14T03:47:49.697 INFO:tasks.workunit.client.0.smithi070.stdout: "[16777216, 33554431]", 2024-09-14T03:47:49.697 INFO:tasks.workunit.client.0.smithi070.stdout: "[33554432, 67108863]", 2024-09-14T03:47:49.697 INFO:tasks.workunit.client.0.smithi070.stdout: "[67108864, 134217727]", 2024-09-14T03:47:49.697 INFO:tasks.workunit.client.0.smithi070.stdout: "[134217728, 268435455]", 2024-09-14T03:47:49.697 INFO:tasks.workunit.client.0.smithi070.stdout: "[268435456, 536870911]", 2024-09-14T03:47:49.698 INFO:tasks.workunit.client.0.smithi070.stdout: "[536870912, 1073741823]", 2024-09-14T03:47:49.698 INFO:tasks.workunit.client.0.smithi070.stdout: "[1073741824, 2147483647]", 2024-09-14T03:47:49.698 INFO:tasks.workunit.client.0.smithi070.stdout: "[2147483648, 4294967295]", 2024-09-14T03:47:49.698 INFO:tasks.workunit.client.0.smithi070.stdout: "[4294967296, 8589934591]", 2024-09-14T03:47:49.698 INFO:tasks.workunit.client.0.smithi070.stdout: "[8589934592, 17179869183]", 2024-09-14T03:47:49.698 INFO:tasks.workunit.client.0.smithi070.stdout: "[17179869184, 34359738367]", 2024-09-14T03:47:49.698 INFO:tasks.workunit.client.0.smithi070.stdout: "[34359738368, 68719476735]", 2024-09-14T03:47:49.699 INFO:tasks.workunit.client.0.smithi070.stdout: "[68719476736, 137438953471]", 2024-09-14T03:47:49.699 INFO:tasks.workunit.client.0.smithi070.stdout: "[137438953472, 274877906943]", 2024-09-14T03:47:49.699 INFO:tasks.workunit.client.0.smithi070.stdout: "[274877906944, None]" 2024-09-14T03:47:49.699 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:49.699 INFO:tasks.workunit.client.0.smithi070.stdout: "scale_type": "log2" 2024-09-14T03:47:49.699 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.699 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:49.699 INFO:tasks.workunit.client.0.smithi070.stdout: "osds": [ 2024-09-14T03:47:49.699 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.699 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 0, 2024-09-14T03:47:49.699 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:49.700 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.700 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.700 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.700 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.700 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.700 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.700 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.700 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.700 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.700 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.700 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.700 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.701 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.701 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.701 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.701 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.701 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.701 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.701 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.701 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.701 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.701 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.701 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.702 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.702 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.702 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.702 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.702 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.702 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.702 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.702 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.702 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:49.702 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.703 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.703 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.703 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 1, 2024-09-14T03:47:49.703 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:49.703 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.703 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.703 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.703 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.703 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.703 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.703 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.703 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.703 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.703 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.703 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.704 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.704 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.704 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.704 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.704 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.704 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.704 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.704 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.704 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.704 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.704 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.704 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.704 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.704 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.704 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.705 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.705 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.705 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.705 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.705 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.705 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:49.705 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.705 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.705 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.705 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 2, 2024-09-14T03:47:49.705 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:49.705 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.705 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.706 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.706 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.706 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.706 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.706 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.706 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.706 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.706 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.706 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.706 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.706 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.706 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.706 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.706 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.707 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.707 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.707 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.707 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.707 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.707 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.707 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.707 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.707 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.707 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.707 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.707 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.708 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.708 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.708 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.708 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:49.708 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.708 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.708 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.708 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 3, 2024-09-14T03:47:49.708 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:49.708 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.708 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.708 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.708 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.709 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.709 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.709 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.709 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.709 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.709 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.709 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.709 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.709 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.709 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.709 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.710 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.710 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.710 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.710 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.710 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.710 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.710 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.710 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.710 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.710 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.710 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.710 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.710 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.710 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.711 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.711 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.711 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:49.711 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.711 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.711 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.711 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 4, 2024-09-14T03:47:49.711 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:49.711 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.711 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.711 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.711 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.711 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.711 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.712 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.712 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.712 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.712 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.712 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.712 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.712 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.712 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.712 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.712 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.713 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.713 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.713 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.713 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.713 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.713 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.713 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.713 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.713 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.713 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.713 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.714 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.714 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.714 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.714 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.714 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:49.714 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.714 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.714 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.714 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 5, 2024-09-14T03:47:49.714 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:49.714 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.714 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.714 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.714 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.715 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.715 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.715 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.715 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.715 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.715 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.715 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.715 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.715 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.715 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.715 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.715 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.715 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.716 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.716 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.716 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.716 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.716 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.716 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.716 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.716 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.716 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.716 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.716 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.716 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.716 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.717 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.717 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:49.717 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.717 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.717 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.717 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 6, 2024-09-14T03:47:49.717 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:49.717 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.717 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.717 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.717 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.717 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.717 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.717 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.718 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.718 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.718 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.718 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.718 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.718 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.718 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.718 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.718 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.718 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.718 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.718 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.719 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.719 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.719 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.719 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.719 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.719 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.719 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.719 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.719 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.719 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.719 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.720 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.720 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:49.720 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.720 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.720 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.720 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 7, 2024-09-14T03:47:49.720 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:49.720 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.720 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.720 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.721 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.721 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.721 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.721 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.721 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.721 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.721 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.721 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.721 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.721 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.721 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.721 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.721 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.722 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.722 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.722 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.722 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.722 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.722 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.722 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.722 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.722 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.722 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.722 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.722 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.722 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.722 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.723 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.723 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:49.723 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.723 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.723 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.723 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.723 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:47:49.723 INFO:tasks.workunit.client.0.smithi070.stdout: "axes": [ 2024-09-14T03:47:49.723 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.723 INFO:tasks.workunit.client.0.smithi070.stdout: "buckets": 32, 2024-09-14T03:47:49.723 INFO:tasks.workunit.client.0.smithi070.stdout: "min": 0, 2024-09-14T03:47:49.723 INFO:tasks.workunit.client.0.smithi070.stdout: "name": "Latency (usec)", 2024-09-14T03:47:49.723 INFO:tasks.workunit.client.0.smithi070.stdout: "quant_size": 100000, 2024-09-14T03:47:49.723 INFO:tasks.workunit.client.0.smithi070.stdout: "ranges": [ 2024-09-14T03:47:49.724 INFO:tasks.workunit.client.0.smithi070.stdout: "[None, -1]", 2024-09-14T03:47:49.724 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 99999]", 2024-09-14T03:47:49.724 INFO:tasks.workunit.client.0.smithi070.stdout: "[100000, 199999]", 2024-09-14T03:47:49.724 INFO:tasks.workunit.client.0.smithi070.stdout: "[200000, 399999]", 2024-09-14T03:47:49.724 INFO:tasks.workunit.client.0.smithi070.stdout: "[400000, 799999]", 2024-09-14T03:47:49.724 INFO:tasks.workunit.client.0.smithi070.stdout: "[800000, 1599999]", 2024-09-14T03:47:49.724 INFO:tasks.workunit.client.0.smithi070.stdout: "[1600000, 3199999]", 2024-09-14T03:47:49.724 INFO:tasks.workunit.client.0.smithi070.stdout: "[3200000, 6399999]", 2024-09-14T03:47:49.724 INFO:tasks.workunit.client.0.smithi070.stdout: "[6400000, 12799999]", 2024-09-14T03:47:49.724 INFO:tasks.workunit.client.0.smithi070.stdout: "[12800000, 25599999]", 2024-09-14T03:47:49.724 INFO:tasks.workunit.client.0.smithi070.stdout: "[25600000, 51199999]", 2024-09-14T03:47:49.724 INFO:tasks.workunit.client.0.smithi070.stdout: "[51200000, 102399999]", 2024-09-14T03:47:49.724 INFO:tasks.workunit.client.0.smithi070.stdout: "[102400000, 204799999]", 2024-09-14T03:47:49.724 INFO:tasks.workunit.client.0.smithi070.stdout: "[204800000, 409599999]", 2024-09-14T03:47:49.725 INFO:tasks.workunit.client.0.smithi070.stdout: "[409600000, 819199999]", 2024-09-14T03:47:49.725 INFO:tasks.workunit.client.0.smithi070.stdout: "[819200000, 1638399999]", 2024-09-14T03:47:49.725 INFO:tasks.workunit.client.0.smithi070.stdout: "[1638400000, 3276799999]", 2024-09-14T03:47:49.725 INFO:tasks.workunit.client.0.smithi070.stdout: "[3276800000, 6553599999]", 2024-09-14T03:47:49.725 INFO:tasks.workunit.client.0.smithi070.stdout: "[6553600000, 13107199999]", 2024-09-14T03:47:49.725 INFO:tasks.workunit.client.0.smithi070.stdout: "[13107200000, 26214399999]", 2024-09-14T03:47:49.725 INFO:tasks.workunit.client.0.smithi070.stdout: "[26214400000, 52428799999]", 2024-09-14T03:47:49.725 INFO:tasks.workunit.client.0.smithi070.stdout: "[52428800000, 104857599999]", 2024-09-14T03:47:49.725 INFO:tasks.workunit.client.0.smithi070.stdout: "[104857600000, 209715199999]", 2024-09-14T03:47:49.725 INFO:tasks.workunit.client.0.smithi070.stdout: "[209715200000, 419430399999]", 2024-09-14T03:47:49.725 INFO:tasks.workunit.client.0.smithi070.stdout: "[419430400000, 838860799999]", 2024-09-14T03:47:49.725 INFO:tasks.workunit.client.0.smithi070.stdout: "[838860800000, 1677721599999]", 2024-09-14T03:47:49.725 INFO:tasks.workunit.client.0.smithi070.stdout: "[1677721600000, 3355443199999]", 2024-09-14T03:47:49.726 INFO:tasks.workunit.client.0.smithi070.stdout: "[3355443200000, 6710886399999]", 2024-09-14T03:47:49.726 INFO:tasks.workunit.client.0.smithi070.stdout: "[6710886400000, 13421772799999]", 2024-09-14T03:47:49.726 INFO:tasks.workunit.client.0.smithi070.stdout: "[13421772800000, 26843545599999]", 2024-09-14T03:47:49.726 INFO:tasks.workunit.client.0.smithi070.stdout: "[26843545600000, 53687091199999]", 2024-09-14T03:47:49.726 INFO:tasks.workunit.client.0.smithi070.stdout: "[53687091200000, None]" 2024-09-14T03:47:49.726 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:49.726 INFO:tasks.workunit.client.0.smithi070.stdout: "scale_type": "log2" 2024-09-14T03:47:49.726 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.726 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.726 INFO:tasks.workunit.client.0.smithi070.stdout: "buckets": 32, 2024-09-14T03:47:49.726 INFO:tasks.workunit.client.0.smithi070.stdout: "min": 0, 2024-09-14T03:47:49.726 INFO:tasks.workunit.client.0.smithi070.stdout: "name": "Request size (bytes)", 2024-09-14T03:47:49.727 INFO:tasks.workunit.client.0.smithi070.stdout: "quant_size": 512, 2024-09-14T03:47:49.727 INFO:tasks.workunit.client.0.smithi070.stdout: "ranges": [ 2024-09-14T03:47:49.727 INFO:tasks.workunit.client.0.smithi070.stdout: "[None, -1]", 2024-09-14T03:47:49.727 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 511]", 2024-09-14T03:47:49.727 INFO:tasks.workunit.client.0.smithi070.stdout: "[512, 1023]", 2024-09-14T03:47:49.727 INFO:tasks.workunit.client.0.smithi070.stdout: "[1024, 2047]", 2024-09-14T03:47:49.727 INFO:tasks.workunit.client.0.smithi070.stdout: "[2048, 4095]", 2024-09-14T03:47:49.727 INFO:tasks.workunit.client.0.smithi070.stdout: "[4096, 8191]", 2024-09-14T03:47:49.727 INFO:tasks.workunit.client.0.smithi070.stdout: "[8192, 16383]", 2024-09-14T03:47:49.727 INFO:tasks.workunit.client.0.smithi070.stdout: "[16384, 32767]", 2024-09-14T03:47:49.727 INFO:tasks.workunit.client.0.smithi070.stdout: "[32768, 65535]", 2024-09-14T03:47:49.727 INFO:tasks.workunit.client.0.smithi070.stdout: "[65536, 131071]", 2024-09-14T03:47:49.727 INFO:tasks.workunit.client.0.smithi070.stdout: "[131072, 262143]", 2024-09-14T03:47:49.728 INFO:tasks.workunit.client.0.smithi070.stdout: "[262144, 524287]", 2024-09-14T03:47:49.728 INFO:tasks.workunit.client.0.smithi070.stdout: "[524288, 1048575]", 2024-09-14T03:47:49.728 INFO:tasks.workunit.client.0.smithi070.stdout: "[1048576, 2097151]", 2024-09-14T03:47:49.728 INFO:tasks.workunit.client.0.smithi070.stdout: "[2097152, 4194303]", 2024-09-14T03:47:49.728 INFO:tasks.workunit.client.0.smithi070.stdout: "[4194304, 8388607]", 2024-09-14T03:47:49.728 INFO:tasks.workunit.client.0.smithi070.stdout: "[8388608, 16777215]", 2024-09-14T03:47:49.728 INFO:tasks.workunit.client.0.smithi070.stdout: "[16777216, 33554431]", 2024-09-14T03:47:49.728 INFO:tasks.workunit.client.0.smithi070.stdout: "[33554432, 67108863]", 2024-09-14T03:47:49.728 INFO:tasks.workunit.client.0.smithi070.stdout: "[67108864, 134217727]", 2024-09-14T03:47:49.728 INFO:tasks.workunit.client.0.smithi070.stdout: "[134217728, 268435455]", 2024-09-14T03:47:49.728 INFO:tasks.workunit.client.0.smithi070.stdout: "[268435456, 536870911]", 2024-09-14T03:47:49.728 INFO:tasks.workunit.client.0.smithi070.stdout: "[536870912, 1073741823]", 2024-09-14T03:47:49.728 INFO:tasks.workunit.client.0.smithi070.stdout: "[1073741824, 2147483647]", 2024-09-14T03:47:49.729 INFO:tasks.workunit.client.0.smithi070.stdout: "[2147483648, 4294967295]", 2024-09-14T03:47:49.729 INFO:tasks.workunit.client.0.smithi070.stdout: "[4294967296, 8589934591]", 2024-09-14T03:47:49.729 INFO:tasks.workunit.client.0.smithi070.stdout: "[8589934592, 17179869183]", 2024-09-14T03:47:49.729 INFO:tasks.workunit.client.0.smithi070.stdout: "[17179869184, 34359738367]", 2024-09-14T03:47:49.729 INFO:tasks.workunit.client.0.smithi070.stdout: "[34359738368, 68719476735]", 2024-09-14T03:47:49.729 INFO:tasks.workunit.client.0.smithi070.stdout: "[68719476736, 137438953471]", 2024-09-14T03:47:49.729 INFO:tasks.workunit.client.0.smithi070.stdout: "[137438953472, 274877906943]", 2024-09-14T03:47:49.729 INFO:tasks.workunit.client.0.smithi070.stdout: "[274877906944, None]" 2024-09-14T03:47:49.729 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:49.729 INFO:tasks.workunit.client.0.smithi070.stdout: "scale_type": "log2" 2024-09-14T03:47:49.729 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.729 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:49.729 INFO:tasks.workunit.client.0.smithi070.stdout: "osds": [ 2024-09-14T03:47:49.730 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.730 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 0, 2024-09-14T03:47:49.730 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:49.730 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.730 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.730 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.730 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.730 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.730 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.730 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.730 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.730 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.730 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.731 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.731 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.731 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.731 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.731 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.731 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.731 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.731 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.731 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.731 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.732 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.732 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.732 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.732 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.732 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.732 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.732 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.732 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.732 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.732 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.732 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.732 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:49.733 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.733 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.733 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.733 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 1, 2024-09-14T03:47:49.733 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:49.733 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.733 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.733 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.733 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.733 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.733 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.733 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.734 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.734 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.734 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.734 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.734 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.734 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.734 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.734 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.734 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.734 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.734 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.734 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.734 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.735 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.735 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.735 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.735 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.735 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.735 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.735 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.735 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.735 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.735 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.735 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.735 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:49.736 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.736 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.736 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.736 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 2, 2024-09-14T03:47:49.736 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:49.736 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.736 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.736 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.736 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.737 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.737 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.737 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.737 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.737 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.737 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.737 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.737 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.737 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.737 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.737 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.737 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.738 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.738 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.738 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.738 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.738 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.738 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.738 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.738 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.738 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.738 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.738 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.738 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.738 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.739 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.739 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.739 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:49.739 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.739 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.739 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.739 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 3, 2024-09-14T03:47:49.739 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:49.739 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.739 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.739 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.739 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.740 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.740 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.740 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.740 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.740 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.740 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.740 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.740 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.740 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.740 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.740 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.741 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.741 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.741 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.741 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.741 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.741 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.741 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.741 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.741 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.741 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.741 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.741 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.741 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.742 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.742 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.742 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.742 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:49.742 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.742 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.742 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.742 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 4, 2024-09-14T03:47:49.742 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:49.742 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.742 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.742 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.742 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.743 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.743 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.743 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.743 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.743 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.743 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.743 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.743 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.743 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.743 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.743 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.743 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.743 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.744 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.744 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.744 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.744 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.744 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.744 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.744 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.744 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.744 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.744 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.744 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.744 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.744 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.745 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.745 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:49.745 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.745 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.745 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.745 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 5, 2024-09-14T03:47:49.745 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:49.746 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.746 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.746 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.746 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.746 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.746 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.746 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.746 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.747 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.747 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.747 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.747 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.747 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.747 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.747 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.747 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.747 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.747 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.747 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.747 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.748 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.748 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.748 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.748 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.748 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.749 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.749 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.749 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.749 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.749 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.749 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.750 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:49.750 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.750 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.750 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.750 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 6, 2024-09-14T03:47:49.750 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:49.751 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.751 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.751 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.751 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.751 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.751 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.751 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.751 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.751 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.751 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.752 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.752 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.752 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.752 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.752 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.753 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.753 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.753 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.753 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.753 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.753 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.753 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.753 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.753 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.753 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.753 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.753 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.754 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.754 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.754 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.754 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.754 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:49.754 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.754 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.754 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.754 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 7, 2024-09-14T03:47:49.754 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:49.754 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.755 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.755 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.755 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.755 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.755 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.755 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.755 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.755 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.755 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.755 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.755 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.755 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.756 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.756 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.756 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.756 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.756 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.756 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.756 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.756 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.756 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.756 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.756 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.756 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.756 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.757 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.757 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.757 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.757 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.757 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.757 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:49.757 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.757 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.757 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.757 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.757 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:47:49.757 INFO:tasks.workunit.client.0.smithi070.stdout: "axes": [ 2024-09-14T03:47:49.757 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.757 INFO:tasks.workunit.client.0.smithi070.stdout: "buckets": 32, 2024-09-14T03:47:49.758 INFO:tasks.workunit.client.0.smithi070.stdout: "min": 0, 2024-09-14T03:47:49.758 INFO:tasks.workunit.client.0.smithi070.stdout: "name": "Latency (usec)", 2024-09-14T03:47:49.758 INFO:tasks.workunit.client.0.smithi070.stdout: "quant_size": 100000, 2024-09-14T03:47:49.758 INFO:tasks.workunit.client.0.smithi070.stdout: "ranges": [ 2024-09-14T03:47:49.758 INFO:tasks.workunit.client.0.smithi070.stdout: "[None, -1]", 2024-09-14T03:47:49.758 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 99999]", 2024-09-14T03:47:49.758 INFO:tasks.workunit.client.0.smithi070.stdout: "[100000, 199999]", 2024-09-14T03:47:49.758 INFO:tasks.workunit.client.0.smithi070.stdout: "[200000, 399999]", 2024-09-14T03:47:49.758 INFO:tasks.workunit.client.0.smithi070.stdout: "[400000, 799999]", 2024-09-14T03:47:49.758 INFO:tasks.workunit.client.0.smithi070.stdout: "[800000, 1599999]", 2024-09-14T03:47:49.758 INFO:tasks.workunit.client.0.smithi070.stdout: "[1600000, 3199999]", 2024-09-14T03:47:49.758 INFO:tasks.workunit.client.0.smithi070.stdout: "[3200000, 6399999]", 2024-09-14T03:47:49.758 INFO:tasks.workunit.client.0.smithi070.stdout: "[6400000, 12799999]", 2024-09-14T03:47:49.758 INFO:tasks.workunit.client.0.smithi070.stdout: "[12800000, 25599999]", 2024-09-14T03:47:49.758 INFO:tasks.workunit.client.0.smithi070.stdout: "[25600000, 51199999]", 2024-09-14T03:47:49.759 INFO:tasks.workunit.client.0.smithi070.stdout: "[51200000, 102399999]", 2024-09-14T03:47:49.759 INFO:tasks.workunit.client.0.smithi070.stdout: "[102400000, 204799999]", 2024-09-14T03:47:49.759 INFO:tasks.workunit.client.0.smithi070.stdout: "[204800000, 409599999]", 2024-09-14T03:47:49.759 INFO:tasks.workunit.client.0.smithi070.stdout: "[409600000, 819199999]", 2024-09-14T03:47:49.759 INFO:tasks.workunit.client.0.smithi070.stdout: "[819200000, 1638399999]", 2024-09-14T03:47:49.759 INFO:tasks.workunit.client.0.smithi070.stdout: "[1638400000, 3276799999]", 2024-09-14T03:47:49.759 INFO:tasks.workunit.client.0.smithi070.stdout: "[3276800000, 6553599999]", 2024-09-14T03:47:49.759 INFO:tasks.workunit.client.0.smithi070.stdout: "[6553600000, 13107199999]", 2024-09-14T03:47:49.759 INFO:tasks.workunit.client.0.smithi070.stdout: "[13107200000, 26214399999]", 2024-09-14T03:47:49.759 INFO:tasks.workunit.client.0.smithi070.stdout: "[26214400000, 52428799999]", 2024-09-14T03:47:49.759 INFO:tasks.workunit.client.0.smithi070.stdout: "[52428800000, 104857599999]", 2024-09-14T03:47:49.759 INFO:tasks.workunit.client.0.smithi070.stdout: "[104857600000, 209715199999]", 2024-09-14T03:47:49.759 INFO:tasks.workunit.client.0.smithi070.stdout: "[209715200000, 419430399999]", 2024-09-14T03:47:49.759 INFO:tasks.workunit.client.0.smithi070.stdout: "[419430400000, 838860799999]", 2024-09-14T03:47:49.759 INFO:tasks.workunit.client.0.smithi070.stdout: "[838860800000, 1677721599999]", 2024-09-14T03:47:49.760 INFO:tasks.workunit.client.0.smithi070.stdout: "[1677721600000, 3355443199999]", 2024-09-14T03:47:49.760 INFO:tasks.workunit.client.0.smithi070.stdout: "[3355443200000, 6710886399999]", 2024-09-14T03:47:49.760 INFO:tasks.workunit.client.0.smithi070.stdout: "[6710886400000, 13421772799999]", 2024-09-14T03:47:49.760 INFO:tasks.workunit.client.0.smithi070.stdout: "[13421772800000, 26843545599999]", 2024-09-14T03:47:49.760 INFO:tasks.workunit.client.0.smithi070.stdout: "[26843545600000, 53687091199999]", 2024-09-14T03:47:49.760 INFO:tasks.workunit.client.0.smithi070.stdout: "[53687091200000, None]" 2024-09-14T03:47:49.760 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:49.760 INFO:tasks.workunit.client.0.smithi070.stdout: "scale_type": "log2" 2024-09-14T03:47:49.760 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.760 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.760 INFO:tasks.workunit.client.0.smithi070.stdout: "buckets": 32, 2024-09-14T03:47:49.760 INFO:tasks.workunit.client.0.smithi070.stdout: "min": 0, 2024-09-14T03:47:49.760 INFO:tasks.workunit.client.0.smithi070.stdout: "name": "Request size (bytes)", 2024-09-14T03:47:49.760 INFO:tasks.workunit.client.0.smithi070.stdout: "quant_size": 512, 2024-09-14T03:47:49.761 INFO:tasks.workunit.client.0.smithi070.stdout: "ranges": [ 2024-09-14T03:47:49.761 INFO:tasks.workunit.client.0.smithi070.stdout: "[None, -1]", 2024-09-14T03:47:49.761 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 511]", 2024-09-14T03:47:49.761 INFO:tasks.workunit.client.0.smithi070.stdout: "[512, 1023]", 2024-09-14T03:47:49.761 INFO:tasks.workunit.client.0.smithi070.stdout: "[1024, 2047]", 2024-09-14T03:47:49.761 INFO:tasks.workunit.client.0.smithi070.stdout: "[2048, 4095]", 2024-09-14T03:47:49.761 INFO:tasks.workunit.client.0.smithi070.stdout: "[4096, 8191]", 2024-09-14T03:47:49.761 INFO:tasks.workunit.client.0.smithi070.stdout: "[8192, 16383]", 2024-09-14T03:47:49.761 INFO:tasks.workunit.client.0.smithi070.stdout: "[16384, 32767]", 2024-09-14T03:47:49.761 INFO:tasks.workunit.client.0.smithi070.stdout: "[32768, 65535]", 2024-09-14T03:47:49.761 INFO:tasks.workunit.client.0.smithi070.stdout: "[65536, 131071]", 2024-09-14T03:47:49.761 INFO:tasks.workunit.client.0.smithi070.stdout: "[131072, 262143]", 2024-09-14T03:47:49.761 INFO:tasks.workunit.client.0.smithi070.stdout: "[262144, 524287]", 2024-09-14T03:47:49.761 INFO:tasks.workunit.client.0.smithi070.stdout: "[524288, 1048575]", 2024-09-14T03:47:49.762 INFO:tasks.workunit.client.0.smithi070.stdout: "[1048576, 2097151]", 2024-09-14T03:47:49.762 INFO:tasks.workunit.client.0.smithi070.stdout: "[2097152, 4194303]", 2024-09-14T03:47:49.762 INFO:tasks.workunit.client.0.smithi070.stdout: "[4194304, 8388607]", 2024-09-14T03:47:49.762 INFO:tasks.workunit.client.0.smithi070.stdout: "[8388608, 16777215]", 2024-09-14T03:47:49.762 INFO:tasks.workunit.client.0.smithi070.stdout: "[16777216, 33554431]", 2024-09-14T03:47:49.762 INFO:tasks.workunit.client.0.smithi070.stdout: "[33554432, 67108863]", 2024-09-14T03:47:49.762 INFO:tasks.workunit.client.0.smithi070.stdout: "[67108864, 134217727]", 2024-09-14T03:47:49.762 INFO:tasks.workunit.client.0.smithi070.stdout: "[134217728, 268435455]", 2024-09-14T03:47:49.762 INFO:tasks.workunit.client.0.smithi070.stdout: "[268435456, 536870911]", 2024-09-14T03:47:49.762 INFO:tasks.workunit.client.0.smithi070.stdout: "[536870912, 1073741823]", 2024-09-14T03:47:49.762 INFO:tasks.workunit.client.0.smithi070.stdout: "[1073741824, 2147483647]", 2024-09-14T03:47:49.762 INFO:tasks.workunit.client.0.smithi070.stdout: "[2147483648, 4294967295]", 2024-09-14T03:47:49.762 INFO:tasks.workunit.client.0.smithi070.stdout: "[4294967296, 8589934591]", 2024-09-14T03:47:49.763 INFO:tasks.workunit.client.0.smithi070.stdout: "[8589934592, 17179869183]", 2024-09-14T03:47:49.763 INFO:tasks.workunit.client.0.smithi070.stdout: "[17179869184, 34359738367]", 2024-09-14T03:47:49.763 INFO:tasks.workunit.client.0.smithi070.stdout: "[34359738368, 68719476735]", 2024-09-14T03:47:49.763 INFO:tasks.workunit.client.0.smithi070.stdout: "[68719476736, 137438953471]", 2024-09-14T03:47:49.763 INFO:tasks.workunit.client.0.smithi070.stdout: "[137438953472, 274877906943]", 2024-09-14T03:47:49.763 INFO:tasks.workunit.client.0.smithi070.stdout: "[274877906944, None]" 2024-09-14T03:47:49.763 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:49.763 INFO:tasks.workunit.client.0.smithi070.stdout: "scale_type": "log2" 2024-09-14T03:47:49.763 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.763 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:49.763 INFO:tasks.workunit.client.0.smithi070.stdout: "osds": [ 2024-09-14T03:47:49.763 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.763 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 0, 2024-09-14T03:47:49.764 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:49.764 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.764 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.764 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.764 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.764 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.764 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.764 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.764 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.764 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.764 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.764 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.765 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.765 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.765 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.765 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.765 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.765 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.765 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.765 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.765 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.765 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.765 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.765 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.766 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.766 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.766 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.766 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.766 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.766 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.766 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.766 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.766 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:49.766 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.766 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.767 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.767 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 1, 2024-09-14T03:47:49.767 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:49.767 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.767 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.767 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.767 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.767 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.767 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 4, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.767 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 6, 2, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.767 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 11, 1, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.767 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 9, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.768 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.768 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.768 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.768 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.768 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.768 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.768 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.768 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.768 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.768 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.769 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.769 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.769 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.769 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.769 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.769 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.769 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.769 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.769 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.769 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.769 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.770 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.770 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:49.770 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.770 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.770 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.770 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 2, 2024-09-14T03:47:49.770 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:49.770 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.770 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.770 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.770 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.770 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.770 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.770 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.771 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.771 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.771 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.771 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.771 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.771 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.771 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.771 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.771 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.771 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.772 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.772 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.772 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.772 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.772 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.772 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.772 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.772 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.772 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.772 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.772 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.772 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.772 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.772 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.773 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:49.773 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.773 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.773 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.773 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 3, 2024-09-14T03:47:49.773 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:49.773 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.773 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.773 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.773 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.773 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.773 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.774 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.774 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.774 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.774 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.774 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.774 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.774 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.774 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.774 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.774 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.774 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.774 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.775 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.775 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.775 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.775 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.775 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.775 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.775 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.775 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.775 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.775 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.775 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.775 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.775 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.776 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:49.776 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.776 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.776 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.776 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 4, 2024-09-14T03:47:49.776 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:49.776 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.776 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.776 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.776 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.776 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.776 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.776 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.776 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.777 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.777 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.777 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.777 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.777 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.777 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.777 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.777 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.777 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.778 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.778 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.778 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.778 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.778 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.778 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.778 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.778 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.778 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.778 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.778 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.778 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.779 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.779 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.779 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:49.779 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.779 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.779 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.779 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 5, 2024-09-14T03:47:49.780 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:49.780 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.780 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.780 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.780 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.780 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.780 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.780 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.780 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.780 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.781 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.781 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.781 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.781 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.781 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.781 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.781 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.781 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.781 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.782 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.782 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.782 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.782 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.782 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.782 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.783 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.783 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.783 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.783 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.783 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.784 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.784 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.784 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:49.784 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.784 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.785 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.785 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 6, 2024-09-14T03:47:49.785 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:49.785 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.785 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.785 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.785 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.785 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.785 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.785 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.786 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.786 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.786 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.786 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.786 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.786 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.786 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.786 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.786 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.786 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.786 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.786 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.786 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.786 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.787 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.787 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.787 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.787 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.787 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.787 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.787 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.787 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.787 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.787 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.787 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:49.787 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.787 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.787 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:49.787 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 7, 2024-09-14T03:47:49.787 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:49.788 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.788 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.788 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.788 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.788 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.788 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.788 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.788 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.788 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.788 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.788 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.788 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.789 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.789 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.789 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.789 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.789 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.789 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.789 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.789 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.789 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.789 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.789 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.790 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.790 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.790 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.790 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.790 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.790 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.790 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.790 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:49.790 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:49.791 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.791 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.791 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:49.791 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.791 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.791 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:49.791 INFO:tasks.workunit.client.0.smithi070.stdout: "perf_counters": { 2024-09-14T03:47:49.791 INFO:tasks.workunit.client.0.smithi070.stdout: "mds.a8e17c6c2bb7d35df7944161574528c366e6a388": { 2024-09-14T03:47:49.791 INFO:tasks.workunit.client.0.smithi070.stdout: "mds": { 2024-09-14T03:47:49.791 INFO:tasks.workunit.client.0.smithi070.stdout: "caps": { 2024-09-14T03:47:49.791 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.791 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.791 INFO:tasks.workunit.client.0.smithi070.stdout: "ceph_cap_op_flush_ack": { 2024-09-14T03:47:49.791 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.792 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.792 INFO:tasks.workunit.client.0.smithi070.stdout: "ceph_cap_op_flushsnap_ack": { 2024-09-14T03:47:49.792 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.792 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.792 INFO:tasks.workunit.client.0.smithi070.stdout: "ceph_cap_op_grant": { 2024-09-14T03:47:49.792 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.792 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.792 INFO:tasks.workunit.client.0.smithi070.stdout: "ceph_cap_op_revoke": { 2024-09-14T03:47:49.792 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.792 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.792 INFO:tasks.workunit.client.0.smithi070.stdout: "ceph_cap_op_trunc": { 2024-09-14T03:47:49.792 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.792 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.792 INFO:tasks.workunit.client.0.smithi070.stdout: "dir_commit": { 2024-09-14T03:47:49.792 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12 2024-09-14T03:47:49.793 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.793 INFO:tasks.workunit.client.0.smithi070.stdout: "dir_fetch": { 2024-09-14T03:47:49.793 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.793 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.793 INFO:tasks.workunit.client.0.smithi070.stdout: "dir_merge": { 2024-09-14T03:47:49.793 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.793 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.793 INFO:tasks.workunit.client.0.smithi070.stdout: "dir_split": { 2024-09-14T03:47:49.793 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.793 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.793 INFO:tasks.workunit.client.0.smithi070.stdout: "exported_inodes": { 2024-09-14T03:47:49.793 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.793 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.794 INFO:tasks.workunit.client.0.smithi070.stdout: "forward": { 2024-09-14T03:47:49.794 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.794 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.794 INFO:tasks.workunit.client.0.smithi070.stdout: "handle_client_cap_release": { 2024-09-14T03:47:49.794 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.794 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.794 INFO:tasks.workunit.client.0.smithi070.stdout: "handle_client_caps": { 2024-09-14T03:47:49.794 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.794 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.794 INFO:tasks.workunit.client.0.smithi070.stdout: "handle_client_caps_dirty": { 2024-09-14T03:47:49.794 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.794 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.794 INFO:tasks.workunit.client.0.smithi070.stdout: "handle_inode_file_caps": { 2024-09-14T03:47:49.794 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.795 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.795 INFO:tasks.workunit.client.0.smithi070.stdout: "imported_inodes": { 2024-09-14T03:47:49.795 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.795 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.795 INFO:tasks.workunit.client.0.smithi070.stdout: "inodes": { 2024-09-14T03:47:49.795 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 10 2024-09-14T03:47:49.795 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.796 INFO:tasks.workunit.client.0.smithi070.stdout: "inodes_expired": { 2024-09-14T03:47:49.796 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.796 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.796 INFO:tasks.workunit.client.0.smithi070.stdout: "inodes_pinned": { 2024-09-14T03:47:49.796 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 10 2024-09-14T03:47:49.797 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.797 INFO:tasks.workunit.client.0.smithi070.stdout: "inodes_with_caps": { 2024-09-14T03:47:49.797 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.797 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.797 INFO:tasks.workunit.client.0.smithi070.stdout: "load_cent": { 2024-09-14T03:47:49.797 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.797 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.797 INFO:tasks.workunit.client.0.smithi070.stdout: "openino_dir_fetch": { 2024-09-14T03:47:49.797 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.797 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.797 INFO:tasks.workunit.client.0.smithi070.stdout: "process_request_cap_release": { 2024-09-14T03:47:49.798 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.798 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.798 INFO:tasks.workunit.client.0.smithi070.stdout: "reply_latency": { 2024-09-14T03:47:49.798 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:49.798 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 125024 2024-09-14T03:47:49.798 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.798 INFO:tasks.workunit.client.0.smithi070.stdout: "request": { 2024-09-14T03:47:49.798 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:49.798 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.798 INFO:tasks.workunit.client.0.smithi070.stdout: "root_rbytes": { 2024-09-14T03:47:49.798 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.798 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.798 INFO:tasks.workunit.client.0.smithi070.stdout: "root_rfiles": { 2024-09-14T03:47:49.798 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.799 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.799 INFO:tasks.workunit.client.0.smithi070.stdout: "root_rsnaps": { 2024-09-14T03:47:49.799 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.799 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.799 INFO:tasks.workunit.client.0.smithi070.stdout: "slow_reply": { 2024-09-14T03:47:49.799 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.799 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.799 INFO:tasks.workunit.client.0.smithi070.stdout: "subtrees": { 2024-09-14T03:47:49.800 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:49.800 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.800 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.800 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_cache": { 2024-09-14T03:47:49.800 INFO:tasks.workunit.client.0.smithi070.stdout: "ireq_enqueue_scrub": { 2024-09-14T03:47:49.800 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.800 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.800 INFO:tasks.workunit.client.0.smithi070.stdout: "ireq_exportdir": { 2024-09-14T03:47:49.800 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.800 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.800 INFO:tasks.workunit.client.0.smithi070.stdout: "ireq_flush": { 2024-09-14T03:47:49.800 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.800 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.800 INFO:tasks.workunit.client.0.smithi070.stdout: "ireq_fragmentdir": { 2024-09-14T03:47:49.801 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.801 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.801 INFO:tasks.workunit.client.0.smithi070.stdout: "ireq_fragstats": { 2024-09-14T03:47:49.801 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.801 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.801 INFO:tasks.workunit.client.0.smithi070.stdout: "ireq_inodestats": { 2024-09-14T03:47:49.801 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.801 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.801 INFO:tasks.workunit.client.0.smithi070.stdout: "num_recovering_enqueued": { 2024-09-14T03:47:49.801 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.801 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.801 INFO:tasks.workunit.client.0.smithi070.stdout: "num_recovering_prioritized": { 2024-09-14T03:47:49.801 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.802 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.802 INFO:tasks.workunit.client.0.smithi070.stdout: "num_recovering_processing": { 2024-09-14T03:47:49.802 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.802 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.802 INFO:tasks.workunit.client.0.smithi070.stdout: "num_strays": { 2024-09-14T03:47:49.802 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.802 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.802 INFO:tasks.workunit.client.0.smithi070.stdout: "num_strays_delayed": { 2024-09-14T03:47:49.802 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.802 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.802 INFO:tasks.workunit.client.0.smithi070.stdout: "num_strays_enqueuing": { 2024-09-14T03:47:49.802 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.802 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.802 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_completed": { 2024-09-14T03:47:49.802 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.803 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.803 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_started": { 2024-09-14T03:47:49.803 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.803 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.803 INFO:tasks.workunit.client.0.smithi070.stdout: "strays_created": { 2024-09-14T03:47:49.803 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.803 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.803 INFO:tasks.workunit.client.0.smithi070.stdout: "strays_enqueued": { 2024-09-14T03:47:49.803 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.803 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.803 INFO:tasks.workunit.client.0.smithi070.stdout: "strays_migrated": { 2024-09-14T03:47:49.803 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.803 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.803 INFO:tasks.workunit.client.0.smithi070.stdout: "strays_reintegrated": { 2024-09-14T03:47:49.803 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.804 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.804 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.804 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_log": { 2024-09-14T03:47:49.804 INFO:tasks.workunit.client.0.smithi070.stdout: "ev": { 2024-09-14T03:47:49.804 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5 2024-09-14T03:47:49.804 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.804 INFO:tasks.workunit.client.0.smithi070.stdout: "evadd": { 2024-09-14T03:47:49.804 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5 2024-09-14T03:47:49.804 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.804 INFO:tasks.workunit.client.0.smithi070.stdout: "evex": { 2024-09-14T03:47:49.804 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.804 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.804 INFO:tasks.workunit.client.0.smithi070.stdout: "evexd": { 2024-09-14T03:47:49.804 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.804 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.805 INFO:tasks.workunit.client.0.smithi070.stdout: "evexg": { 2024-09-14T03:47:49.805 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.805 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.805 INFO:tasks.workunit.client.0.smithi070.stdout: "evtrm": { 2024-09-14T03:47:49.805 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.805 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.805 INFO:tasks.workunit.client.0.smithi070.stdout: "jlat": { 2024-09-14T03:47:49.805 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 5, 2024-09-14T03:47:49.805 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12472377 2024-09-14T03:47:49.806 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.806 INFO:tasks.workunit.client.0.smithi070.stdout: "replayed": { 2024-09-14T03:47:49.806 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.806 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.806 INFO:tasks.workunit.client.0.smithi070.stdout: "seg": { 2024-09-14T03:47:49.806 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1 2024-09-14T03:47:49.807 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.807 INFO:tasks.workunit.client.0.smithi070.stdout: "segadd": { 2024-09-14T03:47:49.807 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1 2024-09-14T03:47:49.807 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.807 INFO:tasks.workunit.client.0.smithi070.stdout: "segex": { 2024-09-14T03:47:49.807 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.807 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.807 INFO:tasks.workunit.client.0.smithi070.stdout: "segexd": { 2024-09-14T03:47:49.807 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.807 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.807 INFO:tasks.workunit.client.0.smithi070.stdout: "segexg": { 2024-09-14T03:47:49.807 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.808 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.808 INFO:tasks.workunit.client.0.smithi070.stdout: "segtrm": { 2024-09-14T03:47:49.808 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.808 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.808 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.808 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_mem": { 2024-09-14T03:47:49.808 INFO:tasks.workunit.client.0.smithi070.stdout: "cap": { 2024-09-14T03:47:49.808 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.808 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.808 INFO:tasks.workunit.client.0.smithi070.stdout: "cap+": { 2024-09-14T03:47:49.808 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:49.808 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.809 INFO:tasks.workunit.client.0.smithi070.stdout: "cap-": { 2024-09-14T03:47:49.809 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:49.809 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.809 INFO:tasks.workunit.client.0.smithi070.stdout: "dir": { 2024-09-14T03:47:49.809 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12 2024-09-14T03:47:49.809 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.809 INFO:tasks.workunit.client.0.smithi070.stdout: "dir+": { 2024-09-14T03:47:49.809 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12 2024-09-14T03:47:49.809 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.809 INFO:tasks.workunit.client.0.smithi070.stdout: "dir-": { 2024-09-14T03:47:49.810 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.810 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.810 INFO:tasks.workunit.client.0.smithi070.stdout: "dn": { 2024-09-14T03:47:49.810 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 10 2024-09-14T03:47:49.810 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.810 INFO:tasks.workunit.client.0.smithi070.stdout: "dn+": { 2024-09-14T03:47:49.810 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 10 2024-09-14T03:47:49.810 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.810 INFO:tasks.workunit.client.0.smithi070.stdout: "dn-": { 2024-09-14T03:47:49.810 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.810 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.810 INFO:tasks.workunit.client.0.smithi070.stdout: "heap": { 2024-09-14T03:47:49.811 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 207068 2024-09-14T03:47:49.811 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.811 INFO:tasks.workunit.client.0.smithi070.stdout: "ino": { 2024-09-14T03:47:49.811 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 13 2024-09-14T03:47:49.811 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.811 INFO:tasks.workunit.client.0.smithi070.stdout: "ino+": { 2024-09-14T03:47:49.811 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 13 2024-09-14T03:47:49.811 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.811 INFO:tasks.workunit.client.0.smithi070.stdout: "ino-": { 2024-09-14T03:47:49.811 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.811 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.811 INFO:tasks.workunit.client.0.smithi070.stdout: "rss": { 2024-09-14T03:47:49.811 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 38060 2024-09-14T03:47:49.812 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.812 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.812 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_server": { 2024-09-14T03:47:49.812 INFO:tasks.workunit.client.0.smithi070.stdout: "cap_acquisition_throttle": { 2024-09-14T03:47:49.812 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.812 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.812 INFO:tasks.workunit.client.0.smithi070.stdout: "cap_revoke_eviction": { 2024-09-14T03:47:49.812 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.812 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.812 INFO:tasks.workunit.client.0.smithi070.stdout: "handle_client_request": { 2024-09-14T03:47:49.812 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:49.813 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.813 INFO:tasks.workunit.client.0.smithi070.stdout: "handle_client_session": { 2024-09-14T03:47:49.813 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 16 2024-09-14T03:47:49.813 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.813 INFO:tasks.workunit.client.0.smithi070.stdout: "handle_peer_request": { 2024-09-14T03:47:49.813 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.813 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.813 INFO:tasks.workunit.client.0.smithi070.stdout: "req_create_latency": { 2024-09-14T03:47:49.813 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.813 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.813 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.813 INFO:tasks.workunit.client.0.smithi070.stdout: "req_getattr_latency": { 2024-09-14T03:47:49.814 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:49.814 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 125024 2024-09-14T03:47:49.814 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.814 INFO:tasks.workunit.client.0.smithi070.stdout: "req_getfilelock_latency": { 2024-09-14T03:47:49.814 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.814 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.814 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.814 INFO:tasks.workunit.client.0.smithi070.stdout: "req_getvxattr_latency": { 2024-09-14T03:47:49.814 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.814 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.814 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.815 INFO:tasks.workunit.client.0.smithi070.stdout: "req_link_latency": { 2024-09-14T03:47:49.815 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.815 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.815 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.815 INFO:tasks.workunit.client.0.smithi070.stdout: "req_lookup_latency": { 2024-09-14T03:47:49.815 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.815 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.815 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.815 INFO:tasks.workunit.client.0.smithi070.stdout: "req_lookuphash_latency": { 2024-09-14T03:47:49.815 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.815 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.815 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.815 INFO:tasks.workunit.client.0.smithi070.stdout: "req_lookupino_latency": { 2024-09-14T03:47:49.816 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.816 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.816 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.816 INFO:tasks.workunit.client.0.smithi070.stdout: "req_lookupname_latency": { 2024-09-14T03:47:49.816 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.816 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.816 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.816 INFO:tasks.workunit.client.0.smithi070.stdout: "req_lookupparent_latency": { 2024-09-14T03:47:49.816 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.816 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.816 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.816 INFO:tasks.workunit.client.0.smithi070.stdout: "req_lookupsnap_latency": { 2024-09-14T03:47:49.816 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.817 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.817 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.817 INFO:tasks.workunit.client.0.smithi070.stdout: "req_lssnap_latency": { 2024-09-14T03:47:49.817 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.817 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.817 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.817 INFO:tasks.workunit.client.0.smithi070.stdout: "req_mkdir_latency": { 2024-09-14T03:47:49.817 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.818 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.818 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.818 INFO:tasks.workunit.client.0.smithi070.stdout: "req_mknod_latency": { 2024-09-14T03:47:49.818 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.818 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.818 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.818 INFO:tasks.workunit.client.0.smithi070.stdout: "req_mksnap_latency": { 2024-09-14T03:47:49.818 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.818 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.818 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.818 INFO:tasks.workunit.client.0.smithi070.stdout: "req_open_latency": { 2024-09-14T03:47:49.818 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.818 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.818 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.819 INFO:tasks.workunit.client.0.smithi070.stdout: "req_readdir_latency": { 2024-09-14T03:47:49.819 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.819 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.819 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.819 INFO:tasks.workunit.client.0.smithi070.stdout: "req_rename_latency": { 2024-09-14T03:47:49.819 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.819 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.819 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.819 INFO:tasks.workunit.client.0.smithi070.stdout: "req_renamesnap_latency": { 2024-09-14T03:47:49.819 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.820 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.820 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.820 INFO:tasks.workunit.client.0.smithi070.stdout: "req_rmdir_latency": { 2024-09-14T03:47:49.820 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.820 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.820 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.820 INFO:tasks.workunit.client.0.smithi070.stdout: "req_rmsnap_latency": { 2024-09-14T03:47:49.821 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.821 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.821 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.821 INFO:tasks.workunit.client.0.smithi070.stdout: "req_rmxattr_latency": { 2024-09-14T03:47:49.821 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.821 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.821 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.821 INFO:tasks.workunit.client.0.smithi070.stdout: "req_setattr_latency": { 2024-09-14T03:47:49.821 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.821 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.821 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.821 INFO:tasks.workunit.client.0.smithi070.stdout: "req_setdirlayout_latency": { 2024-09-14T03:47:49.822 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.822 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.822 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.822 INFO:tasks.workunit.client.0.smithi070.stdout: "req_setfilelock_latency": { 2024-09-14T03:47:49.822 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.822 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.822 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.822 INFO:tasks.workunit.client.0.smithi070.stdout: "req_setlayout_latency": { 2024-09-14T03:47:49.822 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.822 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.822 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.822 INFO:tasks.workunit.client.0.smithi070.stdout: "req_setxattr_latency": { 2024-09-14T03:47:49.822 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.822 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.822 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.823 INFO:tasks.workunit.client.0.smithi070.stdout: "req_symlink_latency": { 2024-09-14T03:47:49.823 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.823 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.823 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.823 INFO:tasks.workunit.client.0.smithi070.stdout: "req_unlink_latency": { 2024-09-14T03:47:49.823 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.823 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.823 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.823 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.823 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_sessions": { 2024-09-14T03:47:49.823 INFO:tasks.workunit.client.0.smithi070.stdout: "average_load": { 2024-09-14T03:47:49.823 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1 2024-09-14T03:47:49.823 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.823 INFO:tasks.workunit.client.0.smithi070.stdout: "avg_session_uptime": { 2024-09-14T03:47:49.824 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 89 2024-09-14T03:47:49.824 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.824 INFO:tasks.workunit.client.0.smithi070.stdout: "mdthresh_evicted": { 2024-09-14T03:47:49.824 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.824 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.824 INFO:tasks.workunit.client.0.smithi070.stdout: "session_add": { 2024-09-14T03:47:49.824 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:49.824 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.824 INFO:tasks.workunit.client.0.smithi070.stdout: "session_count": { 2024-09-14T03:47:49.824 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.824 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.824 INFO:tasks.workunit.client.0.smithi070.stdout: "session_remove": { 2024-09-14T03:47:49.824 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:49.825 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.825 INFO:tasks.workunit.client.0.smithi070.stdout: "sessions_open": { 2024-09-14T03:47:49.825 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.825 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.825 INFO:tasks.workunit.client.0.smithi070.stdout: "sessions_stale": { 2024-09-14T03:47:49.825 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.825 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.825 INFO:tasks.workunit.client.0.smithi070.stdout: "total_load": { 2024-09-14T03:47:49.825 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:49.825 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.825 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.825 INFO:tasks.workunit.client.0.smithi070.stdout: "objecter": { 2024-09-14T03:47:49.825 INFO:tasks.workunit.client.0.smithi070.stdout: "op_active": { 2024-09-14T03:47:49.825 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.825 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.826 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r": { 2024-09-14T03:47:49.826 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.826 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.826 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rmw": { 2024-09-14T03:47:49.826 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.826 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.826 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w": { 2024-09-14T03:47:49.826 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 45 2024-09-14T03:47:49.826 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.826 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.826 INFO:tasks.workunit.client.0.smithi070.stdout: "purge_queue": { 2024-09-14T03:47:49.826 INFO:tasks.workunit.client.0.smithi070.stdout: "pq_executed": { 2024-09-14T03:47:49.826 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.826 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.827 INFO:tasks.workunit.client.0.smithi070.stdout: "pq_executing": { 2024-09-14T03:47:49.827 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.827 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.827 INFO:tasks.workunit.client.0.smithi070.stdout: "pq_executing_high_water": { 2024-09-14T03:47:49.827 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.827 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.827 INFO:tasks.workunit.client.0.smithi070.stdout: "pq_executing_ops": { 2024-09-14T03:47:49.827 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.827 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.827 INFO:tasks.workunit.client.0.smithi070.stdout: "pq_executing_ops_high_water": { 2024-09-14T03:47:49.827 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.827 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.827 INFO:tasks.workunit.client.0.smithi070.stdout: "pq_item_in_journal": { 2024-09-14T03:47:49.828 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.828 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.828 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.828 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.828 INFO:tasks.workunit.client.0.smithi070.stdout: "mon.a8e17c6c2bb7d35df7944161574528c366e6a388": { 2024-09-14T03:47:49.828 INFO:tasks.workunit.client.0.smithi070.stdout: "mon": { 2024-09-14T03:47:49.828 INFO:tasks.workunit.client.0.smithi070.stdout: "election_call": { 2024-09-14T03:47:49.828 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:49.828 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.828 INFO:tasks.workunit.client.0.smithi070.stdout: "election_lose": { 2024-09-14T03:47:49.828 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.828 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.828 INFO:tasks.workunit.client.0.smithi070.stdout: "election_win": { 2024-09-14T03:47:49.828 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3 2024-09-14T03:47:49.828 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.829 INFO:tasks.workunit.client.0.smithi070.stdout: "num_elections": { 2024-09-14T03:47:49.829 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 9 2024-09-14T03:47:49.829 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.829 INFO:tasks.workunit.client.0.smithi070.stdout: "num_sessions": { 2024-09-14T03:47:49.829 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 8 2024-09-14T03:47:49.829 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.829 INFO:tasks.workunit.client.0.smithi070.stdout: "session_add": { 2024-09-14T03:47:49.829 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3 2024-09-14T03:47:49.829 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.829 INFO:tasks.workunit.client.0.smithi070.stdout: "session_rm": { 2024-09-14T03:47:49.829 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 179 2024-09-14T03:47:49.829 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.829 INFO:tasks.workunit.client.0.smithi070.stdout: "session_trim": { 2024-09-14T03:47:49.830 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.830 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.830 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.830 INFO:tasks.workunit.client.0.smithi070.stdout: "paxos": { 2024-09-14T03:47:49.830 INFO:tasks.workunit.client.0.smithi070.stdout: "accept_timeout": { 2024-09-14T03:47:49.830 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.830 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.830 INFO:tasks.workunit.client.0.smithi070.stdout: "begin": { 2024-09-14T03:47:49.830 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 962 2024-09-14T03:47:49.830 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.830 INFO:tasks.workunit.client.0.smithi070.stdout: "begin_bytes": { 2024-09-14T03:47:49.830 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 962, 2024-09-14T03:47:49.830 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5106207 2024-09-14T03:47:49.830 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.831 INFO:tasks.workunit.client.0.smithi070.stdout: "begin_keys": { 2024-09-14T03:47:49.831 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 962, 2024-09-14T03:47:49.831 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2886 2024-09-14T03:47:49.831 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.831 INFO:tasks.workunit.client.0.smithi070.stdout: "begin_latency": { 2024-09-14T03:47:49.831 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 962, 2024-09-14T03:47:49.831 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 131996113 2024-09-14T03:47:49.831 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.831 INFO:tasks.workunit.client.0.smithi070.stdout: "collect": { 2024-09-14T03:47:49.831 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.831 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.831 INFO:tasks.workunit.client.0.smithi070.stdout: "collect_bytes": { 2024-09-14T03:47:49.831 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.831 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.831 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.832 INFO:tasks.workunit.client.0.smithi070.stdout: "collect_keys": { 2024-09-14T03:47:49.832 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.832 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.832 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.832 INFO:tasks.workunit.client.0.smithi070.stdout: "collect_latency": { 2024-09-14T03:47:49.832 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.832 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.832 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.832 INFO:tasks.workunit.client.0.smithi070.stdout: "collect_timeout": { 2024-09-14T03:47:49.832 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.832 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.832 INFO:tasks.workunit.client.0.smithi070.stdout: "collect_uncommitted": { 2024-09-14T03:47:49.832 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.833 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.833 INFO:tasks.workunit.client.0.smithi070.stdout: "commit": { 2024-09-14T03:47:49.833 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 962 2024-09-14T03:47:49.833 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.833 INFO:tasks.workunit.client.0.smithi070.stdout: "commit_bytes": { 2024-09-14T03:47:49.833 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 962, 2024-09-14T03:47:49.833 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5037001 2024-09-14T03:47:49.833 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.833 INFO:tasks.workunit.client.0.smithi070.stdout: "commit_keys": { 2024-09-14T03:47:49.833 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 962, 2024-09-14T03:47:49.833 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5480 2024-09-14T03:47:49.833 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.833 INFO:tasks.workunit.client.0.smithi070.stdout: "commit_latency": { 2024-09-14T03:47:49.833 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 962, 2024-09-14T03:47:49.834 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 131023916 2024-09-14T03:47:49.834 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.834 INFO:tasks.workunit.client.0.smithi070.stdout: "lease_ack_timeout": { 2024-09-14T03:47:49.834 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.834 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.834 INFO:tasks.workunit.client.0.smithi070.stdout: "lease_timeout": { 2024-09-14T03:47:49.834 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.834 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.834 INFO:tasks.workunit.client.0.smithi070.stdout: "new_pn": { 2024-09-14T03:47:49.834 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:49.834 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.834 INFO:tasks.workunit.client.0.smithi070.stdout: "new_pn_latency": { 2024-09-14T03:47:49.834 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:49.835 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.835 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.835 INFO:tasks.workunit.client.0.smithi070.stdout: "refresh": { 2024-09-14T03:47:49.835 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 964 2024-09-14T03:47:49.835 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.835 INFO:tasks.workunit.client.0.smithi070.stdout: "refresh_latency": { 2024-09-14T03:47:49.835 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 964, 2024-09-14T03:47:49.835 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2283932853 2024-09-14T03:47:49.835 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.835 INFO:tasks.workunit.client.0.smithi070.stdout: "restart": { 2024-09-14T03:47:49.835 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12 2024-09-14T03:47:49.835 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.835 INFO:tasks.workunit.client.0.smithi070.stdout: "share_state": { 2024-09-14T03:47:49.835 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3 2024-09-14T03:47:49.835 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.836 INFO:tasks.workunit.client.0.smithi070.stdout: "share_state_bytes": { 2024-09-14T03:47:49.836 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 3, 2024-09-14T03:47:49.836 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1627 2024-09-14T03:47:49.836 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.836 INFO:tasks.workunit.client.0.smithi070.stdout: "share_state_keys": { 2024-09-14T03:47:49.836 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 3, 2024-09-14T03:47:49.836 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3 2024-09-14T03:47:49.836 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.836 INFO:tasks.workunit.client.0.smithi070.stdout: "start_leader": { 2024-09-14T03:47:49.836 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3 2024-09-14T03:47:49.837 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.837 INFO:tasks.workunit.client.0.smithi070.stdout: "start_peon": { 2024-09-14T03:47:49.837 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.837 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.837 INFO:tasks.workunit.client.0.smithi070.stdout: "store_state": { 2024-09-14T03:47:49.837 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.837 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.837 INFO:tasks.workunit.client.0.smithi070.stdout: "store_state_bytes": { 2024-09-14T03:47:49.837 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.837 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.837 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.837 INFO:tasks.workunit.client.0.smithi070.stdout: "store_state_keys": { 2024-09-14T03:47:49.837 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.837 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.837 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.838 INFO:tasks.workunit.client.0.smithi070.stdout: "store_state_latency": { 2024-09-14T03:47:49.838 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.838 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.838 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.838 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.838 INFO:tasks.workunit.client.0.smithi070.stdout: "prioritycache": { 2024-09-14T03:47:49.838 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_bytes": { 2024-09-14T03:47:49.838 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1020054731 2024-09-14T03:47:49.838 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.838 INFO:tasks.workunit.client.0.smithi070.stdout: "heap_bytes": { 2024-09-14T03:47:49.838 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 50036736 2024-09-14T03:47:49.838 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.838 INFO:tasks.workunit.client.0.smithi070.stdout: "mapped_bytes": { 2024-09-14T03:47:49.838 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 41148416 2024-09-14T03:47:49.838 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.839 INFO:tasks.workunit.client.0.smithi070.stdout: "target_bytes": { 2024-09-14T03:47:49.839 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2147483648 2024-09-14T03:47:49.839 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.839 INFO:tasks.workunit.client.0.smithi070.stdout: "unmapped_bytes": { 2024-09-14T03:47:49.839 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 8888320 2024-09-14T03:47:49.839 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.839 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.839 INFO:tasks.workunit.client.0.smithi070.stdout: "prioritycache:full": { 2024-09-14T03:47:49.840 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:49.840 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 348127232 2024-09-14T03:47:49.840 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.840 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:49.840 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.840 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.840 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:49.840 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.840 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.840 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:49.840 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 277021771 2024-09-14T03:47:49.841 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.841 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:49.841 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 273495 2024-09-14T03:47:49.841 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.841 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:49.841 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.841 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.841 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:49.841 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.841 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.841 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:49.842 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.842 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.842 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:49.842 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.842 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.842 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:49.843 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.843 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.843 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:49.843 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.843 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.844 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:49.844 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.844 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.844 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:49.844 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.844 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.845 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:49.845 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 70831966 2024-09-14T03:47:49.845 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.845 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.845 INFO:tasks.workunit.client.0.smithi070.stdout: "prioritycache:inc": { 2024-09-14T03:47:49.845 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:49.845 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 348127232 2024-09-14T03:47:49.845 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.845 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:49.845 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.846 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.846 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:49.846 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.846 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.846 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:49.846 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 277021771 2024-09-14T03:47:49.846 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.846 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:49.847 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 38144 2024-09-14T03:47:49.847 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.847 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:49.847 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.847 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.847 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:49.847 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.847 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.847 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:49.847 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.847 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.848 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:49.848 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.848 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.848 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:49.848 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.848 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.848 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:49.848 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.848 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.848 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:49.848 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.848 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.848 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:49.849 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.849 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.849 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:49.849 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71067317 2024-09-14T03:47:49.849 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.849 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.849 INFO:tasks.workunit.client.0.smithi070.stdout: "prioritycache:kv": { 2024-09-14T03:47:49.849 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:49.849 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 322961408 2024-09-14T03:47:49.849 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.849 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:49.849 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:49.849 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.850 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:49.850 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.850 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.850 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:49.850 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 184681181 2024-09-14T03:47:49.850 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.850 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:49.850 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.850 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.850 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:49.850 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.850 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.850 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:49.850 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.850 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.851 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:49.851 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.851 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.851 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:49.851 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.851 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.851 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:49.851 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.851 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.851 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:49.851 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.851 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.851 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:49.852 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.852 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.852 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:49.852 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.852 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.852 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:49.852 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71171363 2024-09-14T03:47:49.852 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.852 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.852 INFO:tasks.workunit.client.0.smithi070.stdout: "rocksdb": { 2024-09-14T03:47:49.852 INFO:tasks.workunit.client.0.smithi070.stdout: "compact": { 2024-09-14T03:47:49.852 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.852 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.852 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_completed": { 2024-09-14T03:47:49.853 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1 2024-09-14T03:47:49.853 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.853 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lasted": { 2024-09-14T03:47:49.853 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 25282850 2024-09-14T03:47:49.853 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.853 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_queue_len": { 2024-09-14T03:47:49.853 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.853 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.853 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_queue_merge": { 2024-09-14T03:47:49.853 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.853 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.853 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_running": { 2024-09-14T03:47:49.853 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.853 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.854 INFO:tasks.workunit.client.0.smithi070.stdout: "get_latency": { 2024-09-14T03:47:49.854 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 43311, 2024-09-14T03:47:49.854 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 163882893 2024-09-14T03:47:49.854 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.854 INFO:tasks.workunit.client.0.smithi070.stdout: "rocksdb_write_delay_time": { 2024-09-14T03:47:49.854 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.854 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.854 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.854 INFO:tasks.workunit.client.0.smithi070.stdout: "rocksdb_write_memtable_time": { 2024-09-14T03:47:49.854 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.854 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.854 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.855 INFO:tasks.workunit.client.0.smithi070.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-14T03:47:49.855 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.855 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.855 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.855 INFO:tasks.workunit.client.0.smithi070.stdout: "rocksdb_write_wal_time": { 2024-09-14T03:47:49.855 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.855 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.855 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.855 INFO:tasks.workunit.client.0.smithi070.stdout: "submit_latency": { 2024-09-14T03:47:49.855 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.855 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.855 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.855 INFO:tasks.workunit.client.0.smithi070.stdout: "submit_sync_latency": { 2024-09-14T03:47:49.855 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2094, 2024-09-14T03:47:49.855 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 276326241 2024-09-14T03:47:49.856 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.856 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.856 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.856 INFO:tasks.workunit.client.0.smithi070.stdout: "mon.b1d2edfb86a8f225d9283baf06ff7497eda48d2d": { 2024-09-14T03:47:49.856 INFO:tasks.workunit.client.0.smithi070.stdout: "mon": { 2024-09-14T03:47:49.856 INFO:tasks.workunit.client.0.smithi070.stdout: "election_call": { 2024-09-14T03:47:49.856 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:49.856 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.856 INFO:tasks.workunit.client.0.smithi070.stdout: "election_lose": { 2024-09-14T03:47:49.856 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:49.856 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.856 INFO:tasks.workunit.client.0.smithi070.stdout: "election_win": { 2024-09-14T03:47:49.856 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.856 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.857 INFO:tasks.workunit.client.0.smithi070.stdout: "num_elections": { 2024-09-14T03:47:49.857 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7 2024-09-14T03:47:49.857 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.857 INFO:tasks.workunit.client.0.smithi070.stdout: "num_sessions": { 2024-09-14T03:47:49.857 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 9 2024-09-14T03:47:49.857 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.857 INFO:tasks.workunit.client.0.smithi070.stdout: "session_add": { 2024-09-14T03:47:49.857 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4 2024-09-14T03:47:49.857 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.857 INFO:tasks.workunit.client.0.smithi070.stdout: "session_rm": { 2024-09-14T03:47:49.857 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 54 2024-09-14T03:47:49.857 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.857 INFO:tasks.workunit.client.0.smithi070.stdout: "session_trim": { 2024-09-14T03:47:49.858 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.858 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.858 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.858 INFO:tasks.workunit.client.0.smithi070.stdout: "paxos": { 2024-09-14T03:47:49.858 INFO:tasks.workunit.client.0.smithi070.stdout: "accept_timeout": { 2024-09-14T03:47:49.858 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.858 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.858 INFO:tasks.workunit.client.0.smithi070.stdout: "begin": { 2024-09-14T03:47:49.858 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 834 2024-09-14T03:47:49.858 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.858 INFO:tasks.workunit.client.0.smithi070.stdout: "begin_bytes": { 2024-09-14T03:47:49.858 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 834, 2024-09-14T03:47:49.858 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4057273 2024-09-14T03:47:49.859 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.859 INFO:tasks.workunit.client.0.smithi070.stdout: "begin_keys": { 2024-09-14T03:47:49.859 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.859 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.859 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.859 INFO:tasks.workunit.client.0.smithi070.stdout: "begin_latency": { 2024-09-14T03:47:49.859 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 834, 2024-09-14T03:47:49.859 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 103998304 2024-09-14T03:47:49.859 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.859 INFO:tasks.workunit.client.0.smithi070.stdout: "collect": { 2024-09-14T03:47:49.859 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:49.859 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.860 INFO:tasks.workunit.client.0.smithi070.stdout: "collect_bytes": { 2024-09-14T03:47:49.860 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:49.860 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 146 2024-09-14T03:47:49.860 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.860 INFO:tasks.workunit.client.0.smithi070.stdout: "collect_keys": { 2024-09-14T03:47:49.860 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:49.860 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:49.860 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.860 INFO:tasks.workunit.client.0.smithi070.stdout: "collect_latency": { 2024-09-14T03:47:49.860 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:49.860 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.860 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.860 INFO:tasks.workunit.client.0.smithi070.stdout: "collect_timeout": { 2024-09-14T03:47:49.860 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.861 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.861 INFO:tasks.workunit.client.0.smithi070.stdout: "collect_uncommitted": { 2024-09-14T03:47:49.861 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.861 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.861 INFO:tasks.workunit.client.0.smithi070.stdout: "commit": { 2024-09-14T03:47:49.861 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 835 2024-09-14T03:47:49.861 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.861 INFO:tasks.workunit.client.0.smithi070.stdout: "commit_bytes": { 2024-09-14T03:47:49.861 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.861 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.861 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.861 INFO:tasks.workunit.client.0.smithi070.stdout: "commit_keys": { 2024-09-14T03:47:49.861 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.861 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.862 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.862 INFO:tasks.workunit.client.0.smithi070.stdout: "commit_latency": { 2024-09-14T03:47:49.862 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.862 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.862 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.862 INFO:tasks.workunit.client.0.smithi070.stdout: "lease_ack_timeout": { 2024-09-14T03:47:49.862 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.862 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.862 INFO:tasks.workunit.client.0.smithi070.stdout: "lease_timeout": { 2024-09-14T03:47:49.862 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.862 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.862 INFO:tasks.workunit.client.0.smithi070.stdout: "new_pn": { 2024-09-14T03:47:49.862 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.862 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.862 INFO:tasks.workunit.client.0.smithi070.stdout: "new_pn_latency": { 2024-09-14T03:47:49.863 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.863 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.863 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.863 INFO:tasks.workunit.client.0.smithi070.stdout: "refresh": { 2024-09-14T03:47:49.863 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 835 2024-09-14T03:47:49.863 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.863 INFO:tasks.workunit.client.0.smithi070.stdout: "refresh_latency": { 2024-09-14T03:47:49.863 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 835, 2024-09-14T03:47:49.863 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1665972924 2024-09-14T03:47:49.863 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.863 INFO:tasks.workunit.client.0.smithi070.stdout: "restart": { 2024-09-14T03:47:49.863 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 13 2024-09-14T03:47:49.863 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.863 INFO:tasks.workunit.client.0.smithi070.stdout: "share_state": { 2024-09-14T03:47:49.864 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.864 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.864 INFO:tasks.workunit.client.0.smithi070.stdout: "share_state_bytes": { 2024-09-14T03:47:49.864 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.864 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.864 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.864 INFO:tasks.workunit.client.0.smithi070.stdout: "share_state_keys": { 2024-09-14T03:47:49.864 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.864 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.864 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.864 INFO:tasks.workunit.client.0.smithi070.stdout: "start_leader": { 2024-09-14T03:47:49.864 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.864 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.864 INFO:tasks.workunit.client.0.smithi070.stdout: "start_peon": { 2024-09-14T03:47:49.865 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:49.865 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.865 INFO:tasks.workunit.client.0.smithi070.stdout: "store_state": { 2024-09-14T03:47:49.865 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 835 2024-09-14T03:47:49.865 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.865 INFO:tasks.workunit.client.0.smithi070.stdout: "store_state_bytes": { 2024-09-14T03:47:49.865 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 835, 2024-09-14T03:47:49.865 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7957909 2024-09-14T03:47:49.865 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.865 INFO:tasks.workunit.client.0.smithi070.stdout: "store_state_keys": { 2024-09-14T03:47:49.865 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 835, 2024-09-14T03:47:49.865 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5627 2024-09-14T03:47:49.865 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.866 INFO:tasks.workunit.client.0.smithi070.stdout: "store_state_latency": { 2024-09-14T03:47:49.866 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 835, 2024-09-14T03:47:49.866 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 114998126 2024-09-14T03:47:49.866 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.866 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.866 INFO:tasks.workunit.client.0.smithi070.stdout: "prioritycache": { 2024-09-14T03:47:49.866 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_bytes": { 2024-09-14T03:47:49.866 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1020054731 2024-09-14T03:47:49.866 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.866 INFO:tasks.workunit.client.0.smithi070.stdout: "heap_bytes": { 2024-09-14T03:47:49.866 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 49078272 2024-09-14T03:47:49.867 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.867 INFO:tasks.workunit.client.0.smithi070.stdout: "mapped_bytes": { 2024-09-14T03:47:49.867 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 33202176 2024-09-14T03:47:49.867 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.867 INFO:tasks.workunit.client.0.smithi070.stdout: "target_bytes": { 2024-09-14T03:47:49.867 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2147483648 2024-09-14T03:47:49.867 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.867 INFO:tasks.workunit.client.0.smithi070.stdout: "unmapped_bytes": { 2024-09-14T03:47:49.867 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 15876096 2024-09-14T03:47:49.867 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.867 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.867 INFO:tasks.workunit.client.0.smithi070.stdout: "prioritycache:full": { 2024-09-14T03:47:49.867 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:49.868 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 348127232 2024-09-14T03:47:49.868 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.868 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:49.868 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.868 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.868 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:49.868 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.868 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.868 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:49.868 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 277020366 2024-09-14T03:47:49.868 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.869 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:49.869 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 273925 2024-09-14T03:47:49.869 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.869 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:49.869 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.869 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.870 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:49.870 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.870 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.870 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:49.870 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.870 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.870 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:49.871 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.871 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.871 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:49.871 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.871 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.871 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:49.871 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.871 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.871 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:49.872 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.872 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.872 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:49.872 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.872 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.872 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:49.872 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 70832941 2024-09-14T03:47:49.872 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.872 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.872 INFO:tasks.workunit.client.0.smithi070.stdout: "prioritycache:inc": { 2024-09-14T03:47:49.872 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:49.872 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 348127232 2024-09-14T03:47:49.873 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.873 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:49.873 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.873 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.873 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:49.873 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.873 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.873 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:49.873 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 277020366 2024-09-14T03:47:49.873 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.873 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:49.874 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 41460 2024-09-14T03:47:49.874 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.874 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:49.874 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.874 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.874 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:49.874 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.874 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.874 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:49.874 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.874 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.874 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:49.874 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.875 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.875 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:49.875 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.875 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.875 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:49.875 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.875 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.875 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:49.875 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.875 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.875 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:49.875 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.876 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.876 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:49.876 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71065406 2024-09-14T03:47:49.876 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.876 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.876 INFO:tasks.workunit.client.0.smithi070.stdout: "prioritycache:kv": { 2024-09-14T03:47:49.876 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:49.877 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 322961408 2024-09-14T03:47:49.877 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.877 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:49.877 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:49.877 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.877 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:49.877 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.878 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.878 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:49.878 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 184680244 2024-09-14T03:47:49.878 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.878 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:49.878 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.878 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.878 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:49.878 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.878 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.879 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:49.879 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.879 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.879 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:49.879 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.879 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.879 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:49.879 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.879 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.879 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:49.879 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.879 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.879 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:49.879 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.880 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.880 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:49.880 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.880 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.880 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:49.880 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.880 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.880 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:49.880 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71172300 2024-09-14T03:47:49.880 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.880 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.880 INFO:tasks.workunit.client.0.smithi070.stdout: "rocksdb": { 2024-09-14T03:47:49.880 INFO:tasks.workunit.client.0.smithi070.stdout: "compact": { 2024-09-14T03:47:49.880 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.881 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.881 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_completed": { 2024-09-14T03:47:49.881 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1 2024-09-14T03:47:49.881 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.881 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lasted": { 2024-09-14T03:47:49.881 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 25235841 2024-09-14T03:47:49.881 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.881 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_queue_len": { 2024-09-14T03:47:49.881 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.881 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.881 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_queue_merge": { 2024-09-14T03:47:49.881 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.881 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.882 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_running": { 2024-09-14T03:47:49.882 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.882 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.882 INFO:tasks.workunit.client.0.smithi070.stdout: "get_latency": { 2024-09-14T03:47:49.882 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 35547, 2024-09-14T03:47:49.882 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 79207408 2024-09-14T03:47:49.882 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.882 INFO:tasks.workunit.client.0.smithi070.stdout: "rocksdb_write_delay_time": { 2024-09-14T03:47:49.882 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.882 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.882 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.882 INFO:tasks.workunit.client.0.smithi070.stdout: "rocksdb_write_memtable_time": { 2024-09-14T03:47:49.882 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.882 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.883 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.883 INFO:tasks.workunit.client.0.smithi070.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-14T03:47:49.883 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.883 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.883 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.883 INFO:tasks.workunit.client.0.smithi070.stdout: "rocksdb_write_wal_time": { 2024-09-14T03:47:49.883 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.883 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.883 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.883 INFO:tasks.workunit.client.0.smithi070.stdout: "submit_latency": { 2024-09-14T03:47:49.883 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.883 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.884 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.884 INFO:tasks.workunit.client.0.smithi070.stdout: "submit_sync_latency": { 2024-09-14T03:47:49.884 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1819, 2024-09-14T03:47:49.884 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 269015584 2024-09-14T03:47:49.884 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.884 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.884 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.884 INFO:tasks.workunit.client.0.smithi070.stdout: "mon.bb10d63b3f8e26c4a0c5da991c12ff03df4a20b0": { 2024-09-14T03:47:49.884 INFO:tasks.workunit.client.0.smithi070.stdout: "mon": { 2024-09-14T03:47:49.885 INFO:tasks.workunit.client.0.smithi070.stdout: "election_call": { 2024-09-14T03:47:49.885 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1 2024-09-14T03:47:49.885 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.885 INFO:tasks.workunit.client.0.smithi070.stdout: "election_lose": { 2024-09-14T03:47:49.885 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1 2024-09-14T03:47:49.885 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.886 INFO:tasks.workunit.client.0.smithi070.stdout: "election_win": { 2024-09-14T03:47:49.886 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.886 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.886 INFO:tasks.workunit.client.0.smithi070.stdout: "num_elections": { 2024-09-14T03:47:49.886 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3 2024-09-14T03:47:49.886 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.886 INFO:tasks.workunit.client.0.smithi070.stdout: "num_sessions": { 2024-09-14T03:47:49.887 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12 2024-09-14T03:47:49.887 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.887 INFO:tasks.workunit.client.0.smithi070.stdout: "session_add": { 2024-09-14T03:47:49.887 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5 2024-09-14T03:47:49.887 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.887 INFO:tasks.workunit.client.0.smithi070.stdout: "session_rm": { 2024-09-14T03:47:49.888 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 86 2024-09-14T03:47:49.888 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.888 INFO:tasks.workunit.client.0.smithi070.stdout: "session_trim": { 2024-09-14T03:47:49.888 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.888 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.888 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.889 INFO:tasks.workunit.client.0.smithi070.stdout: "paxos": { 2024-09-14T03:47:49.889 INFO:tasks.workunit.client.0.smithi070.stdout: "accept_timeout": { 2024-09-14T03:47:49.889 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.889 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.889 INFO:tasks.workunit.client.0.smithi070.stdout: "begin": { 2024-09-14T03:47:49.889 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 824 2024-09-14T03:47:49.890 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.890 INFO:tasks.workunit.client.0.smithi070.stdout: "begin_bytes": { 2024-09-14T03:47:49.890 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 824, 2024-09-14T03:47:49.890 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4030459 2024-09-14T03:47:49.890 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.890 INFO:tasks.workunit.client.0.smithi070.stdout: "begin_keys": { 2024-09-14T03:47:49.891 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.891 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.891 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.891 INFO:tasks.workunit.client.0.smithi070.stdout: "begin_latency": { 2024-09-14T03:47:49.891 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 824, 2024-09-14T03:47:49.891 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 96997151 2024-09-14T03:47:49.892 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.892 INFO:tasks.workunit.client.0.smithi070.stdout: "collect": { 2024-09-14T03:47:49.892 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1 2024-09-14T03:47:49.892 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.892 INFO:tasks.workunit.client.0.smithi070.stdout: "collect_bytes": { 2024-09-14T03:47:49.892 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:49.893 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 73 2024-09-14T03:47:49.893 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.893 INFO:tasks.workunit.client.0.smithi070.stdout: "collect_keys": { 2024-09-14T03:47:49.893 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:49.893 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1 2024-09-14T03:47:49.894 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.894 INFO:tasks.workunit.client.0.smithi070.stdout: "collect_latency": { 2024-09-14T03:47:49.894 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:49.894 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.894 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.895 INFO:tasks.workunit.client.0.smithi070.stdout: "collect_timeout": { 2024-09-14T03:47:49.895 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.895 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.895 INFO:tasks.workunit.client.0.smithi070.stdout: "collect_uncommitted": { 2024-09-14T03:47:49.895 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.895 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.895 INFO:tasks.workunit.client.0.smithi070.stdout: "commit": { 2024-09-14T03:47:49.896 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 826 2024-09-14T03:47:49.896 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.896 INFO:tasks.workunit.client.0.smithi070.stdout: "commit_bytes": { 2024-09-14T03:47:49.896 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.896 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.896 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.897 INFO:tasks.workunit.client.0.smithi070.stdout: "commit_keys": { 2024-09-14T03:47:49.897 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.897 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.897 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.897 INFO:tasks.workunit.client.0.smithi070.stdout: "commit_latency": { 2024-09-14T03:47:49.897 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.898 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.898 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.898 INFO:tasks.workunit.client.0.smithi070.stdout: "lease_ack_timeout": { 2024-09-14T03:47:49.898 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.898 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.898 INFO:tasks.workunit.client.0.smithi070.stdout: "lease_timeout": { 2024-09-14T03:47:49.899 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.899 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.899 INFO:tasks.workunit.client.0.smithi070.stdout: "new_pn": { 2024-09-14T03:47:49.899 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.899 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.899 INFO:tasks.workunit.client.0.smithi070.stdout: "new_pn_latency": { 2024-09-14T03:47:49.900 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.900 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.900 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.900 INFO:tasks.workunit.client.0.smithi070.stdout: "refresh": { 2024-09-14T03:47:49.900 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 826 2024-09-14T03:47:49.900 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.901 INFO:tasks.workunit.client.0.smithi070.stdout: "refresh_latency": { 2024-09-14T03:47:49.901 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 826, 2024-09-14T03:47:49.901 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1856945399 2024-09-14T03:47:49.901 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.901 INFO:tasks.workunit.client.0.smithi070.stdout: "restart": { 2024-09-14T03:47:49.901 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 8 2024-09-14T03:47:49.902 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.902 INFO:tasks.workunit.client.0.smithi070.stdout: "share_state": { 2024-09-14T03:47:49.902 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.902 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.902 INFO:tasks.workunit.client.0.smithi070.stdout: "share_state_bytes": { 2024-09-14T03:47:49.903 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.903 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.903 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.903 INFO:tasks.workunit.client.0.smithi070.stdout: "share_state_keys": { 2024-09-14T03:47:49.903 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.903 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.904 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.904 INFO:tasks.workunit.client.0.smithi070.stdout: "start_leader": { 2024-09-14T03:47:49.904 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.904 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.904 INFO:tasks.workunit.client.0.smithi070.stdout: "start_peon": { 2024-09-14T03:47:49.904 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1 2024-09-14T03:47:49.905 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.905 INFO:tasks.workunit.client.0.smithi070.stdout: "store_state": { 2024-09-14T03:47:49.905 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 825 2024-09-14T03:47:49.905 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.905 INFO:tasks.workunit.client.0.smithi070.stdout: "store_state_bytes": { 2024-09-14T03:47:49.905 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 825, 2024-09-14T03:47:49.906 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7906379 2024-09-14T03:47:49.906 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.906 INFO:tasks.workunit.client.0.smithi070.stdout: "store_state_keys": { 2024-09-14T03:47:49.906 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 825, 2024-09-14T03:47:49.906 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5552 2024-09-14T03:47:49.906 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.907 INFO:tasks.workunit.client.0.smithi070.stdout: "store_state_latency": { 2024-09-14T03:47:49.907 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 825, 2024-09-14T03:47:49.907 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 94997208 2024-09-14T03:47:49.907 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.907 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.907 INFO:tasks.workunit.client.0.smithi070.stdout: "prioritycache": { 2024-09-14T03:47:49.908 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_bytes": { 2024-09-14T03:47:49.908 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1020054731 2024-09-14T03:47:49.908 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.908 INFO:tasks.workunit.client.0.smithi070.stdout: "heap_bytes": { 2024-09-14T03:47:49.908 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 48029696 2024-09-14T03:47:49.909 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.909 INFO:tasks.workunit.client.0.smithi070.stdout: "mapped_bytes": { 2024-09-14T03:47:49.909 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 33873920 2024-09-14T03:47:49.909 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.909 INFO:tasks.workunit.client.0.smithi070.stdout: "target_bytes": { 2024-09-14T03:47:49.909 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2147483648 2024-09-14T03:47:49.910 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.910 INFO:tasks.workunit.client.0.smithi070.stdout: "unmapped_bytes": { 2024-09-14T03:47:49.910 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 14155776 2024-09-14T03:47:49.910 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.910 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.910 INFO:tasks.workunit.client.0.smithi070.stdout: "prioritycache:full": { 2024-09-14T03:47:49.911 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:49.911 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 348127232 2024-09-14T03:47:49.911 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.911 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:49.911 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.911 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.912 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:49.912 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.912 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.912 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:49.912 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 277020366 2024-09-14T03:47:49.912 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.913 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:49.913 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 273925 2024-09-14T03:47:49.913 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.913 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:49.913 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.913 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.914 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:49.914 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.914 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.914 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:49.914 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.915 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.915 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:49.915 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.915 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.915 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:49.915 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.916 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.916 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:49.916 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.916 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.916 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:49.916 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.917 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.917 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:49.917 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.917 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.917 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:49.917 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 70832941 2024-09-14T03:47:49.918 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.918 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.918 INFO:tasks.workunit.client.0.smithi070.stdout: "prioritycache:inc": { 2024-09-14T03:47:49.918 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:49.918 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 348127232 2024-09-14T03:47:49.918 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.919 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:49.919 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.919 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.919 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:49.919 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.919 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.920 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:49.920 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 277020366 2024-09-14T03:47:49.920 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.920 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:49.920 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 41460 2024-09-14T03:47:49.920 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.921 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:49.921 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.921 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.921 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:49.921 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.921 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.922 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:49.922 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.922 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.922 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:49.922 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.922 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.922 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:49.923 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.923 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.923 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:49.923 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.923 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.923 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:49.923 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.923 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.924 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:49.924 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.924 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.924 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:49.924 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71065406 2024-09-14T03:47:49.924 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.924 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.924 INFO:tasks.workunit.client.0.smithi070.stdout: "prioritycache:kv": { 2024-09-14T03:47:49.925 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:49.925 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 322961408 2024-09-14T03:47:49.925 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.925 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:49.925 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:49.925 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.925 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:49.925 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.926 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.926 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:49.926 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 184680244 2024-09-14T03:47:49.926 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.926 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:49.926 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.926 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.927 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:49.927 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.927 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.927 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:49.927 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.927 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.927 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:49.927 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.928 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.928 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:49.928 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.928 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.928 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:49.928 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.928 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.928 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:49.928 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.928 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.928 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:49.929 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.929 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.929 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:49.929 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.929 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.929 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:49.929 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71172300 2024-09-14T03:47:49.929 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.929 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.929 INFO:tasks.workunit.client.0.smithi070.stdout: "rocksdb": { 2024-09-14T03:47:49.929 INFO:tasks.workunit.client.0.smithi070.stdout: "compact": { 2024-09-14T03:47:49.929 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.929 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.929 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_completed": { 2024-09-14T03:47:49.930 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1 2024-09-14T03:47:49.930 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.930 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lasted": { 2024-09-14T03:47:49.930 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 24169414 2024-09-14T03:47:49.930 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.930 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_queue_len": { 2024-09-14T03:47:49.930 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.930 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.930 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_queue_merge": { 2024-09-14T03:47:49.930 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.930 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.930 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_running": { 2024-09-14T03:47:49.930 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.930 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.931 INFO:tasks.workunit.client.0.smithi070.stdout: "get_latency": { 2024-09-14T03:47:49.931 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 35177, 2024-09-14T03:47:49.931 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 85337748 2024-09-14T03:47:49.931 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.931 INFO:tasks.workunit.client.0.smithi070.stdout: "rocksdb_write_delay_time": { 2024-09-14T03:47:49.931 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.931 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.931 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.931 INFO:tasks.workunit.client.0.smithi070.stdout: "rocksdb_write_memtable_time": { 2024-09-14T03:47:49.931 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.931 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.931 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.931 INFO:tasks.workunit.client.0.smithi070.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-14T03:47:49.932 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.932 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.932 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.932 INFO:tasks.workunit.client.0.smithi070.stdout: "rocksdb_write_wal_time": { 2024-09-14T03:47:49.932 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.932 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.932 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.932 INFO:tasks.workunit.client.0.smithi070.stdout: "submit_latency": { 2024-09-14T03:47:49.932 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.932 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.932 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.933 INFO:tasks.workunit.client.0.smithi070.stdout: "submit_sync_latency": { 2024-09-14T03:47:49.933 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1771, 2024-09-14T03:47:49.933 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 203625534 2024-09-14T03:47:49.933 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.933 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.933 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.933 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.0": { 2024-09-14T03:47:49.933 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:49.933 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_slow_fallback": { 2024-09-14T03:47:49.933 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.933 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.933 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_slow_size_fallback": { 2024-09-14T03:47:49.933 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.934 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.934 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_db": { 2024-09-14T03:47:49.934 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 65536 2024-09-14T03:47:49.934 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.934 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_main": { 2024-09-14T03:47:49.934 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.934 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.934 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_wal": { 2024-09-14T03:47:49.934 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.934 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.935 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_slow": { 2024-09-14T03:47:49.935 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.935 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.935 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_sst": { 2024-09-14T03:47:49.935 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12288 2024-09-14T03:47:49.935 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.935 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_wal": { 2024-09-14T03:47:49.935 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2306048 2024-09-14T03:47:49.935 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.935 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lat": { 2024-09-14T03:47:49.935 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.935 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.935 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.936 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lock_lat": { 2024-09-14T03:47:49.936 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.936 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.936 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.936 INFO:tasks.workunit.client.0.smithi070.stdout: "db_total_bytes": { 2024-09-14T03:47:49.936 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 95995035648 2024-09-14T03:47:49.936 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.936 INFO:tasks.workunit.client.0.smithi070.stdout: "db_used_bytes": { 2024-09-14T03:47:49.936 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 304480256 2024-09-14T03:47:49.936 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.936 INFO:tasks.workunit.client.0.smithi070.stdout: "log_bytes": { 2024-09-14T03:47:49.936 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1552384 2024-09-14T03:47:49.936 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.937 INFO:tasks.workunit.client.0.smithi070.stdout: "logged_bytes": { 2024-09-14T03:47:49.937 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1413120 2024-09-14T03:47:49.937 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.937 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_db": { 2024-09-14T03:47:49.937 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 312868864 2024-09-14T03:47:49.937 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.937 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_slow": { 2024-09-14T03:47:49.937 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.937 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.937 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_wal": { 2024-09-14T03:47:49.937 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.938 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.938 INFO:tasks.workunit.client.0.smithi070.stdout: "num_files": { 2024-09-14T03:47:49.938 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 14 2024-09-14T03:47:49.938 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.938 INFO:tasks.workunit.client.0.smithi070.stdout: "read_bytes": { 2024-09-14T03:47:49.938 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 195697 2024-09-14T03:47:49.938 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.938 INFO:tasks.workunit.client.0.smithi070.stdout: "read_count": { 2024-09-14T03:47:49.938 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 23 2024-09-14T03:47:49.938 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.938 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes": { 2024-09-14T03:47:49.938 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1269760 2024-09-14T03:47:49.938 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.939 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_db": { 2024-09-14T03:47:49.939 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1273856 2024-09-14T03:47:49.939 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.939 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_slow": { 2024-09-14T03:47:49.939 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.939 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.939 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_wal": { 2024-09-14T03:47:49.939 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.939 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.939 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_count": { 2024-09-14T03:47:49.939 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 15 2024-09-14T03:47:49.939 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.939 INFO:tasks.workunit.client.0.smithi070.stdout: "read_prefetch_bytes": { 2024-09-14T03:47:49.939 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4789 2024-09-14T03:47:49.940 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.940 INFO:tasks.workunit.client.0.smithi070.stdout: "read_prefetch_count": { 2024-09-14T03:47:49.940 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4 2024-09-14T03:47:49.940 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.940 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_buffer_bytes": { 2024-09-14T03:47:49.940 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5589 2024-09-14T03:47:49.940 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.940 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_buffer_count": { 2024-09-14T03:47:49.940 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 24 2024-09-14T03:47:49.940 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.940 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_bytes": { 2024-09-14T03:47:49.940 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5589 2024-09-14T03:47:49.940 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.941 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_count": { 2024-09-14T03:47:49.941 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 24 2024-09-14T03:47:49.941 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.941 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes": { 2024-09-14T03:47:49.941 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.941 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.941 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_db": { 2024-09-14T03:47:49.941 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.941 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.941 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_slow": { 2024-09-14T03:47:49.941 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.941 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.941 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_wal": { 2024-09-14T03:47:49.942 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.942 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.942 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_count": { 2024-09-14T03:47:49.942 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.942 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.942 INFO:tasks.workunit.client.0.smithi070.stdout: "slow_total_bytes": { 2024-09-14T03:47:49.942 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.942 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.942 INFO:tasks.workunit.client.0.smithi070.stdout: "slow_used_bytes": { 2024-09-14T03:47:49.942 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.942 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.942 INFO:tasks.workunit.client.0.smithi070.stdout: "wal_total_bytes": { 2024-09-14T03:47:49.942 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.942 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.943 INFO:tasks.workunit.client.0.smithi070.stdout: "wal_used_bytes": { 2024-09-14T03:47:49.943 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.943 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.943 INFO:tasks.workunit.client.0.smithi070.stdout: "write_bytes": { 2024-09-14T03:47:49.943 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3792896 2024-09-14T03:47:49.943 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.943 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.943 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore": { 2024-09-14T03:47:49.943 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit": { 2024-09-14T03:47:49.943 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4096 2024-09-14T03:47:49.943 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.943 INFO:tasks.workunit.client.0.smithi070.stdout: "allocated": { 2024-09-14T03:47:49.943 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1302528 2024-09-14T03:47:49.943 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.944 INFO:tasks.workunit.client.0.smithi070.stdout: "clist_lat": { 2024-09-14T03:47:49.944 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 4, 2024-09-14T03:47:49.944 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 152555 2024-09-14T03:47:49.944 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.944 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_lat": { 2024-09-14T03:47:49.944 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.944 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.944 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.944 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed": { 2024-09-14T03:47:49.944 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.944 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.944 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed_allocated": { 2024-09-14T03:47:49.944 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.945 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.945 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed_original": { 2024-09-14T03:47:49.945 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.945 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.945 INFO:tasks.workunit.client.0.smithi070.stdout: "csum_lat": { 2024-09-14T03:47:49.945 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 24, 2024-09-14T03:47:49.945 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 73787 2024-09-14T03:47:49.945 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.945 INFO:tasks.workunit.client.0.smithi070.stdout: "decompress_lat": { 2024-09-14T03:47:49.945 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.946 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.946 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.946 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_commit_lat": { 2024-09-14T03:47:49.946 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 328, 2024-09-14T03:47:49.946 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 59340322 2024-09-14T03:47:49.946 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.947 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_final_lat": { 2024-09-14T03:47:49.947 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 321, 2024-09-14T03:47:49.947 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7807995 2024-09-14T03:47:49.947 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.947 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_flush_lat": { 2024-09-14T03:47:49.947 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 328, 2024-09-14T03:47:49.947 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 452089 2024-09-14T03:47:49.948 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.948 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_sync_lat": { 2024-09-14T03:47:49.948 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 328, 2024-09-14T03:47:49.948 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 59792411 2024-09-14T03:47:49.948 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.948 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_get_keys_lat": { 2024-09-14T03:47:49.948 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.949 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.949 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.949 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_get_values_lat": { 2024-09-14T03:47:49.949 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:49.949 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 26965 2024-09-14T03:47:49.949 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.949 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_lower_bound_lat": { 2024-09-14T03:47:49.950 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 5, 2024-09-14T03:47:49.950 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 11764 2024-09-14T03:47:49.950 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.950 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_next_lat": { 2024-09-14T03:47:49.950 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.950 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.950 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.950 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_seek_to_first_lat": { 2024-09-14T03:47:49.951 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.951 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.951 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.951 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_upper_bound_lat": { 2024-09-14T03:47:49.951 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 4, 2024-09-14T03:47:49.951 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 10295 2024-09-14T03:47:49.951 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.952 INFO:tasks.workunit.client.0.smithi070.stdout: "onode_hits": { 2024-09-14T03:47:49.952 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 689 2024-09-14T03:47:49.952 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.952 INFO:tasks.workunit.client.0.smithi070.stdout: "onode_misses": { 2024-09-14T03:47:49.952 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 228 2024-09-14T03:47:49.952 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.953 INFO:tasks.workunit.client.0.smithi070.stdout: "read_lat": { 2024-09-14T03:47:49.953 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 111, 2024-09-14T03:47:49.953 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1647160 2024-09-14T03:47:49.953 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.953 INFO:tasks.workunit.client.0.smithi070.stdout: "read_onode_meta_lat": { 2024-09-14T03:47:49.953 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 262, 2024-09-14T03:47:49.954 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 266412 2024-09-14T03:47:49.954 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.954 INFO:tasks.workunit.client.0.smithi070.stdout: "read_wait_aio_lat": { 2024-09-14T03:47:49.954 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 151, 2024-09-14T03:47:49.954 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 997696 2024-09-14T03:47:49.954 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.954 INFO:tasks.workunit.client.0.smithi070.stdout: "reads_with_retries": { 2024-09-14T03:47:49.954 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.954 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.955 INFO:tasks.workunit.client.0.smithi070.stdout: "remove_lat": { 2024-09-14T03:47:49.955 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:49.955 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 31068 2024-09-14T03:47:49.955 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.955 INFO:tasks.workunit.client.0.smithi070.stdout: "state_aio_wait_lat": { 2024-09-14T03:47:49.955 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 420, 2024-09-14T03:47:49.955 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 9066587 2024-09-14T03:47:49.955 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.955 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_aio_wait_lat": { 2024-09-14T03:47:49.955 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 96, 2024-09-14T03:47:49.955 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7418949 2024-09-14T03:47:49.955 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.955 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_cleanup_lat": { 2024-09-14T03:47:49.955 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 96, 2024-09-14T03:47:49.956 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 474004274278 2024-09-14T03:47:49.956 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.956 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_queued_lat": { 2024-09-14T03:47:49.956 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 96, 2024-09-14T03:47:49.956 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 160665384289 2024-09-14T03:47:49.956 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.956 INFO:tasks.workunit.client.0.smithi070.stdout: "state_done_lat": { 2024-09-14T03:47:49.956 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 420, 2024-09-14T03:47:49.956 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 64892973381 2024-09-14T03:47:49.956 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.956 INFO:tasks.workunit.client.0.smithi070.stdout: "state_finishing_lat": { 2024-09-14T03:47:49.956 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 420, 2024-09-14T03:47:49.956 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 43338 2024-09-14T03:47:49.957 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.957 INFO:tasks.workunit.client.0.smithi070.stdout: "state_io_done_lat": { 2024-09-14T03:47:49.957 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 420, 2024-09-14T03:47:49.957 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 796428 2024-09-14T03:47:49.957 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.957 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_commiting_lat": { 2024-09-14T03:47:49.957 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 420, 2024-09-14T03:47:49.957 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 119571365 2024-09-14T03:47:49.957 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.957 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_done_lat": { 2024-09-14T03:47:49.957 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 420, 2024-09-14T03:47:49.957 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 130554 2024-09-14T03:47:49.958 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.958 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_queued_lat": { 2024-09-14T03:47:49.958 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 420, 2024-09-14T03:47:49.958 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 86629140 2024-09-14T03:47:49.958 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.958 INFO:tasks.workunit.client.0.smithi070.stdout: "state_prepare_lat": { 2024-09-14T03:47:49.958 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 420, 2024-09-14T03:47:49.958 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 17467553 2024-09-14T03:47:49.958 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.958 INFO:tasks.workunit.client.0.smithi070.stdout: "stored": { 2024-09-14T03:47:49.958 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 903336 2024-09-14T03:47:49.958 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.958 INFO:tasks.workunit.client.0.smithi070.stdout: "truncate_lat": { 2024-09-14T03:47:49.958 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 56, 2024-09-14T03:47:49.959 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 264416 2024-09-14T03:47:49.959 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.959 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_commit_lat": { 2024-09-14T03:47:49.959 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 420, 2024-09-14T03:47:49.959 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 233587502 2024-09-14T03:47:49.959 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.959 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_submit_lat": { 2024-09-14T03:47:49.959 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 420, 2024-09-14T03:47:49.959 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 21081399 2024-09-14T03:47:49.959 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.959 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_throttle_lat": { 2024-09-14T03:47:49.959 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 420, 2024-09-14T03:47:49.959 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2755231 2024-09-14T03:47:49.959 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.960 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.960 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache": { 2024-09-14T03:47:49.960 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_bytes": { 2024-09-14T03:47:49.960 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1861444387 2024-09-14T03:47:49.960 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.960 INFO:tasks.workunit.client.0.smithi070.stdout: "heap_bytes": { 2024-09-14T03:47:49.960 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 272023552 2024-09-14T03:47:49.960 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.960 INFO:tasks.workunit.client.0.smithi070.stdout: "mapped_bytes": { 2024-09-14T03:47:49.960 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 271007744 2024-09-14T03:47:49.960 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.960 INFO:tasks.workunit.client.0.smithi070.stdout: "target_bytes": { 2024-09-14T03:47:49.960 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3137353830 2024-09-14T03:47:49.961 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.961 INFO:tasks.workunit.client.0.smithi070.stdout: "unmapped_bytes": { 2024-09-14T03:47:49.961 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1015808 2024-09-14T03:47:49.961 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.961 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.961 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:data": { 2024-09-14T03:47:49.961 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:49.961 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 159383552 2024-09-14T03:47:49.961 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.961 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:49.961 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.961 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.962 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:49.962 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.962 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.962 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:49.962 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 85496610 2024-09-14T03:47:49.962 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.962 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:49.962 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.962 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.962 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:49.962 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.962 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.962 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:49.962 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.963 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.963 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:49.963 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.963 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.963 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:49.963 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.963 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.963 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:49.963 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 8192 2024-09-14T03:47:49.963 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.963 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:49.963 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.963 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.963 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:49.964 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.964 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.964 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:49.964 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.964 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.964 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:49.964 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 73878750 2024-09-14T03:47:49.964 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.964 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.964 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:kv": { 2024-09-14T03:47:49.964 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:49.964 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 780140544 2024-09-14T03:47:49.964 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.965 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:49.965 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:49.965 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.965 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:49.965 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.965 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.965 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:49.965 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 641224575 2024-09-14T03:47:49.965 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.965 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:49.965 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.965 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.965 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:49.966 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.966 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.966 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:49.966 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.966 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.966 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:49.966 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.966 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.966 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:49.966 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.966 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.967 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:49.967 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2128 2024-09-14T03:47:49.967 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.967 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:49.967 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.967 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.967 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:49.967 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.967 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.967 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:49.967 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.967 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.967 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:49.967 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71804977 2024-09-14T03:47:49.968 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.968 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.968 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:kv_onode": { 2024-09-14T03:47:49.968 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:49.968 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 192937984 2024-09-14T03:47:49.968 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.968 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:49.968 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:49.968 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.968 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:49.968 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 464 2024-09-14T03:47:49.968 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.968 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:49.968 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 56997740 2024-09-14T03:47:49.969 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.969 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:49.969 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.969 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.969 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:49.969 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.969 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.969 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:49.969 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.969 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.969 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:49.969 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.970 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.970 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:49.970 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.970 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.970 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:49.970 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.970 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.970 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:49.970 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.970 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.970 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:49.970 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.971 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.971 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:49.971 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.971 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.971 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:49.971 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 68830916 2024-09-14T03:47:49.971 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.971 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.971 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:meta": { 2024-09-14T03:47:49.971 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:49.971 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 713031680 2024-09-14T03:47:49.972 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.972 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:49.972 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.972 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.972 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:49.972 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.972 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.972 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:49.972 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 641224575 2024-09-14T03:47:49.972 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.972 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:49.972 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.972 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.972 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:49.973 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2839 2024-09-14T03:47:49.973 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.973 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:49.973 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7097 2024-09-14T03:47:49.973 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.973 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:49.973 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.973 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.973 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:49.973 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.973 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.973 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:49.974 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 272550 2024-09-14T03:47:49.974 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.974 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:49.974 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.974 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.974 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:49.974 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.974 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.974 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:49.974 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.974 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.974 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:49.974 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71524619 2024-09-14T03:47:49.974 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.975 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.975 INFO:tasks.workunit.client.0.smithi070.stdout: "objecter": { 2024-09-14T03:47:49.975 INFO:tasks.workunit.client.0.smithi070.stdout: "op_active": { 2024-09-14T03:47:49.975 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.975 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.975 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r": { 2024-09-14T03:47:49.975 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.975 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.975 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rmw": { 2024-09-14T03:47:49.975 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.975 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.975 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w": { 2024-09-14T03:47:49.976 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.976 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.976 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.976 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:49.976 INFO:tasks.workunit.client.0.smithi070.stdout: "numpg": { 2024-09-14T03:47:49.976 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 45 2024-09-14T03:47:49.976 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.976 INFO:tasks.workunit.client.0.smithi070.stdout: "numpg_removing": { 2024-09-14T03:47:49.976 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.976 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.976 INFO:tasks.workunit.client.0.smithi070.stdout: "op": { 2024-09-14T03:47:49.976 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 6 2024-09-14T03:47:49.976 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.977 INFO:tasks.workunit.client.0.smithi070.stdout: "op_delayed_degraded": { 2024-09-14T03:47:49.977 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.977 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.977 INFO:tasks.workunit.client.0.smithi070.stdout: "op_delayed_unreadable": { 2024-09-14T03:47:49.977 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.977 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.977 INFO:tasks.workunit.client.0.smithi070.stdout: "op_in_bytes": { 2024-09-14T03:47:49.977 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 604 2024-09-14T03:47:49.977 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.977 INFO:tasks.workunit.client.0.smithi070.stdout: "op_latency": { 2024-09-14T03:47:49.977 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 6, 2024-09-14T03:47:49.977 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 49065134 2024-09-14T03:47:49.977 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.977 INFO:tasks.workunit.client.0.smithi070.stdout: "op_out_bytes": { 2024-09-14T03:47:49.978 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.978 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.978 INFO:tasks.workunit.client.0.smithi070.stdout: "op_prepare_latency": { 2024-09-14T03:47:49.978 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 7, 2024-09-14T03:47:49.978 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4089275 2024-09-14T03:47:49.978 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.978 INFO:tasks.workunit.client.0.smithi070.stdout: "op_process_latency": { 2024-09-14T03:47:49.978 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 6, 2024-09-14T03:47:49.979 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 15715463 2024-09-14T03:47:49.979 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.979 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r": { 2024-09-14T03:47:49.979 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.979 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.979 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_latency": { 2024-09-14T03:47:49.979 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.979 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.979 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.979 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:47:49.979 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.979 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.979 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_out_bytes": { 2024-09-14T03:47:49.980 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.980 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.980 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_prepare_latency": { 2024-09-14T03:47:49.980 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.980 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.980 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.980 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_process_latency": { 2024-09-14T03:47:49.980 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.980 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.980 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.980 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw": { 2024-09-14T03:47:49.980 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.980 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.981 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_in_bytes": { 2024-09-14T03:47:49.981 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.981 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.981 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency": { 2024-09-14T03:47:49.981 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.981 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.981 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.981 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:47:49.981 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.981 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.981 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:47:49.981 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.982 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.982 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_out_bytes": { 2024-09-14T03:47:49.982 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.982 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.982 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_prepare_latency": { 2024-09-14T03:47:49.982 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.983 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.984 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.984 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_process_latency": { 2024-09-14T03:47:49.984 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.984 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.984 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.984 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w": { 2024-09-14T03:47:49.984 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 6 2024-09-14T03:47:49.984 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.984 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_in_bytes": { 2024-09-14T03:47:49.984 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 604 2024-09-14T03:47:49.984 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.984 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_latency": { 2024-09-14T03:47:49.984 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 6, 2024-09-14T03:47:49.985 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 49065134 2024-09-14T03:47:49.985 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.985 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:47:49.985 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.985 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.985 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_prepare_latency": { 2024-09-14T03:47:49.985 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 7, 2024-09-14T03:47:49.985 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4089275 2024-09-14T03:47:49.985 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.985 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_process_latency": { 2024-09-14T03:47:49.985 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 6, 2024-09-14T03:47:49.985 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 15715463 2024-09-14T03:47:49.986 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.986 INFO:tasks.workunit.client.0.smithi070.stdout: "op_wip": { 2024-09-14T03:47:49.986 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.986 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.986 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_bytes": { 2024-09-14T03:47:49.986 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.986 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.986 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_ops": { 2024-09-14T03:47:49.986 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.986 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.986 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_bytes": { 2024-09-14T03:47:49.986 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 95995035648 2024-09-14T03:47:49.986 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.986 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_bytes_used": { 2024-09-14T03:47:49.986 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 305790976 2024-09-14T03:47:49.987 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.987 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.987 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.987 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.1": { 2024-09-14T03:47:49.987 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:49.987 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_slow_fallback": { 2024-09-14T03:47:49.987 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.987 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.987 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_slow_size_fallback": { 2024-09-14T03:47:49.987 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.987 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.987 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_db": { 2024-09-14T03:47:49.987 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 65536 2024-09-14T03:47:49.987 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.987 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_main": { 2024-09-14T03:47:49.988 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.988 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.988 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_wal": { 2024-09-14T03:47:49.988 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.988 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.988 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_slow": { 2024-09-14T03:47:49.988 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.988 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.988 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_sst": { 2024-09-14T03:47:49.988 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12288 2024-09-14T03:47:49.988 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.988 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_wal": { 2024-09-14T03:47:49.988 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1970176 2024-09-14T03:47:49.989 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.989 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lat": { 2024-09-14T03:47:49.989 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.989 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.989 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.989 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lock_lat": { 2024-09-14T03:47:49.989 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.989 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.989 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.989 INFO:tasks.workunit.client.0.smithi070.stdout: "db_total_bytes": { 2024-09-14T03:47:49.989 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 95995035648 2024-09-14T03:47:49.989 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.989 INFO:tasks.workunit.client.0.smithi070.stdout: "db_used_bytes": { 2024-09-14T03:47:49.990 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 304480256 2024-09-14T03:47:49.990 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.990 INFO:tasks.workunit.client.0.smithi070.stdout: "log_bytes": { 2024-09-14T03:47:49.990 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1306624 2024-09-14T03:47:49.990 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.990 INFO:tasks.workunit.client.0.smithi070.stdout: "logged_bytes": { 2024-09-14T03:47:49.990 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1167360 2024-09-14T03:47:49.990 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.990 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_db": { 2024-09-14T03:47:49.990 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 312868864 2024-09-14T03:47:49.990 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.990 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_slow": { 2024-09-14T03:47:49.990 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.991 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.991 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_wal": { 2024-09-14T03:47:49.991 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.991 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.991 INFO:tasks.workunit.client.0.smithi070.stdout: "num_files": { 2024-09-14T03:47:49.991 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 14 2024-09-14T03:47:49.991 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.991 INFO:tasks.workunit.client.0.smithi070.stdout: "read_bytes": { 2024-09-14T03:47:49.991 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 195697 2024-09-14T03:47:49.991 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.991 INFO:tasks.workunit.client.0.smithi070.stdout: "read_count": { 2024-09-14T03:47:49.991 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 23 2024-09-14T03:47:49.991 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.991 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes": { 2024-09-14T03:47:49.992 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1269760 2024-09-14T03:47:49.992 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.992 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_db": { 2024-09-14T03:47:49.992 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1273856 2024-09-14T03:47:49.992 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.992 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_slow": { 2024-09-14T03:47:49.992 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.992 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.992 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_wal": { 2024-09-14T03:47:49.992 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.992 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.992 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_count": { 2024-09-14T03:47:49.992 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 15 2024-09-14T03:47:49.993 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.993 INFO:tasks.workunit.client.0.smithi070.stdout: "read_prefetch_bytes": { 2024-09-14T03:47:49.993 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4789 2024-09-14T03:47:49.993 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.993 INFO:tasks.workunit.client.0.smithi070.stdout: "read_prefetch_count": { 2024-09-14T03:47:49.993 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4 2024-09-14T03:47:49.993 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.993 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_buffer_bytes": { 2024-09-14T03:47:49.993 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5589 2024-09-14T03:47:49.993 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.993 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_buffer_count": { 2024-09-14T03:47:49.993 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 24 2024-09-14T03:47:49.993 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.994 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_bytes": { 2024-09-14T03:47:49.994 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5589 2024-09-14T03:47:49.994 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.994 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_count": { 2024-09-14T03:47:49.994 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 24 2024-09-14T03:47:49.994 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.994 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes": { 2024-09-14T03:47:49.994 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.994 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.994 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_db": { 2024-09-14T03:47:49.994 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.994 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.994 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_slow": { 2024-09-14T03:47:49.994 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.995 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.995 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_wal": { 2024-09-14T03:47:49.995 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.995 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.995 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_count": { 2024-09-14T03:47:49.995 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.995 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.995 INFO:tasks.workunit.client.0.smithi070.stdout: "slow_total_bytes": { 2024-09-14T03:47:49.995 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.995 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.995 INFO:tasks.workunit.client.0.smithi070.stdout: "slow_used_bytes": { 2024-09-14T03:47:49.995 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.995 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.996 INFO:tasks.workunit.client.0.smithi070.stdout: "wal_total_bytes": { 2024-09-14T03:47:49.996 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.996 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.996 INFO:tasks.workunit.client.0.smithi070.stdout: "wal_used_bytes": { 2024-09-14T03:47:49.996 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.996 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.996 INFO:tasks.workunit.client.0.smithi070.stdout: "write_bytes": { 2024-09-14T03:47:49.996 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3211264 2024-09-14T03:47:49.996 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:49.996 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.996 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore": { 2024-09-14T03:47:49.996 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit": { 2024-09-14T03:47:49.996 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4096 2024-09-14T03:47:49.996 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.997 INFO:tasks.workunit.client.0.smithi070.stdout: "allocated": { 2024-09-14T03:47:49.997 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 696320 2024-09-14T03:47:49.997 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.997 INFO:tasks.workunit.client.0.smithi070.stdout: "clist_lat": { 2024-09-14T03:47:49.997 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 5, 2024-09-14T03:47:49.997 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 266520 2024-09-14T03:47:49.997 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.997 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_lat": { 2024-09-14T03:47:49.997 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.997 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.997 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.997 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed": { 2024-09-14T03:47:49.997 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.998 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.998 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed_allocated": { 2024-09-14T03:47:49.998 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.998 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.998 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed_original": { 2024-09-14T03:47:49.998 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.998 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.998 INFO:tasks.workunit.client.0.smithi070.stdout: "csum_lat": { 2024-09-14T03:47:49.998 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 22, 2024-09-14T03:47:49.998 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 70568 2024-09-14T03:47:49.998 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.998 INFO:tasks.workunit.client.0.smithi070.stdout: "decompress_lat": { 2024-09-14T03:47:49.999 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:49.999 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:49.999 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.999 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_commit_lat": { 2024-09-14T03:47:49.999 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 268, 2024-09-14T03:47:49.999 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 53199400 2024-09-14T03:47:49.999 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.999 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_final_lat": { 2024-09-14T03:47:49.999 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 265, 2024-09-14T03:47:49.999 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7667292 2024-09-14T03:47:49.999 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:49.999 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_flush_lat": { 2024-09-14T03:47:49.999 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 268, 2024-09-14T03:47:50.000 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 361726 2024-09-14T03:47:50.000 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.000 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_sync_lat": { 2024-09-14T03:47:50.000 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 268, 2024-09-14T03:47:50.000 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 53561126 2024-09-14T03:47:50.000 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.000 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_get_keys_lat": { 2024-09-14T03:47:50.000 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.000 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.000 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.000 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_get_values_lat": { 2024-09-14T03:47:50.000 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 3, 2024-09-14T03:47:50.000 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 34171 2024-09-14T03:47:50.000 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.001 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_lower_bound_lat": { 2024-09-14T03:47:50.001 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 6, 2024-09-14T03:47:50.001 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 15713 2024-09-14T03:47:50.001 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.001 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_next_lat": { 2024-09-14T03:47:50.001 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.001 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.001 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.001 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_seek_to_first_lat": { 2024-09-14T03:47:50.001 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.002 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.002 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.002 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_upper_bound_lat": { 2024-09-14T03:47:50.002 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 5, 2024-09-14T03:47:50.002 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 13193 2024-09-14T03:47:50.002 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.002 INFO:tasks.workunit.client.0.smithi070.stdout: "onode_hits": { 2024-09-14T03:47:50.002 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 682 2024-09-14T03:47:50.002 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.002 INFO:tasks.workunit.client.0.smithi070.stdout: "onode_misses": { 2024-09-14T03:47:50.002 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 228 2024-09-14T03:47:50.002 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.002 INFO:tasks.workunit.client.0.smithi070.stdout: "read_lat": { 2024-09-14T03:47:50.003 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 120, 2024-09-14T03:47:50.003 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1733260 2024-09-14T03:47:50.003 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.003 INFO:tasks.workunit.client.0.smithi070.stdout: "read_onode_meta_lat": { 2024-09-14T03:47:50.003 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 279, 2024-09-14T03:47:50.003 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 251501 2024-09-14T03:47:50.003 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.003 INFO:tasks.workunit.client.0.smithi070.stdout: "read_wait_aio_lat": { 2024-09-14T03:47:50.003 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 159, 2024-09-14T03:47:50.003 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 988213 2024-09-14T03:47:50.003 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.003 INFO:tasks.workunit.client.0.smithi070.stdout: "reads_with_retries": { 2024-09-14T03:47:50.003 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.003 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.004 INFO:tasks.workunit.client.0.smithi070.stdout: "remove_lat": { 2024-09-14T03:47:50.004 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 3, 2024-09-14T03:47:50.004 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 102148 2024-09-14T03:47:50.004 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.004 INFO:tasks.workunit.client.0.smithi070.stdout: "state_aio_wait_lat": { 2024-09-14T03:47:50.004 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 361, 2024-09-14T03:47:50.004 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5214151 2024-09-14T03:47:50.004 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.004 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_aio_wait_lat": { 2024-09-14T03:47:50.004 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 95, 2024-09-14T03:47:50.004 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 6796471 2024-09-14T03:47:50.004 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.004 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_cleanup_lat": { 2024-09-14T03:47:50.005 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 94, 2024-09-14T03:47:50.005 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 271815987479 2024-09-14T03:47:50.005 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.005 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_queued_lat": { 2024-09-14T03:47:50.005 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 95, 2024-09-14T03:47:50.005 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 138625203385 2024-09-14T03:47:50.005 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.005 INFO:tasks.workunit.client.0.smithi070.stdout: "state_done_lat": { 2024-09-14T03:47:50.005 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 360, 2024-09-14T03:47:50.005 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 31863399419 2024-09-14T03:47:50.005 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.005 INFO:tasks.workunit.client.0.smithi070.stdout: "state_finishing_lat": { 2024-09-14T03:47:50.005 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 360, 2024-09-14T03:47:50.006 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 36839 2024-09-14T03:47:50.006 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.006 INFO:tasks.workunit.client.0.smithi070.stdout: "state_io_done_lat": { 2024-09-14T03:47:50.006 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 361, 2024-09-14T03:47:50.006 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 358838 2024-09-14T03:47:50.006 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.006 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_commiting_lat": { 2024-09-14T03:47:50.006 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 361, 2024-09-14T03:47:50.006 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 95831917 2024-09-14T03:47:50.006 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.006 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_done_lat": { 2024-09-14T03:47:50.006 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 361, 2024-09-14T03:47:50.006 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 109066 2024-09-14T03:47:50.007 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.007 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_queued_lat": { 2024-09-14T03:47:50.007 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 361, 2024-09-14T03:47:50.007 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 83392148 2024-09-14T03:47:50.007 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.007 INFO:tasks.workunit.client.0.smithi070.stdout: "state_prepare_lat": { 2024-09-14T03:47:50.007 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 361, 2024-09-14T03:47:50.007 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 17285706 2024-09-14T03:47:50.007 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.007 INFO:tasks.workunit.client.0.smithi070.stdout: "stored": { 2024-09-14T03:47:50.007 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 311801 2024-09-14T03:47:50.007 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.007 INFO:tasks.workunit.client.0.smithi070.stdout: "truncate_lat": { 2024-09-14T03:47:50.007 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 54, 2024-09-14T03:47:50.008 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 279365 2024-09-14T03:47:50.008 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.008 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_commit_lat": { 2024-09-14T03:47:50.008 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 361, 2024-09-14T03:47:50.008 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 202128554 2024-09-14T03:47:50.008 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.008 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_submit_lat": { 2024-09-14T03:47:50.008 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 361, 2024-09-14T03:47:50.008 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 22012732 2024-09-14T03:47:50.008 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.008 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_throttle_lat": { 2024-09-14T03:47:50.008 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 361, 2024-09-14T03:47:50.008 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2404877 2024-09-14T03:47:50.009 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.009 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.009 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache": { 2024-09-14T03:47:50.009 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_bytes": { 2024-09-14T03:47:50.009 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1861444387 2024-09-14T03:47:50.009 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.009 INFO:tasks.workunit.client.0.smithi070.stdout: "heap_bytes": { 2024-09-14T03:47:50.009 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 270909440 2024-09-14T03:47:50.009 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.009 INFO:tasks.workunit.client.0.smithi070.stdout: "mapped_bytes": { 2024-09-14T03:47:50.009 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 270409728 2024-09-14T03:47:50.009 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.009 INFO:tasks.workunit.client.0.smithi070.stdout: "target_bytes": { 2024-09-14T03:47:50.010 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3137353830 2024-09-14T03:47:50.010 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.010 INFO:tasks.workunit.client.0.smithi070.stdout: "unmapped_bytes": { 2024-09-14T03:47:50.010 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 499712 2024-09-14T03:47:50.010 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.010 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.010 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:data": { 2024-09-14T03:47:50.010 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:50.010 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 159383552 2024-09-14T03:47:50.010 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.010 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:50.010 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.010 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.011 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:50.011 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.011 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.011 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:50.011 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 85497890 2024-09-14T03:47:50.011 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.011 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:50.011 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.011 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.011 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:50.011 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.011 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.011 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:50.012 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.012 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.012 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:50.012 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.012 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.012 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:50.012 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.012 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.012 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:50.012 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12288 2024-09-14T03:47:50.012 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.012 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:50.012 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.013 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.013 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:50.013 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.013 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.013 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:50.013 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.013 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.013 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:50.013 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 73873374 2024-09-14T03:47:50.013 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.013 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.013 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:kv": { 2024-09-14T03:47:50.013 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:50.014 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 780140544 2024-09-14T03:47:50.014 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.014 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:50.014 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:50.014 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.014 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:50.014 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.014 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.014 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:50.014 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 641234175 2024-09-14T03:47:50.014 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.014 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:50.014 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.015 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.015 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:50.015 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.015 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.015 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:50.015 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.015 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.015 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:50.015 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.015 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.015 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:50.016 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.016 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.016 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:50.016 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2128 2024-09-14T03:47:50.016 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.016 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:50.016 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.016 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.016 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:50.016 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.016 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.016 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:50.016 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.017 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.017 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:50.017 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71795377 2024-09-14T03:47:50.017 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.017 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.017 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:kv_onode": { 2024-09-14T03:47:50.017 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:50.017 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 192937984 2024-09-14T03:47:50.017 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.017 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:50.017 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:50.017 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.017 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:50.018 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 464 2024-09-14T03:47:50.018 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.018 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:50.018 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 56998593 2024-09-14T03:47:50.018 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.018 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:50.018 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.018 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.018 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:50.018 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.018 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.018 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:50.018 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.018 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.019 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:50.019 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.019 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.019 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:50.019 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.019 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.019 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:50.019 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.019 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.019 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:50.019 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.019 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.020 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:50.020 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.020 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.020 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:50.020 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.020 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.020 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:50.020 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 68830063 2024-09-14T03:47:50.020 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.020 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.020 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:meta": { 2024-09-14T03:47:50.020 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:50.020 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 713031680 2024-09-14T03:47:50.020 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.021 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:50.021 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.021 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.021 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:50.021 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.021 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.021 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:50.021 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 641234175 2024-09-14T03:47:50.022 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.022 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:50.022 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.022 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.022 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:50.022 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.022 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.022 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:50.022 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.022 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.022 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:50.022 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.022 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.022 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:50.022 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.023 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.023 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:50.023 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 254262 2024-09-14T03:47:50.023 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.023 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:50.023 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.023 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.023 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:50.023 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.023 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.023 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:50.023 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.023 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.024 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:50.024 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71543243 2024-09-14T03:47:50.024 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.024 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.024 INFO:tasks.workunit.client.0.smithi070.stdout: "objecter": { 2024-09-14T03:47:50.024 INFO:tasks.workunit.client.0.smithi070.stdout: "op_active": { 2024-09-14T03:47:50.024 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.024 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.024 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r": { 2024-09-14T03:47:50.024 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.024 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.024 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rmw": { 2024-09-14T03:47:50.024 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.025 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.025 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w": { 2024-09-14T03:47:50.025 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.025 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.025 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.025 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:50.025 INFO:tasks.workunit.client.0.smithi070.stdout: "numpg": { 2024-09-14T03:47:50.025 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 38 2024-09-14T03:47:50.025 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.025 INFO:tasks.workunit.client.0.smithi070.stdout: "numpg_removing": { 2024-09-14T03:47:50.025 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.026 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.026 INFO:tasks.workunit.client.0.smithi070.stdout: "op": { 2024-09-14T03:47:50.026 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 97 2024-09-14T03:47:50.026 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.026 INFO:tasks.workunit.client.0.smithi070.stdout: "op_delayed_degraded": { 2024-09-14T03:47:50.026 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.026 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.026 INFO:tasks.workunit.client.0.smithi070.stdout: "op_delayed_unreadable": { 2024-09-14T03:47:50.026 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.026 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.026 INFO:tasks.workunit.client.0.smithi070.stdout: "op_in_bytes": { 2024-09-14T03:47:50.027 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1640827 2024-09-14T03:47:50.027 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.027 INFO:tasks.workunit.client.0.smithi070.stdout: "op_latency": { 2024-09-14T03:47:50.027 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 97, 2024-09-14T03:47:50.027 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 456375647 2024-09-14T03:47:50.027 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.027 INFO:tasks.workunit.client.0.smithi070.stdout: "op_out_bytes": { 2024-09-14T03:47:50.027 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 602 2024-09-14T03:47:50.027 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.027 INFO:tasks.workunit.client.0.smithi070.stdout: "op_prepare_latency": { 2024-09-14T03:47:50.027 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 97, 2024-09-14T03:47:50.027 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 33994934 2024-09-14T03:47:50.027 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.028 INFO:tasks.workunit.client.0.smithi070.stdout: "op_process_latency": { 2024-09-14T03:47:50.028 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 97, 2024-09-14T03:47:50.028 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 105513799 2024-09-14T03:47:50.028 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.028 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r": { 2024-09-14T03:47:50.028 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 18 2024-09-14T03:47:50.028 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.028 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_latency": { 2024-09-14T03:47:50.028 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 18, 2024-09-14T03:47:50.028 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2687087 2024-09-14T03:47:50.028 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.028 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:47:50.029 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.029 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.029 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_out_bytes": { 2024-09-14T03:47:50.029 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 602 2024-09-14T03:47:50.029 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.029 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_prepare_latency": { 2024-09-14T03:47:50.029 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 18, 2024-09-14T03:47:50.029 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2366109 2024-09-14T03:47:50.029 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.029 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_process_latency": { 2024-09-14T03:47:50.029 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 18, 2024-09-14T03:47:50.029 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1914640 2024-09-14T03:47:50.029 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.029 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw": { 2024-09-14T03:47:50.030 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12 2024-09-14T03:47:50.030 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.030 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_in_bytes": { 2024-09-14T03:47:50.030 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.030 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.030 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency": { 2024-09-14T03:47:50.030 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 12, 2024-09-14T03:47:50.030 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 10442901 2024-09-14T03:47:50.030 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.030 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:47:50.030 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.030 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.030 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:47:50.030 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.030 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.031 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_out_bytes": { 2024-09-14T03:47:50.031 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.031 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.031 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_prepare_latency": { 2024-09-14T03:47:50.031 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 12, 2024-09-14T03:47:50.031 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4934481 2024-09-14T03:47:50.031 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.031 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_process_latency": { 2024-09-14T03:47:50.031 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 12, 2024-09-14T03:47:50.031 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 9786014 2024-09-14T03:47:50.031 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.031 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w": { 2024-09-14T03:47:50.031 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67 2024-09-14T03:47:50.031 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.031 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_in_bytes": { 2024-09-14T03:47:50.032 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1640827 2024-09-14T03:47:50.032 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.032 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_latency": { 2024-09-14T03:47:50.032 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 67, 2024-09-14T03:47:50.032 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 443245659 2024-09-14T03:47:50.032 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.032 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:47:50.032 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.033 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.033 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_prepare_latency": { 2024-09-14T03:47:50.033 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 67, 2024-09-14T03:47:50.033 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 26694344 2024-09-14T03:47:50.033 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.033 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_process_latency": { 2024-09-14T03:47:50.033 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 67, 2024-09-14T03:47:50.033 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 93813145 2024-09-14T03:47:50.033 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.033 INFO:tasks.workunit.client.0.smithi070.stdout: "op_wip": { 2024-09-14T03:47:50.033 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.033 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.033 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_bytes": { 2024-09-14T03:47:50.034 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 590368 2024-09-14T03:47:50.034 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.034 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_ops": { 2024-09-14T03:47:50.034 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:50.034 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.034 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_bytes": { 2024-09-14T03:47:50.034 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 95995035648 2024-09-14T03:47:50.034 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.034 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_bytes_used": { 2024-09-14T03:47:50.034 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 305188864 2024-09-14T03:47:50.034 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.034 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.034 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.034 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.2": { 2024-09-14T03:47:50.035 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:50.035 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_slow_fallback": { 2024-09-14T03:47:50.035 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.035 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.035 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_slow_size_fallback": { 2024-09-14T03:47:50.035 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.035 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.035 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_db": { 2024-09-14T03:47:50.035 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 65536 2024-09-14T03:47:50.035 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.035 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_main": { 2024-09-14T03:47:50.035 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.035 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.036 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_wal": { 2024-09-14T03:47:50.036 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.036 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.036 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_slow": { 2024-09-14T03:47:50.036 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.036 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.036 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_sst": { 2024-09-14T03:47:50.036 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12288 2024-09-14T03:47:50.036 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.036 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_wal": { 2024-09-14T03:47:50.036 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1724416 2024-09-14T03:47:50.036 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.036 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lat": { 2024-09-14T03:47:50.037 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.037 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.037 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.037 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lock_lat": { 2024-09-14T03:47:50.037 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.037 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.037 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.037 INFO:tasks.workunit.client.0.smithi070.stdout: "db_total_bytes": { 2024-09-14T03:47:50.037 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 95995035648 2024-09-14T03:47:50.037 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.037 INFO:tasks.workunit.client.0.smithi070.stdout: "db_used_bytes": { 2024-09-14T03:47:50.037 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 304480256 2024-09-14T03:47:50.037 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.038 INFO:tasks.workunit.client.0.smithi070.stdout: "log_bytes": { 2024-09-14T03:47:50.038 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1171456 2024-09-14T03:47:50.038 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.038 INFO:tasks.workunit.client.0.smithi070.stdout: "logged_bytes": { 2024-09-14T03:47:50.038 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1032192 2024-09-14T03:47:50.038 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.038 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_db": { 2024-09-14T03:47:50.038 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 312868864 2024-09-14T03:47:50.038 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.039 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_slow": { 2024-09-14T03:47:50.039 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.039 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.039 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_wal": { 2024-09-14T03:47:50.039 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.039 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.039 INFO:tasks.workunit.client.0.smithi070.stdout: "num_files": { 2024-09-14T03:47:50.039 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 14 2024-09-14T03:47:50.039 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.039 INFO:tasks.workunit.client.0.smithi070.stdout: "read_bytes": { 2024-09-14T03:47:50.039 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 195697 2024-09-14T03:47:50.039 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.039 INFO:tasks.workunit.client.0.smithi070.stdout: "read_count": { 2024-09-14T03:47:50.039 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 23 2024-09-14T03:47:50.040 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.040 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes": { 2024-09-14T03:47:50.040 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1269760 2024-09-14T03:47:50.040 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.040 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_db": { 2024-09-14T03:47:50.040 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1273856 2024-09-14T03:47:50.040 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.040 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_slow": { 2024-09-14T03:47:50.040 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.040 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.041 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_wal": { 2024-09-14T03:47:50.041 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.041 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.041 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_count": { 2024-09-14T03:47:50.041 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 15 2024-09-14T03:47:50.041 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.041 INFO:tasks.workunit.client.0.smithi070.stdout: "read_prefetch_bytes": { 2024-09-14T03:47:50.041 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4789 2024-09-14T03:47:50.041 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.041 INFO:tasks.workunit.client.0.smithi070.stdout: "read_prefetch_count": { 2024-09-14T03:47:50.041 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4 2024-09-14T03:47:50.041 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.041 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_buffer_bytes": { 2024-09-14T03:47:50.042 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5589 2024-09-14T03:47:50.042 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.042 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_buffer_count": { 2024-09-14T03:47:50.042 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 24 2024-09-14T03:47:50.042 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.042 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_bytes": { 2024-09-14T03:47:50.042 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5589 2024-09-14T03:47:50.042 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.042 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_count": { 2024-09-14T03:47:50.042 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 24 2024-09-14T03:47:50.042 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.042 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes": { 2024-09-14T03:47:50.042 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.042 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.043 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_db": { 2024-09-14T03:47:50.043 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.043 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.043 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_slow": { 2024-09-14T03:47:50.043 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.043 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.043 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_wal": { 2024-09-14T03:47:50.043 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.043 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.043 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_count": { 2024-09-14T03:47:50.043 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.043 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.043 INFO:tasks.workunit.client.0.smithi070.stdout: "slow_total_bytes": { 2024-09-14T03:47:50.043 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.043 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.044 INFO:tasks.workunit.client.0.smithi070.stdout: "slow_used_bytes": { 2024-09-14T03:47:50.044 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.044 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.044 INFO:tasks.workunit.client.0.smithi070.stdout: "wal_total_bytes": { 2024-09-14T03:47:50.044 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.044 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.044 INFO:tasks.workunit.client.0.smithi070.stdout: "wal_used_bytes": { 2024-09-14T03:47:50.044 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.044 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.044 INFO:tasks.workunit.client.0.smithi070.stdout: "write_bytes": { 2024-09-14T03:47:50.044 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2830336 2024-09-14T03:47:50.044 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.044 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.044 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore": { 2024-09-14T03:47:50.045 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit": { 2024-09-14T03:47:50.045 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4096 2024-09-14T03:47:50.045 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.045 INFO:tasks.workunit.client.0.smithi070.stdout: "allocated": { 2024-09-14T03:47:50.045 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 692224 2024-09-14T03:47:50.045 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.045 INFO:tasks.workunit.client.0.smithi070.stdout: "clist_lat": { 2024-09-14T03:47:50.045 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 4, 2024-09-14T03:47:50.045 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 200380 2024-09-14T03:47:50.045 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.045 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_lat": { 2024-09-14T03:47:50.045 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.045 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.046 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.046 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed": { 2024-09-14T03:47:50.046 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.046 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.046 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed_allocated": { 2024-09-14T03:47:50.046 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.046 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.046 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed_original": { 2024-09-14T03:47:50.046 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.046 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.046 INFO:tasks.workunit.client.0.smithi070.stdout: "csum_lat": { 2024-09-14T03:47:50.046 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 9, 2024-09-14T03:47:50.046 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 14449 2024-09-14T03:47:50.046 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.046 INFO:tasks.workunit.client.0.smithi070.stdout: "decompress_lat": { 2024-09-14T03:47:50.047 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.047 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.047 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.047 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_commit_lat": { 2024-09-14T03:47:50.047 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 235, 2024-09-14T03:47:50.047 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 37816316 2024-09-14T03:47:50.047 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.047 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_final_lat": { 2024-09-14T03:47:50.047 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 227, 2024-09-14T03:47:50.047 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 8138689 2024-09-14T03:47:50.047 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.047 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_flush_lat": { 2024-09-14T03:47:50.047 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 235, 2024-09-14T03:47:50.047 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 323761 2024-09-14T03:47:50.048 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.048 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_sync_lat": { 2024-09-14T03:47:50.048 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 235, 2024-09-14T03:47:50.048 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 38140077 2024-09-14T03:47:50.048 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.048 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_get_keys_lat": { 2024-09-14T03:47:50.048 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.048 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.048 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.048 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_get_values_lat": { 2024-09-14T03:47:50.048 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 3, 2024-09-14T03:47:50.048 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 31316 2024-09-14T03:47:50.048 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.048 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_lower_bound_lat": { 2024-09-14T03:47:50.049 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:50.049 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1597 2024-09-14T03:47:50.049 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.049 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_next_lat": { 2024-09-14T03:47:50.049 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.049 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.049 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.049 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_seek_to_first_lat": { 2024-09-14T03:47:50.049 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.049 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.049 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.049 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_upper_bound_lat": { 2024-09-14T03:47:50.049 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:50.050 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3729 2024-09-14T03:47:50.050 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.050 INFO:tasks.workunit.client.0.smithi070.stdout: "onode_hits": { 2024-09-14T03:47:50.050 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 492 2024-09-14T03:47:50.050 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.050 INFO:tasks.workunit.client.0.smithi070.stdout: "onode_misses": { 2024-09-14T03:47:50.050 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 210 2024-09-14T03:47:50.050 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.050 INFO:tasks.workunit.client.0.smithi070.stdout: "read_lat": { 2024-09-14T03:47:50.050 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 89, 2024-09-14T03:47:50.050 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 881508 2024-09-14T03:47:50.050 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.050 INFO:tasks.workunit.client.0.smithi070.stdout: "read_onode_meta_lat": { 2024-09-14T03:47:50.050 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 210, 2024-09-14T03:47:50.050 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 249409 2024-09-14T03:47:50.051 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.051 INFO:tasks.workunit.client.0.smithi070.stdout: "read_wait_aio_lat": { 2024-09-14T03:47:50.051 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 121, 2024-09-14T03:47:50.051 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 417565 2024-09-14T03:47:50.051 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.051 INFO:tasks.workunit.client.0.smithi070.stdout: "reads_with_retries": { 2024-09-14T03:47:50.051 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.051 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.051 INFO:tasks.workunit.client.0.smithi070.stdout: "remove_lat": { 2024-09-14T03:47:50.051 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 3, 2024-09-14T03:47:50.051 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 90347 2024-09-14T03:47:50.051 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.051 INFO:tasks.workunit.client.0.smithi070.stdout: "state_aio_wait_lat": { 2024-09-14T03:47:50.051 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 285, 2024-09-14T03:47:50.051 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5063422 2024-09-14T03:47:50.052 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.052 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_aio_wait_lat": { 2024-09-14T03:47:50.052 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 87, 2024-09-14T03:47:50.052 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5592582 2024-09-14T03:47:50.052 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.052 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_cleanup_lat": { 2024-09-14T03:47:50.052 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 86, 2024-09-14T03:47:50.052 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 257667352068 2024-09-14T03:47:50.052 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.052 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_queued_lat": { 2024-09-14T03:47:50.052 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 87, 2024-09-14T03:47:50.052 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 145354073905 2024-09-14T03:47:50.052 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.052 INFO:tasks.workunit.client.0.smithi070.stdout: "state_done_lat": { 2024-09-14T03:47:50.052 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 284, 2024-09-14T03:47:50.053 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 32818108531 2024-09-14T03:47:50.053 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.053 INFO:tasks.workunit.client.0.smithi070.stdout: "state_finishing_lat": { 2024-09-14T03:47:50.053 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 284, 2024-09-14T03:47:50.053 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 38139 2024-09-14T03:47:50.053 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.053 INFO:tasks.workunit.client.0.smithi070.stdout: "state_io_done_lat": { 2024-09-14T03:47:50.053 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 285, 2024-09-14T03:47:50.053 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 344294 2024-09-14T03:47:50.053 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.053 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_commiting_lat": { 2024-09-14T03:47:50.053 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 285, 2024-09-14T03:47:50.053 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 83170160 2024-09-14T03:47:50.054 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.054 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_done_lat": { 2024-09-14T03:47:50.054 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 285, 2024-09-14T03:47:50.054 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 94531 2024-09-14T03:47:50.054 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.054 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_queued_lat": { 2024-09-14T03:47:50.054 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 285, 2024-09-14T03:47:50.054 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 57059253 2024-09-14T03:47:50.054 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.054 INFO:tasks.workunit.client.0.smithi070.stdout: "state_prepare_lat": { 2024-09-14T03:47:50.054 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 285, 2024-09-14T03:47:50.054 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 14890234 2024-09-14T03:47:50.054 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.054 INFO:tasks.workunit.client.0.smithi070.stdout: "stored": { 2024-09-14T03:47:50.054 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 311956 2024-09-14T03:47:50.055 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.055 INFO:tasks.workunit.client.0.smithi070.stdout: "truncate_lat": { 2024-09-14T03:47:50.055 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 47, 2024-09-14T03:47:50.055 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 228940 2024-09-14T03:47:50.055 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.055 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_commit_lat": { 2024-09-14T03:47:50.055 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 285, 2024-09-14T03:47:50.055 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 160570020 2024-09-14T03:47:50.055 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.055 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_submit_lat": { 2024-09-14T03:47:50.055 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 285, 2024-09-14T03:47:50.055 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 18307697 2024-09-14T03:47:50.055 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.055 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_throttle_lat": { 2024-09-14T03:47:50.055 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 285, 2024-09-14T03:47:50.056 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1921189 2024-09-14T03:47:50.056 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.056 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.056 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache": { 2024-09-14T03:47:50.056 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_bytes": { 2024-09-14T03:47:50.056 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1861444387 2024-09-14T03:47:50.056 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.056 INFO:tasks.workunit.client.0.smithi070.stdout: "heap_bytes": { 2024-09-14T03:47:50.056 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 269860864 2024-09-14T03:47:50.056 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.056 INFO:tasks.workunit.client.0.smithi070.stdout: "mapped_bytes": { 2024-09-14T03:47:50.056 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 269131776 2024-09-14T03:47:50.056 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.056 INFO:tasks.workunit.client.0.smithi070.stdout: "target_bytes": { 2024-09-14T03:47:50.056 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3137353830 2024-09-14T03:47:50.057 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.057 INFO:tasks.workunit.client.0.smithi070.stdout: "unmapped_bytes": { 2024-09-14T03:47:50.057 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 729088 2024-09-14T03:47:50.057 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.057 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.057 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:data": { 2024-09-14T03:47:50.057 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:50.057 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 159383552 2024-09-14T03:47:50.057 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.057 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:50.057 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.057 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.057 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:50.058 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.058 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.058 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:50.058 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 85498746 2024-09-14T03:47:50.058 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.058 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:50.058 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.058 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.058 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:50.058 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.058 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.058 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:50.058 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.058 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.059 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:50.059 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.059 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.059 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:50.059 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.059 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.059 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:50.059 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 8192 2024-09-14T03:47:50.059 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.059 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:50.059 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.059 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.059 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:50.060 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.060 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.060 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:50.060 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.060 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.060 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:50.060 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 73876614 2024-09-14T03:47:50.060 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.060 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.060 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:kv": { 2024-09-14T03:47:50.060 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:50.060 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 780140544 2024-09-14T03:47:50.060 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.060 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:50.060 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:50.061 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.061 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:50.061 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.061 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.061 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:50.061 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 641240600 2024-09-14T03:47:50.061 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.061 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:50.061 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.061 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.062 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:50.062 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.062 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.062 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:50.062 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.062 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.062 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:50.062 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.062 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.062 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:50.062 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.062 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.062 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:50.062 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2128 2024-09-14T03:47:50.063 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.063 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:50.063 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.063 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.063 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:50.063 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.063 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.063 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:50.063 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.063 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.063 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:50.063 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71788952 2024-09-14T03:47:50.063 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.063 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.063 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:kv_onode": { 2024-09-14T03:47:50.063 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:50.064 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 192937984 2024-09-14T03:47:50.064 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.064 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:50.064 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:50.064 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.064 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:50.064 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 464 2024-09-14T03:47:50.064 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.064 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:50.064 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 56999164 2024-09-14T03:47:50.064 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.064 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:50.064 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.065 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.065 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:50.065 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.065 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.065 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:50.065 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.065 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.065 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:50.065 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.065 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.065 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:50.065 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.065 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.065 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:50.066 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.066 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.066 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:50.066 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.066 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.066 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:50.066 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.066 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.066 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:50.066 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.066 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.066 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:50.066 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 68829492 2024-09-14T03:47:50.067 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.067 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.067 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:meta": { 2024-09-14T03:47:50.067 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:50.067 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 713031680 2024-09-14T03:47:50.067 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.067 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:50.067 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.067 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.067 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:50.068 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.068 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.068 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:50.068 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 641240600 2024-09-14T03:47:50.068 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.068 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:50.068 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.068 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.068 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:50.068 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.068 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.068 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:50.068 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.068 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.069 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:50.069 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.069 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.069 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:50.069 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.069 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.069 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:50.069 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 244085 2024-09-14T03:47:50.069 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.069 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:50.069 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.069 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.069 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:50.070 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.070 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.070 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:50.070 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.070 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.070 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:50.070 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71546995 2024-09-14T03:47:50.070 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.070 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.070 INFO:tasks.workunit.client.0.smithi070.stdout: "objecter": { 2024-09-14T03:47:50.070 INFO:tasks.workunit.client.0.smithi070.stdout: "op_active": { 2024-09-14T03:47:50.070 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.070 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.070 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r": { 2024-09-14T03:47:50.070 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.071 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.071 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rmw": { 2024-09-14T03:47:50.071 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.071 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.071 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w": { 2024-09-14T03:47:50.071 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.071 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.071 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.071 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:50.071 INFO:tasks.workunit.client.0.smithi070.stdout: "numpg": { 2024-09-14T03:47:50.071 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 31 2024-09-14T03:47:50.071 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.071 INFO:tasks.workunit.client.0.smithi070.stdout: "numpg_removing": { 2024-09-14T03:47:50.071 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.071 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.071 INFO:tasks.workunit.client.0.smithi070.stdout: "op": { 2024-09-14T03:47:50.072 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 10 2024-09-14T03:47:50.072 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.072 INFO:tasks.workunit.client.0.smithi070.stdout: "op_delayed_degraded": { 2024-09-14T03:47:50.072 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.072 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.072 INFO:tasks.workunit.client.0.smithi070.stdout: "op_delayed_unreadable": { 2024-09-14T03:47:50.072 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.072 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.072 INFO:tasks.workunit.client.0.smithi070.stdout: "op_in_bytes": { 2024-09-14T03:47:50.072 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.072 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.072 INFO:tasks.workunit.client.0.smithi070.stdout: "op_latency": { 2024-09-14T03:47:50.072 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 10, 2024-09-14T03:47:50.072 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 90683861 2024-09-14T03:47:50.073 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.073 INFO:tasks.workunit.client.0.smithi070.stdout: "op_out_bytes": { 2024-09-14T03:47:50.073 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.073 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.073 INFO:tasks.workunit.client.0.smithi070.stdout: "op_prepare_latency": { 2024-09-14T03:47:50.073 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 10, 2024-09-14T03:47:50.073 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5772820 2024-09-14T03:47:50.073 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.073 INFO:tasks.workunit.client.0.smithi070.stdout: "op_process_latency": { 2024-09-14T03:47:50.073 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 10, 2024-09-14T03:47:50.073 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 72345145 2024-09-14T03:47:50.073 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.073 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r": { 2024-09-14T03:47:50.074 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.074 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.074 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_latency": { 2024-09-14T03:47:50.074 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.074 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.074 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.074 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:47:50.074 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.074 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.074 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_out_bytes": { 2024-09-14T03:47:50.074 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.074 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.074 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_prepare_latency": { 2024-09-14T03:47:50.075 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.075 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.075 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.075 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_process_latency": { 2024-09-14T03:47:50.075 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.075 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.075 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.075 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw": { 2024-09-14T03:47:50.075 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.075 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.075 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_in_bytes": { 2024-09-14T03:47:50.075 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.075 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.075 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency": { 2024-09-14T03:47:50.076 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.076 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.076 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.076 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:47:50.076 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.076 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.076 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:47:50.076 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.076 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.076 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_out_bytes": { 2024-09-14T03:47:50.076 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.076 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.077 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_prepare_latency": { 2024-09-14T03:47:50.077 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.077 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.077 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.077 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_process_latency": { 2024-09-14T03:47:50.077 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.077 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.077 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.077 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w": { 2024-09-14T03:47:50.077 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 10 2024-09-14T03:47:50.077 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.077 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_in_bytes": { 2024-09-14T03:47:50.077 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.078 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.078 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_latency": { 2024-09-14T03:47:50.078 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 10, 2024-09-14T03:47:50.078 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 90683861 2024-09-14T03:47:50.078 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.078 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:47:50.078 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.078 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.078 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_prepare_latency": { 2024-09-14T03:47:50.078 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 10, 2024-09-14T03:47:50.079 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5772820 2024-09-14T03:47:50.079 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.079 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_process_latency": { 2024-09-14T03:47:50.079 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 10, 2024-09-14T03:47:50.079 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 72345145 2024-09-14T03:47:50.079 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.079 INFO:tasks.workunit.client.0.smithi070.stdout: "op_wip": { 2024-09-14T03:47:50.079 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.079 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.079 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_bytes": { 2024-09-14T03:47:50.079 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.079 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.080 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_ops": { 2024-09-14T03:47:50.080 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.080 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.080 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_bytes": { 2024-09-14T03:47:50.080 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 95995035648 2024-09-14T03:47:50.080 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.080 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_bytes_used": { 2024-09-14T03:47:50.080 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 305184768 2024-09-14T03:47:50.080 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.080 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.080 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.080 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.3": { 2024-09-14T03:47:50.080 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:50.080 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_slow_fallback": { 2024-09-14T03:47:50.080 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.081 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.081 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_slow_size_fallback": { 2024-09-14T03:47:50.081 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.081 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.081 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_db": { 2024-09-14T03:47:50.081 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 65536 2024-09-14T03:47:50.081 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.081 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_main": { 2024-09-14T03:47:50.081 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.081 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.081 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_wal": { 2024-09-14T03:47:50.081 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.081 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.081 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_slow": { 2024-09-14T03:47:50.082 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.082 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.082 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_sst": { 2024-09-14T03:47:50.082 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12288 2024-09-14T03:47:50.082 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.082 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_wal": { 2024-09-14T03:47:50.082 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1159168 2024-09-14T03:47:50.082 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.082 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lat": { 2024-09-14T03:47:50.082 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.082 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.082 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.082 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lock_lat": { 2024-09-14T03:47:50.082 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.082 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.083 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.083 INFO:tasks.workunit.client.0.smithi070.stdout: "db_total_bytes": { 2024-09-14T03:47:50.083 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 95995035648 2024-09-14T03:47:50.083 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.083 INFO:tasks.workunit.client.0.smithi070.stdout: "db_used_bytes": { 2024-09-14T03:47:50.083 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 304480256 2024-09-14T03:47:50.083 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.083 INFO:tasks.workunit.client.0.smithi070.stdout: "log_bytes": { 2024-09-14T03:47:50.083 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 892928 2024-09-14T03:47:50.083 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.083 INFO:tasks.workunit.client.0.smithi070.stdout: "logged_bytes": { 2024-09-14T03:47:50.083 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 753664 2024-09-14T03:47:50.083 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.083 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_db": { 2024-09-14T03:47:50.083 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 312868864 2024-09-14T03:47:50.083 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.084 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_slow": { 2024-09-14T03:47:50.084 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.084 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.084 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_wal": { 2024-09-14T03:47:50.084 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.084 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.084 INFO:tasks.workunit.client.0.smithi070.stdout: "num_files": { 2024-09-14T03:47:50.084 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 14 2024-09-14T03:47:50.084 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.084 INFO:tasks.workunit.client.0.smithi070.stdout: "read_bytes": { 2024-09-14T03:47:50.084 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 195697 2024-09-14T03:47:50.084 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.085 INFO:tasks.workunit.client.0.smithi070.stdout: "read_count": { 2024-09-14T03:47:50.085 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 23 2024-09-14T03:47:50.085 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.085 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes": { 2024-09-14T03:47:50.085 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1269760 2024-09-14T03:47:50.085 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.085 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_db": { 2024-09-14T03:47:50.085 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1273856 2024-09-14T03:47:50.085 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.085 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_slow": { 2024-09-14T03:47:50.085 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.085 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.085 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_wal": { 2024-09-14T03:47:50.086 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.086 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.086 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_count": { 2024-09-14T03:47:50.086 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 15 2024-09-14T03:47:50.086 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.086 INFO:tasks.workunit.client.0.smithi070.stdout: "read_prefetch_bytes": { 2024-09-14T03:47:50.086 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4789 2024-09-14T03:47:50.086 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.086 INFO:tasks.workunit.client.0.smithi070.stdout: "read_prefetch_count": { 2024-09-14T03:47:50.086 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4 2024-09-14T03:47:50.086 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.086 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_buffer_bytes": { 2024-09-14T03:47:50.086 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5589 2024-09-14T03:47:50.086 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.086 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_buffer_count": { 2024-09-14T03:47:50.087 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 24 2024-09-14T03:47:50.087 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.087 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_bytes": { 2024-09-14T03:47:50.087 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5589 2024-09-14T03:47:50.087 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.087 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_count": { 2024-09-14T03:47:50.087 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 24 2024-09-14T03:47:50.087 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.087 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes": { 2024-09-14T03:47:50.087 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.087 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.087 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_db": { 2024-09-14T03:47:50.087 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.088 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.088 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_slow": { 2024-09-14T03:47:50.088 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.088 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.088 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_wal": { 2024-09-14T03:47:50.088 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.088 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.088 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_count": { 2024-09-14T03:47:50.088 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.088 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.088 INFO:tasks.workunit.client.0.smithi070.stdout: "slow_total_bytes": { 2024-09-14T03:47:50.088 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.088 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.089 INFO:tasks.workunit.client.0.smithi070.stdout: "slow_used_bytes": { 2024-09-14T03:47:50.089 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.089 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.089 INFO:tasks.workunit.client.0.smithi070.stdout: "wal_total_bytes": { 2024-09-14T03:47:50.089 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.089 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.089 INFO:tasks.workunit.client.0.smithi070.stdout: "wal_used_bytes": { 2024-09-14T03:47:50.089 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.090 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.090 INFO:tasks.workunit.client.0.smithi070.stdout: "write_bytes": { 2024-09-14T03:47:50.090 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1986560 2024-09-14T03:47:50.090 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.090 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.090 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore": { 2024-09-14T03:47:50.090 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit": { 2024-09-14T03:47:50.090 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4096 2024-09-14T03:47:50.090 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.090 INFO:tasks.workunit.client.0.smithi070.stdout: "allocated": { 2024-09-14T03:47:50.090 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 712704 2024-09-14T03:47:50.091 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.091 INFO:tasks.workunit.client.0.smithi070.stdout: "clist_lat": { 2024-09-14T03:47:50.091 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.091 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.091 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.091 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_lat": { 2024-09-14T03:47:50.091 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.091 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.091 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.091 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed": { 2024-09-14T03:47:50.091 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.091 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.091 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed_allocated": { 2024-09-14T03:47:50.091 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.091 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.092 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed_original": { 2024-09-14T03:47:50.092 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.092 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.092 INFO:tasks.workunit.client.0.smithi070.stdout: "csum_lat": { 2024-09-14T03:47:50.092 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 3, 2024-09-14T03:47:50.092 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5222 2024-09-14T03:47:50.092 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.092 INFO:tasks.workunit.client.0.smithi070.stdout: "decompress_lat": { 2024-09-14T03:47:50.092 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.092 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.092 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.092 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_commit_lat": { 2024-09-14T03:47:50.092 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 167, 2024-09-14T03:47:50.092 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 39046400 2024-09-14T03:47:50.093 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.093 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_final_lat": { 2024-09-14T03:47:50.093 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 164, 2024-09-14T03:47:50.093 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 8628994 2024-09-14T03:47:50.093 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.093 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_flush_lat": { 2024-09-14T03:47:50.093 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 167, 2024-09-14T03:47:50.093 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 241305 2024-09-14T03:47:50.093 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.093 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_sync_lat": { 2024-09-14T03:47:50.093 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 167, 2024-09-14T03:47:50.093 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 39287705 2024-09-14T03:47:50.093 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.093 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_get_keys_lat": { 2024-09-14T03:47:50.094 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.094 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.094 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.094 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_get_values_lat": { 2024-09-14T03:47:50.094 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:50.094 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 35574 2024-09-14T03:47:50.094 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.094 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_lower_bound_lat": { 2024-09-14T03:47:50.094 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:50.094 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1902 2024-09-14T03:47:50.094 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.094 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_next_lat": { 2024-09-14T03:47:50.094 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.095 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.095 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.095 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_seek_to_first_lat": { 2024-09-14T03:47:50.095 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.095 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.095 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.095 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_upper_bound_lat": { 2024-09-14T03:47:50.095 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 3, 2024-09-14T03:47:50.095 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 8538 2024-09-14T03:47:50.095 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.095 INFO:tasks.workunit.client.0.smithi070.stdout: "onode_hits": { 2024-09-14T03:47:50.095 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 332 2024-09-14T03:47:50.095 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.095 INFO:tasks.workunit.client.0.smithi070.stdout: "onode_misses": { 2024-09-14T03:47:50.095 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 218 2024-09-14T03:47:50.096 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.096 INFO:tasks.workunit.client.0.smithi070.stdout: "read_lat": { 2024-09-14T03:47:50.096 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 86, 2024-09-14T03:47:50.096 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 723816 2024-09-14T03:47:50.096 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.096 INFO:tasks.workunit.client.0.smithi070.stdout: "read_onode_meta_lat": { 2024-09-14T03:47:50.096 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 148, 2024-09-14T03:47:50.096 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 268277 2024-09-14T03:47:50.096 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.096 INFO:tasks.workunit.client.0.smithi070.stdout: "read_wait_aio_lat": { 2024-09-14T03:47:50.096 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 62, 2024-09-14T03:47:50.096 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 137814 2024-09-14T03:47:50.096 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.096 INFO:tasks.workunit.client.0.smithi070.stdout: "reads_with_retries": { 2024-09-14T03:47:50.096 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.097 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.097 INFO:tasks.workunit.client.0.smithi070.stdout: "remove_lat": { 2024-09-14T03:47:50.097 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.097 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.097 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.097 INFO:tasks.workunit.client.0.smithi070.stdout: "state_aio_wait_lat": { 2024-09-14T03:47:50.097 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 236, 2024-09-14T03:47:50.097 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3014988 2024-09-14T03:47:50.097 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.097 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_aio_wait_lat": { 2024-09-14T03:47:50.097 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 49, 2024-09-14T03:47:50.097 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3669955 2024-09-14T03:47:50.097 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.097 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_cleanup_lat": { 2024-09-14T03:47:50.098 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 46, 2024-09-14T03:47:50.098 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 324567165234 2024-09-14T03:47:50.098 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.098 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_queued_lat": { 2024-09-14T03:47:50.098 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 49, 2024-09-14T03:47:50.098 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 83532140038 2024-09-14T03:47:50.098 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.098 INFO:tasks.workunit.client.0.smithi070.stdout: "state_done_lat": { 2024-09-14T03:47:50.098 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 233, 2024-09-14T03:47:50.098 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 467320 2024-09-14T03:47:50.098 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.098 INFO:tasks.workunit.client.0.smithi070.stdout: "state_finishing_lat": { 2024-09-14T03:47:50.098 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 233, 2024-09-14T03:47:50.098 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 26633 2024-09-14T03:47:50.099 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.099 INFO:tasks.workunit.client.0.smithi070.stdout: "state_io_done_lat": { 2024-09-14T03:47:50.099 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 236, 2024-09-14T03:47:50.099 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 208789 2024-09-14T03:47:50.099 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.099 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_commiting_lat": { 2024-09-14T03:47:50.099 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 236, 2024-09-14T03:47:50.099 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 129093950 2024-09-14T03:47:50.100 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.100 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_done_lat": { 2024-09-14T03:47:50.100 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 236, 2024-09-14T03:47:50.100 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67412 2024-09-14T03:47:50.100 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.100 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_queued_lat": { 2024-09-14T03:47:50.100 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 236, 2024-09-14T03:47:50.100 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 75539469 2024-09-14T03:47:50.100 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.100 INFO:tasks.workunit.client.0.smithi070.stdout: "state_prepare_lat": { 2024-09-14T03:47:50.100 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 236, 2024-09-14T03:47:50.100 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 10861153 2024-09-14T03:47:50.101 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.101 INFO:tasks.workunit.client.0.smithi070.stdout: "stored": { 2024-09-14T03:47:50.101 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 314980 2024-09-14T03:47:50.101 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.101 INFO:tasks.workunit.client.0.smithi070.stdout: "truncate_lat": { 2024-09-14T03:47:50.101 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 40, 2024-09-14T03:47:50.101 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 219052 2024-09-14T03:47:50.101 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.101 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_commit_lat": { 2024-09-14T03:47:50.101 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 236, 2024-09-14T03:47:50.101 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 218746204 2024-09-14T03:47:50.101 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.102 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_submit_lat": { 2024-09-14T03:47:50.102 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 236, 2024-09-14T03:47:50.102 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 14422419 2024-09-14T03:47:50.102 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.102 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_throttle_lat": { 2024-09-14T03:47:50.102 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 236, 2024-09-14T03:47:50.102 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1491296 2024-09-14T03:47:50.102 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.102 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.102 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache": { 2024-09-14T03:47:50.102 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_bytes": { 2024-09-14T03:47:50.102 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1861444387 2024-09-14T03:47:50.102 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.103 INFO:tasks.workunit.client.0.smithi070.stdout: "heap_bytes": { 2024-09-14T03:47:50.103 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 269926400 2024-09-14T03:47:50.103 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.103 INFO:tasks.workunit.client.0.smithi070.stdout: "mapped_bytes": { 2024-09-14T03:47:50.103 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 268845056 2024-09-14T03:47:50.103 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.103 INFO:tasks.workunit.client.0.smithi070.stdout: "target_bytes": { 2024-09-14T03:47:50.103 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3137353830 2024-09-14T03:47:50.103 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.103 INFO:tasks.workunit.client.0.smithi070.stdout: "unmapped_bytes": { 2024-09-14T03:47:50.103 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1081344 2024-09-14T03:47:50.104 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.104 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.104 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:data": { 2024-09-14T03:47:50.104 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:50.104 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 159383552 2024-09-14T03:47:50.104 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.104 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:50.104 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.104 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.104 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:50.104 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.105 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.105 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:50.105 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 85498699 2024-09-14T03:47:50.105 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.105 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:50.105 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.105 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.105 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:50.105 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.105 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.105 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:50.105 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.105 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.105 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:50.106 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.106 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.106 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:50.106 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.106 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.106 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:50.106 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.106 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.106 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:50.106 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.106 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.106 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:50.106 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.107 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.107 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:50.107 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.107 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.107 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:50.107 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 73884853 2024-09-14T03:47:50.107 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.107 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.107 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:kv": { 2024-09-14T03:47:50.107 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:50.107 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 780140544 2024-09-14T03:47:50.107 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.107 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:50.107 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:50.107 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.107 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:50.108 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.108 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.108 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:50.108 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 641240246 2024-09-14T03:47:50.108 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.108 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:50.108 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.108 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.108 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:50.109 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.109 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.109 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:50.109 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.109 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.109 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:50.109 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.109 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.109 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:50.109 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.109 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.109 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:50.109 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2128 2024-09-14T03:47:50.109 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.110 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:50.110 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.110 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.110 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:50.110 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.110 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.110 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:50.110 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.110 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.110 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:50.110 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71789306 2024-09-14T03:47:50.110 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.111 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.111 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:kv_onode": { 2024-09-14T03:47:50.111 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:50.111 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 192937984 2024-09-14T03:47:50.111 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.111 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:50.111 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:50.111 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.111 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:50.111 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 464 2024-09-14T03:47:50.111 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.111 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:50.112 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 56999133 2024-09-14T03:47:50.112 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.112 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:50.112 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.112 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.112 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:50.112 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.112 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.112 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:50.112 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.112 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.112 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:50.112 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.113 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.113 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:50.113 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.113 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.113 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:50.113 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.113 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.113 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:50.113 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.113 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.113 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:50.113 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.113 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.114 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:50.114 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.114 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.114 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:50.114 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 68829523 2024-09-14T03:47:50.114 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.114 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.114 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:meta": { 2024-09-14T03:47:50.114 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:50.114 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 713031680 2024-09-14T03:47:50.114 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.114 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:50.114 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.114 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.115 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:50.115 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.115 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.115 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:50.115 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 641240246 2024-09-14T03:47:50.115 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.115 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:50.115 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.115 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.116 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:50.116 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.116 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.116 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:50.116 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.116 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.116 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:50.116 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.116 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.116 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:50.117 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 6952 2024-09-14T03:47:50.117 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.117 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:50.117 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 248902 2024-09-14T03:47:50.117 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.117 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:50.117 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.117 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.117 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:50.117 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.117 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.117 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:50.117 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.117 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.118 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:50.118 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71535580 2024-09-14T03:47:50.118 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.118 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.118 INFO:tasks.workunit.client.0.smithi070.stdout: "objecter": { 2024-09-14T03:47:50.118 INFO:tasks.workunit.client.0.smithi070.stdout: "op_active": { 2024-09-14T03:47:50.118 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.118 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.118 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r": { 2024-09-14T03:47:50.118 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.118 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.118 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rmw": { 2024-09-14T03:47:50.118 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.119 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.119 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w": { 2024-09-14T03:47:50.119 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.119 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.119 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.119 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:50.119 INFO:tasks.workunit.client.0.smithi070.stdout: "numpg": { 2024-09-14T03:47:50.119 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 37 2024-09-14T03:47:50.119 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.119 INFO:tasks.workunit.client.0.smithi070.stdout: "numpg_removing": { 2024-09-14T03:47:50.119 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.119 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.119 INFO:tasks.workunit.client.0.smithi070.stdout: "op": { 2024-09-14T03:47:50.119 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.119 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.120 INFO:tasks.workunit.client.0.smithi070.stdout: "op_delayed_degraded": { 2024-09-14T03:47:50.120 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.120 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.120 INFO:tasks.workunit.client.0.smithi070.stdout: "op_delayed_unreadable": { 2024-09-14T03:47:50.120 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.120 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.120 INFO:tasks.workunit.client.0.smithi070.stdout: "op_in_bytes": { 2024-09-14T03:47:50.120 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.120 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.120 INFO:tasks.workunit.client.0.smithi070.stdout: "op_latency": { 2024-09-14T03:47:50.120 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.120 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.120 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.121 INFO:tasks.workunit.client.0.smithi070.stdout: "op_out_bytes": { 2024-09-14T03:47:50.121 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.121 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.121 INFO:tasks.workunit.client.0.smithi070.stdout: "op_prepare_latency": { 2024-09-14T03:47:50.121 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:50.121 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 652050 2024-09-14T03:47:50.121 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.121 INFO:tasks.workunit.client.0.smithi070.stdout: "op_process_latency": { 2024-09-14T03:47:50.121 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.121 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.121 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.122 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r": { 2024-09-14T03:47:50.122 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.122 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.122 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_latency": { 2024-09-14T03:47:50.122 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.122 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.122 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.122 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:47:50.122 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.122 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.123 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_out_bytes": { 2024-09-14T03:47:50.123 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.123 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.123 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_prepare_latency": { 2024-09-14T03:47:50.123 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.123 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.123 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.123 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_process_latency": { 2024-09-14T03:47:50.123 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.123 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.123 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.123 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw": { 2024-09-14T03:47:50.123 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.123 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.123 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_in_bytes": { 2024-09-14T03:47:50.124 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.124 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.124 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency": { 2024-09-14T03:47:50.124 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.124 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.124 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.124 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:47:50.124 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.124 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.124 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:47:50.124 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.124 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.124 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_out_bytes": { 2024-09-14T03:47:50.125 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.125 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.125 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_prepare_latency": { 2024-09-14T03:47:50.125 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.125 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.125 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.125 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_process_latency": { 2024-09-14T03:47:50.125 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.125 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.125 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.125 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w": { 2024-09-14T03:47:50.125 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.125 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.126 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_in_bytes": { 2024-09-14T03:47:50.126 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.126 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.126 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_latency": { 2024-09-14T03:47:50.126 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.126 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.126 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.126 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:47:50.126 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.126 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.126 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_prepare_latency": { 2024-09-14T03:47:50.126 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:50.126 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 652050 2024-09-14T03:47:50.127 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.127 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_process_latency": { 2024-09-14T03:47:50.127 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.127 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.127 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.127 INFO:tasks.workunit.client.0.smithi070.stdout: "op_wip": { 2024-09-14T03:47:50.127 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.127 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.127 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_bytes": { 2024-09-14T03:47:50.127 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.127 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.127 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_ops": { 2024-09-14T03:47:50.127 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.127 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.128 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_bytes": { 2024-09-14T03:47:50.128 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 95995035648 2024-09-14T03:47:50.128 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.128 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_bytes_used": { 2024-09-14T03:47:50.128 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 305201152 2024-09-14T03:47:50.128 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.128 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.128 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.128 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.4": { 2024-09-14T03:47:50.128 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:50.128 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_slow_fallback": { 2024-09-14T03:47:50.128 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.128 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.129 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_slow_size_fallback": { 2024-09-14T03:47:50.129 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.129 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.129 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_db": { 2024-09-14T03:47:50.129 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 65536 2024-09-14T03:47:50.129 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.129 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_main": { 2024-09-14T03:47:50.129 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.129 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.129 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_wal": { 2024-09-14T03:47:50.129 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.129 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.129 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_slow": { 2024-09-14T03:47:50.129 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.130 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.130 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_sst": { 2024-09-14T03:47:50.130 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12288 2024-09-14T03:47:50.130 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.130 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_wal": { 2024-09-14T03:47:50.130 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1249280 2024-09-14T03:47:50.130 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.130 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lat": { 2024-09-14T03:47:50.130 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.130 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.130 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.130 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lock_lat": { 2024-09-14T03:47:50.130 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.130 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.131 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.131 INFO:tasks.workunit.client.0.smithi070.stdout: "db_total_bytes": { 2024-09-14T03:47:50.131 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 95995035648 2024-09-14T03:47:50.131 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.131 INFO:tasks.workunit.client.0.smithi070.stdout: "db_used_bytes": { 2024-09-14T03:47:50.131 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 304480256 2024-09-14T03:47:50.131 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.131 INFO:tasks.workunit.client.0.smithi070.stdout: "log_bytes": { 2024-09-14T03:47:50.131 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 958464 2024-09-14T03:47:50.131 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.131 INFO:tasks.workunit.client.0.smithi070.stdout: "logged_bytes": { 2024-09-14T03:47:50.131 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 819200 2024-09-14T03:47:50.131 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.132 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_db": { 2024-09-14T03:47:50.132 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 312868864 2024-09-14T03:47:50.132 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.132 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_slow": { 2024-09-14T03:47:50.132 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.132 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.132 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_wal": { 2024-09-14T03:47:50.132 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.132 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.132 INFO:tasks.workunit.client.0.smithi070.stdout: "num_files": { 2024-09-14T03:47:50.132 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 14 2024-09-14T03:47:50.132 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.133 INFO:tasks.workunit.client.0.smithi070.stdout: "read_bytes": { 2024-09-14T03:47:50.133 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 195697 2024-09-14T03:47:50.133 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.133 INFO:tasks.workunit.client.0.smithi070.stdout: "read_count": { 2024-09-14T03:47:50.133 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 23 2024-09-14T03:47:50.133 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.133 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes": { 2024-09-14T03:47:50.133 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1269760 2024-09-14T03:47:50.133 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.133 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_db": { 2024-09-14T03:47:50.133 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1273856 2024-09-14T03:47:50.133 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.134 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_slow": { 2024-09-14T03:47:50.134 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.134 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.134 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_wal": { 2024-09-14T03:47:50.134 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.134 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.134 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_count": { 2024-09-14T03:47:50.134 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 15 2024-09-14T03:47:50.134 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.134 INFO:tasks.workunit.client.0.smithi070.stdout: "read_prefetch_bytes": { 2024-09-14T03:47:50.134 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4789 2024-09-14T03:47:50.134 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.134 INFO:tasks.workunit.client.0.smithi070.stdout: "read_prefetch_count": { 2024-09-14T03:47:50.134 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4 2024-09-14T03:47:50.135 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.135 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_buffer_bytes": { 2024-09-14T03:47:50.135 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5589 2024-09-14T03:47:50.135 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.135 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_buffer_count": { 2024-09-14T03:47:50.135 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 24 2024-09-14T03:47:50.135 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.135 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_bytes": { 2024-09-14T03:47:50.135 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5589 2024-09-14T03:47:50.135 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.135 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_count": { 2024-09-14T03:47:50.135 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 24 2024-09-14T03:47:50.135 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.135 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes": { 2024-09-14T03:47:50.136 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.136 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.136 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_db": { 2024-09-14T03:47:50.136 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.136 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.136 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_slow": { 2024-09-14T03:47:50.136 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.136 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.136 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_wal": { 2024-09-14T03:47:50.136 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.136 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.136 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_count": { 2024-09-14T03:47:50.136 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.137 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.137 INFO:tasks.workunit.client.0.smithi070.stdout: "slow_total_bytes": { 2024-09-14T03:47:50.137 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.137 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.137 INFO:tasks.workunit.client.0.smithi070.stdout: "slow_used_bytes": { 2024-09-14T03:47:50.137 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.137 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.137 INFO:tasks.workunit.client.0.smithi070.stdout: "wal_total_bytes": { 2024-09-14T03:47:50.137 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.137 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.137 INFO:tasks.workunit.client.0.smithi070.stdout: "wal_used_bytes": { 2024-09-14T03:47:50.137 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.137 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.137 INFO:tasks.workunit.client.0.smithi070.stdout: "write_bytes": { 2024-09-14T03:47:50.138 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2142208 2024-09-14T03:47:50.138 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.138 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.138 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore": { 2024-09-14T03:47:50.138 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit": { 2024-09-14T03:47:50.138 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4096 2024-09-14T03:47:50.138 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.138 INFO:tasks.workunit.client.0.smithi070.stdout: "allocated": { 2024-09-14T03:47:50.138 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 716800 2024-09-14T03:47:50.138 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.139 INFO:tasks.workunit.client.0.smithi070.stdout: "clist_lat": { 2024-09-14T03:47:50.139 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 8, 2024-09-14T03:47:50.139 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 345825 2024-09-14T03:47:50.139 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.139 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_lat": { 2024-09-14T03:47:50.139 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.139 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.139 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.139 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed": { 2024-09-14T03:47:50.139 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.139 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.139 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed_allocated": { 2024-09-14T03:47:50.140 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.140 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.140 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed_original": { 2024-09-14T03:47:50.140 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.140 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.140 INFO:tasks.workunit.client.0.smithi070.stdout: "csum_lat": { 2024-09-14T03:47:50.140 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:50.140 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2359 2024-09-14T03:47:50.140 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.140 INFO:tasks.workunit.client.0.smithi070.stdout: "decompress_lat": { 2024-09-14T03:47:50.140 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.141 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.141 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.141 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_commit_lat": { 2024-09-14T03:47:50.141 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 183, 2024-09-14T03:47:50.141 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 40111570 2024-09-14T03:47:50.141 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.141 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_final_lat": { 2024-09-14T03:47:50.141 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 168, 2024-09-14T03:47:50.141 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 9160436 2024-09-14T03:47:50.141 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.141 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_flush_lat": { 2024-09-14T03:47:50.141 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 183, 2024-09-14T03:47:50.141 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 222326 2024-09-14T03:47:50.142 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.142 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_sync_lat": { 2024-09-14T03:47:50.142 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 183, 2024-09-14T03:47:50.142 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 40333896 2024-09-14T03:47:50.142 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.142 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_get_keys_lat": { 2024-09-14T03:47:50.142 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.142 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.142 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.142 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_get_values_lat": { 2024-09-14T03:47:50.142 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 3, 2024-09-14T03:47:50.142 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 29663 2024-09-14T03:47:50.142 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.142 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_lower_bound_lat": { 2024-09-14T03:47:50.142 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:50.143 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1276 2024-09-14T03:47:50.143 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.143 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_next_lat": { 2024-09-14T03:47:50.143 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.143 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.143 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.143 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_seek_to_first_lat": { 2024-09-14T03:47:50.143 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.143 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.143 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.143 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_upper_bound_lat": { 2024-09-14T03:47:50.143 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 3, 2024-09-14T03:47:50.143 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7892 2024-09-14T03:47:50.143 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.144 INFO:tasks.workunit.client.0.smithi070.stdout: "onode_hits": { 2024-09-14T03:47:50.144 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 357 2024-09-14T03:47:50.144 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.144 INFO:tasks.workunit.client.0.smithi070.stdout: "onode_misses": { 2024-09-14T03:47:50.144 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 237 2024-09-14T03:47:50.144 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.144 INFO:tasks.workunit.client.0.smithi070.stdout: "read_lat": { 2024-09-14T03:47:50.144 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 94, 2024-09-14T03:47:50.144 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 721550 2024-09-14T03:47:50.144 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.144 INFO:tasks.workunit.client.0.smithi070.stdout: "read_onode_meta_lat": { 2024-09-14T03:47:50.144 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 155, 2024-09-14T03:47:50.144 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 238486 2024-09-14T03:47:50.145 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.145 INFO:tasks.workunit.client.0.smithi070.stdout: "read_wait_aio_lat": { 2024-09-14T03:47:50.145 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 61, 2024-09-14T03:47:50.145 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 69229 2024-09-14T03:47:50.145 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.145 INFO:tasks.workunit.client.0.smithi070.stdout: "reads_with_retries": { 2024-09-14T03:47:50.145 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.145 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.145 INFO:tasks.workunit.client.0.smithi070.stdout: "remove_lat": { 2024-09-14T03:47:50.145 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 5, 2024-09-14T03:47:50.145 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 135227 2024-09-14T03:47:50.146 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.146 INFO:tasks.workunit.client.0.smithi070.stdout: "state_aio_wait_lat": { 2024-09-14T03:47:50.146 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 265, 2024-09-14T03:47:50.146 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3460408 2024-09-14T03:47:50.146 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.146 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_aio_wait_lat": { 2024-09-14T03:47:50.146 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 39, 2024-09-14T03:47:50.146 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2901292 2024-09-14T03:47:50.146 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.146 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_cleanup_lat": { 2024-09-14T03:47:50.146 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 38, 2024-09-14T03:47:50.146 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 228341686886 2024-09-14T03:47:50.147 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.147 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_queued_lat": { 2024-09-14T03:47:50.147 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 39, 2024-09-14T03:47:50.147 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 64303300754 2024-09-14T03:47:50.147 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.147 INFO:tasks.workunit.client.0.smithi070.stdout: "state_done_lat": { 2024-09-14T03:47:50.147 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 264, 2024-09-14T03:47:50.147 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 501637 2024-09-14T03:47:50.147 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.148 INFO:tasks.workunit.client.0.smithi070.stdout: "state_finishing_lat": { 2024-09-14T03:47:50.148 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 264, 2024-09-14T03:47:50.148 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 29802 2024-09-14T03:47:50.148 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.148 INFO:tasks.workunit.client.0.smithi070.stdout: "state_io_done_lat": { 2024-09-14T03:47:50.148 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 265, 2024-09-14T03:47:50.149 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 252745 2024-09-14T03:47:50.149 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.149 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_commiting_lat": { 2024-09-14T03:47:50.149 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 265, 2024-09-14T03:47:50.149 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 104003707 2024-09-14T03:47:50.150 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.150 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_done_lat": { 2024-09-14T03:47:50.150 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 265, 2024-09-14T03:47:50.150 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 77912 2024-09-14T03:47:50.150 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.150 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_queued_lat": { 2024-09-14T03:47:50.151 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 265, 2024-09-14T03:47:50.151 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 36691311 2024-09-14T03:47:50.151 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.151 INFO:tasks.workunit.client.0.smithi070.stdout: "state_prepare_lat": { 2024-09-14T03:47:50.151 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 265, 2024-09-14T03:47:50.152 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12786127 2024-09-14T03:47:50.152 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.152 INFO:tasks.workunit.client.0.smithi070.stdout: "stored": { 2024-09-14T03:47:50.152 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 313282 2024-09-14T03:47:50.152 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.152 INFO:tasks.workunit.client.0.smithi070.stdout: "truncate_lat": { 2024-09-14T03:47:50.152 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 37, 2024-09-14T03:47:50.152 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 184889 2024-09-14T03:47:50.152 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.152 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_commit_lat": { 2024-09-14T03:47:50.153 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 265, 2024-09-14T03:47:50.153 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 157227847 2024-09-14T03:47:50.153 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.153 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_submit_lat": { 2024-09-14T03:47:50.153 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 265, 2024-09-14T03:47:50.153 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 15411335 2024-09-14T03:47:50.153 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.153 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_throttle_lat": { 2024-09-14T03:47:50.153 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 265, 2024-09-14T03:47:50.153 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2156304 2024-09-14T03:47:50.153 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.153 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.153 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache": { 2024-09-14T03:47:50.154 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_bytes": { 2024-09-14T03:47:50.154 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1861444387 2024-09-14T03:47:50.154 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.154 INFO:tasks.workunit.client.0.smithi070.stdout: "heap_bytes": { 2024-09-14T03:47:50.154 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 269926400 2024-09-14T03:47:50.154 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.154 INFO:tasks.workunit.client.0.smithi070.stdout: "mapped_bytes": { 2024-09-14T03:47:50.154 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 269852672 2024-09-14T03:47:50.155 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.155 INFO:tasks.workunit.client.0.smithi070.stdout: "target_bytes": { 2024-09-14T03:47:50.155 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3137353830 2024-09-14T03:47:50.155 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.155 INFO:tasks.workunit.client.0.smithi070.stdout: "unmapped_bytes": { 2024-09-14T03:47:50.155 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 73728 2024-09-14T03:47:50.155 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.156 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.156 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:data": { 2024-09-14T03:47:50.156 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:50.156 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 159383552 2024-09-14T03:47:50.156 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.156 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:50.156 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.156 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.156 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:50.156 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.156 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.156 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:50.156 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 85498353 2024-09-14T03:47:50.157 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.157 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:50.157 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.157 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.157 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:50.157 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.157 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.157 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:50.157 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.157 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.158 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:50.158 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.158 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.158 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:50.158 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.158 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.158 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:50.158 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.158 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.158 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:50.158 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.158 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.158 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:50.158 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.159 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.159 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:50.159 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.159 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.159 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:50.159 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 73885199 2024-09-14T03:47:50.159 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.159 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.159 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:kv": { 2024-09-14T03:47:50.159 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:50.159 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 780140544 2024-09-14T03:47:50.159 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.159 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:50.159 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:50.160 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.160 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:50.160 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.160 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.160 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:50.160 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 641237651 2024-09-14T03:47:50.160 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.160 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:50.160 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.160 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.160 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:50.160 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.160 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.160 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:50.161 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.161 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.161 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:50.161 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.161 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.161 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:50.161 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.161 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.162 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:50.162 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2128 2024-09-14T03:47:50.162 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.162 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:50.162 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.162 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.162 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:50.162 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.162 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.162 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:50.163 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.163 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.163 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:50.163 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71791901 2024-09-14T03:47:50.163 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.163 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.163 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:kv_onode": { 2024-09-14T03:47:50.163 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:50.163 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 192937984 2024-09-14T03:47:50.163 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.163 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:50.163 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:50.163 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.163 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:50.164 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 464 2024-09-14T03:47:50.164 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.164 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:50.164 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 56998902 2024-09-14T03:47:50.164 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.164 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:50.164 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.164 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.164 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:50.164 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.164 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.164 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:50.164 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.165 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.165 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:50.165 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.165 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.165 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:50.165 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.165 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.165 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:50.165 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.165 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.165 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:50.165 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.165 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.165 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:50.166 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.166 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.166 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:50.166 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.166 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.166 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:50.166 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 68829754 2024-09-14T03:47:50.167 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.167 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.167 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:meta": { 2024-09-14T03:47:50.167 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:50.167 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 713031680 2024-09-14T03:47:50.167 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.167 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:50.168 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.168 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.168 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:50.168 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.168 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.168 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:50.168 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 641237651 2024-09-14T03:47:50.168 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.168 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:50.168 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.168 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.168 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:50.168 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.169 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.169 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:50.169 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.169 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.169 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:50.169 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.169 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.169 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:50.169 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.169 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.169 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:50.169 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 256053 2024-09-14T03:47:50.169 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.170 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:50.170 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.170 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.170 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:50.170 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.170 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.170 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:50.170 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.170 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.170 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:50.170 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71537976 2024-09-14T03:47:50.170 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.170 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.170 INFO:tasks.workunit.client.0.smithi070.stdout: "objecter": { 2024-09-14T03:47:50.171 INFO:tasks.workunit.client.0.smithi070.stdout: "op_active": { 2024-09-14T03:47:50.171 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.171 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.171 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r": { 2024-09-14T03:47:50.171 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.171 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.171 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rmw": { 2024-09-14T03:47:50.171 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.171 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.171 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w": { 2024-09-14T03:47:50.171 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.171 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.172 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.172 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:50.172 INFO:tasks.workunit.client.0.smithi070.stdout: "numpg": { 2024-09-14T03:47:50.172 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 39 2024-09-14T03:47:50.172 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.172 INFO:tasks.workunit.client.0.smithi070.stdout: "numpg_removing": { 2024-09-14T03:47:50.172 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.172 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.172 INFO:tasks.workunit.client.0.smithi070.stdout: "op": { 2024-09-14T03:47:50.172 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1 2024-09-14T03:47:50.172 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.172 INFO:tasks.workunit.client.0.smithi070.stdout: "op_delayed_degraded": { 2024-09-14T03:47:50.172 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.172 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.173 INFO:tasks.workunit.client.0.smithi070.stdout: "op_delayed_unreadable": { 2024-09-14T03:47:50.173 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.173 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.173 INFO:tasks.workunit.client.0.smithi070.stdout: "op_in_bytes": { 2024-09-14T03:47:50.173 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 22 2024-09-14T03:47:50.173 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.173 INFO:tasks.workunit.client.0.smithi070.stdout: "op_latency": { 2024-09-14T03:47:50.173 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:50.173 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7596677 2024-09-14T03:47:50.173 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.173 INFO:tasks.workunit.client.0.smithi070.stdout: "op_out_bytes": { 2024-09-14T03:47:50.173 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.174 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.174 INFO:tasks.workunit.client.0.smithi070.stdout: "op_prepare_latency": { 2024-09-14T03:47:50.174 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:50.174 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 730201 2024-09-14T03:47:50.174 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.174 INFO:tasks.workunit.client.0.smithi070.stdout: "op_process_latency": { 2024-09-14T03:47:50.174 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:50.174 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7516157 2024-09-14T03:47:50.174 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.174 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r": { 2024-09-14T03:47:50.175 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.175 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.175 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_latency": { 2024-09-14T03:47:50.175 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.175 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.175 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.175 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:47:50.175 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.175 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.175 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_out_bytes": { 2024-09-14T03:47:50.175 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.175 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.175 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_prepare_latency": { 2024-09-14T03:47:50.175 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.175 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.176 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.176 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_process_latency": { 2024-09-14T03:47:50.176 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.176 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.176 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.176 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw": { 2024-09-14T03:47:50.176 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.176 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.176 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_in_bytes": { 2024-09-14T03:47:50.176 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.176 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.176 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency": { 2024-09-14T03:47:50.177 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.177 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.177 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.177 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:47:50.177 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.177 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.177 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:47:50.177 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.177 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.177 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_out_bytes": { 2024-09-14T03:47:50.177 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.177 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.177 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_prepare_latency": { 2024-09-14T03:47:50.178 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.178 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.178 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.178 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_process_latency": { 2024-09-14T03:47:50.178 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.178 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.178 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.178 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w": { 2024-09-14T03:47:50.178 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1 2024-09-14T03:47:50.178 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.178 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_in_bytes": { 2024-09-14T03:47:50.178 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 22 2024-09-14T03:47:50.179 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.179 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_latency": { 2024-09-14T03:47:50.179 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:50.179 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7596677 2024-09-14T03:47:50.179 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.179 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:47:50.179 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.179 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.179 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_prepare_latency": { 2024-09-14T03:47:50.180 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:50.180 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 730201 2024-09-14T03:47:50.180 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.180 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_process_latency": { 2024-09-14T03:47:50.180 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:50.180 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7516157 2024-09-14T03:47:50.180 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.180 INFO:tasks.workunit.client.0.smithi070.stdout: "op_wip": { 2024-09-14T03:47:50.180 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.180 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.180 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_bytes": { 2024-09-14T03:47:50.180 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.180 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.181 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_ops": { 2024-09-14T03:47:50.181 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.181 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.181 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_bytes": { 2024-09-14T03:47:50.181 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 95995035648 2024-09-14T03:47:50.181 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.181 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_bytes_used": { 2024-09-14T03:47:50.181 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 305209344 2024-09-14T03:47:50.181 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.181 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.181 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.181 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.5": { 2024-09-14T03:47:50.181 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:50.182 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_slow_fallback": { 2024-09-14T03:47:50.182 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.182 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.182 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_slow_size_fallback": { 2024-09-14T03:47:50.182 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.182 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.182 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_db": { 2024-09-14T03:47:50.182 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 65536 2024-09-14T03:47:50.182 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.182 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_main": { 2024-09-14T03:47:50.182 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.182 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.182 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_wal": { 2024-09-14T03:47:50.183 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.183 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.183 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_slow": { 2024-09-14T03:47:50.183 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.183 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.183 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_sst": { 2024-09-14T03:47:50.183 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12288 2024-09-14T03:47:50.183 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.183 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_wal": { 2024-09-14T03:47:50.183 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1179648 2024-09-14T03:47:50.183 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.183 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lat": { 2024-09-14T03:47:50.183 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.184 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.184 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.184 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lock_lat": { 2024-09-14T03:47:50.184 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.184 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.184 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.184 INFO:tasks.workunit.client.0.smithi070.stdout: "db_total_bytes": { 2024-09-14T03:47:50.184 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 95995035648 2024-09-14T03:47:50.184 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.184 INFO:tasks.workunit.client.0.smithi070.stdout: "db_used_bytes": { 2024-09-14T03:47:50.184 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 304480256 2024-09-14T03:47:50.184 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.184 INFO:tasks.workunit.client.0.smithi070.stdout: "log_bytes": { 2024-09-14T03:47:50.184 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 880640 2024-09-14T03:47:50.185 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.185 INFO:tasks.workunit.client.0.smithi070.stdout: "logged_bytes": { 2024-09-14T03:47:50.185 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 741376 2024-09-14T03:47:50.185 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.185 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_db": { 2024-09-14T03:47:50.185 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 312868864 2024-09-14T03:47:50.185 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.185 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_slow": { 2024-09-14T03:47:50.185 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.185 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.185 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_wal": { 2024-09-14T03:47:50.185 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.185 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.186 INFO:tasks.workunit.client.0.smithi070.stdout: "num_files": { 2024-09-14T03:47:50.186 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 14 2024-09-14T03:47:50.186 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.186 INFO:tasks.workunit.client.0.smithi070.stdout: "read_bytes": { 2024-09-14T03:47:50.186 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 195697 2024-09-14T03:47:50.186 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.186 INFO:tasks.workunit.client.0.smithi070.stdout: "read_count": { 2024-09-14T03:47:50.186 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 23 2024-09-14T03:47:50.186 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.186 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes": { 2024-09-14T03:47:50.186 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1269760 2024-09-14T03:47:50.186 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.186 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_db": { 2024-09-14T03:47:50.186 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1273856 2024-09-14T03:47:50.186 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.187 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_slow": { 2024-09-14T03:47:50.187 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.187 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.187 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_wal": { 2024-09-14T03:47:50.187 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.187 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.187 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_count": { 2024-09-14T03:47:50.187 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 15 2024-09-14T03:47:50.187 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.187 INFO:tasks.workunit.client.0.smithi070.stdout: "read_prefetch_bytes": { 2024-09-14T03:47:50.187 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4789 2024-09-14T03:47:50.187 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.187 INFO:tasks.workunit.client.0.smithi070.stdout: "read_prefetch_count": { 2024-09-14T03:47:50.187 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4 2024-09-14T03:47:50.188 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.188 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_buffer_bytes": { 2024-09-14T03:47:50.188 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5589 2024-09-14T03:47:50.188 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.188 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_buffer_count": { 2024-09-14T03:47:50.188 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 24 2024-09-14T03:47:50.188 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.188 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_bytes": { 2024-09-14T03:47:50.188 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5589 2024-09-14T03:47:50.188 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.189 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_count": { 2024-09-14T03:47:50.189 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 24 2024-09-14T03:47:50.189 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.189 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes": { 2024-09-14T03:47:50.189 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.189 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.189 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_db": { 2024-09-14T03:47:50.189 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.189 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.189 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_slow": { 2024-09-14T03:47:50.189 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.189 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.189 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_wal": { 2024-09-14T03:47:50.189 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.190 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.190 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_count": { 2024-09-14T03:47:50.190 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.190 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.190 INFO:tasks.workunit.client.0.smithi070.stdout: "slow_total_bytes": { 2024-09-14T03:47:50.190 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.190 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.190 INFO:tasks.workunit.client.0.smithi070.stdout: "slow_used_bytes": { 2024-09-14T03:47:50.190 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.190 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.190 INFO:tasks.workunit.client.0.smithi070.stdout: "wal_total_bytes": { 2024-09-14T03:47:50.190 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.190 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.191 INFO:tasks.workunit.client.0.smithi070.stdout: "wal_used_bytes": { 2024-09-14T03:47:50.191 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.191 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.191 INFO:tasks.workunit.client.0.smithi070.stdout: "write_bytes": { 2024-09-14T03:47:50.191 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1994752 2024-09-14T03:47:50.191 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.191 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.191 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore": { 2024-09-14T03:47:50.191 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit": { 2024-09-14T03:47:50.191 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4096 2024-09-14T03:47:50.191 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.191 INFO:tasks.workunit.client.0.smithi070.stdout: "allocated": { 2024-09-14T03:47:50.191 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 712704 2024-09-14T03:47:50.191 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.191 INFO:tasks.workunit.client.0.smithi070.stdout: "clist_lat": { 2024-09-14T03:47:50.192 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 6, 2024-09-14T03:47:50.192 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 229396 2024-09-14T03:47:50.192 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.192 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_lat": { 2024-09-14T03:47:50.192 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.192 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.192 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.192 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed": { 2024-09-14T03:47:50.192 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.192 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.193 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed_allocated": { 2024-09-14T03:47:50.193 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.193 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.193 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed_original": { 2024-09-14T03:47:50.193 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.193 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.193 INFO:tasks.workunit.client.0.smithi070.stdout: "csum_lat": { 2024-09-14T03:47:50.193 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 3, 2024-09-14T03:47:50.193 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4234 2024-09-14T03:47:50.193 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.193 INFO:tasks.workunit.client.0.smithi070.stdout: "decompress_lat": { 2024-09-14T03:47:50.193 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.193 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.193 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.194 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_commit_lat": { 2024-09-14T03:47:50.194 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 164, 2024-09-14T03:47:50.194 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 47839247 2024-09-14T03:47:50.194 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.194 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_final_lat": { 2024-09-14T03:47:50.194 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 164, 2024-09-14T03:47:50.194 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4151205 2024-09-14T03:47:50.194 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.194 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_flush_lat": { 2024-09-14T03:47:50.194 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 164, 2024-09-14T03:47:50.194 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 175925 2024-09-14T03:47:50.194 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.194 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_sync_lat": { 2024-09-14T03:47:50.194 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 164, 2024-09-14T03:47:50.195 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 48015172 2024-09-14T03:47:50.195 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.195 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_get_keys_lat": { 2024-09-14T03:47:50.195 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.195 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.195 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.195 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_get_values_lat": { 2024-09-14T03:47:50.195 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 3, 2024-09-14T03:47:50.195 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 36969 2024-09-14T03:47:50.196 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.196 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_lower_bound_lat": { 2024-09-14T03:47:50.196 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:50.196 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1026 2024-09-14T03:47:50.196 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.196 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_next_lat": { 2024-09-14T03:47:50.197 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.197 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.197 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.197 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_seek_to_first_lat": { 2024-09-14T03:47:50.197 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.197 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.197 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.198 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_upper_bound_lat": { 2024-09-14T03:47:50.198 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:50.198 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7136 2024-09-14T03:47:50.198 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.198 INFO:tasks.workunit.client.0.smithi070.stdout: "onode_hits": { 2024-09-14T03:47:50.198 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 344 2024-09-14T03:47:50.198 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.198 INFO:tasks.workunit.client.0.smithi070.stdout: "onode_misses": { 2024-09-14T03:47:50.198 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 240 2024-09-14T03:47:50.198 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.198 INFO:tasks.workunit.client.0.smithi070.stdout: "read_lat": { 2024-09-14T03:47:50.198 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 90, 2024-09-14T03:47:50.198 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 690701 2024-09-14T03:47:50.199 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.199 INFO:tasks.workunit.client.0.smithi070.stdout: "read_onode_meta_lat": { 2024-09-14T03:47:50.199 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 146, 2024-09-14T03:47:50.199 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 261511 2024-09-14T03:47:50.199 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.199 INFO:tasks.workunit.client.0.smithi070.stdout: "read_wait_aio_lat": { 2024-09-14T03:47:50.199 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 56, 2024-09-14T03:47:50.199 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 171611 2024-09-14T03:47:50.199 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.199 INFO:tasks.workunit.client.0.smithi070.stdout: "reads_with_retries": { 2024-09-14T03:47:50.199 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.199 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.199 INFO:tasks.workunit.client.0.smithi070.stdout: "remove_lat": { 2024-09-14T03:47:50.199 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 4, 2024-09-14T03:47:50.200 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 97676 2024-09-14T03:47:50.200 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.200 INFO:tasks.workunit.client.0.smithi070.stdout: "state_aio_wait_lat": { 2024-09-14T03:47:50.200 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 274, 2024-09-14T03:47:50.200 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2801500 2024-09-14T03:47:50.200 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.200 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_aio_wait_lat": { 2024-09-14T03:47:50.200 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 41, 2024-09-14T03:47:50.200 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4051654 2024-09-14T03:47:50.200 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.201 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_cleanup_lat": { 2024-09-14T03:47:50.201 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 38, 2024-09-14T03:47:50.201 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 297657276129 2024-09-14T03:47:50.201 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.201 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_queued_lat": { 2024-09-14T03:47:50.201 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 41, 2024-09-14T03:47:50.201 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 65658052773 2024-09-14T03:47:50.201 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.201 INFO:tasks.workunit.client.0.smithi070.stdout: "state_done_lat": { 2024-09-14T03:47:50.201 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 271, 2024-09-14T03:47:50.201 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 6002782444 2024-09-14T03:47:50.201 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.201 INFO:tasks.workunit.client.0.smithi070.stdout: "state_finishing_lat": { 2024-09-14T03:47:50.201 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 271, 2024-09-14T03:47:50.202 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 30580 2024-09-14T03:47:50.202 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.202 INFO:tasks.workunit.client.0.smithi070.stdout: "state_io_done_lat": { 2024-09-14T03:47:50.202 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 274, 2024-09-14T03:47:50.202 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 274880 2024-09-14T03:47:50.202 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.202 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_commiting_lat": { 2024-09-14T03:47:50.202 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 274, 2024-09-14T03:47:50.202 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 107158317 2024-09-14T03:47:50.202 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.202 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_done_lat": { 2024-09-14T03:47:50.202 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 274, 2024-09-14T03:47:50.202 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 74514 2024-09-14T03:47:50.202 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.203 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_queued_lat": { 2024-09-14T03:47:50.203 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 274, 2024-09-14T03:47:50.203 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 102524241 2024-09-14T03:47:50.203 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.203 INFO:tasks.workunit.client.0.smithi070.stdout: "state_prepare_lat": { 2024-09-14T03:47:50.203 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 274, 2024-09-14T03:47:50.203 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 13236763 2024-09-14T03:47:50.203 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.203 INFO:tasks.workunit.client.0.smithi070.stdout: "stored": { 2024-09-14T03:47:50.203 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 314985 2024-09-14T03:47:50.203 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.203 INFO:tasks.workunit.client.0.smithi070.stdout: "truncate_lat": { 2024-09-14T03:47:50.203 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 33, 2024-09-14T03:47:50.204 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 134125 2024-09-14T03:47:50.204 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.204 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_commit_lat": { 2024-09-14T03:47:50.204 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 274, 2024-09-14T03:47:50.204 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 226027726 2024-09-14T03:47:50.204 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.204 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_submit_lat": { 2024-09-14T03:47:50.204 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 274, 2024-09-14T03:47:50.204 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 16481402 2024-09-14T03:47:50.205 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.205 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_throttle_lat": { 2024-09-14T03:47:50.205 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 274, 2024-09-14T03:47:50.205 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2140962 2024-09-14T03:47:50.205 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.205 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.205 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache": { 2024-09-14T03:47:50.206 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_bytes": { 2024-09-14T03:47:50.206 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1861444387 2024-09-14T03:47:50.206 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.206 INFO:tasks.workunit.client.0.smithi070.stdout: "heap_bytes": { 2024-09-14T03:47:50.206 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 269926400 2024-09-14T03:47:50.206 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.206 INFO:tasks.workunit.client.0.smithi070.stdout: "mapped_bytes": { 2024-09-14T03:47:50.206 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 269615104 2024-09-14T03:47:50.206 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.206 INFO:tasks.workunit.client.0.smithi070.stdout: "target_bytes": { 2024-09-14T03:47:50.206 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3137353830 2024-09-14T03:47:50.206 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.206 INFO:tasks.workunit.client.0.smithi070.stdout: "unmapped_bytes": { 2024-09-14T03:47:50.207 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 311296 2024-09-14T03:47:50.207 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.207 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.207 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:data": { 2024-09-14T03:47:50.207 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:50.207 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 159383552 2024-09-14T03:47:50.207 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.207 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:50.207 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.207 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.207 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:50.207 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.207 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.208 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:50.208 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 85498195 2024-09-14T03:47:50.208 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.208 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:50.208 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.208 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.208 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:50.208 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.208 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.209 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:50.209 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.209 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.209 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:50.209 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.209 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.210 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:50.210 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.210 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.210 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:50.210 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.210 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.210 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:50.210 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.211 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.211 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:50.211 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.211 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.211 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:50.211 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.211 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.211 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:50.211 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 73885357 2024-09-14T03:47:50.211 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.211 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.212 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:kv": { 2024-09-14T03:47:50.212 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:50.212 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 780140544 2024-09-14T03:47:50.212 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.212 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:50.212 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:50.212 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.212 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:50.212 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.212 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.212 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:50.212 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 641236466 2024-09-14T03:47:50.212 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.213 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:50.213 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.213 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.213 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:50.213 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.213 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.213 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:50.213 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.213 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.213 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:50.213 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.213 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.214 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:50.214 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.214 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.214 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:50.214 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2128 2024-09-14T03:47:50.214 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.214 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:50.214 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.214 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.214 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:50.214 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.215 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.215 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:50.215 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.215 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.215 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:50.215 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71793086 2024-09-14T03:47:50.215 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.215 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.216 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:kv_onode": { 2024-09-14T03:47:50.216 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:50.216 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 192937984 2024-09-14T03:47:50.216 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.216 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:50.216 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:50.216 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.216 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:50.216 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 464 2024-09-14T03:47:50.216 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.216 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:50.216 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 56998797 2024-09-14T03:47:50.216 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.217 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:50.217 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.217 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.217 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:50.217 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.217 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.217 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:50.217 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.217 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.217 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:50.217 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.217 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.217 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:50.218 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.218 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.218 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:50.218 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.218 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.218 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:50.218 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.218 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.218 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:50.218 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.218 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.218 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:50.218 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.218 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.219 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:50.219 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 68829859 2024-09-14T03:47:50.219 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.219 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.219 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:meta": { 2024-09-14T03:47:50.219 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:50.219 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 713031680 2024-09-14T03:47:50.219 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.219 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:50.219 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.219 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.220 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:50.220 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.220 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.220 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:50.220 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 641236466 2024-09-14T03:47:50.220 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.220 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:50.220 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.221 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.221 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:50.221 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.221 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.221 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:50.221 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.221 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.221 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:50.221 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.221 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.221 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:50.221 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 6954 2024-09-14T03:47:50.222 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.222 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:50.222 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 257300 2024-09-14T03:47:50.222 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.222 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:50.222 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.222 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.223 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:50.223 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.223 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.223 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:50.223 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.223 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.223 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:50.223 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71530960 2024-09-14T03:47:50.223 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.223 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.223 INFO:tasks.workunit.client.0.smithi070.stdout: "objecter": { 2024-09-14T03:47:50.224 INFO:tasks.workunit.client.0.smithi070.stdout: "op_active": { 2024-09-14T03:47:50.224 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.224 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.224 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r": { 2024-09-14T03:47:50.224 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.224 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.224 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rmw": { 2024-09-14T03:47:50.224 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.224 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.224 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w": { 2024-09-14T03:47:50.225 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.225 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.225 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.225 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:50.225 INFO:tasks.workunit.client.0.smithi070.stdout: "numpg": { 2024-09-14T03:47:50.225 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 41 2024-09-14T03:47:50.225 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.225 INFO:tasks.workunit.client.0.smithi070.stdout: "numpg_removing": { 2024-09-14T03:47:50.225 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.225 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.225 INFO:tasks.workunit.client.0.smithi070.stdout: "op": { 2024-09-14T03:47:50.226 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.226 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.226 INFO:tasks.workunit.client.0.smithi070.stdout: "op_delayed_degraded": { 2024-09-14T03:47:50.226 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.226 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.226 INFO:tasks.workunit.client.0.smithi070.stdout: "op_delayed_unreadable": { 2024-09-14T03:47:50.226 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.226 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.226 INFO:tasks.workunit.client.0.smithi070.stdout: "op_in_bytes": { 2024-09-14T03:47:50.226 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.226 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.226 INFO:tasks.workunit.client.0.smithi070.stdout: "op_latency": { 2024-09-14T03:47:50.226 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.227 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.227 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.227 INFO:tasks.workunit.client.0.smithi070.stdout: "op_out_bytes": { 2024-09-14T03:47:50.227 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.227 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.227 INFO:tasks.workunit.client.0.smithi070.stdout: "op_prepare_latency": { 2024-09-14T03:47:50.227 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.227 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.227 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.227 INFO:tasks.workunit.client.0.smithi070.stdout: "op_process_latency": { 2024-09-14T03:47:50.227 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.227 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.227 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.227 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r": { 2024-09-14T03:47:50.228 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.228 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.228 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_latency": { 2024-09-14T03:47:50.228 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.228 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.228 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.228 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:47:50.228 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.228 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.228 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_out_bytes": { 2024-09-14T03:47:50.228 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.228 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.228 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_prepare_latency": { 2024-09-14T03:47:50.228 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.229 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.229 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.229 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_process_latency": { 2024-09-14T03:47:50.229 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.229 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.229 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.229 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw": { 2024-09-14T03:47:50.229 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.229 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.229 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_in_bytes": { 2024-09-14T03:47:50.229 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.229 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.229 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency": { 2024-09-14T03:47:50.230 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.230 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.230 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.230 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:47:50.230 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.230 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.230 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:47:50.230 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.230 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.230 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_out_bytes": { 2024-09-14T03:47:50.230 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.230 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.230 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_prepare_latency": { 2024-09-14T03:47:50.230 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.230 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.231 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.231 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_process_latency": { 2024-09-14T03:47:50.231 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.231 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.231 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.231 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w": { 2024-09-14T03:47:50.231 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.231 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.231 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_in_bytes": { 2024-09-14T03:47:50.231 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.231 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.231 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_latency": { 2024-09-14T03:47:50.231 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.231 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.232 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.232 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:47:50.232 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.232 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.232 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_prepare_latency": { 2024-09-14T03:47:50.232 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.232 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.232 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.232 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_process_latency": { 2024-09-14T03:47:50.232 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.232 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.232 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.232 INFO:tasks.workunit.client.0.smithi070.stdout: "op_wip": { 2024-09-14T03:47:50.232 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.233 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.233 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_bytes": { 2024-09-14T03:47:50.233 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 590368 2024-09-14T03:47:50.233 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.233 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_ops": { 2024-09-14T03:47:50.233 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:50.233 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.233 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_bytes": { 2024-09-14T03:47:50.233 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 95995035648 2024-09-14T03:47:50.233 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.234 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_bytes_used": { 2024-09-14T03:47:50.234 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 305201152 2024-09-14T03:47:50.234 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.234 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.234 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.234 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.6": { 2024-09-14T03:47:50.234 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:50.235 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_slow_fallback": { 2024-09-14T03:47:50.235 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.235 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.235 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_slow_size_fallback": { 2024-09-14T03:47:50.235 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.235 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.235 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_db": { 2024-09-14T03:47:50.235 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 65536 2024-09-14T03:47:50.235 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.235 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_main": { 2024-09-14T03:47:50.235 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.235 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.235 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_wal": { 2024-09-14T03:47:50.235 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.236 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.236 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_slow": { 2024-09-14T03:47:50.236 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.236 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.236 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_sst": { 2024-09-14T03:47:50.236 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12288 2024-09-14T03:47:50.236 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.236 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_wal": { 2024-09-14T03:47:50.236 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1245184 2024-09-14T03:47:50.236 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.236 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lat": { 2024-09-14T03:47:50.237 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.237 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.237 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.237 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lock_lat": { 2024-09-14T03:47:50.237 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.237 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.237 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.237 INFO:tasks.workunit.client.0.smithi070.stdout: "db_total_bytes": { 2024-09-14T03:47:50.237 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 95995035648 2024-09-14T03:47:50.237 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.238 INFO:tasks.workunit.client.0.smithi070.stdout: "db_used_bytes": { 2024-09-14T03:47:50.238 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 304480256 2024-09-14T03:47:50.238 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.238 INFO:tasks.workunit.client.0.smithi070.stdout: "log_bytes": { 2024-09-14T03:47:50.238 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 962560 2024-09-14T03:47:50.238 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.238 INFO:tasks.workunit.client.0.smithi070.stdout: "logged_bytes": { 2024-09-14T03:47:50.238 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 823296 2024-09-14T03:47:50.238 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.238 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_db": { 2024-09-14T03:47:50.238 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 312868864 2024-09-14T03:47:50.238 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.238 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_slow": { 2024-09-14T03:47:50.238 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.239 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.239 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_wal": { 2024-09-14T03:47:50.239 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.239 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.239 INFO:tasks.workunit.client.0.smithi070.stdout: "num_files": { 2024-09-14T03:47:50.239 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 14 2024-09-14T03:47:50.239 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.239 INFO:tasks.workunit.client.0.smithi070.stdout: "read_bytes": { 2024-09-14T03:47:50.239 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 195697 2024-09-14T03:47:50.239 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.240 INFO:tasks.workunit.client.0.smithi070.stdout: "read_count": { 2024-09-14T03:47:50.240 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 23 2024-09-14T03:47:50.240 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.240 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes": { 2024-09-14T03:47:50.240 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1269760 2024-09-14T03:47:50.240 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.240 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_db": { 2024-09-14T03:47:50.240 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1273856 2024-09-14T03:47:50.240 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.240 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_slow": { 2024-09-14T03:47:50.241 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.241 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.241 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_wal": { 2024-09-14T03:47:50.241 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.241 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.241 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_count": { 2024-09-14T03:47:50.241 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 15 2024-09-14T03:47:50.241 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.241 INFO:tasks.workunit.client.0.smithi070.stdout: "read_prefetch_bytes": { 2024-09-14T03:47:50.242 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4789 2024-09-14T03:47:50.242 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.242 INFO:tasks.workunit.client.0.smithi070.stdout: "read_prefetch_count": { 2024-09-14T03:47:50.242 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4 2024-09-14T03:47:50.242 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.242 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_buffer_bytes": { 2024-09-14T03:47:50.242 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5589 2024-09-14T03:47:50.242 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.242 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_buffer_count": { 2024-09-14T03:47:50.242 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 24 2024-09-14T03:47:50.242 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.242 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_bytes": { 2024-09-14T03:47:50.242 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5589 2024-09-14T03:47:50.243 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.243 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_count": { 2024-09-14T03:47:50.243 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 24 2024-09-14T03:47:50.243 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.243 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes": { 2024-09-14T03:47:50.243 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.243 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.243 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_db": { 2024-09-14T03:47:50.243 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.243 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.243 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_slow": { 2024-09-14T03:47:50.243 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.243 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.244 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_wal": { 2024-09-14T03:47:50.244 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.244 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.244 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_count": { 2024-09-14T03:47:50.244 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.244 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.244 INFO:tasks.workunit.client.0.smithi070.stdout: "slow_total_bytes": { 2024-09-14T03:47:50.244 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.244 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.244 INFO:tasks.workunit.client.0.smithi070.stdout: "slow_used_bytes": { 2024-09-14T03:47:50.244 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.244 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.245 INFO:tasks.workunit.client.0.smithi070.stdout: "wal_total_bytes": { 2024-09-14T03:47:50.245 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.245 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.245 INFO:tasks.workunit.client.0.smithi070.stdout: "wal_used_bytes": { 2024-09-14T03:47:50.245 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.245 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.245 INFO:tasks.workunit.client.0.smithi070.stdout: "write_bytes": { 2024-09-14T03:47:50.245 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2142208 2024-09-14T03:47:50.245 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.245 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.245 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore": { 2024-09-14T03:47:50.245 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit": { 2024-09-14T03:47:50.245 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4096 2024-09-14T03:47:50.245 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.246 INFO:tasks.workunit.client.0.smithi070.stdout: "allocated": { 2024-09-14T03:47:50.246 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1298432 2024-09-14T03:47:50.246 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.246 INFO:tasks.workunit.client.0.smithi070.stdout: "clist_lat": { 2024-09-14T03:47:50.246 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:50.246 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 110641 2024-09-14T03:47:50.246 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.246 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_lat": { 2024-09-14T03:47:50.246 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.246 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.246 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.246 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed": { 2024-09-14T03:47:50.246 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.246 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.247 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed_allocated": { 2024-09-14T03:47:50.247 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.247 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.247 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed_original": { 2024-09-14T03:47:50.247 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.247 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.247 INFO:tasks.workunit.client.0.smithi070.stdout: "csum_lat": { 2024-09-14T03:47:50.247 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 10, 2024-09-14T03:47:50.247 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 33348 2024-09-14T03:47:50.247 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.247 INFO:tasks.workunit.client.0.smithi070.stdout: "decompress_lat": { 2024-09-14T03:47:50.247 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.247 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.248 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.248 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_commit_lat": { 2024-09-14T03:47:50.248 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 184, 2024-09-14T03:47:50.248 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 38425516 2024-09-14T03:47:50.248 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.248 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_final_lat": { 2024-09-14T03:47:50.248 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 171, 2024-09-14T03:47:50.248 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 16844763 2024-09-14T03:47:50.248 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.248 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_flush_lat": { 2024-09-14T03:47:50.248 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 184, 2024-09-14T03:47:50.249 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 156025 2024-09-14T03:47:50.249 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.249 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_sync_lat": { 2024-09-14T03:47:50.249 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 184, 2024-09-14T03:47:50.249 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 38581541 2024-09-14T03:47:50.249 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.249 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_get_keys_lat": { 2024-09-14T03:47:50.249 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.249 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.249 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.249 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_get_values_lat": { 2024-09-14T03:47:50.249 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:50.249 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 30588 2024-09-14T03:47:50.250 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.250 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_lower_bound_lat": { 2024-09-14T03:47:50.250 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 3, 2024-09-14T03:47:50.250 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4687 2024-09-14T03:47:50.250 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.250 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_next_lat": { 2024-09-14T03:47:50.250 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.250 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.250 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.250 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_seek_to_first_lat": { 2024-09-14T03:47:50.250 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.250 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.250 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.250 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_upper_bound_lat": { 2024-09-14T03:47:50.251 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 3, 2024-09-14T03:47:50.251 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 8549 2024-09-14T03:47:50.251 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.251 INFO:tasks.workunit.client.0.smithi070.stdout: "onode_hits": { 2024-09-14T03:47:50.251 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 353 2024-09-14T03:47:50.251 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.251 INFO:tasks.workunit.client.0.smithi070.stdout: "onode_misses": { 2024-09-14T03:47:50.251 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 244 2024-09-14T03:47:50.251 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.251 INFO:tasks.workunit.client.0.smithi070.stdout: "read_lat": { 2024-09-14T03:47:50.251 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 88, 2024-09-14T03:47:50.251 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 924026 2024-09-14T03:47:50.251 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.251 INFO:tasks.workunit.client.0.smithi070.stdout: "read_onode_meta_lat": { 2024-09-14T03:47:50.252 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 137, 2024-09-14T03:47:50.252 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 210059 2024-09-14T03:47:50.252 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.252 INFO:tasks.workunit.client.0.smithi070.stdout: "read_wait_aio_lat": { 2024-09-14T03:47:50.252 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 49, 2024-09-14T03:47:50.252 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 509522 2024-09-14T03:47:50.252 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.252 INFO:tasks.workunit.client.0.smithi070.stdout: "reads_with_retries": { 2024-09-14T03:47:50.252 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.252 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.252 INFO:tasks.workunit.client.0.smithi070.stdout: "remove_lat": { 2024-09-14T03:47:50.252 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:50.252 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 26070 2024-09-14T03:47:50.253 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.253 INFO:tasks.workunit.client.0.smithi070.stdout: "state_aio_wait_lat": { 2024-09-14T03:47:50.253 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 272, 2024-09-14T03:47:50.253 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2833419 2024-09-14T03:47:50.253 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.253 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_aio_wait_lat": { 2024-09-14T03:47:50.253 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 34, 2024-09-14T03:47:50.253 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2521038 2024-09-14T03:47:50.253 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.253 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_cleanup_lat": { 2024-09-14T03:47:50.253 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 34, 2024-09-14T03:47:50.253 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 449358501559 2024-09-14T03:47:50.253 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.253 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_queued_lat": { 2024-09-14T03:47:50.254 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 34, 2024-09-14T03:47:50.254 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 53644807972 2024-09-14T03:47:50.254 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.254 INFO:tasks.workunit.client.0.smithi070.stdout: "state_done_lat": { 2024-09-14T03:47:50.254 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 272, 2024-09-14T03:47:50.254 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 495568 2024-09-14T03:47:50.254 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.254 INFO:tasks.workunit.client.0.smithi070.stdout: "state_finishing_lat": { 2024-09-14T03:47:50.254 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 272, 2024-09-14T03:47:50.254 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 38270 2024-09-14T03:47:50.254 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.254 INFO:tasks.workunit.client.0.smithi070.stdout: "state_io_done_lat": { 2024-09-14T03:47:50.254 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 272, 2024-09-14T03:47:50.255 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 205836 2024-09-14T03:47:50.255 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.255 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_commiting_lat": { 2024-09-14T03:47:50.255 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 272, 2024-09-14T03:47:50.255 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 169304171 2024-09-14T03:47:50.255 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.255 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_done_lat": { 2024-09-14T03:47:50.255 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 272, 2024-09-14T03:47:50.255 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 84725 2024-09-14T03:47:50.255 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.255 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_queued_lat": { 2024-09-14T03:47:50.256 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 272, 2024-09-14T03:47:50.256 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 108517973 2024-09-14T03:47:50.256 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.256 INFO:tasks.workunit.client.0.smithi070.stdout: "state_prepare_lat": { 2024-09-14T03:47:50.256 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 272, 2024-09-14T03:47:50.256 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 11043922 2024-09-14T03:47:50.256 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.256 INFO:tasks.workunit.client.0.smithi070.stdout: "stored": { 2024-09-14T03:47:50.256 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 905244 2024-09-14T03:47:50.256 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.257 INFO:tasks.workunit.client.0.smithi070.stdout: "truncate_lat": { 2024-09-14T03:47:50.257 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 27, 2024-09-14T03:47:50.257 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 114486 2024-09-14T03:47:50.257 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.257 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_commit_lat": { 2024-09-14T03:47:50.257 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 272, 2024-09-14T03:47:50.257 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 291943815 2024-09-14T03:47:50.257 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.257 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_submit_lat": { 2024-09-14T03:47:50.257 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 272, 2024-09-14T03:47:50.257 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 13196129 2024-09-14T03:47:50.258 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.258 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_throttle_lat": { 2024-09-14T03:47:50.258 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 272, 2024-09-14T03:47:50.258 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1892202 2024-09-14T03:47:50.258 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.258 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.258 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache": { 2024-09-14T03:47:50.258 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_bytes": { 2024-09-14T03:47:50.258 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1861444387 2024-09-14T03:47:50.258 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.258 INFO:tasks.workunit.client.0.smithi070.stdout: "heap_bytes": { 2024-09-14T03:47:50.258 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 270974976 2024-09-14T03:47:50.258 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.259 INFO:tasks.workunit.client.0.smithi070.stdout: "mapped_bytes": { 2024-09-14T03:47:50.259 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 269418496 2024-09-14T03:47:50.259 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.259 INFO:tasks.workunit.client.0.smithi070.stdout: "target_bytes": { 2024-09-14T03:47:50.259 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3137353830 2024-09-14T03:47:50.259 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.259 INFO:tasks.workunit.client.0.smithi070.stdout: "unmapped_bytes": { 2024-09-14T03:47:50.259 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1556480 2024-09-14T03:47:50.259 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.259 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.259 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:data": { 2024-09-14T03:47:50.259 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:50.259 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 159383552 2024-09-14T03:47:50.259 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.260 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:50.260 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.260 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.260 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:50.260 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.260 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.260 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:50.260 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 85497699 2024-09-14T03:47:50.260 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.260 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:50.260 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.260 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.260 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:50.260 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.261 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.261 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:50.261 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.261 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.261 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:50.261 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.261 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.261 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:50.261 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.261 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.261 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:50.261 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 8192 2024-09-14T03:47:50.261 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.262 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:50.262 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.262 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.262 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:50.262 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.262 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.262 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:50.262 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.262 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.262 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:50.262 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 73877661 2024-09-14T03:47:50.262 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.263 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.263 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:kv": { 2024-09-14T03:47:50.263 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:50.263 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 780140544 2024-09-14T03:47:50.263 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.263 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:50.263 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:50.263 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.263 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:50.263 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.263 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.263 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:50.263 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 641232746 2024-09-14T03:47:50.263 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.264 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:50.264 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.264 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.264 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:50.264 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.264 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.264 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:50.264 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.264 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.264 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:50.264 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.264 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.264 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:50.265 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.265 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.265 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:50.265 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2128 2024-09-14T03:47:50.265 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.265 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:50.265 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.265 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.265 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:50.265 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.265 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.265 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:50.265 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.266 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.266 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:50.266 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71796806 2024-09-14T03:47:50.266 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.266 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.266 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:kv_onode": { 2024-09-14T03:47:50.266 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:50.266 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 192937984 2024-09-14T03:47:50.266 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.266 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:50.266 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:50.266 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.266 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:50.267 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 464 2024-09-14T03:47:50.267 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.267 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:50.267 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 56998466 2024-09-14T03:47:50.267 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.267 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:50.267 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.268 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.268 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:50.268 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.268 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.268 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:50.268 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.269 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.269 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:50.269 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.269 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.269 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:50.269 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.269 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.269 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:50.269 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.269 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.269 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:50.269 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.269 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.270 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:50.270 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.270 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.270 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:50.270 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.270 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.270 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:50.270 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 68830190 2024-09-14T03:47:50.270 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.270 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.270 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:meta": { 2024-09-14T03:47:50.270 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:50.270 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 713031680 2024-09-14T03:47:50.270 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.271 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:50.271 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.271 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.271 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:50.271 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.271 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.271 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:50.271 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 641232746 2024-09-14T03:47:50.271 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.271 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:50.271 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.272 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.272 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:50.272 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.272 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.272 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:50.272 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.272 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.272 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:50.272 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.272 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.272 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:50.272 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7105 2024-09-14T03:47:50.272 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.273 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:50.273 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 257223 2024-09-14T03:47:50.273 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.273 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:50.273 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.273 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.273 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:50.273 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.273 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.273 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:50.273 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.273 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.273 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:50.274 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71534606 2024-09-14T03:47:50.274 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.274 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.274 INFO:tasks.workunit.client.0.smithi070.stdout: "objecter": { 2024-09-14T03:47:50.274 INFO:tasks.workunit.client.0.smithi070.stdout: "op_active": { 2024-09-14T03:47:50.274 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.274 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.274 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r": { 2024-09-14T03:47:50.274 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.274 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.274 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rmw": { 2024-09-14T03:47:50.274 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.275 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.275 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w": { 2024-09-14T03:47:50.275 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.275 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.275 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.275 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:50.275 INFO:tasks.workunit.client.0.smithi070.stdout: "numpg": { 2024-09-14T03:47:50.275 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 38 2024-09-14T03:47:50.275 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.275 INFO:tasks.workunit.client.0.smithi070.stdout: "numpg_removing": { 2024-09-14T03:47:50.275 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.276 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.276 INFO:tasks.workunit.client.0.smithi070.stdout: "op": { 2024-09-14T03:47:50.276 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 13 2024-09-14T03:47:50.276 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.276 INFO:tasks.workunit.client.0.smithi070.stdout: "op_delayed_degraded": { 2024-09-14T03:47:50.276 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.276 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.276 INFO:tasks.workunit.client.0.smithi070.stdout: "op_delayed_unreadable": { 2024-09-14T03:47:50.276 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.276 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.276 INFO:tasks.workunit.client.0.smithi070.stdout: "op_in_bytes": { 2024-09-14T03:47:50.276 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2804 2024-09-14T03:47:50.277 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.277 INFO:tasks.workunit.client.0.smithi070.stdout: "op_latency": { 2024-09-14T03:47:50.277 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 13, 2024-09-14T03:47:50.277 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 73359531 2024-09-14T03:47:50.277 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.277 INFO:tasks.workunit.client.0.smithi070.stdout: "op_out_bytes": { 2024-09-14T03:47:50.277 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.277 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.277 INFO:tasks.workunit.client.0.smithi070.stdout: "op_prepare_latency": { 2024-09-14T03:47:50.277 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 15, 2024-09-14T03:47:50.277 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7673717 2024-09-14T03:47:50.277 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.277 INFO:tasks.workunit.client.0.smithi070.stdout: "op_process_latency": { 2024-09-14T03:47:50.277 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 13, 2024-09-14T03:47:50.278 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 26473448 2024-09-14T03:47:50.278 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.278 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r": { 2024-09-14T03:47:50.278 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.278 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.278 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_latency": { 2024-09-14T03:47:50.278 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.278 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.278 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.278 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:47:50.278 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.278 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.279 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_out_bytes": { 2024-09-14T03:47:50.279 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.279 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.279 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_prepare_latency": { 2024-09-14T03:47:50.279 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.279 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.279 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.279 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_process_latency": { 2024-09-14T03:47:50.279 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.280 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.280 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.280 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw": { 2024-09-14T03:47:50.280 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.280 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.280 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_in_bytes": { 2024-09-14T03:47:50.280 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.280 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.280 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency": { 2024-09-14T03:47:50.280 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.280 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.280 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.281 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:47:50.281 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.281 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.281 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:47:50.281 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.281 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.281 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_out_bytes": { 2024-09-14T03:47:50.281 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.281 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.281 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_prepare_latency": { 2024-09-14T03:47:50.281 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.281 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.281 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.281 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_process_latency": { 2024-09-14T03:47:50.282 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.282 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.282 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.282 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w": { 2024-09-14T03:47:50.282 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 13 2024-09-14T03:47:50.282 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.282 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_in_bytes": { 2024-09-14T03:47:50.282 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2804 2024-09-14T03:47:50.282 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.282 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_latency": { 2024-09-14T03:47:50.282 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 13, 2024-09-14T03:47:50.282 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 73359531 2024-09-14T03:47:50.282 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.282 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:47:50.283 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.283 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.283 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_prepare_latency": { 2024-09-14T03:47:50.283 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 15, 2024-09-14T03:47:50.283 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7673717 2024-09-14T03:47:50.283 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.283 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_process_latency": { 2024-09-14T03:47:50.283 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 13, 2024-09-14T03:47:50.283 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 26473448 2024-09-14T03:47:50.283 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.283 INFO:tasks.workunit.client.0.smithi070.stdout: "op_wip": { 2024-09-14T03:47:50.284 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.284 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.284 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_bytes": { 2024-09-14T03:47:50.284 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 590368 2024-09-14T03:47:50.284 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.284 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_ops": { 2024-09-14T03:47:50.284 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:50.284 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.284 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_bytes": { 2024-09-14T03:47:50.284 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 95995035648 2024-09-14T03:47:50.284 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.284 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_bytes_used": { 2024-09-14T03:47:50.284 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 305786880 2024-09-14T03:47:50.285 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.285 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.285 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.285 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.7": { 2024-09-14T03:47:50.285 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:50.285 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_slow_fallback": { 2024-09-14T03:47:50.285 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.285 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.285 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_slow_size_fallback": { 2024-09-14T03:47:50.285 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.285 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.285 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_db": { 2024-09-14T03:47:50.285 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 65536 2024-09-14T03:47:50.285 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.286 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_main": { 2024-09-14T03:47:50.286 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.286 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.286 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_wal": { 2024-09-14T03:47:50.286 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.286 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.286 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_slow": { 2024-09-14T03:47:50.286 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.286 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.286 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_sst": { 2024-09-14T03:47:50.286 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12288 2024-09-14T03:47:50.286 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.286 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_wal": { 2024-09-14T03:47:50.286 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1245184 2024-09-14T03:47:50.287 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.287 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lat": { 2024-09-14T03:47:50.287 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.287 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.287 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.287 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lock_lat": { 2024-09-14T03:47:50.287 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.287 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.287 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.287 INFO:tasks.workunit.client.0.smithi070.stdout: "db_total_bytes": { 2024-09-14T03:47:50.287 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 95995035648 2024-09-14T03:47:50.287 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.287 INFO:tasks.workunit.client.0.smithi070.stdout: "db_used_bytes": { 2024-09-14T03:47:50.287 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 304480256 2024-09-14T03:47:50.287 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.288 INFO:tasks.workunit.client.0.smithi070.stdout: "log_bytes": { 2024-09-14T03:47:50.288 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 983040 2024-09-14T03:47:50.288 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.288 INFO:tasks.workunit.client.0.smithi070.stdout: "logged_bytes": { 2024-09-14T03:47:50.288 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 843776 2024-09-14T03:47:50.288 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.288 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_db": { 2024-09-14T03:47:50.289 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 312868864 2024-09-14T03:47:50.289 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.289 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_slow": { 2024-09-14T03:47:50.289 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.289 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.289 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_wal": { 2024-09-14T03:47:50.289 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.289 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.289 INFO:tasks.workunit.client.0.smithi070.stdout: "num_files": { 2024-09-14T03:47:50.289 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 14 2024-09-14T03:47:50.290 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.290 INFO:tasks.workunit.client.0.smithi070.stdout: "read_bytes": { 2024-09-14T03:47:50.290 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 195697 2024-09-14T03:47:50.290 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.290 INFO:tasks.workunit.client.0.smithi070.stdout: "read_count": { 2024-09-14T03:47:50.290 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 23 2024-09-14T03:47:50.290 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.290 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes": { 2024-09-14T03:47:50.290 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1269760 2024-09-14T03:47:50.290 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.290 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_db": { 2024-09-14T03:47:50.290 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1273856 2024-09-14T03:47:50.290 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.291 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_slow": { 2024-09-14T03:47:50.291 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.291 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.291 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_wal": { 2024-09-14T03:47:50.291 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.291 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.291 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_count": { 2024-09-14T03:47:50.291 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 15 2024-09-14T03:47:50.291 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.291 INFO:tasks.workunit.client.0.smithi070.stdout: "read_prefetch_bytes": { 2024-09-14T03:47:50.291 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4789 2024-09-14T03:47:50.291 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.291 INFO:tasks.workunit.client.0.smithi070.stdout: "read_prefetch_count": { 2024-09-14T03:47:50.291 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4 2024-09-14T03:47:50.291 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.292 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_buffer_bytes": { 2024-09-14T03:47:50.292 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5589 2024-09-14T03:47:50.292 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.292 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_buffer_count": { 2024-09-14T03:47:50.292 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 24 2024-09-14T03:47:50.292 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.292 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_bytes": { 2024-09-14T03:47:50.292 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5589 2024-09-14T03:47:50.292 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.292 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_count": { 2024-09-14T03:47:50.292 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 24 2024-09-14T03:47:50.293 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.293 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes": { 2024-09-14T03:47:50.293 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.293 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.293 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_db": { 2024-09-14T03:47:50.293 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.293 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.294 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_slow": { 2024-09-14T03:47:50.294 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.294 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.294 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_wal": { 2024-09-14T03:47:50.294 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.294 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.294 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_count": { 2024-09-14T03:47:50.294 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.294 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.294 INFO:tasks.workunit.client.0.smithi070.stdout: "slow_total_bytes": { 2024-09-14T03:47:50.294 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.294 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.294 INFO:tasks.workunit.client.0.smithi070.stdout: "slow_used_bytes": { 2024-09-14T03:47:50.294 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.295 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.295 INFO:tasks.workunit.client.0.smithi070.stdout: "wal_total_bytes": { 2024-09-14T03:47:50.295 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.295 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.295 INFO:tasks.workunit.client.0.smithi070.stdout: "wal_used_bytes": { 2024-09-14T03:47:50.295 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.295 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.295 INFO:tasks.workunit.client.0.smithi070.stdout: "write_bytes": { 2024-09-14T03:47:50.295 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2162688 2024-09-14T03:47:50.295 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.295 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.295 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore": { 2024-09-14T03:47:50.295 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit": { 2024-09-14T03:47:50.296 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4096 2024-09-14T03:47:50.296 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.296 INFO:tasks.workunit.client.0.smithi070.stdout: "allocated": { 2024-09-14T03:47:50.296 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1298432 2024-09-14T03:47:50.296 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.296 INFO:tasks.workunit.client.0.smithi070.stdout: "clist_lat": { 2024-09-14T03:47:50.296 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 3, 2024-09-14T03:47:50.296 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 165845 2024-09-14T03:47:50.296 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.296 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_lat": { 2024-09-14T03:47:50.296 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.296 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.296 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.296 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed": { 2024-09-14T03:47:50.297 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.297 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.297 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed_allocated": { 2024-09-14T03:47:50.297 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.297 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.297 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed_original": { 2024-09-14T03:47:50.297 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.297 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.297 INFO:tasks.workunit.client.0.smithi070.stdout: "csum_lat": { 2024-09-14T03:47:50.297 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 12, 2024-09-14T03:47:50.298 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 26784 2024-09-14T03:47:50.298 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.298 INFO:tasks.workunit.client.0.smithi070.stdout: "decompress_lat": { 2024-09-14T03:47:50.298 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.298 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.298 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.298 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_commit_lat": { 2024-09-14T03:47:50.298 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 189, 2024-09-14T03:47:50.298 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 44860059 2024-09-14T03:47:50.298 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.298 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_final_lat": { 2024-09-14T03:47:50.298 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 179, 2024-09-14T03:47:50.299 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 8737744 2024-09-14T03:47:50.299 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.299 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_flush_lat": { 2024-09-14T03:47:50.299 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 189, 2024-09-14T03:47:50.299 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 277025 2024-09-14T03:47:50.299 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.299 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_sync_lat": { 2024-09-14T03:47:50.299 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 189, 2024-09-14T03:47:50.299 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 45137084 2024-09-14T03:47:50.299 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.299 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_get_keys_lat": { 2024-09-14T03:47:50.299 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.299 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.299 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.300 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_get_values_lat": { 2024-09-14T03:47:50.300 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:50.300 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 33804 2024-09-14T03:47:50.300 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.300 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_lower_bound_lat": { 2024-09-14T03:47:50.300 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:50.300 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1288 2024-09-14T03:47:50.300 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.300 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_next_lat": { 2024-09-14T03:47:50.300 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.300 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.300 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.300 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_seek_to_first_lat": { 2024-09-14T03:47:50.301 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:50.301 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.301 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.301 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_upper_bound_lat": { 2024-09-14T03:47:50.301 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:50.301 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4334 2024-09-14T03:47:50.301 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.301 INFO:tasks.workunit.client.0.smithi070.stdout: "onode_hits": { 2024-09-14T03:47:50.301 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 542 2024-09-14T03:47:50.301 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.301 INFO:tasks.workunit.client.0.smithi070.stdout: "onode_misses": { 2024-09-14T03:47:50.301 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 268 2024-09-14T03:47:50.301 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.301 INFO:tasks.workunit.client.0.smithi070.stdout: "read_lat": { 2024-09-14T03:47:50.302 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 113, 2024-09-14T03:47:50.302 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1292204 2024-09-14T03:47:50.302 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.302 INFO:tasks.workunit.client.0.smithi070.stdout: "read_onode_meta_lat": { 2024-09-14T03:47:50.302 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 179, 2024-09-14T03:47:50.302 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 212712 2024-09-14T03:47:50.302 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.302 INFO:tasks.workunit.client.0.smithi070.stdout: "read_wait_aio_lat": { 2024-09-14T03:47:50.302 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 66, 2024-09-14T03:47:50.302 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 695603 2024-09-14T03:47:50.302 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.302 INFO:tasks.workunit.client.0.smithi070.stdout: "reads_with_retries": { 2024-09-14T03:47:50.302 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.302 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.303 INFO:tasks.workunit.client.0.smithi070.stdout: "remove_lat": { 2024-09-14T03:47:50.303 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:50.303 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 26374 2024-09-14T03:47:50.303 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.303 INFO:tasks.workunit.client.0.smithi070.stdout: "state_aio_wait_lat": { 2024-09-14T03:47:50.303 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 288, 2024-09-14T03:47:50.303 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2190213 2024-09-14T03:47:50.303 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.303 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_aio_wait_lat": { 2024-09-14T03:47:50.303 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 24, 2024-09-14T03:47:50.303 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1694874 2024-09-14T03:47:50.303 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.303 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_cleanup_lat": { 2024-09-14T03:47:50.303 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 24, 2024-09-14T03:47:50.304 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 298495756417 2024-09-14T03:47:50.304 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.304 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_queued_lat": { 2024-09-14T03:47:50.304 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 24, 2024-09-14T03:47:50.304 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 36302536737 2024-09-14T03:47:50.304 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.304 INFO:tasks.workunit.client.0.smithi070.stdout: "state_done_lat": { 2024-09-14T03:47:50.304 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 288, 2024-09-14T03:47:50.304 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 448360 2024-09-14T03:47:50.304 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.304 INFO:tasks.workunit.client.0.smithi070.stdout: "state_finishing_lat": { 2024-09-14T03:47:50.304 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 288, 2024-09-14T03:47:50.304 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 33332 2024-09-14T03:47:50.305 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.305 INFO:tasks.workunit.client.0.smithi070.stdout: "state_io_done_lat": { 2024-09-14T03:47:50.305 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 288, 2024-09-14T03:47:50.305 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 219535 2024-09-14T03:47:50.305 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.305 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_commiting_lat": { 2024-09-14T03:47:50.305 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 288, 2024-09-14T03:47:50.305 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 127434141 2024-09-14T03:47:50.305 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.305 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_done_lat": { 2024-09-14T03:47:50.305 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 288, 2024-09-14T03:47:50.305 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 80980 2024-09-14T03:47:50.306 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.306 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_queued_lat": { 2024-09-14T03:47:50.306 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 288, 2024-09-14T03:47:50.306 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 92350704 2024-09-14T03:47:50.306 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.306 INFO:tasks.workunit.client.0.smithi070.stdout: "state_prepare_lat": { 2024-09-14T03:47:50.306 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 288, 2024-09-14T03:47:50.306 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12175385 2024-09-14T03:47:50.306 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.306 INFO:tasks.workunit.client.0.smithi070.stdout: "stored": { 2024-09-14T03:47:50.306 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 903527 2024-09-14T03:47:50.306 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.306 INFO:tasks.workunit.client.0.smithi070.stdout: "truncate_lat": { 2024-09-14T03:47:50.306 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 23, 2024-09-14T03:47:50.307 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 103984 2024-09-14T03:47:50.307 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.307 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_commit_lat": { 2024-09-14T03:47:50.307 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 288, 2024-09-14T03:47:50.307 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 234404958 2024-09-14T03:47:50.307 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.307 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_submit_lat": { 2024-09-14T03:47:50.307 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 288, 2024-09-14T03:47:50.307 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 16408932 2024-09-14T03:47:50.307 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.307 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_throttle_lat": { 2024-09-14T03:47:50.307 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 288, 2024-09-14T03:47:50.307 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1971853 2024-09-14T03:47:50.307 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.308 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.308 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache": { 2024-09-14T03:47:50.308 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_bytes": { 2024-09-14T03:47:50.308 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1861444387 2024-09-14T03:47:50.308 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.308 INFO:tasks.workunit.client.0.smithi070.stdout: "heap_bytes": { 2024-09-14T03:47:50.308 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 271958016 2024-09-14T03:47:50.308 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.308 INFO:tasks.workunit.client.0.smithi070.stdout: "mapped_bytes": { 2024-09-14T03:47:50.308 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 270417920 2024-09-14T03:47:50.308 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.308 INFO:tasks.workunit.client.0.smithi070.stdout: "target_bytes": { 2024-09-14T03:47:50.309 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3137353830 2024-09-14T03:47:50.309 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.309 INFO:tasks.workunit.client.0.smithi070.stdout: "unmapped_bytes": { 2024-09-14T03:47:50.309 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1540096 2024-09-14T03:47:50.309 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.309 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.309 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:data": { 2024-09-14T03:47:50.309 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:50.309 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 159383552 2024-09-14T03:47:50.309 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.309 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:50.310 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.310 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.310 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:50.310 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.310 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.310 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:50.310 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 85480021 2024-09-14T03:47:50.310 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.310 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:50.310 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.310 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.310 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:50.310 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 151552 2024-09-14T03:47:50.311 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.311 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:50.311 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.311 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.311 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:50.311 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.311 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.311 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:50.311 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.311 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.312 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:50.312 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 135168 2024-09-14T03:47:50.312 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.312 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:50.312 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.312 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.312 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:50.312 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.312 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.312 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:50.312 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.312 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.312 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:50.313 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 73616811 2024-09-14T03:47:50.313 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.313 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.313 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:kv": { 2024-09-14T03:47:50.313 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:50.313 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 780140544 2024-09-14T03:47:50.313 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.313 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:50.313 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:50.313 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.313 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:50.313 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.313 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.314 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:50.314 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 641100162 2024-09-14T03:47:50.314 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.314 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:50.314 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.314 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.314 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:50.314 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.314 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.314 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:50.314 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.314 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.314 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:50.315 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.315 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.315 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:50.315 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.315 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.315 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:50.315 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2128 2024-09-14T03:47:50.315 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.315 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:50.315 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.316 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.316 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:50.316 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.316 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.316 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:50.316 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.316 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.316 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:50.317 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71929390 2024-09-14T03:47:50.317 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.317 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.317 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:kv_onode": { 2024-09-14T03:47:50.317 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:50.317 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 192937984 2024-09-14T03:47:50.317 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.317 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:50.317 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:50.317 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.317 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:50.317 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 464 2024-09-14T03:47:50.317 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.318 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:50.318 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 56986681 2024-09-14T03:47:50.318 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.318 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:50.318 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.318 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.318 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:50.318 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.319 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.319 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:50.319 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.319 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.319 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:50.319 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.319 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.319 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:50.319 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.319 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.319 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:50.319 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.319 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.320 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:50.320 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.320 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.320 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:50.320 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.320 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.321 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:50.321 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.321 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.321 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:50.321 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 68841975 2024-09-14T03:47:50.321 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.321 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.321 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:meta": { 2024-09-14T03:47:50.321 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:50.321 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 713031680 2024-09-14T03:47:50.321 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.321 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:50.321 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.322 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.322 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:50.322 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.322 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.322 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:50.322 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 641100162 2024-09-14T03:47:50.322 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.322 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:50.322 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.322 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.322 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:50.323 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 15658 2024-09-14T03:47:50.323 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.323 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:50.323 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.323 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.323 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:50.323 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.323 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.323 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:50.323 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.323 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.323 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:50.323 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 264773 2024-09-14T03:47:50.323 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.324 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:50.324 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.324 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.324 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:50.324 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.324 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.324 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:50.324 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.324 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.324 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:50.324 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71651087 2024-09-14T03:47:50.324 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.324 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.324 INFO:tasks.workunit.client.0.smithi070.stdout: "objecter": { 2024-09-14T03:47:50.325 INFO:tasks.workunit.client.0.smithi070.stdout: "op_active": { 2024-09-14T03:47:50.325 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.325 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.325 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r": { 2024-09-14T03:47:50.325 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.325 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.325 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rmw": { 2024-09-14T03:47:50.325 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.325 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.325 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w": { 2024-09-14T03:47:50.325 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.325 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.325 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.325 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:50.326 INFO:tasks.workunit.client.0.smithi070.stdout: "numpg": { 2024-09-14T03:47:50.326 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 46 2024-09-14T03:47:50.326 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.326 INFO:tasks.workunit.client.0.smithi070.stdout: "numpg_removing": { 2024-09-14T03:47:50.326 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.326 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.326 INFO:tasks.workunit.client.0.smithi070.stdout: "op": { 2024-09-14T03:47:50.326 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 87 2024-09-14T03:47:50.326 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.326 INFO:tasks.workunit.client.0.smithi070.stdout: "op_delayed_degraded": { 2024-09-14T03:47:50.326 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.326 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.326 INFO:tasks.workunit.client.0.smithi070.stdout: "op_delayed_unreadable": { 2024-09-14T03:47:50.327 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.327 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.327 INFO:tasks.workunit.client.0.smithi070.stdout: "op_in_bytes": { 2024-09-14T03:47:50.327 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.327 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.327 INFO:tasks.workunit.client.0.smithi070.stdout: "op_latency": { 2024-09-14T03:47:50.327 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 87, 2024-09-14T03:47:50.327 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 66490697 2024-09-14T03:47:50.327 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.327 INFO:tasks.workunit.client.0.smithi070.stdout: "op_out_bytes": { 2024-09-14T03:47:50.327 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 264311 2024-09-14T03:47:50.327 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.327 INFO:tasks.workunit.client.0.smithi070.stdout: "op_prepare_latency": { 2024-09-14T03:47:50.328 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 89, 2024-09-14T03:47:50.328 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 17864970 2024-09-14T03:47:50.328 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.328 INFO:tasks.workunit.client.0.smithi070.stdout: "op_process_latency": { 2024-09-14T03:47:50.328 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 87, 2024-09-14T03:47:50.328 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 49784785 2024-09-14T03:47:50.328 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.328 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r": { 2024-09-14T03:47:50.328 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 60 2024-09-14T03:47:50.328 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.328 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_latency": { 2024-09-14T03:47:50.328 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 60, 2024-09-14T03:47:50.328 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7178504 2024-09-14T03:47:50.329 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.329 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:47:50.329 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.329 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.329 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_out_bytes": { 2024-09-14T03:47:50.329 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 264311 2024-09-14T03:47:50.329 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.329 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_prepare_latency": { 2024-09-14T03:47:50.329 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 60, 2024-09-14T03:47:50.329 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 6462750 2024-09-14T03:47:50.329 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.329 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_process_latency": { 2024-09-14T03:47:50.329 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 60, 2024-09-14T03:47:50.329 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5238184 2024-09-14T03:47:50.330 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.330 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw": { 2024-09-14T03:47:50.330 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 22 2024-09-14T03:47:50.330 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.330 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_in_bytes": { 2024-09-14T03:47:50.330 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.330 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.330 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency": { 2024-09-14T03:47:50.330 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 22, 2024-09-14T03:47:50.330 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 18338763 2024-09-14T03:47:50.330 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.330 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:47:50.330 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.330 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.331 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:47:50.331 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.331 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.331 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_out_bytes": { 2024-09-14T03:47:50.331 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.331 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.331 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_prepare_latency": { 2024-09-14T03:47:50.331 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 22, 2024-09-14T03:47:50.331 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7585805 2024-09-14T03:47:50.331 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.331 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_process_latency": { 2024-09-14T03:47:50.331 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 22, 2024-09-14T03:47:50.331 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 17315288 2024-09-14T03:47:50.331 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.331 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w": { 2024-09-14T03:47:50.332 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5 2024-09-14T03:47:50.332 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.332 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_in_bytes": { 2024-09-14T03:47:50.332 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.332 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.332 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_latency": { 2024-09-14T03:47:50.332 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 5, 2024-09-14T03:47:50.332 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 40973430 2024-09-14T03:47:50.332 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.332 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:47:50.332 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.332 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.332 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_prepare_latency": { 2024-09-14T03:47:50.333 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 7, 2024-09-14T03:47:50.333 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3816415 2024-09-14T03:47:50.333 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.333 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_process_latency": { 2024-09-14T03:47:50.333 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 5, 2024-09-14T03:47:50.333 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 27231313 2024-09-14T03:47:50.333 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.333 INFO:tasks.workunit.client.0.smithi070.stdout: "op_wip": { 2024-09-14T03:47:50.333 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:50.333 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.333 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_bytes": { 2024-09-14T03:47:50.334 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 590368 2024-09-14T03:47:50.334 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.334 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_ops": { 2024-09-14T03:47:50.334 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:50.334 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.334 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_bytes": { 2024-09-14T03:47:50.334 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 95995035648 2024-09-14T03:47:50.334 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.334 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_bytes_used": { 2024-09-14T03:47:50.334 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 305786880 2024-09-14T03:47:50.334 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.334 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.335 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.335 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.335 INFO:tasks.workunit.client.0.smithi070.stdout: "pools": [ 2024-09-14T03:47:50.335 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.335 INFO:tasks.workunit.client.0.smithi070.stdout: "application": [ 2024-09-14T03:47:50.335 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr" 2024-09-14T03:47:50.335 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.335 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_mode": "none", 2024-09-14T03:47:50.335 INFO:tasks.workunit.client.0.smithi070.stdout: "erasure_code_profile": {}, 2024-09-14T03:47:50.335 INFO:tasks.workunit.client.0.smithi070.stdout: "min_size": 2, 2024-09-14T03:47:50.336 INFO:tasks.workunit.client.0.smithi070.stdout: "options": {}, 2024-09-14T03:47:50.336 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_autoscale_mode": "off", 2024-09-14T03:47:50.336 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_num": 1, 2024-09-14T03:47:50.336 INFO:tasks.workunit.client.0.smithi070.stdout: "pgp_num": 1, 2024-09-14T03:47:50.336 INFO:tasks.workunit.client.0.smithi070.stdout: "pool": 1, 2024-09-14T03:47:50.336 INFO:tasks.workunit.client.0.smithi070.stdout: "size": 3, 2024-09-14T03:47:50.336 INFO:tasks.workunit.client.0.smithi070.stdout: "stats": { 2024-09-14T03:47:50.336 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_raw": 727115933962, 2024-09-14T03:47:50.336 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_used": 1781760, 2024-09-14T03:47:50.336 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_bytes_used": 0, 2024-09-14T03:47:50.336 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_under_bytes": 0, 2024-09-14T03:47:50.337 INFO:tasks.workunit.client.0.smithi070.stdout: "data_bytes_used": 1781760, 2024-09-14T03:47:50.337 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty": 0, 2024-09-14T03:47:50.337 INFO:tasks.workunit.client.0.smithi070.stdout: "max_avail": 242371985408, 2024-09-14T03:47:50.337 INFO:tasks.workunit.client.0.smithi070.stdout: "objects": 2, 2024-09-14T03:47:50.337 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_bytes_used": 0, 2024-09-14T03:47:50.337 INFO:tasks.workunit.client.0.smithi070.stdout: "percent_used": 2.4504422526661074e-06, 2024-09-14T03:47:50.337 INFO:tasks.workunit.client.0.smithi070.stdout: "quota_bytes": 0, 2024-09-14T03:47:50.337 INFO:tasks.workunit.client.0.smithi070.stdout: "quota_objects": 0, 2024-09-14T03:47:50.337 INFO:tasks.workunit.client.0.smithi070.stdout: "rd": 272, 2024-09-14T03:47:50.337 INFO:tasks.workunit.client.0.smithi070.stdout: "rd_bytes": 490496, 2024-09-14T03:47:50.337 INFO:tasks.workunit.client.0.smithi070.stdout: "stored": 590368, 2024-09-14T03:47:50.337 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_data": 590368, 2024-09-14T03:47:50.338 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_omap": 0, 2024-09-14T03:47:50.338 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_raw": 1771104, 2024-09-14T03:47:50.338 INFO:tasks.workunit.client.0.smithi070.stdout: "wr": 145, 2024-09-14T03:47:50.338 INFO:tasks.workunit.client.0.smithi070.stdout: "wr_bytes": 1667072 2024-09-14T03:47:50.338 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.338 INFO:tasks.workunit.client.0.smithi070.stdout: "target_max_bytes": 0, 2024-09-14T03:47:50.338 INFO:tasks.workunit.client.0.smithi070.stdout: "target_max_objects": 0, 2024-09-14T03:47:50.338 INFO:tasks.workunit.client.0.smithi070.stdout: "type": "replicated" 2024-09-14T03:47:50.338 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.338 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.338 INFO:tasks.workunit.client.0.smithi070.stdout: "application": [ 2024-09-14T03:47:50.339 INFO:tasks.workunit.client.0.smithi070.stdout: "cephfs" 2024-09-14T03:47:50.339 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.339 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_mode": "none", 2024-09-14T03:47:50.339 INFO:tasks.workunit.client.0.smithi070.stdout: "erasure_code_profile": {}, 2024-09-14T03:47:50.339 INFO:tasks.workunit.client.0.smithi070.stdout: "min_size": 2, 2024-09-14T03:47:50.339 INFO:tasks.workunit.client.0.smithi070.stdout: "options": {}, 2024-09-14T03:47:50.339 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_autoscale_mode": "off", 2024-09-14T03:47:50.339 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_num": 32, 2024-09-14T03:47:50.339 INFO:tasks.workunit.client.0.smithi070.stdout: "pgp_num": 32, 2024-09-14T03:47:50.339 INFO:tasks.workunit.client.0.smithi070.stdout: "pool": 2, 2024-09-14T03:47:50.339 INFO:tasks.workunit.client.0.smithi070.stdout: "size": 3, 2024-09-14T03:47:50.339 INFO:tasks.workunit.client.0.smithi070.stdout: "stats": { 2024-09-14T03:47:50.339 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_raw": 727115933962, 2024-09-14T03:47:50.339 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_used": 98304, 2024-09-14T03:47:50.340 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_bytes_used": 0, 2024-09-14T03:47:50.340 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_under_bytes": 0, 2024-09-14T03:47:50.340 INFO:tasks.workunit.client.0.smithi070.stdout: "data_bytes_used": 98304, 2024-09-14T03:47:50.340 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty": 0, 2024-09-14T03:47:50.340 INFO:tasks.workunit.client.0.smithi070.stdout: "max_avail": 242371985408, 2024-09-14T03:47:50.340 INFO:tasks.workunit.client.0.smithi070.stdout: "objects": 22, 2024-09-14T03:47:50.340 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_bytes_used": 0, 2024-09-14T03:47:50.340 INFO:tasks.workunit.client.0.smithi070.stdout: "percent_used": 1.3519712638299097e-07, 2024-09-14T03:47:50.340 INFO:tasks.workunit.client.0.smithi070.stdout: "quota_bytes": 0, 2024-09-14T03:47:50.340 INFO:tasks.workunit.client.0.smithi070.stdout: "quota_objects": 0, 2024-09-14T03:47:50.340 INFO:tasks.workunit.client.0.smithi070.stdout: "rd": 0, 2024-09-14T03:47:50.340 INFO:tasks.workunit.client.0.smithi070.stdout: "rd_bytes": 0, 2024-09-14T03:47:50.340 INFO:tasks.workunit.client.0.smithi070.stdout: "stored": 3410, 2024-09-14T03:47:50.341 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_data": 3410, 2024-09-14T03:47:50.341 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_omap": 0, 2024-09-14T03:47:50.341 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_raw": 10230, 2024-09-14T03:47:50.341 INFO:tasks.workunit.client.0.smithi070.stdout: "wr": 51, 2024-09-14T03:47:50.341 INFO:tasks.workunit.client.0.smithi070.stdout: "wr_bytes": 19456 2024-09-14T03:47:50.341 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.341 INFO:tasks.workunit.client.0.smithi070.stdout: "target_max_bytes": 0, 2024-09-14T03:47:50.341 INFO:tasks.workunit.client.0.smithi070.stdout: "target_max_objects": 0, 2024-09-14T03:47:50.341 INFO:tasks.workunit.client.0.smithi070.stdout: "type": "replicated" 2024-09-14T03:47:50.341 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.341 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.342 INFO:tasks.workunit.client.0.smithi070.stdout: "application": [ 2024-09-14T03:47:50.342 INFO:tasks.workunit.client.0.smithi070.stdout: "cephfs" 2024-09-14T03:47:50.342 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.342 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_mode": "none", 2024-09-14T03:47:50.342 INFO:tasks.workunit.client.0.smithi070.stdout: "erasure_code_profile": {}, 2024-09-14T03:47:50.342 INFO:tasks.workunit.client.0.smithi070.stdout: "min_size": 2, 2024-09-14T03:47:50.342 INFO:tasks.workunit.client.0.smithi070.stdout: "options": {}, 2024-09-14T03:47:50.342 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_autoscale_mode": "off", 2024-09-14T03:47:50.342 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_num": 64, 2024-09-14T03:47:50.342 INFO:tasks.workunit.client.0.smithi070.stdout: "pgp_num": 64, 2024-09-14T03:47:50.342 INFO:tasks.workunit.client.0.smithi070.stdout: "pool": 3, 2024-09-14T03:47:50.342 INFO:tasks.workunit.client.0.smithi070.stdout: "size": 3, 2024-09-14T03:47:50.342 INFO:tasks.workunit.client.0.smithi070.stdout: "stats": { 2024-09-14T03:47:50.343 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_raw": 727115933962, 2024-09-14T03:47:50.343 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_used": 0, 2024-09-14T03:47:50.343 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_bytes_used": 0, 2024-09-14T03:47:50.343 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_under_bytes": 0, 2024-09-14T03:47:50.343 INFO:tasks.workunit.client.0.smithi070.stdout: "data_bytes_used": 0, 2024-09-14T03:47:50.343 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty": 0, 2024-09-14T03:47:50.343 INFO:tasks.workunit.client.0.smithi070.stdout: "max_avail": 242371985408, 2024-09-14T03:47:50.343 INFO:tasks.workunit.client.0.smithi070.stdout: "objects": 0, 2024-09-14T03:47:50.343 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_bytes_used": 0, 2024-09-14T03:47:50.343 INFO:tasks.workunit.client.0.smithi070.stdout: "percent_used": 0.0, 2024-09-14T03:47:50.343 INFO:tasks.workunit.client.0.smithi070.stdout: "quota_bytes": 0, 2024-09-14T03:47:50.343 INFO:tasks.workunit.client.0.smithi070.stdout: "quota_objects": 0, 2024-09-14T03:47:50.343 INFO:tasks.workunit.client.0.smithi070.stdout: "rd": 0, 2024-09-14T03:47:50.343 INFO:tasks.workunit.client.0.smithi070.stdout: "rd_bytes": 0, 2024-09-14T03:47:50.344 INFO:tasks.workunit.client.0.smithi070.stdout: "stored": 0, 2024-09-14T03:47:50.344 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_data": 0, 2024-09-14T03:47:50.344 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_omap": 0, 2024-09-14T03:47:50.344 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_raw": 0, 2024-09-14T03:47:50.344 INFO:tasks.workunit.client.0.smithi070.stdout: "wr": 0, 2024-09-14T03:47:50.344 INFO:tasks.workunit.client.0.smithi070.stdout: "wr_bytes": 0 2024-09-14T03:47:50.344 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.344 INFO:tasks.workunit.client.0.smithi070.stdout: "target_max_bytes": 0, 2024-09-14T03:47:50.344 INFO:tasks.workunit.client.0.smithi070.stdout: "target_max_objects": 0, 2024-09-14T03:47:50.344 INFO:tasks.workunit.client.0.smithi070.stdout: "type": "replicated" 2024-09-14T03:47:50.344 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.344 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.344 INFO:tasks.workunit.client.0.smithi070.stdout: "application": [ 2024-09-14T03:47:50.345 INFO:tasks.workunit.client.0.smithi070.stdout: "rbd" 2024-09-14T03:47:50.345 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.345 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_mode": "none", 2024-09-14T03:47:50.345 INFO:tasks.workunit.client.0.smithi070.stdout: "erasure_code_profile": {}, 2024-09-14T03:47:50.345 INFO:tasks.workunit.client.0.smithi070.stdout: "min_size": 2, 2024-09-14T03:47:50.345 INFO:tasks.workunit.client.0.smithi070.stdout: "options": {}, 2024-09-14T03:47:50.345 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_autoscale_mode": "off", 2024-09-14T03:47:50.345 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_num": 8, 2024-09-14T03:47:50.345 INFO:tasks.workunit.client.0.smithi070.stdout: "pgp_num": 8, 2024-09-14T03:47:50.345 INFO:tasks.workunit.client.0.smithi070.stdout: "pool": 4, 2024-09-14T03:47:50.345 INFO:tasks.workunit.client.0.smithi070.stdout: "size": 3, 2024-09-14T03:47:50.345 INFO:tasks.workunit.client.0.smithi070.stdout: "stats": { 2024-09-14T03:47:50.345 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_raw": 727115933962, 2024-09-14T03:47:50.346 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_used": 12288, 2024-09-14T03:47:50.346 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_bytes_used": 0, 2024-09-14T03:47:50.346 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_under_bytes": 0, 2024-09-14T03:47:50.346 INFO:tasks.workunit.client.0.smithi070.stdout: "data_bytes_used": 12288, 2024-09-14T03:47:50.346 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty": 0, 2024-09-14T03:47:50.346 INFO:tasks.workunit.client.0.smithi070.stdout: "max_avail": 242371985408, 2024-09-14T03:47:50.346 INFO:tasks.workunit.client.0.smithi070.stdout: "objects": 2, 2024-09-14T03:47:50.346 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_bytes_used": 0, 2024-09-14T03:47:50.346 INFO:tasks.workunit.client.0.smithi070.stdout: "percent_used": 1.689964257423071e-08, 2024-09-14T03:47:50.346 INFO:tasks.workunit.client.0.smithi070.stdout: "quota_bytes": 0, 2024-09-14T03:47:50.346 INFO:tasks.workunit.client.0.smithi070.stdout: "quota_objects": 0, 2024-09-14T03:47:50.346 INFO:tasks.workunit.client.0.smithi070.stdout: "rd": 0, 2024-09-14T03:47:50.346 INFO:tasks.workunit.client.0.smithi070.stdout: "rd_bytes": 0, 2024-09-14T03:47:50.346 INFO:tasks.workunit.client.0.smithi070.stdout: "stored": 19, 2024-09-14T03:47:50.346 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_data": 19, 2024-09-14T03:47:50.347 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_omap": 0, 2024-09-14T03:47:50.347 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_raw": 57, 2024-09-14T03:47:50.347 INFO:tasks.workunit.client.0.smithi070.stdout: "wr": 2, 2024-09-14T03:47:50.347 INFO:tasks.workunit.client.0.smithi070.stdout: "wr_bytes": 2048 2024-09-14T03:47:50.347 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.347 INFO:tasks.workunit.client.0.smithi070.stdout: "target_max_bytes": 0, 2024-09-14T03:47:50.347 INFO:tasks.workunit.client.0.smithi070.stdout: "target_max_objects": 0, 2024-09-14T03:47:50.347 INFO:tasks.workunit.client.0.smithi070.stdout: "type": "replicated" 2024-09-14T03:47:50.347 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.347 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.347 INFO:tasks.workunit.client.0.smithi070.stdout: "rbd": { 2024-09-14T03:47:50.347 INFO:tasks.workunit.client.0.smithi070.stdout: "mirroring_by_pool": [ 2024-09-14T03:47:50.347 INFO:tasks.workunit.client.0.smithi070.stdout: false 2024-09-14T03:47:50.347 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.348 INFO:tasks.workunit.client.0.smithi070.stdout: "num_images_by_pool": [ 2024-09-14T03:47:50.348 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:50.348 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.348 INFO:tasks.workunit.client.0.smithi070.stdout: "num_pools": 1 2024-09-14T03:47:50.348 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.348 INFO:tasks.workunit.client.0.smithi070.stdout: "report_id": "04c26ace-6e52-41dd-9ad2-e917daf8e93d", 2024-09-14T03:47:50.348 INFO:tasks.workunit.client.0.smithi070.stdout: "report_timestamp": "2024-09-14T03:47:49.371043", 2024-09-14T03:47:50.348 INFO:tasks.workunit.client.0.smithi070.stdout: "report_version": 1, 2024-09-14T03:47:50.348 INFO:tasks.workunit.client.0.smithi070.stdout: "rocksdb_stats": { 2024-09-14T03:47:50.348 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "6.15.5" 2024-09-14T03:47:50.348 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.348 INFO:tasks.workunit.client.0.smithi070.stdout: "rook": { 2024-09-14T03:47:50.348 INFO:tasks.workunit.client.0.smithi070.stdout: "cluster": { 2024-09-14T03:47:50.348 INFO:tasks.workunit.client.0.smithi070.stdout: "external-mode": null, 2024-09-14T03:47:50.349 INFO:tasks.workunit.client.0.smithi070.stdout: "mon": { 2024-09-14T03:47:50.349 INFO:tasks.workunit.client.0.smithi070.stdout: "allow-multiple-per-node": null, 2024-09-14T03:47:50.349 INFO:tasks.workunit.client.0.smithi070.stdout: "count": null, 2024-09-14T03:47:50.349 INFO:tasks.workunit.client.0.smithi070.stdout: "max-id": null, 2024-09-14T03:47:50.349 INFO:tasks.workunit.client.0.smithi070.stdout: "pvc": { 2024-09-14T03:47:50.349 INFO:tasks.workunit.client.0.smithi070.stdout: "enabled": null 2024-09-14T03:47:50.349 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.349 INFO:tasks.workunit.client.0.smithi070.stdout: "stretch": { 2024-09-14T03:47:50.349 INFO:tasks.workunit.client.0.smithi070.stdout: "enabled": null 2024-09-14T03:47:50.349 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.349 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.349 INFO:tasks.workunit.client.0.smithi070.stdout: "network": { 2024-09-14T03:47:50.350 INFO:tasks.workunit.client.0.smithi070.stdout: "provider": null 2024-09-14T03:47:50.350 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.350 INFO:tasks.workunit.client.0.smithi070.stdout: "storage": { 2024-09-14T03:47:50.350 INFO:tasks.workunit.client.0.smithi070.stdout: "device-set": { 2024-09-14T03:47:50.350 INFO:tasks.workunit.client.0.smithi070.stdout: "count": { 2024-09-14T03:47:50.350 INFO:tasks.workunit.client.0.smithi070.stdout: "non-portable": null, 2024-09-14T03:47:50.350 INFO:tasks.workunit.client.0.smithi070.stdout: "portable": null, 2024-09-14T03:47:50.350 INFO:tasks.workunit.client.0.smithi070.stdout: "total": null 2024-09-14T03:47:50.350 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.350 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.350 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.351 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.351 INFO:tasks.workunit.client.0.smithi070.stdout: "csi": { 2024-09-14T03:47:50.351 INFO:tasks.workunit.client.0.smithi070.stdout: "version": null 2024-09-14T03:47:50.351 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.351 INFO:tasks.workunit.client.0.smithi070.stdout: "kubernetes": { 2024-09-14T03:47:50.351 INFO:tasks.workunit.client.0.smithi070.stdout: "version": null 2024-09-14T03:47:50.351 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.351 INFO:tasks.workunit.client.0.smithi070.stdout: "node": { 2024-09-14T03:47:50.351 INFO:tasks.workunit.client.0.smithi070.stdout: "count": { 2024-09-14T03:47:50.351 INFO:tasks.workunit.client.0.smithi070.stdout: "kubernetes-total": null, 2024-09-14T03:47:50.351 INFO:tasks.workunit.client.0.smithi070.stdout: "with-ceph-daemons": null, 2024-09-14T03:47:50.351 INFO:tasks.workunit.client.0.smithi070.stdout: "with-csi-cephfs-plugin": null, 2024-09-14T03:47:50.351 INFO:tasks.workunit.client.0.smithi070.stdout: "with-csi-nfs-plugin": null, 2024-09-14T03:47:50.351 INFO:tasks.workunit.client.0.smithi070.stdout: "with-csi-rbd-plugin": null 2024-09-14T03:47:50.352 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.352 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.352 INFO:tasks.workunit.client.0.smithi070.stdout: "usage": { 2024-09-14T03:47:50.352 INFO:tasks.workunit.client.0.smithi070.stdout: "storage-class": { 2024-09-14T03:47:50.352 INFO:tasks.workunit.client.0.smithi070.stdout: "count": { 2024-09-14T03:47:50.352 INFO:tasks.workunit.client.0.smithi070.stdout: "bucket": null, 2024-09-14T03:47:50.352 INFO:tasks.workunit.client.0.smithi070.stdout: "cephfs": null, 2024-09-14T03:47:50.352 INFO:tasks.workunit.client.0.smithi070.stdout: "nfs": null, 2024-09-14T03:47:50.352 INFO:tasks.workunit.client.0.smithi070.stdout: "rbd": null, 2024-09-14T03:47:50.352 INFO:tasks.workunit.client.0.smithi070.stdout: "total": null 2024-09-14T03:47:50.352 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.353 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:50.353 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.353 INFO:tasks.workunit.client.0.smithi070.stdout: "version": null 2024-09-14T03:47:50.353 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.353 INFO:tasks.workunit.client.0.smithi070.stdout: "services": {}, 2024-09-14T03:47:50.353 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_per_pg": [ 2024-09-14T03:47:50.353 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.353 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.353 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.353 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:50.353 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:50.354 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.354 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:50.354 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.354 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.354 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.354 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.354 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.354 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.354 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.661024+0000", 2024-09-14T03:47:50.354 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.882481+0000", 2024-09-14T03:47:50.354 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.882481+0000", 2024-09-14T03:47:50.354 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.882774+0000", 2024-09-14T03:47:50.354 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.661024+0000", 2024-09-14T03:47:50.355 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.355 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.355 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.355 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.355 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.661024+0000", 2024-09-14T03:47:50.355 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.661024+0000", 2024-09-14T03:47:50.355 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.661024+0000", 2024-09-14T03:47:50.355 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.355 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.355 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.355 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.661024+0000", 2024-09-14T03:47:50.355 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.661024+0000", 2024-09-14T03:47:50.356 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.356 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.356 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.356 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.356 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.356 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.356 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.356 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.356 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.356 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.356 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.357 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.357 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.357 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.3f", 2024-09-14T03:47:50.357 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.357 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.357 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.357 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:50.357 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.357 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:49:25.209573+0000", 2024-09-14T03:47:50.357 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.357 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.358 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.358 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.358 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.358 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.358 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.358 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.358 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.358 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.358 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.358 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.358 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.358 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.358 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.358 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.359 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.359 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.359 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.359 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.359 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.359 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.359 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.359 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.359 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.359 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.359 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.359 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.359 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.359 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.359 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.360 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.360 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.360 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.360 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.360 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.360 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.360 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.360 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.360 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.360 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.360 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.360 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.360 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.360 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.360 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.361 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.361 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.361 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:50.361 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:50.361 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.361 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:50.361 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.361 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.361 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.361 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.361 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:50.361 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.362 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:50.362 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.362 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 1, 2024-09-14T03:47:50.362 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.362 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.362 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.362 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.362 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.362 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.362 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.458995+0000", 2024-09-14T03:47:50.362 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.868152+0000", 2024-09-14T03:47:50.362 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.868152+0000", 2024-09-14T03:47:50.362 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.868415+0000", 2024-09-14T03:47:50.363 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.458995+0000", 2024-09-14T03:47:50.363 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.363 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.363 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.363 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.363 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.458995+0000", 2024-09-14T03:47:50.363 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.458995+0000", 2024-09-14T03:47:50.363 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.458995+0000", 2024-09-14T03:47:50.363 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.363 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.363 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.363 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.458995+0000", 2024-09-14T03:47:50.364 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.458995+0000", 2024-09-14T03:47:50.364 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.364 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.364 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.364 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.364 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.364 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.364 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.364 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.364 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.364 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.364 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.364 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.365 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.365 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.3e", 2024-09-14T03:47:50.365 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.365 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.365 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.365 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:50.365 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.365 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T03:56:00.270070+0000", 2024-09-14T03:47:50.365 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.365 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.365 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.365 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.365 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.365 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.366 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.366 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.366 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.366 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.366 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.366 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.366 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.366 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.366 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.366 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.366 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.366 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.366 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.367 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.367 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.367 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.367 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.367 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.367 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.367 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.367 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.367 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.367 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.367 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.367 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.368 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.368 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.368 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.368 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.368 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.368 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.368 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.368 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.368 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.368 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.368 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.368 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.369 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.369 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.369 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.369 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.369 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:50.369 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.369 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:50.369 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.369 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 1, 2024-09-14T03:47:50.369 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.369 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.369 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.369 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.370 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.370 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:50.370 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:50.370 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.370 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 0, 2024-09-14T03:47:50.370 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.370 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.370 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.370 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.370 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.370 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.370 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.684432+0000", 2024-09-14T03:47:50.370 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.870749+0000", 2024-09-14T03:47:50.370 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.870749+0000", 2024-09-14T03:47:50.371 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.871459+0000", 2024-09-14T03:47:50.371 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.684432+0000", 2024-09-14T03:47:50.371 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.371 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.371 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.371 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.371 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.684432+0000", 2024-09-14T03:47:50.371 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.684432+0000", 2024-09-14T03:47:50.371 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.684432+0000", 2024-09-14T03:47:50.371 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.371 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.371 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.371 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.684432+0000", 2024-09-14T03:47:50.372 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.684432+0000", 2024-09-14T03:47:50.372 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.372 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.372 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.372 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.372 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.372 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.372 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.372 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.372 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.373 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.373 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.373 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.373 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.373 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.3d", 2024-09-14T03:47:50.373 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.373 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.373 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.373 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:50.373 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.373 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:40:54.391363+0000", 2024-09-14T03:47:50.373 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.373 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.373 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.374 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.374 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.374 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.374 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.374 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.374 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.374 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.374 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.374 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.374 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.374 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.374 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.375 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.375 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.375 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.375 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.375 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.375 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.375 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.375 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.375 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.375 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.375 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.375 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.376 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.376 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.376 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.376 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.376 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.376 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.376 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.376 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.376 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.376 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.376 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.376 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.376 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.377 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.377 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.377 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.377 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.377 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.377 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.377 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.377 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.377 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:50.377 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:50.377 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.377 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 0, 2024-09-14T03:47:50.377 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.378 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.378 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.378 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.378 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.378 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.378 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:50.378 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.378 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 5, 2024-09-14T03:47:50.378 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.378 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.378 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.378 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.378 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.379 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.379 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.461823+0000", 2024-09-14T03:47:50.379 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.882250+0000", 2024-09-14T03:47:50.379 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.882250+0000", 2024-09-14T03:47:50.379 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.882503+0000", 2024-09-14T03:47:50.379 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.461823+0000", 2024-09-14T03:47:50.379 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.379 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.379 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.379 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.380 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.461823+0000", 2024-09-14T03:47:50.380 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.461823+0000", 2024-09-14T03:47:50.380 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.461823+0000", 2024-09-14T03:47:50.380 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.380 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.380 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.380 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.461823+0000", 2024-09-14T03:47:50.380 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.461823+0000", 2024-09-14T03:47:50.380 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.380 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.381 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.381 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.381 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.381 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.381 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.381 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.381 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.381 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.381 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.381 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.381 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.381 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.3c", 2024-09-14T03:47:50.381 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.381 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.381 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.382 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:50.382 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.382 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:46:29.551285+0000", 2024-09-14T03:47:50.382 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.382 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.382 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.382 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.382 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.382 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.382 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.382 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.382 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.382 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.382 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.382 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.383 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.383 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.383 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.383 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.383 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.383 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.383 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.383 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.383 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.383 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.383 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.383 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.383 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.383 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.383 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.384 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.384 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.384 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.384 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.384 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.384 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.384 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.384 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.384 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.384 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.384 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.384 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.384 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.384 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.385 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.385 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.385 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.385 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.385 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.385 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.385 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.385 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.385 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:50.385 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.385 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 5, 2024-09-14T03:47:50.385 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.385 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.385 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.385 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.386 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.386 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:50.386 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:50.386 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.386 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:50.386 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.386 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.386 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.386 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.386 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.386 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.386 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.660633+0000", 2024-09-14T03:47:50.386 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.880010+0000", 2024-09-14T03:47:50.386 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.880010+0000", 2024-09-14T03:47:50.386 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.880333+0000", 2024-09-14T03:47:50.387 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.660633+0000", 2024-09-14T03:47:50.387 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.387 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.387 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.387 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.387 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.660633+0000", 2024-09-14T03:47:50.387 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.660633+0000", 2024-09-14T03:47:50.387 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.660633+0000", 2024-09-14T03:47:50.387 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.387 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.387 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.387 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.660633+0000", 2024-09-14T03:47:50.387 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.660633+0000", 2024-09-14T03:47:50.387 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.387 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.388 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.388 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.388 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.388 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.388 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.388 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.388 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.388 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.388 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.388 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.388 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.388 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.3b", 2024-09-14T03:47:50.388 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.388 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.388 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.389 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:50.389 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.389 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:45:02.741425+0000", 2024-09-14T03:47:50.389 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.389 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.389 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.389 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.389 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.389 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.389 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.390 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.390 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.390 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.390 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.390 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.390 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.390 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.390 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.390 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.390 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.390 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.390 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.390 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.390 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.390 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.391 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.391 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.391 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.391 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.391 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.391 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.391 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.391 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.391 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.391 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.391 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.391 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.391 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.391 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.392 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.392 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.392 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.392 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.392 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.392 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.392 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.392 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.392 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.392 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.392 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.392 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.392 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:50.392 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:50.393 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.393 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:50.393 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.393 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.393 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.393 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.393 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.393 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.394 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:50.394 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.394 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 5, 2024-09-14T03:47:50.394 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.394 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.394 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.394 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.394 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.394 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.394 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.461594+0000", 2024-09-14T03:47:50.394 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.880237+0000", 2024-09-14T03:47:50.395 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.880237+0000", 2024-09-14T03:47:50.395 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.880710+0000", 2024-09-14T03:47:50.395 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.461594+0000", 2024-09-14T03:47:50.395 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.395 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.395 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.395 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.395 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.461594+0000", 2024-09-14T03:47:50.395 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.461594+0000", 2024-09-14T03:47:50.395 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.461594+0000", 2024-09-14T03:47:50.395 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.395 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.395 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.395 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.461594+0000", 2024-09-14T03:47:50.396 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.461594+0000", 2024-09-14T03:47:50.396 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.396 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.396 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.396 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.396 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.396 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.396 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.396 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.396 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.396 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.396 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.396 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.397 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.397 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.3a", 2024-09-14T03:47:50.397 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.397 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.397 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.397 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:50.397 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.397 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T03:56:49.808707+0000", 2024-09-14T03:47:50.397 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.397 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.397 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.397 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.397 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.397 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.397 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.397 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.398 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.398 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.398 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.398 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.398 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.398 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.398 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.398 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.398 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.398 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.398 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.398 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.399 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.399 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.399 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.399 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.399 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.399 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.399 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.399 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.399 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.399 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.399 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.399 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.399 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.399 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.399 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.400 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.400 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.400 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.400 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.400 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.400 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.400 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.400 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.400 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.400 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.400 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.400 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.400 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.400 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.400 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:50.401 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.401 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 5, 2024-09-14T03:47:50.401 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.401 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.401 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.401 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.401 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.401 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.401 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:50.401 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.401 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:50.401 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.401 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.401 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.402 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.402 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.402 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.402 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.660415+0000", 2024-09-14T03:47:50.402 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.878281+0000", 2024-09-14T03:47:50.402 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.878281+0000", 2024-09-14T03:47:50.402 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.878590+0000", 2024-09-14T03:47:50.402 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.660415+0000", 2024-09-14T03:47:50.402 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.402 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.402 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.402 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.402 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.660415+0000", 2024-09-14T03:47:50.402 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.660415+0000", 2024-09-14T03:47:50.402 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.660415+0000", 2024-09-14T03:47:50.403 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.403 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.403 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.403 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.660415+0000", 2024-09-14T03:47:50.403 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.660415+0000", 2024-09-14T03:47:50.403 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.403 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.403 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.403 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.403 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.403 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.403 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.403 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.403 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.404 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.404 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.404 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.404 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.404 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.39", 2024-09-14T03:47:50.404 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.404 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.404 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.404 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:50.404 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.404 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:41:57.164227+0000", 2024-09-14T03:47:50.404 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.404 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.404 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.405 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.405 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.405 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.405 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.405 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.405 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.405 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.405 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.405 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.405 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.405 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.405 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.405 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.405 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.406 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.406 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.406 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.406 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.406 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.406 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.406 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.406 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.406 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.406 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.406 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.406 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.406 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.407 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.407 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.407 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.407 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.407 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.407 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.407 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.407 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.407 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.407 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.407 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.407 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.407 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.408 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.408 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.408 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.408 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.408 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.408 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.408 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:50.408 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.408 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:50.408 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.408 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.408 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.408 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.408 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.409 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.409 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:50.409 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.409 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 3, 2024-09-14T03:47:50.409 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.409 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.409 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.409 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.409 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.409 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.409 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.857411+0000", 2024-09-14T03:47:50.410 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.876727+0000", 2024-09-14T03:47:50.410 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.876727+0000", 2024-09-14T03:47:50.410 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.877057+0000", 2024-09-14T03:47:50.410 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.857411+0000", 2024-09-14T03:47:50.410 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.410 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.410 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.410 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.410 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.857411+0000", 2024-09-14T03:47:50.410 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.857411+0000", 2024-09-14T03:47:50.410 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.857411+0000", 2024-09-14T03:47:50.410 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.411 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.411 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.411 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.857411+0000", 2024-09-14T03:47:50.411 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.857411+0000", 2024-09-14T03:47:50.411 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.411 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.411 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.411 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.411 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.411 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.411 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.411 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.411 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.412 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.412 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.412 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.412 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.412 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.38", 2024-09-14T03:47:50.412 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.412 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.412 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.412 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:50.412 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.412 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:31:24.716316+0000", 2024-09-14T03:47:50.412 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.413 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.413 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.413 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.413 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.413 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.413 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.413 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.413 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.413 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.413 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.413 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.413 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.413 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.414 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.414 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.414 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.414 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.414 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.414 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.414 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.414 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.414 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.414 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.414 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.414 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.414 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.414 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.415 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.415 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.415 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.415 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.415 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.415 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.415 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.415 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.415 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.415 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.415 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.415 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.415 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.416 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.416 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.416 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.416 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.416 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.416 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.416 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.416 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.416 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:50.416 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.417 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 3, 2024-09-14T03:47:50.417 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.417 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.417 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.417 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.417 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.417 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.417 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:50.417 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.417 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:50.417 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.417 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.417 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.418 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.418 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.418 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.418 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.661211+0000", 2024-09-14T03:47:50.418 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.884136+0000", 2024-09-14T03:47:50.418 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.884136+0000", 2024-09-14T03:47:50.418 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.884865+0000", 2024-09-14T03:47:50.418 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.661211+0000", 2024-09-14T03:47:50.418 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.418 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.418 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.419 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.419 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.661211+0000", 2024-09-14T03:47:50.419 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.661211+0000", 2024-09-14T03:47:50.419 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.661211+0000", 2024-09-14T03:47:50.419 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.419 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.419 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.419 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.661211+0000", 2024-09-14T03:47:50.419 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.661211+0000", 2024-09-14T03:47:50.419 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.419 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.420 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.420 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.420 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.420 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.420 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.420 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.420 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.420 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.420 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.420 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.421 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.421 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.37", 2024-09-14T03:47:50.421 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.421 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.421 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.421 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:50.421 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.421 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:14:59.903382+0000", 2024-09-14T03:47:50.421 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.421 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.421 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.421 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.421 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.422 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.422 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.422 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.422 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.422 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.422 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.422 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.422 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.422 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.422 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.422 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.422 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.422 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.422 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.423 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.423 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.423 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.423 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.423 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.423 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.423 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.424 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.424 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.424 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.424 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.424 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.424 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.425 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.425 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.425 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.425 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.425 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.425 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.425 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.425 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.425 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.425 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.425 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.425 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.425 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.425 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.425 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.426 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.426 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.426 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:50.426 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.426 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:50.426 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.426 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.426 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.426 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.426 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:50.426 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:50.426 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:50.426 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.426 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 6, 2024-09-14T03:47:50.426 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.427 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.427 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.427 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.427 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.427 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.427 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.544938+0000", 2024-09-14T03:47:50.427 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:26.429658+0000", 2024-09-14T03:47:50.427 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:26.429658+0000", 2024-09-14T03:47:50.427 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:26.430052+0000", 2024-09-14T03:47:50.427 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.544938+0000", 2024-09-14T03:47:50.427 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.427 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.427 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.427 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 60, 2024-09-14T03:47:50.428 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.544938+0000", 2024-09-14T03:47:50.428 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.544938+0000", 2024-09-14T03:47:50.428 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.544938+0000", 2024-09-14T03:47:50.428 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.428 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.428 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.428 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.544938+0000", 2024-09-14T03:47:50.428 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.544938+0000", 2024-09-14T03:47:50.428 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.428 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.428 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.428 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.428 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 59, 2024-09-14T03:47:50.429 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.429 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.429 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.429 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.429 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.429 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.429 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.429 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.429 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.36", 2024-09-14T03:47:50.429 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.429 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.429 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.429 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 30, 2024-09-14T03:47:50.429 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.430 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T03:59:34.312039+0000", 2024-09-14T03:47:50.430 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.430 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.430 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.430 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.430 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.430 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.430 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.430 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.430 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.430 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.430 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.430 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.430 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.430 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.431 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.431 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.431 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.431 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.431 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.431 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.431 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.431 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.431 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.431 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.431 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.431 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.431 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.431 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.432 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.432 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.432 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.432 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.432 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.432 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.432 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.432 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.432 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.432 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.432 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.432 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.432 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.432 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.432 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.432 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.433 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.433 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.433 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.433 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:50.433 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:50.433 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:50.433 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.433 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 6, 2024-09-14T03:47:50.433 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.433 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.433 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.433 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.433 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:50.433 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.433 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:50.434 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.434 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 6, 2024-09-14T03:47:50.434 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.434 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.434 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.434 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.434 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.434 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.434 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.544941+0000", 2024-09-14T03:47:50.434 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.874687+0000", 2024-09-14T03:47:50.434 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.874687+0000", 2024-09-14T03:47:50.435 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.875017+0000", 2024-09-14T03:47:50.435 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.544941+0000", 2024-09-14T03:47:50.435 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.435 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.435 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.435 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.435 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.544941+0000", 2024-09-14T03:47:50.435 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.544941+0000", 2024-09-14T03:47:50.435 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.544941+0000", 2024-09-14T03:47:50.435 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.435 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.435 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.435 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.544941+0000", 2024-09-14T03:47:50.435 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.544941+0000", 2024-09-14T03:47:50.435 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.436 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.436 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.436 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.436 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.436 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.436 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.436 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.436 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.436 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.436 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.436 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.436 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.437 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.35", 2024-09-14T03:47:50.437 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.437 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.437 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.437 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:50.437 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.437 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:29:09.035342+0000", 2024-09-14T03:47:50.437 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.437 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.437 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.438 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.438 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.438 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.438 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.438 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.438 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.438 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.438 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.438 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.438 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.438 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.438 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.438 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.438 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.438 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.439 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.439 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.439 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.439 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.439 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.439 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.439 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.439 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.439 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.439 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.439 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.439 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.440 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.440 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.440 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.440 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.440 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.440 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.440 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.440 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.440 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.440 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.440 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.440 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.440 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.441 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.441 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.441 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.441 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.441 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:50.441 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.441 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:50.441 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.441 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 6, 2024-09-14T03:47:50.441 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.441 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.442 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.442 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.442 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.442 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:50.442 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:50.442 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.442 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 4, 2024-09-14T03:47:50.442 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.442 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.442 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.442 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.442 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.442 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.442 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.458577+0000", 2024-09-14T03:47:50.443 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.882770+0000", 2024-09-14T03:47:50.443 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.882770+0000", 2024-09-14T03:47:50.443 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.883313+0000", 2024-09-14T03:47:50.443 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.458577+0000", 2024-09-14T03:47:50.443 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.443 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.443 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.443 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.443 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.458577+0000", 2024-09-14T03:47:50.443 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.458577+0000", 2024-09-14T03:47:50.443 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.458577+0000", 2024-09-14T03:47:50.443 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.444 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.444 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.444 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.458577+0000", 2024-09-14T03:47:50.444 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.458577+0000", 2024-09-14T03:47:50.444 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.444 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.444 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.444 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.444 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.445 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.445 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.445 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.445 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.445 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.445 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.445 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.445 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.445 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.34", 2024-09-14T03:47:50.445 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.445 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.445 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.445 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:50.446 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.446 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T04:57:19.402319+0000", 2024-09-14T03:47:50.446 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.446 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.446 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.446 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.446 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.446 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.446 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.446 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.446 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.446 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.446 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.446 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.447 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.447 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.447 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.447 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.447 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.447 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.447 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.447 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.447 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.447 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.447 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.447 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.447 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.447 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.448 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.448 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.448 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.448 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.448 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.448 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.448 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.448 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.448 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.448 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.448 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.448 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.448 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.448 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.449 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.449 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.449 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.449 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.449 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.449 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.449 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.449 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.449 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:50.449 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:50.449 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.449 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 4, 2024-09-14T03:47:50.449 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.449 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.449 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.450 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.450 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.450 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:50.450 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:50.450 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.450 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 3, 2024-09-14T03:47:50.450 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.450 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.450 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.450 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.450 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.451 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.451 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.857631+0000", 2024-09-14T03:47:50.451 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.877861+0000", 2024-09-14T03:47:50.451 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.877861+0000", 2024-09-14T03:47:50.451 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.878245+0000", 2024-09-14T03:47:50.451 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.857631+0000", 2024-09-14T03:47:50.451 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.451 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.451 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.451 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.451 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.857631+0000", 2024-09-14T03:47:50.451 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.857631+0000", 2024-09-14T03:47:50.451 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.857631+0000", 2024-09-14T03:47:50.451 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.452 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.452 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.452 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.857631+0000", 2024-09-14T03:47:50.452 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.857631+0000", 2024-09-14T03:47:50.452 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.452 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.452 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.452 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.452 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.452 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.452 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.452 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.452 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.453 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.453 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.453 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.453 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.453 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.33", 2024-09-14T03:47:50.453 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.453 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.453 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.453 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:50.453 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.453 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:08:29.902090+0000", 2024-09-14T03:47:50.453 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.453 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.454 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.454 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.454 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.454 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.454 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.454 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.454 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.454 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.454 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.454 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.454 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.454 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.454 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.455 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.455 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.455 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.455 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.455 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.455 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.455 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.455 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.455 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.455 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.455 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.455 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.455 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.455 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.456 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.456 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.456 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.456 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.456 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.456 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.456 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.456 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.456 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.456 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.456 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.456 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.456 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.457 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.457 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.457 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.457 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.457 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.457 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.457 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:50.457 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:50.457 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.457 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 3, 2024-09-14T03:47:50.458 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.458 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.458 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.458 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.458 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.458 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:50.458 INFO:tasks.workunit.client.0.smithi070.stdout: 5 2024-09-14T03:47:50.458 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.458 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 0, 2024-09-14T03:47:50.458 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.458 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.458 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.458 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.459 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.459 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.459 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.684034+0000", 2024-09-14T03:47:50.459 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.876797+0000", 2024-09-14T03:47:50.459 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.876797+0000", 2024-09-14T03:47:50.459 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.877391+0000", 2024-09-14T03:47:50.459 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.684034+0000", 2024-09-14T03:47:50.459 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.459 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.459 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.459 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.459 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.684034+0000", 2024-09-14T03:47:50.459 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.684034+0000", 2024-09-14T03:47:50.460 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.684034+0000", 2024-09-14T03:47:50.460 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.460 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.460 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.460 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.684034+0000", 2024-09-14T03:47:50.460 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.684034+0000", 2024-09-14T03:47:50.460 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.460 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.460 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.460 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.460 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.460 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.460 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.461 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.461 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.461 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.461 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.461 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.461 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.461 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.32", 2024-09-14T03:47:50.461 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.461 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.461 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.461 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:50.461 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.461 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:06:02.891762+0000", 2024-09-14T03:47:50.462 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.462 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.462 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.462 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.462 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.462 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.462 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.462 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.462 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.462 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.462 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.462 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.462 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.462 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.463 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.463 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.463 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.463 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.463 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.463 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.463 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.463 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.463 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.463 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.463 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.463 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.463 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.463 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.464 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.464 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.464 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.464 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.464 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.464 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.464 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.464 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.464 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.464 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.464 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.465 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.465 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.465 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.465 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.465 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.465 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.465 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.465 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.465 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.465 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:50.465 INFO:tasks.workunit.client.0.smithi070.stdout: 5 2024-09-14T03:47:50.465 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.465 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 0, 2024-09-14T03:47:50.466 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.466 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.466 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.466 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.466 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:50.466 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.466 INFO:tasks.workunit.client.0.smithi070.stdout: 5 2024-09-14T03:47:50.466 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.466 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 1, 2024-09-14T03:47:50.466 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.466 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.466 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.466 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.467 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.467 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.467 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.458588+0000", 2024-09-14T03:47:50.467 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.876091+0000", 2024-09-14T03:47:50.467 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.876091+0000", 2024-09-14T03:47:50.467 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.876700+0000", 2024-09-14T03:47:50.467 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.458588+0000", 2024-09-14T03:47:50.467 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.468 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.468 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.468 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.468 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.458588+0000", 2024-09-14T03:47:50.468 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.458588+0000", 2024-09-14T03:47:50.468 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.458588+0000", 2024-09-14T03:47:50.468 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.468 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.468 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.469 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.458588+0000", 2024-09-14T03:47:50.469 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.458588+0000", 2024-09-14T03:47:50.469 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.469 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.469 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.469 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.469 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.470 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.470 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.470 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.470 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.470 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.470 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.470 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.470 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.470 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.31", 2024-09-14T03:47:50.470 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.470 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.470 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.470 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:50.471 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.471 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:25:41.750127+0000", 2024-09-14T03:47:50.471 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.471 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.471 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.471 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.471 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.471 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.471 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.471 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.471 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.471 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.471 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.471 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.471 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.472 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.472 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.472 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.472 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.472 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.472 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.472 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.472 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.472 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.472 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.472 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.473 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.473 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.473 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.473 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.473 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.473 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.473 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.473 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.473 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.473 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.473 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.473 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.473 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.473 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.473 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.474 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.474 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.474 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.474 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.474 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.474 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.474 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.474 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.474 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:50.474 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.474 INFO:tasks.workunit.client.0.smithi070.stdout: 5 2024-09-14T03:47:50.474 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.474 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 1, 2024-09-14T03:47:50.474 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.475 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.475 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.475 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.475 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.475 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.475 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:50.475 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.475 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 5, 2024-09-14T03:47:50.475 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.475 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.475 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.475 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.476 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.476 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.476 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.461481+0000", 2024-09-14T03:47:50.476 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.876793+0000", 2024-09-14T03:47:50.476 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.876793+0000", 2024-09-14T03:47:50.476 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.877120+0000", 2024-09-14T03:47:50.476 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.461481+0000", 2024-09-14T03:47:50.476 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.476 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.476 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.476 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.476 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.461481+0000", 2024-09-14T03:47:50.476 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.461481+0000", 2024-09-14T03:47:50.476 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.461481+0000", 2024-09-14T03:47:50.477 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.477 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.477 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.477 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.461481+0000", 2024-09-14T03:47:50.477 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.461481+0000", 2024-09-14T03:47:50.477 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.477 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.477 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.477 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.477 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.477 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.477 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.477 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.477 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.478 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.478 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.478 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.478 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.478 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.30", 2024-09-14T03:47:50.478 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.478 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.478 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.478 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:50.478 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.478 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T05:23:56.450728+0000", 2024-09-14T03:47:50.478 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.478 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.478 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.479 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.479 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.479 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.479 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.479 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.479 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.479 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.479 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.479 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.479 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.479 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.479 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.479 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.480 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.480 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.480 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.480 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.480 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.480 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.480 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.480 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.481 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.481 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.481 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.481 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.481 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.481 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.481 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.481 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.481 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.481 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.481 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.481 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.481 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.481 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.481 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.482 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.482 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.482 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.482 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.482 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.482 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.482 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.482 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.482 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.482 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.482 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:50.482 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.482 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 5, 2024-09-14T03:47:50.482 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.483 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.483 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.483 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.483 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.483 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.483 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:50.483 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.483 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 5, 2024-09-14T03:47:50.483 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.483 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.483 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.483 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.483 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.483 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.484 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.462251+0000", 2024-09-14T03:47:50.484 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.884857+0000", 2024-09-14T03:47:50.484 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.884857+0000", 2024-09-14T03:47:50.484 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.885108+0000", 2024-09-14T03:47:50.484 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.462251+0000", 2024-09-14T03:47:50.484 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.484 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.484 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.484 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.484 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.462251+0000", 2024-09-14T03:47:50.484 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.462251+0000", 2024-09-14T03:47:50.484 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.462251+0000", 2024-09-14T03:47:50.484 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.484 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.484 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.485 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.462251+0000", 2024-09-14T03:47:50.485 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.462251+0000", 2024-09-14T03:47:50.485 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.485 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.485 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.485 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.485 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.485 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.485 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.485 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.485 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.485 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.485 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.485 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.485 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.485 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.2f", 2024-09-14T03:47:50.486 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.486 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.486 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.486 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:50.486 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.486 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T07:09:31.448617+0000", 2024-09-14T03:47:50.486 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.486 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.486 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.486 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.486 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.486 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.486 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.487 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.487 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.487 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.487 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.487 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.487 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.487 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.487 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.487 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.487 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.487 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.487 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.488 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.488 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.488 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.488 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.488 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.488 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.488 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.488 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.488 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.488 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.488 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.488 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.488 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.489 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.489 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.489 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.489 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.489 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.489 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.489 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.489 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.489 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.489 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.489 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.490 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.490 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.490 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.490 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.490 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.490 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.490 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:50.490 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.490 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 5, 2024-09-14T03:47:50.490 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.490 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.490 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.490 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.490 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:50.491 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:50.491 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:50.491 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.491 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 2, 2024-09-14T03:47:50.491 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.491 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.491 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.491 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.491 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.491 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.491 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:15.135790+0000", 2024-09-14T03:47:50.491 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.876227+0000", 2024-09-14T03:47:50.491 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.876227+0000", 2024-09-14T03:47:50.492 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.876503+0000", 2024-09-14T03:47:50.492 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:15.135790+0000", 2024-09-14T03:47:50.492 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.492 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.492 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.492 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.492 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:15.135790+0000", 2024-09-14T03:47:50.492 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:15.135790+0000", 2024-09-14T03:47:50.492 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:15.135790+0000", 2024-09-14T03:47:50.492 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.492 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.492 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.492 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:15.135790+0000", 2024-09-14T03:47:50.492 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:15.135790+0000", 2024-09-14T03:47:50.493 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.493 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.493 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.493 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.493 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.493 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.493 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.493 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.493 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.494 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.494 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.494 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.494 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.494 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.2e", 2024-09-14T03:47:50.494 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.494 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.494 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.495 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:50.495 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.495 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T04:44:50.132152+0000", 2024-09-14T03:47:50.495 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.495 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.495 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.495 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.495 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.495 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.495 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.495 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.496 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.496 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.496 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.496 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.496 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.496 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.496 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.496 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.496 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.496 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.496 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.496 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.496 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.497 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.497 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.497 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.497 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.497 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.497 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.497 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.497 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.497 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.497 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.497 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.497 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.497 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.497 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.497 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.498 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.498 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.498 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.498 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.498 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.498 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.498 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.498 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.498 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.498 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.498 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.499 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:50.499 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:50.499 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:50.499 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.499 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 2, 2024-09-14T03:47:50.499 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.499 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.499 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.499 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.499 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.499 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:50.499 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:50.499 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.499 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:50.500 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.500 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.500 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.500 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.500 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.500 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.500 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.661079+0000", 2024-09-14T03:47:50.500 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.883440+0000", 2024-09-14T03:47:50.500 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.883440+0000", 2024-09-14T03:47:50.500 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.883708+0000", 2024-09-14T03:47:50.500 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.661079+0000", 2024-09-14T03:47:50.500 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.500 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.501 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.501 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.501 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.661079+0000", 2024-09-14T03:47:50.501 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.661079+0000", 2024-09-14T03:47:50.501 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.661079+0000", 2024-09-14T03:47:50.501 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.501 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.501 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.501 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.661079+0000", 2024-09-14T03:47:50.501 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.661079+0000", 2024-09-14T03:47:50.501 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.501 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.501 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.501 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.502 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.502 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.502 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.502 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.502 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.502 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.502 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.502 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.502 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.502 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.2d", 2024-09-14T03:47:50.502 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.502 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.502 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.503 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:50.503 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.503 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:10:34.658062+0000", 2024-09-14T03:47:50.503 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.503 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.503 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.503 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.503 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.503 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.503 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.503 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.503 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.503 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.503 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.504 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.504 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.504 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.504 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.504 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.504 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.504 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.504 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.504 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.504 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.504 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.504 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.504 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.504 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.505 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.505 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.505 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.505 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.505 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.505 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.505 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.505 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.505 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.505 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.505 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.505 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.505 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.505 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.506 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.506 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.506 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.506 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.506 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.506 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.506 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.506 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.506 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.506 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:50.506 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:50.506 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.506 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:50.507 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.507 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.507 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.507 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.507 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:50.507 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.507 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:50.507 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.507 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 6, 2024-09-14T03:47:50.507 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.507 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.507 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.507 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.507 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.508 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.508 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.544854+0000", 2024-09-14T03:47:50.508 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.888191+0000", 2024-09-14T03:47:50.508 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.888191+0000", 2024-09-14T03:47:50.508 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.888565+0000", 2024-09-14T03:47:50.508 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.544854+0000", 2024-09-14T03:47:50.508 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.508 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.508 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.508 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.508 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.544854+0000", 2024-09-14T03:47:50.508 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.544854+0000", 2024-09-14T03:47:50.508 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.544854+0000", 2024-09-14T03:47:50.508 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.509 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.509 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.509 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.544854+0000", 2024-09-14T03:47:50.509 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.544854+0000", 2024-09-14T03:47:50.509 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.509 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.509 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.509 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.509 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.510 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.510 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.510 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.510 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.510 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.510 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.511 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.511 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.511 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.2c", 2024-09-14T03:47:50.511 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.511 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.511 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.511 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:50.511 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.511 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T07:52:23.782664+0000", 2024-09-14T03:47:50.511 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.511 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.511 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.511 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.512 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.512 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.512 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.512 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.512 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.513 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.513 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.513 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.513 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.513 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.513 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.514 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.514 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.514 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.514 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.514 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.514 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.515 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.515 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.515 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.515 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.515 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.515 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.515 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.515 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.515 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.515 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.515 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.515 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.515 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.516 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.516 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.516 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.516 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.516 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.517 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.517 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.517 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.517 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.517 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.517 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.517 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.517 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.517 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:50.517 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.518 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:50.518 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.518 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 6, 2024-09-14T03:47:50.518 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.518 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.518 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.518 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.518 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.518 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.518 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:50.518 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.518 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 0, 2024-09-14T03:47:50.518 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.518 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.518 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.519 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.519 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.519 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.519 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.684144+0000", 2024-09-14T03:47:50.519 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.874384+0000", 2024-09-14T03:47:50.519 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.874384+0000", 2024-09-14T03:47:50.519 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.874642+0000", 2024-09-14T03:47:50.519 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.684144+0000", 2024-09-14T03:47:50.519 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.519 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.519 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.519 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.519 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.684144+0000", 2024-09-14T03:47:50.520 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.684144+0000", 2024-09-14T03:47:50.520 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.684144+0000", 2024-09-14T03:47:50.520 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.520 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.520 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.520 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.684144+0000", 2024-09-14T03:47:50.520 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.684144+0000", 2024-09-14T03:47:50.520 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.520 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.520 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.520 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.520 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.520 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.520 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.521 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.521 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.521 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.521 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.521 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.521 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.521 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.2b", 2024-09-14T03:47:50.521 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.521 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.521 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.521 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:50.522 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.522 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T15:35:05.535173+0000", 2024-09-14T03:47:50.522 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.522 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.522 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.522 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.522 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.522 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.522 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.522 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.522 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.522 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.523 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.523 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.523 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.523 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.523 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.523 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.523 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.523 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.523 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.523 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.523 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.524 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.524 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.524 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.524 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.524 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.524 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.524 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.524 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.524 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.524 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.524 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.524 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.524 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.525 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.525 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.525 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.525 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.525 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.525 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.525 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.525 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.525 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.525 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.525 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.525 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.525 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.526 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.526 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.526 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:50.526 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.526 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 0, 2024-09-14T03:47:50.526 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.526 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.526 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.526 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.526 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.526 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.526 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:50.526 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.527 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 5, 2024-09-14T03:47:50.527 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.527 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.527 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.527 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.527 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.527 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.527 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.461689+0000", 2024-09-14T03:47:50.527 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.886594+0000", 2024-09-14T03:47:50.527 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.886594+0000", 2024-09-14T03:47:50.527 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.886893+0000", 2024-09-14T03:47:50.527 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.461689+0000", 2024-09-14T03:47:50.527 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.527 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.528 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.528 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.528 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.461689+0000", 2024-09-14T03:47:50.528 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.461689+0000", 2024-09-14T03:47:50.528 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.461689+0000", 2024-09-14T03:47:50.528 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.528 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.528 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.528 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.461689+0000", 2024-09-14T03:47:50.528 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.461689+0000", 2024-09-14T03:47:50.528 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.528 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.528 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.529 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.529 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.529 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.529 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.529 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.529 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.529 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.529 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.529 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.529 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.529 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.2a", 2024-09-14T03:47:50.529 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.529 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.530 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.530 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:50.530 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.530 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:07:37.154358+0000", 2024-09-14T03:47:50.530 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.530 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.530 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.530 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.530 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.530 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.530 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.530 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.530 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.530 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.530 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.531 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.531 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.531 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.531 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.531 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.531 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.531 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.531 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.531 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.531 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.531 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.531 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.531 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.531 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.531 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.532 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.532 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.532 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.532 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.532 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.532 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.532 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.532 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.532 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.532 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.532 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.532 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.532 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.532 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.532 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.533 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.533 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.533 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.533 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.533 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.533 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.533 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.533 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.533 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:50.533 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.533 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 5, 2024-09-14T03:47:50.533 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.533 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.533 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.534 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.534 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.534 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.534 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:50.534 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.534 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 3, 2024-09-14T03:47:50.534 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.534 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.534 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.534 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.534 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.535 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.535 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.857420+0000", 2024-09-14T03:47:50.535 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.868442+0000", 2024-09-14T03:47:50.535 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.868442+0000", 2024-09-14T03:47:50.535 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.868698+0000", 2024-09-14T03:47:50.535 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.857420+0000", 2024-09-14T03:47:50.535 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.535 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.535 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.535 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.535 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.857420+0000", 2024-09-14T03:47:50.535 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.857420+0000", 2024-09-14T03:47:50.535 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.857420+0000", 2024-09-14T03:47:50.535 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.536 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.536 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.536 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.857420+0000", 2024-09-14T03:47:50.536 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.857420+0000", 2024-09-14T03:47:50.536 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.536 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.536 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.536 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.536 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.536 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.536 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.536 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.536 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.536 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.537 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.537 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.537 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.537 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.29", 2024-09-14T03:47:50.537 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.537 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.537 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.537 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:50.537 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.537 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:41:18.296424+0000", 2024-09-14T03:47:50.537 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.537 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.537 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.538 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.538 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.538 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.538 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.538 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.538 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.538 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.538 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.538 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.538 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.538 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.539 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.539 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.539 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.539 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.539 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.539 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.539 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.539 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.539 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.539 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.539 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.540 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.540 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.540 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.540 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.540 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.540 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.540 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.540 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.540 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.540 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.540 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.540 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.540 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.540 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.541 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.541 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.541 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.541 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.541 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.541 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.541 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.541 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.541 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.541 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.541 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:50.541 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.541 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 3, 2024-09-14T03:47:50.542 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.542 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.542 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.542 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.542 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.542 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.542 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:50.542 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.542 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 5, 2024-09-14T03:47:50.542 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.542 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.543 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.543 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.543 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.543 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.543 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.461524+0000", 2024-09-14T03:47:50.543 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.882443+0000", 2024-09-14T03:47:50.543 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.882443+0000", 2024-09-14T03:47:50.543 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.882708+0000", 2024-09-14T03:47:50.543 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.461524+0000", 2024-09-14T03:47:50.543 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.543 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.543 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.543 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.543 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.461524+0000", 2024-09-14T03:47:50.543 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.461524+0000", 2024-09-14T03:47:50.544 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.461524+0000", 2024-09-14T03:47:50.544 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.544 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.544 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.544 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.461524+0000", 2024-09-14T03:47:50.544 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.461524+0000", 2024-09-14T03:47:50.544 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.544 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.544 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.544 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.544 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.544 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.544 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.545 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.545 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.545 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.545 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.545 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.545 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.545 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.28", 2024-09-14T03:47:50.545 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.545 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.545 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.545 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:50.545 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.545 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:46:48.306263+0000", 2024-09-14T03:47:50.545 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.546 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.546 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.546 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.546 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.546 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.546 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.546 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.546 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.546 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.546 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.546 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.546 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.546 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.546 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.547 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.547 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.547 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.547 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.547 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.547 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.547 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.547 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.547 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.547 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.547 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.547 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.547 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.547 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.548 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.548 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.548 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.548 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.548 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.548 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.548 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.548 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.548 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.548 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.548 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.548 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.548 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.548 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.549 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.549 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.549 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.549 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.549 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.549 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.549 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:50.549 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.549 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 5, 2024-09-14T03:47:50.549 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.549 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.549 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.549 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.549 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.550 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.550 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:50.550 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.550 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 3, 2024-09-14T03:47:50.550 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.550 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.550 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.550 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.550 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.550 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.550 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.857790+0000", 2024-09-14T03:47:50.550 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.884304+0000", 2024-09-14T03:47:50.550 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.884304+0000", 2024-09-14T03:47:50.551 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.884940+0000", 2024-09-14T03:47:50.551 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.857790+0000", 2024-09-14T03:47:50.551 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.551 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.551 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.551 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.551 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.857790+0000", 2024-09-14T03:47:50.551 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.857790+0000", 2024-09-14T03:47:50.551 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.857790+0000", 2024-09-14T03:47:50.551 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.551 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.551 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.551 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.857790+0000", 2024-09-14T03:47:50.552 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.857790+0000", 2024-09-14T03:47:50.552 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.552 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.552 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.552 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.552 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.552 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.552 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.552 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.552 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.552 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.552 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.552 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.553 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.553 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.27", 2024-09-14T03:47:50.553 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.553 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.553 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.553 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:50.553 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.554 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:01:49.061637+0000", 2024-09-14T03:47:50.554 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.554 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.554 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.554 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.554 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.554 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.554 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.554 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.554 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.554 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.555 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.555 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.555 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.555 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.555 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.555 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.555 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.555 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.555 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.555 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.555 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.555 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.555 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.555 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.555 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.556 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.556 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.556 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.556 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.556 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.556 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.556 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.556 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.556 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.556 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.556 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.556 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.557 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.557 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.557 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.557 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.557 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.557 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.557 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.557 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.557 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.557 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.557 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.558 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.558 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:50.558 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.558 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 3, 2024-09-14T03:47:50.558 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.558 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.558 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.558 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.558 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:50.558 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.558 INFO:tasks.workunit.client.0.smithi070.stdout: 3 2024-09-14T03:47:50.558 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.559 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 6, 2024-09-14T03:47:50.559 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.559 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.559 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.559 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.559 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.559 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.559 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.544999+0000", 2024-09-14T03:47:50.559 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:26.430426+0000", 2024-09-14T03:47:50.559 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:26.430426+0000", 2024-09-14T03:47:50.559 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:26.430891+0000", 2024-09-14T03:47:50.559 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.544999+0000", 2024-09-14T03:47:50.559 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.559 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.560 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.560 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 60, 2024-09-14T03:47:50.560 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.544999+0000", 2024-09-14T03:47:50.560 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.544999+0000", 2024-09-14T03:47:50.560 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.544999+0000", 2024-09-14T03:47:50.560 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.560 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.560 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.560 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.544999+0000", 2024-09-14T03:47:50.560 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.544999+0000", 2024-09-14T03:47:50.560 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.560 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.560 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.560 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.561 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 59, 2024-09-14T03:47:50.561 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.561 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.561 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.561 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.561 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.561 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.561 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.561 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.561 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.26", 2024-09-14T03:47:50.561 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.561 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.561 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.561 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 30, 2024-09-14T03:47:50.562 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.562 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T05:00:53.758328+0000", 2024-09-14T03:47:50.562 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.562 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.562 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.562 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.562 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.562 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.562 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.562 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.562 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.562 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.562 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.563 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.563 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.563 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.563 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.563 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.563 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.563 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.563 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.563 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.563 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.563 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.563 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.563 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.563 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.563 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.564 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.564 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.564 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.564 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.564 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.564 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.564 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.564 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.564 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.564 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.564 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.564 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.565 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.565 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.565 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.565 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.565 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.565 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.565 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.565 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.565 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.565 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:50.565 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.565 INFO:tasks.workunit.client.0.smithi070.stdout: 3 2024-09-14T03:47:50.565 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.565 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 6, 2024-09-14T03:47:50.566 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.566 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.566 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.566 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.566 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:50.566 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.566 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:50.566 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.566 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 1, 2024-09-14T03:47:50.566 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.566 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.566 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.566 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.566 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.567 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.567 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.458426+0000", 2024-09-14T03:47:50.567 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.874297+0000", 2024-09-14T03:47:50.567 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.874297+0000", 2024-09-14T03:47:50.567 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.874564+0000", 2024-09-14T03:47:50.567 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.458426+0000", 2024-09-14T03:47:50.567 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.567 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.567 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.567 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.567 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.458426+0000", 2024-09-14T03:47:50.567 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.458426+0000", 2024-09-14T03:47:50.567 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.458426+0000", 2024-09-14T03:47:50.567 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.568 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.568 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.568 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.458426+0000", 2024-09-14T03:47:50.568 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.458426+0000", 2024-09-14T03:47:50.568 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.568 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.568 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.568 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.568 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.568 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.568 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.568 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.568 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.568 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.569 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.569 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.569 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.569 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.25", 2024-09-14T03:47:50.569 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.569 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.569 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.569 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:50.569 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.569 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:28:07.027610+0000", 2024-09-14T03:47:50.569 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.569 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.569 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.570 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.570 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.570 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.570 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.570 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.570 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.570 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.570 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.570 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.570 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.571 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.571 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.571 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.571 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.571 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.571 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.571 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.571 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.571 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.571 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.571 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.571 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.571 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.571 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.572 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.572 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.572 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.572 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.572 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.572 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.572 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.572 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.572 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.572 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.572 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.573 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.573 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.573 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.573 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.573 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.573 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.573 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.573 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.573 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.573 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:50.573 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.573 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:50.573 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.573 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 1, 2024-09-14T03:47:50.574 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.574 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.574 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.574 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.574 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.574 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.574 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:50.574 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.574 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 3, 2024-09-14T03:47:50.574 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.574 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.574 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.574 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.575 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.575 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.575 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.857969+0000", 2024-09-14T03:47:50.575 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.884218+0000", 2024-09-14T03:47:50.575 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.884218+0000", 2024-09-14T03:47:50.575 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.884486+0000", 2024-09-14T03:47:50.575 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.857969+0000", 2024-09-14T03:47:50.575 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.575 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.575 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.575 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.575 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.857969+0000", 2024-09-14T03:47:50.576 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.857969+0000", 2024-09-14T03:47:50.576 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.857969+0000", 2024-09-14T03:47:50.576 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.576 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.576 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.576 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.857969+0000", 2024-09-14T03:47:50.576 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.857969+0000", 2024-09-14T03:47:50.576 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.576 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.576 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.576 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.576 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.576 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.577 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.577 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.577 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.577 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.577 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.577 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.577 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.577 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.24", 2024-09-14T03:47:50.577 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.577 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.577 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.577 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:50.577 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.577 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T07:59:18.153239+0000", 2024-09-14T03:47:50.577 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.578 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.578 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.578 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.578 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.578 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.578 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.578 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.578 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.578 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.578 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.578 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.578 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.578 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.578 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.579 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.579 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.579 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.579 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.579 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.579 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.579 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.579 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.579 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.579 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.579 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.580 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.580 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.580 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.580 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.580 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.580 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.580 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.580 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.580 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.580 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.581 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.581 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.581 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.581 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.581 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.581 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.581 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.581 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.581 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.581 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.581 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.581 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.582 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.582 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:50.582 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.582 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 3, 2024-09-14T03:47:50.582 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.582 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.582 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.582 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.582 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.582 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.582 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:50.582 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.582 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 0, 2024-09-14T03:47:50.582 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.582 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.582 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.583 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.583 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.583 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.583 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.684242+0000", 2024-09-14T03:47:50.583 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.878504+0000", 2024-09-14T03:47:50.583 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.878504+0000", 2024-09-14T03:47:50.583 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.878737+0000", 2024-09-14T03:47:50.583 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.684242+0000", 2024-09-14T03:47:50.583 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.583 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.583 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.583 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.583 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.684242+0000", 2024-09-14T03:47:50.583 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.684242+0000", 2024-09-14T03:47:50.584 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.684242+0000", 2024-09-14T03:47:50.584 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.584 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.584 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.584 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.684242+0000", 2024-09-14T03:47:50.584 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.684242+0000", 2024-09-14T03:47:50.584 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.584 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.584 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.584 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.584 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.584 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.584 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.584 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.584 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.585 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.585 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.585 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.585 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.585 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.23", 2024-09-14T03:47:50.585 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.585 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.585 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.585 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:50.585 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.585 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T04:41:35.420957+0000", 2024-09-14T03:47:50.585 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.585 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.585 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.585 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.586 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.586 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.586 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.586 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.586 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.586 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.586 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.586 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.586 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.586 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.586 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.586 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.587 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.587 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.587 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.587 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.587 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.587 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.587 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.587 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.588 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.588 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.588 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.588 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.588 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.588 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.588 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.588 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.588 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.588 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.588 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.588 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.588 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.589 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.589 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.589 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.589 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.589 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.589 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.589 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.589 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.589 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.589 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.589 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.589 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.589 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:50.590 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.590 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 0, 2024-09-14T03:47:50.590 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.590 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.590 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.590 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.590 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.590 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.590 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:50.590 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.590 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 5, 2024-09-14T03:47:50.590 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.590 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.591 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.591 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.591 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.591 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.591 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.461691+0000", 2024-09-14T03:47:50.591 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.879855+0000", 2024-09-14T03:47:50.591 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.879855+0000", 2024-09-14T03:47:50.591 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.880509+0000", 2024-09-14T03:47:50.591 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.461691+0000", 2024-09-14T03:47:50.591 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.591 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.591 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.591 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.591 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.461691+0000", 2024-09-14T03:47:50.592 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.461691+0000", 2024-09-14T03:47:50.592 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.461691+0000", 2024-09-14T03:47:50.592 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.592 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.592 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.592 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.461691+0000", 2024-09-14T03:47:50.592 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.461691+0000", 2024-09-14T03:47:50.592 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.592 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.592 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.592 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.592 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.593 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.593 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.593 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.593 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.593 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.593 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.593 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.593 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.593 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.22", 2024-09-14T03:47:50.593 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.593 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.593 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.593 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:50.593 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.594 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T05:05:31.166902+0000", 2024-09-14T03:47:50.594 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.594 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.594 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.594 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.594 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.594 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.594 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.594 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.594 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.594 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.594 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.594 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.594 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.595 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.595 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.595 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.595 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.595 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.595 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.595 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.595 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.595 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.595 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.595 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.595 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.595 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.596 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.596 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.596 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.596 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.596 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.596 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.596 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.596 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.596 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.596 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.596 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.596 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.596 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.596 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.597 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.597 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.597 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.597 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.597 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.597 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.597 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.597 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.597 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.597 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:50.597 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.597 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 5, 2024-09-14T03:47:50.597 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.598 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.598 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.598 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.598 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:50.598 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.598 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:50.598 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.598 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 2, 2024-09-14T03:47:50.598 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.598 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.598 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.598 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.598 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.598 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.599 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:15.135594+0000", 2024-09-14T03:47:50.599 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.869225+0000", 2024-09-14T03:47:50.599 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.869225+0000", 2024-09-14T03:47:50.599 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.869523+0000", 2024-09-14T03:47:50.599 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:15.135594+0000", 2024-09-14T03:47:50.599 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.599 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.599 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.599 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.599 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:15.135594+0000", 2024-09-14T03:47:50.599 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:15.135594+0000", 2024-09-14T03:47:50.599 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:15.135594+0000", 2024-09-14T03:47:50.600 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.600 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.600 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.600 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:15.135594+0000", 2024-09-14T03:47:50.600 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:15.135594+0000", 2024-09-14T03:47:50.600 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.600 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.600 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.600 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.600 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.600 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.600 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.600 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.601 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.601 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.601 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.601 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.601 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.601 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.21", 2024-09-14T03:47:50.601 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.601 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.601 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.601 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:50.601 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.601 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T10:43:15.638187+0000", 2024-09-14T03:47:50.601 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.601 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.601 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.602 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.602 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.602 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.602 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.602 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.602 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.602 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.602 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.602 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.602 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.602 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.602 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.602 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.602 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.603 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.603 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.603 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.603 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.603 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.603 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.603 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.603 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.603 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.603 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.603 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.603 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.603 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.603 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.604 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.604 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.604 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.604 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.604 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.604 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.604 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.604 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.604 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.604 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.604 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.604 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.604 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.604 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.605 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.605 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.605 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:50.605 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.605 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:50.605 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.605 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 2, 2024-09-14T03:47:50.605 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.605 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.605 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.605 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.605 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:50.605 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.605 INFO:tasks.workunit.client.0.smithi070.stdout: 5 2024-09-14T03:47:50.605 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.606 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 6, 2024-09-14T03:47:50.606 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.606 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.606 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.606 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.606 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.606 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.606 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.544409+0000", 2024-09-14T03:47:50.606 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.875496+0000", 2024-09-14T03:47:50.606 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.875496+0000", 2024-09-14T03:47:50.606 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.876014+0000", 2024-09-14T03:47:50.606 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.544409+0000", 2024-09-14T03:47:50.606 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.607 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.607 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.607 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.607 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.544409+0000", 2024-09-14T03:47:50.607 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.544409+0000", 2024-09-14T03:47:50.607 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.544409+0000", 2024-09-14T03:47:50.607 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.607 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.607 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.607 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.544409+0000", 2024-09-14T03:47:50.607 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.544409+0000", 2024-09-14T03:47:50.607 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.607 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.608 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.608 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.608 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.608 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.608 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.608 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.608 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.608 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.608 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.608 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.608 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.608 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.20", 2024-09-14T03:47:50.608 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.608 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.609 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.609 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:50.609 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.609 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:03:45.344199+0000", 2024-09-14T03:47:50.609 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.609 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.609 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.609 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.609 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.609 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.609 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.609 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.609 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.609 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.609 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.610 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.610 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.610 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.610 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.610 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.610 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.610 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.610 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.610 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.610 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.610 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.610 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.610 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.611 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.611 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.611 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.611 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.611 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.611 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.611 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.611 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.611 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.611 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.612 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.612 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.612 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.612 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.612 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.612 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.612 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.612 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.612 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.613 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.613 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.613 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.613 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.613 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:50.613 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.613 INFO:tasks.workunit.client.0.smithi070.stdout: 5 2024-09-14T03:47:50.613 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.613 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 6, 2024-09-14T03:47:50.613 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.613 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.614 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.614 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.614 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.614 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:50.614 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:50.614 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.614 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 4, 2024-09-14T03:47:50.614 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.614 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.614 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.614 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.614 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.614 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.614 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.458305+0000", 2024-09-14T03:47:50.614 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.865190+0000", 2024-09-14T03:47:50.615 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.865190+0000", 2024-09-14T03:47:50.615 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.865496+0000", 2024-09-14T03:47:50.615 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.458305+0000", 2024-09-14T03:47:50.615 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.615 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.615 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.615 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.615 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.458305+0000", 2024-09-14T03:47:50.615 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.458305+0000", 2024-09-14T03:47:50.615 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.458305+0000", 2024-09-14T03:47:50.615 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.615 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.615 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.615 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.458305+0000", 2024-09-14T03:47:50.616 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.458305+0000", 2024-09-14T03:47:50.616 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.616 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.616 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.616 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.616 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.616 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.616 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.616 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.616 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.616 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.617 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.617 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.617 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.617 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.1a", 2024-09-14T03:47:50.617 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.617 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.618 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.618 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:50.618 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.618 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:18:50.037675+0000", 2024-09-14T03:47:50.618 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.618 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.619 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.619 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.619 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.619 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.619 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.619 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.619 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.619 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.619 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.619 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.619 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.620 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.620 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.620 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.620 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.620 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.620 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.620 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.620 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.620 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.620 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.620 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.620 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.621 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.621 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.621 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.621 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.621 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.622 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.622 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.622 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.622 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.622 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.622 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.623 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.623 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.623 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.623 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.623 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.624 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.624 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.624 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.624 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.624 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.624 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.625 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.625 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:50.625 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:50.625 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.625 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 4, 2024-09-14T03:47:50.626 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.626 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.626 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.626 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.626 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.626 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.627 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:50.627 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.627 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 3, 2024-09-14T03:47:50.627 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.627 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.627 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:50.628 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.628 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.628 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.628 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.857788+0000", 2024-09-14T03:47:50.628 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.728927+0000", 2024-09-14T03:47:50.629 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.728927+0000", 2024-09-14T03:47:50.629 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.729188+0000", 2024-09-14T03:47:50.629 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.857788+0000", 2024-09-14T03:47:50.629 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.629 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.630 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.630 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:50.630 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.857788+0000", 2024-09-14T03:47:50.630 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.857788+0000", 2024-09-14T03:47:50.630 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.857788+0000", 2024-09-14T03:47:50.630 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.631 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.631 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.631 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.857788+0000", 2024-09-14T03:47:50.631 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.857788+0000", 2024-09-14T03:47:50.631 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.631 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 1, 2024-09-14T03:47:50.632 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.632 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.632 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:50.632 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.632 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.633 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.633 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.633 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 1, 2024-09-14T03:47:50.633 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.633 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.633 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.633 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.1b", 2024-09-14T03:47:50.633 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.633 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.633 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.633 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 24, 2024-09-14T03:47:50.633 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.633 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:13:16.419701+0000", 2024-09-14T03:47:50.634 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.634 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.634 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.634 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.634 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.634 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.634 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.634 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.634 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.634 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.634 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.634 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.634 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.634 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.635 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.635 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.635 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.635 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.635 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.635 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.635 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.635 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.635 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.635 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.635 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.635 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.636 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.636 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.636 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.636 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.636 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.636 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.636 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.636 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.636 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.636 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.636 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.636 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.636 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.637 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.637 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.637 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.637 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.637 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.637 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.637 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.637 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.637 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.638 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.638 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:50.638 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.638 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 3, 2024-09-14T03:47:50.638 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "58'1" 2024-09-14T03:47:50.638 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.638 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.638 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.638 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.638 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.639 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:50.639 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.639 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 0, 2024-09-14T03:47:50.639 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.639 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.639 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.639 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.639 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.639 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.639 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.684318+0000", 2024-09-14T03:47:50.639 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.878801+0000", 2024-09-14T03:47:50.639 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.878801+0000", 2024-09-14T03:47:50.639 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.879101+0000", 2024-09-14T03:47:50.640 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.684318+0000", 2024-09-14T03:47:50.640 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.640 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.640 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.640 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.640 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.684318+0000", 2024-09-14T03:47:50.640 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.684318+0000", 2024-09-14T03:47:50.640 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.684318+0000", 2024-09-14T03:47:50.640 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.640 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.640 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.641 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.684318+0000", 2024-09-14T03:47:50.641 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.684318+0000", 2024-09-14T03:47:50.641 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.641 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.641 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.641 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.641 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.641 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.641 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.641 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.641 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.641 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.641 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.641 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.641 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.641 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.1b", 2024-09-14T03:47:50.642 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.642 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.642 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.642 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:50.642 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.642 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:59:13.447251+0000", 2024-09-14T03:47:50.642 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.642 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.642 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.642 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.642 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.643 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.643 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.643 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.643 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.643 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.643 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.643 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.643 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.643 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.643 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.643 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.643 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.644 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.644 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.644 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.644 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.644 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.644 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.644 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.644 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.644 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.644 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.644 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.644 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.644 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.645 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.645 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.645 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.645 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.645 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.645 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.645 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.645 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.645 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.645 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.645 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.645 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.645 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.645 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.646 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.646 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.646 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.646 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.646 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.646 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:50.646 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.646 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 0, 2024-09-14T03:47:50.646 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.646 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.646 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.646 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.646 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:50.646 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.647 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:50.647 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.647 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 6, 2024-09-14T03:47:50.647 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.647 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.647 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:50.647 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.647 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.647 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.647 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.544625+0000", 2024-09-14T03:47:50.647 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.730172+0000", 2024-09-14T03:47:50.647 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.730172+0000", 2024-09-14T03:47:50.647 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.730444+0000", 2024-09-14T03:47:50.648 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.544625+0000", 2024-09-14T03:47:50.648 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.648 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.648 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.648 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:50.648 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.544625+0000", 2024-09-14T03:47:50.648 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.544625+0000", 2024-09-14T03:47:50.648 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.544625+0000", 2024-09-14T03:47:50.648 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.648 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.648 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.648 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.544625+0000", 2024-09-14T03:47:50.648 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.544625+0000", 2024-09-14T03:47:50.648 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.649 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 1, 2024-09-14T03:47:50.649 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.649 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.649 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:50.649 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.649 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.649 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.649 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.649 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 1, 2024-09-14T03:47:50.649 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.649 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.649 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.649 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.1a", 2024-09-14T03:47:50.650 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.650 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.650 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.650 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 24, 2024-09-14T03:47:50.650 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.650 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T10:03:44.810562+0000", 2024-09-14T03:47:50.650 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.650 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.650 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.650 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.650 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.650 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.650 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.650 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.651 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.651 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.651 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.651 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.651 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.651 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.651 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.651 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.651 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.651 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.651 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.651 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.651 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.651 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.652 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.652 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.652 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.652 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.652 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.652 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.652 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.652 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.652 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.652 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.652 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.652 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.652 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.653 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.653 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.653 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.653 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.653 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.653 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.653 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.653 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.653 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.653 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.653 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.654 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.654 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:50.654 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.654 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:50.654 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.654 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 6, 2024-09-14T03:47:50.654 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "58'1" 2024-09-14T03:47:50.654 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.654 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.654 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.654 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.654 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:50.654 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:50.654 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.655 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 3, 2024-09-14T03:47:50.655 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.655 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.655 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:50.655 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.655 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.655 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.655 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.857500+0000", 2024-09-14T03:47:50.655 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.731206+0000", 2024-09-14T03:47:50.655 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.731206+0000", 2024-09-14T03:47:50.655 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.731458+0000", 2024-09-14T03:47:50.655 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.857500+0000", 2024-09-14T03:47:50.655 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.655 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.655 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.656 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:50.656 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.857500+0000", 2024-09-14T03:47:50.656 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.857500+0000", 2024-09-14T03:47:50.656 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.857500+0000", 2024-09-14T03:47:50.656 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.656 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.656 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.656 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.857500+0000", 2024-09-14T03:47:50.656 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.857500+0000", 2024-09-14T03:47:50.656 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.656 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.656 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.656 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.657 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:50.657 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.657 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.657 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.657 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.657 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.657 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.657 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.657 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.657 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.19", 2024-09-14T03:47:50.657 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.657 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.657 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.657 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 23, 2024-09-14T03:47:50.658 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.658 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:37:06.670539+0000", 2024-09-14T03:47:50.658 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.658 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.658 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.658 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.658 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.658 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.658 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.658 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.658 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.658 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.659 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.659 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.659 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.659 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.659 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.659 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.659 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.659 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.659 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.659 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.659 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.659 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.659 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.660 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.660 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.660 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.660 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.660 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.660 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.660 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.660 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.660 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.660 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.660 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.660 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.660 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.661 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.661 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.661 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.661 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.661 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.661 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.661 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.661 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.661 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.661 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.661 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.661 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.661 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:50.661 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:50.662 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.662 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 3, 2024-09-14T03:47:50.662 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.662 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.662 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.662 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.662 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.662 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.662 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:50.662 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.662 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 3, 2024-09-14T03:47:50.662 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.662 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.662 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.663 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.663 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.663 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.663 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.857453+0000", 2024-09-14T03:47:50.663 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.869521+0000", 2024-09-14T03:47:50.663 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.869521+0000", 2024-09-14T03:47:50.663 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.869781+0000", 2024-09-14T03:47:50.664 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.857453+0000", 2024-09-14T03:47:50.664 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.664 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.664 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.664 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.664 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.857453+0000", 2024-09-14T03:47:50.664 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.857453+0000", 2024-09-14T03:47:50.664 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.857453+0000", 2024-09-14T03:47:50.664 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.664 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.664 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.664 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.857453+0000", 2024-09-14T03:47:50.665 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.857453+0000", 2024-09-14T03:47:50.665 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.665 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.665 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.665 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.665 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.665 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.665 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.665 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.665 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.665 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.666 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.666 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.666 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.666 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.18", 2024-09-14T03:47:50.666 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.666 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.666 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.666 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:50.666 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.666 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T04:35:18.565206+0000", 2024-09-14T03:47:50.666 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.666 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.667 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.667 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.667 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.667 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.667 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.667 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.667 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.667 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.667 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.667 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.667 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.667 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.667 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.668 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.668 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.668 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.668 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.668 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.668 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.668 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.668 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.668 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.668 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.668 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.668 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.668 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.669 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.669 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.669 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.669 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.669 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.669 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.669 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.669 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.669 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.669 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.670 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.670 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.670 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.670 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.670 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.670 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.670 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.670 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.670 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.670 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.670 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.670 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:50.670 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.670 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 3, 2024-09-14T03:47:50.671 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.671 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.671 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.671 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.671 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.671 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.671 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:50.671 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.671 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 5, 2024-09-14T03:47:50.671 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.671 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.671 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:50.671 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.672 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.672 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.672 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.461579+0000", 2024-09-14T03:47:50.672 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:25.932288+0000", 2024-09-14T03:47:50.672 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:25.932288+0000", 2024-09-14T03:47:50.672 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:25.932578+0000", 2024-09-14T03:47:50.672 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.461579+0000", 2024-09-14T03:47:50.672 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.672 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.672 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.672 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 60, 2024-09-14T03:47:50.672 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.461579+0000", 2024-09-14T03:47:50.672 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.461579+0000", 2024-09-14T03:47:50.672 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.461579+0000", 2024-09-14T03:47:50.673 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.673 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.673 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.673 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.461579+0000", 2024-09-14T03:47:50.673 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.461579+0000", 2024-09-14T03:47:50.673 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.673 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.673 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.673 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.673 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 59, 2024-09-14T03:47:50.673 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.674 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.674 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.674 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.674 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.674 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.674 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.674 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.674 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.18", 2024-09-14T03:47:50.674 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.674 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.674 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.674 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 33, 2024-09-14T03:47:50.674 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.674 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:24:41.091353+0000", 2024-09-14T03:47:50.675 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.675 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.675 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.675 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.675 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.675 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.675 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.675 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.675 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.675 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.675 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.675 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.675 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.676 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.676 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.676 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.676 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.676 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.676 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.676 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.676 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.676 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.676 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.676 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.676 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.677 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.677 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.677 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.677 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.677 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.677 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.677 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.677 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.677 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.677 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.677 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.677 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.677 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.678 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.678 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.678 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.678 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.678 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.678 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.678 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.678 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.678 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.678 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.678 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.678 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:50.678 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.678 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 5, 2024-09-14T03:47:50.678 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.679 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.679 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.679 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.679 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:50.679 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.679 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:50.679 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.679 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 1, 2024-09-14T03:47:50.679 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.679 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.679 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.679 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.679 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.680 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.680 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.457866+0000", 2024-09-14T03:47:50.680 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.873473+0000", 2024-09-14T03:47:50.680 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.873473+0000", 2024-09-14T03:47:50.680 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.873748+0000", 2024-09-14T03:47:50.680 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.457866+0000", 2024-09-14T03:47:50.680 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.680 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.680 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.680 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.680 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.457866+0000", 2024-09-14T03:47:50.680 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.457866+0000", 2024-09-14T03:47:50.681 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.457866+0000", 2024-09-14T03:47:50.681 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.681 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.681 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.681 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.457866+0000", 2024-09-14T03:47:50.681 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.457866+0000", 2024-09-14T03:47:50.681 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.681 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.681 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.681 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.681 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.682 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.682 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.682 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.682 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.682 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.682 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.682 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.682 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.682 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.19", 2024-09-14T03:47:50.682 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.682 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.682 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.682 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:50.683 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.683 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:04:39.289903+0000", 2024-09-14T03:47:50.683 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.683 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.683 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.683 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.683 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.683 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.683 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.683 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.683 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.683 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.683 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.684 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.684 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.684 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.684 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.684 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.684 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.684 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.684 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.684 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.684 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.684 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.684 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.684 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.685 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.685 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.685 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.685 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.685 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.685 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.685 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.685 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.685 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.685 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.685 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.685 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.685 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.686 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.686 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.686 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.686 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.686 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.686 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.686 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.686 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.686 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.686 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.686 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:50.686 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.686 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:50.686 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.686 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 1, 2024-09-14T03:47:50.687 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.687 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.687 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.687 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.687 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:50.687 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.687 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:50.687 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.687 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 6, 2024-09-14T03:47:50.687 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.687 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.687 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:50.687 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.687 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.688 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.688 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.544953+0000", 2024-09-14T03:47:50.688 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.735301+0000", 2024-09-14T03:47:50.688 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.735301+0000", 2024-09-14T03:47:50.688 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.735608+0000", 2024-09-14T03:47:50.688 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.544953+0000", 2024-09-14T03:47:50.688 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.689 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.689 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.689 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:50.689 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.544953+0000", 2024-09-14T03:47:50.689 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.544953+0000", 2024-09-14T03:47:50.689 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.544953+0000", 2024-09-14T03:47:50.689 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.689 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.689 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.689 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.544953+0000", 2024-09-14T03:47:50.689 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.544953+0000", 2024-09-14T03:47:50.689 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.689 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.689 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.690 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.690 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:50.690 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.690 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.690 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.690 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.690 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.690 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.690 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.690 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.690 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.17", 2024-09-14T03:47:50.690 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.690 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.690 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.690 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 23, 2024-09-14T03:47:50.691 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.691 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:44:56.523073+0000", 2024-09-14T03:47:50.691 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.691 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.691 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.691 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.691 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.691 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.691 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.691 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.691 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.691 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.691 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.691 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.691 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.691 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.692 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.692 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.692 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.692 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.692 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.692 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.692 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.692 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.692 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.692 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.692 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.692 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.692 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.693 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.693 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.693 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.693 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.693 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.693 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.693 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.693 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.693 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.693 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.693 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.694 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.694 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.694 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.694 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.694 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.694 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.694 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.694 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.694 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.694 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:50.694 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.694 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:50.694 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.694 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 6, 2024-09-14T03:47:50.694 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.695 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.695 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.695 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.695 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.695 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.695 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:50.695 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.695 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 5, 2024-09-14T03:47:50.695 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.695 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.695 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.695 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.695 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.695 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.695 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.462078+0000", 2024-09-14T03:47:50.695 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.874473+0000", 2024-09-14T03:47:50.696 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.874473+0000", 2024-09-14T03:47:50.696 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.874777+0000", 2024-09-14T03:47:50.696 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.462078+0000", 2024-09-14T03:47:50.696 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.696 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.696 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.696 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.696 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.462078+0000", 2024-09-14T03:47:50.696 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.462078+0000", 2024-09-14T03:47:50.696 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.462078+0000", 2024-09-14T03:47:50.696 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.696 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.696 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.696 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.462078+0000", 2024-09-14T03:47:50.697 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.462078+0000", 2024-09-14T03:47:50.697 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.697 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.697 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.697 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.697 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.697 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.697 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.697 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.697 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.697 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.698 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.698 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.698 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.698 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.16", 2024-09-14T03:47:50.698 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.698 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.698 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.698 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:50.698 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.698 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T04:49:02.042635+0000", 2024-09-14T03:47:50.699 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.699 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.699 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.699 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.699 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.699 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.699 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.699 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.699 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.699 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.699 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.699 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.700 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.700 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.700 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.700 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.700 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.700 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.700 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.700 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.700 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.700 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.700 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.700 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.700 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.701 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.701 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.701 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.701 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.701 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.701 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.701 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.701 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.701 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.701 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.701 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.701 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.701 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.701 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.702 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.702 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.702 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.702 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.702 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.702 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.702 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.702 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.702 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.702 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.702 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:50.702 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.703 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 5, 2024-09-14T03:47:50.703 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.703 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.703 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.703 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.703 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.703 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:50.703 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:50.703 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.703 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 5, 2024-09-14T03:47:50.703 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.703 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.703 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:50.704 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.704 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.704 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.704 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.462172+0000", 2024-09-14T03:47:50.704 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.732492+0000", 2024-09-14T03:47:50.704 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.732492+0000", 2024-09-14T03:47:50.704 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.732969+0000", 2024-09-14T03:47:50.704 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.462172+0000", 2024-09-14T03:47:50.704 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.704 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.704 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.704 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:50.704 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.462172+0000", 2024-09-14T03:47:50.704 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.462172+0000", 2024-09-14T03:47:50.705 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.462172+0000", 2024-09-14T03:47:50.705 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.705 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.705 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.705 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.462172+0000", 2024-09-14T03:47:50.705 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.462172+0000", 2024-09-14T03:47:50.705 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.705 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.705 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.705 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.705 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:50.705 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.705 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.705 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.706 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.706 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.706 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.706 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.706 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.706 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.16", 2024-09-14T03:47:50.706 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.706 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.706 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.706 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 23, 2024-09-14T03:47:50.706 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.706 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:50:28.719447+0000", 2024-09-14T03:47:50.706 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.706 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.707 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.707 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.707 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.707 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.707 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.707 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.707 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.707 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.707 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.707 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.707 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.707 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.707 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.707 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.708 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.708 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.708 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.708 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.708 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.708 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.708 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.708 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.708 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.708 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.708 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.708 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.709 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.709 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.709 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.709 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.709 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.709 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.709 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.709 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.709 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.709 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.709 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.709 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.709 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.710 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.710 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.710 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.710 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.710 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.710 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.710 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.710 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:50.710 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:50.710 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.710 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 5, 2024-09-14T03:47:50.710 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.710 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.710 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.710 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.711 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.711 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.711 INFO:tasks.workunit.client.0.smithi070.stdout: 3 2024-09-14T03:47:50.711 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.711 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 0, 2024-09-14T03:47:50.711 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.711 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.711 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.711 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.711 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.711 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.712 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.684384+0000", 2024-09-14T03:47:50.712 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.875422+0000", 2024-09-14T03:47:50.712 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.875422+0000", 2024-09-14T03:47:50.712 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.875679+0000", 2024-09-14T03:47:50.712 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.684384+0000", 2024-09-14T03:47:50.712 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.712 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.712 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.712 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.712 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.684384+0000", 2024-09-14T03:47:50.712 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.684384+0000", 2024-09-14T03:47:50.712 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.684384+0000", 2024-09-14T03:47:50.712 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.712 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.713 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.713 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.684384+0000", 2024-09-14T03:47:50.713 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.684384+0000", 2024-09-14T03:47:50.713 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.713 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.713 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.713 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.713 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.713 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.713 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.713 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.713 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.714 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.714 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.714 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.714 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.714 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.17", 2024-09-14T03:47:50.714 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.714 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.714 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.714 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:50.714 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.714 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:13:00.349373+0000", 2024-09-14T03:47:50.714 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.714 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.715 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.715 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.715 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.715 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.715 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.715 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.715 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.715 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.715 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.715 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.715 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.715 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.715 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.715 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.715 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.715 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.716 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.716 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.716 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.716 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.716 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.716 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.716 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.716 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.716 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.716 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.716 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.716 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.716 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.717 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.717 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.717 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.717 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.717 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.717 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.717 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.717 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.717 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.717 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.717 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.718 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.718 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.718 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.718 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.718 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.718 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.718 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.718 INFO:tasks.workunit.client.0.smithi070.stdout: 3 2024-09-14T03:47:50.718 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.718 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 0, 2024-09-14T03:47:50.718 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.718 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.718 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.718 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.718 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:50.719 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.719 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:50.719 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.719 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 1, 2024-09-14T03:47:50.719 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.719 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.719 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:50.719 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.719 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.719 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.719 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.458567+0000", 2024-09-14T03:47:50.719 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.734514+0000", 2024-09-14T03:47:50.719 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.734514+0000", 2024-09-14T03:47:50.719 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.735066+0000", 2024-09-14T03:47:50.719 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.458567+0000", 2024-09-14T03:47:50.720 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.720 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.720 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.720 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:50.720 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.458567+0000", 2024-09-14T03:47:50.720 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.458567+0000", 2024-09-14T03:47:50.720 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.458567+0000", 2024-09-14T03:47:50.720 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.720 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.720 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.720 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.458567+0000", 2024-09-14T03:47:50.720 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.458567+0000", 2024-09-14T03:47:50.720 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.720 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 3, 2024-09-14T03:47:50.721 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.721 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.721 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:50.721 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.721 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.721 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.721 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.721 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 3, 2024-09-14T03:47:50.721 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.721 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.721 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.721 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.15", 2024-09-14T03:47:50.721 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.721 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.722 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.722 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 26, 2024-09-14T03:47:50.722 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.722 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T09:41:55.478295+0000", 2024-09-14T03:47:50.722 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.722 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.722 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.722 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 90, 2024-09-14T03:47:50.722 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.722 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.722 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.722 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.722 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.722 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.722 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.723 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.723 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.723 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.723 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.723 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.723 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.723 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.723 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.723 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 6, 2024-09-14T03:47:50.723 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 2, 2024-09-14T03:47:50.723 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.723 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 2, 2024-09-14T03:47:50.723 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.723 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.723 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.724 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.724 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.724 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 1, 2024-09-14T03:47:50.724 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.724 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.724 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.724 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.724 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.724 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.724 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.724 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.725 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.725 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.725 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.725 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.725 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 4, 2024-09-14T03:47:50.725 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 1 2024-09-14T03:47:50.725 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.725 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.725 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.725 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.725 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:50.725 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.725 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:50.726 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.726 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 1, 2024-09-14T03:47:50.726 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "58'3" 2024-09-14T03:47:50.726 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.726 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.726 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.726 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.726 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.726 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:50.726 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.726 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 4, 2024-09-14T03:47:50.726 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.726 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.726 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.727 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.727 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.727 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.727 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.458619+0000", 2024-09-14T03:47:50.727 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.881032+0000", 2024-09-14T03:47:50.727 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.881032+0000", 2024-09-14T03:47:50.727 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.881319+0000", 2024-09-14T03:47:50.727 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.458619+0000", 2024-09-14T03:47:50.727 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.727 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.727 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.727 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.727 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.458619+0000", 2024-09-14T03:47:50.727 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.458619+0000", 2024-09-14T03:47:50.728 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.458619+0000", 2024-09-14T03:47:50.728 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.728 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.728 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.728 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.458619+0000", 2024-09-14T03:47:50.728 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.458619+0000", 2024-09-14T03:47:50.728 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.728 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.728 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.728 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.728 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.728 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.728 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.729 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.729 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.729 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.729 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.729 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.729 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.729 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.14", 2024-09-14T03:47:50.729 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.729 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.729 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.729 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:50.729 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.729 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T15:35:41.762014+0000", 2024-09-14T03:47:50.730 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.730 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.730 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.730 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.730 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.730 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.730 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.730 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.730 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.730 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.730 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.730 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.730 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.730 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.731 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.731 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.731 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.731 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.731 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.731 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.731 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.731 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.731 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.731 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.731 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.731 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.731 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.732 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.732 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.732 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.732 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.732 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.732 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.732 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.732 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.732 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.732 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.732 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.733 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.733 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.733 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.733 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.733 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.733 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.733 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.733 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.733 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.733 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.734 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.734 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:50.734 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.734 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 4, 2024-09-14T03:47:50.734 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.734 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.734 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.734 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.734 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:50.734 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.734 INFO:tasks.workunit.client.0.smithi070.stdout: 5 2024-09-14T03:47:50.735 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.735 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 6, 2024-09-14T03:47:50.735 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.735 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.735 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:50.735 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.735 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.735 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.735 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:46:36.633586+0000", 2024-09-14T03:47:50.735 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.734380+0000", 2024-09-14T03:47:50.735 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.734380+0000", 2024-09-14T03:47:50.735 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.734778+0000", 2024-09-14T03:47:50.736 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:46:36.633586+0000", 2024-09-14T03:47:50.736 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.736 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.736 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.736 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:50.736 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:46:36.633586+0000", 2024-09-14T03:47:50.736 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:46:36.633586+0000", 2024-09-14T03:47:50.736 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:46:36.633586+0000", 2024-09-14T03:47:50.736 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.736 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.736 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.736 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:46:36.633586+0000", 2024-09-14T03:47:50.736 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:46:36.633586+0000", 2024-09-14T03:47:50.737 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.737 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 12, 2024-09-14T03:47:50.737 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.737 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.737 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:50.737 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.737 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.737 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.737 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.737 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 12, 2024-09-14T03:47:50.737 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.737 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.737 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.738 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.14", 2024-09-14T03:47:50.738 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.738 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.738 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 68, 2024-09-14T03:47:50.738 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 35, 2024-09-14T03:47:50.738 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.738 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:55:13.561065+0000", 2024-09-14T03:47:50.738 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.738 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.738 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.738 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 2624, 2024-09-14T03:47:50.738 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.738 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.738 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.739 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.739 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.739 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.739 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.739 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.739 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.739 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.739 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.739 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.739 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.739 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.740 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.740 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 12, 2024-09-14T03:47:50.740 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 4, 2024-09-14T03:47:50.740 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.740 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 4, 2024-09-14T03:47:50.740 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.740 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.740 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.740 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.740 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.740 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 1, 2024-09-14T03:47:50.740 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.741 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.741 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.741 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.741 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.741 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.741 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.741 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.741 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.741 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.741 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.741 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.741 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 12, 2024-09-14T03:47:50.741 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 9 2024-09-14T03:47:50.742 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.742 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.742 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.742 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.742 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:50.742 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.742 INFO:tasks.workunit.client.0.smithi070.stdout: 5 2024-09-14T03:47:50.742 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.742 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 6, 2024-09-14T03:47:50.742 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "68'12" 2024-09-14T03:47:50.742 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.742 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.742 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.742 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.743 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.743 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:50.743 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.743 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:50.743 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.743 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.743 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.743 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.743 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.743 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.743 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.660824+0000", 2024-09-14T03:47:50.743 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.879840+0000", 2024-09-14T03:47:50.744 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.879840+0000", 2024-09-14T03:47:50.744 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.880139+0000", 2024-09-14T03:47:50.744 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.660824+0000", 2024-09-14T03:47:50.744 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.744 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.744 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.745 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.745 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.660824+0000", 2024-09-14T03:47:50.745 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.660824+0000", 2024-09-14T03:47:50.745 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.660824+0000", 2024-09-14T03:47:50.745 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.745 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.745 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.745 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.660824+0000", 2024-09-14T03:47:50.745 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.660824+0000", 2024-09-14T03:47:50.745 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.745 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.745 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.746 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.746 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.746 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.746 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.746 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.746 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.746 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.746 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.746 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.746 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.746 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.15", 2024-09-14T03:47:50.746 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.746 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.746 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.747 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:50.747 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.747 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:31:34.003676+0000", 2024-09-14T03:47:50.747 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.747 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.747 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.747 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.747 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.747 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.747 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.747 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.747 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.747 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.748 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.748 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.748 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.748 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.748 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.748 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.748 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.748 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.748 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.748 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.748 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.748 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.748 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.749 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.749 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.749 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.749 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.749 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.749 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.749 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.749 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.749 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.749 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.749 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.749 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.749 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.749 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.750 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.750 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.750 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.750 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.750 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.750 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.750 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.750 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.750 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.750 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.750 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.750 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.750 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:50.751 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.751 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:50.751 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.751 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.751 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.751 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.751 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.751 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.751 INFO:tasks.workunit.client.0.smithi070.stdout: 3 2024-09-14T03:47:50.751 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.751 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 0, 2024-09-14T03:47:50.751 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.751 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.752 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:50.752 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.752 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.752 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.752 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.684403+0000", 2024-09-14T03:47:50.752 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.727893+0000", 2024-09-14T03:47:50.752 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.727893+0000", 2024-09-14T03:47:50.752 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.728768+0000", 2024-09-14T03:47:50.752 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.684403+0000", 2024-09-14T03:47:50.752 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.752 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.752 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.753 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:50.753 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.684403+0000", 2024-09-14T03:47:50.753 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.684403+0000", 2024-09-14T03:47:50.753 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.684403+0000", 2024-09-14T03:47:50.753 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.754 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.754 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.754 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.684403+0000", 2024-09-14T03:47:50.754 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.684403+0000", 2024-09-14T03:47:50.754 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.754 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 2, 2024-09-14T03:47:50.755 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.755 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.755 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:50.755 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.755 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.755 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.756 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.756 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 2, 2024-09-14T03:47:50.756 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.756 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.756 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.756 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.13", 2024-09-14T03:47:50.756 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.756 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.756 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.756 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 25, 2024-09-14T03:47:50.757 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.757 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T04:54:48.554486+0000", 2024-09-14T03:47:50.757 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.757 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.757 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.757 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 34, 2024-09-14T03:47:50.758 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.758 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.758 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.758 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.758 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.758 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.758 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.758 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.758 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.758 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.758 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.758 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.758 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.759 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.759 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.759 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 6, 2024-09-14T03:47:50.759 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 2, 2024-09-14T03:47:50.759 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.759 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 2, 2024-09-14T03:47:50.759 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.759 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.759 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.759 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.759 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.759 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 1, 2024-09-14T03:47:50.760 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.760 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.760 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.760 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.760 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.760 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.760 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.760 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.760 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.760 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.760 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.760 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.761 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 3, 2024-09-14T03:47:50.761 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 6 2024-09-14T03:47:50.761 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.761 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.761 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.761 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.761 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.761 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.761 INFO:tasks.workunit.client.0.smithi070.stdout: 3 2024-09-14T03:47:50.761 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.761 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 0, 2024-09-14T03:47:50.761 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "58'2" 2024-09-14T03:47:50.761 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.762 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.762 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.762 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.762 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.762 INFO:tasks.workunit.client.0.smithi070.stdout: 3 2024-09-14T03:47:50.762 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.762 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 0, 2024-09-14T03:47:50.762 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.762 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.762 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.762 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.762 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.762 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.763 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.684119+0000", 2024-09-14T03:47:50.763 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.877147+0000", 2024-09-14T03:47:50.763 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.877147+0000", 2024-09-14T03:47:50.763 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.877540+0000", 2024-09-14T03:47:50.763 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.684119+0000", 2024-09-14T03:47:50.763 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.763 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.763 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.763 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.763 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.684119+0000", 2024-09-14T03:47:50.763 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.684119+0000", 2024-09-14T03:47:50.763 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.684119+0000", 2024-09-14T03:47:50.763 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.763 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.764 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.764 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.684119+0000", 2024-09-14T03:47:50.764 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.684119+0000", 2024-09-14T03:47:50.764 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.764 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.764 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.764 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.764 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.764 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.764 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.764 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.764 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.765 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.765 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.765 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.765 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.765 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.12", 2024-09-14T03:47:50.765 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.765 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.765 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.765 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:50.765 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.765 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T07:17:57.724648+0000", 2024-09-14T03:47:50.765 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.766 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.766 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.766 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.766 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.766 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.766 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.766 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.766 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.766 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.766 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.766 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.766 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.766 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.767 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.767 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.767 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.767 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.767 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.767 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.767 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.767 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.767 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.767 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.767 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.767 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.767 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.768 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.768 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.768 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.768 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.768 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.768 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.768 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.768 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.768 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.768 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.768 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.768 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.768 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.769 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.769 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.769 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.769 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.769 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.769 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.769 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.769 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.769 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.769 INFO:tasks.workunit.client.0.smithi070.stdout: 3 2024-09-14T03:47:50.769 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.769 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 0, 2024-09-14T03:47:50.769 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.769 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.770 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.770 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.770 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.770 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.770 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:50.770 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.770 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 5, 2024-09-14T03:47:50.770 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.770 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.770 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:50.770 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.771 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.771 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.771 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.461815+0000", 2024-09-14T03:47:50.771 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.732537+0000", 2024-09-14T03:47:50.771 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.732537+0000", 2024-09-14T03:47:50.771 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.732812+0000", 2024-09-14T03:47:50.771 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.461815+0000", 2024-09-14T03:47:50.771 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.771 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.771 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.771 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:50.771 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.461815+0000", 2024-09-14T03:47:50.771 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.461815+0000", 2024-09-14T03:47:50.771 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.461815+0000", 2024-09-14T03:47:50.772 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.772 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.772 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.772 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.461815+0000", 2024-09-14T03:47:50.772 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.461815+0000", 2024-09-14T03:47:50.772 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.772 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.773 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.773 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.773 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:50.773 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.773 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.773 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.773 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.773 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.773 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.774 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.774 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.774 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.12", 2024-09-14T03:47:50.774 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.774 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.774 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.774 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 23, 2024-09-14T03:47:50.774 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.774 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:26:11.551182+0000", 2024-09-14T03:47:50.774 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.774 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.774 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.774 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.774 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.775 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.775 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.775 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.775 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.775 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.775 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.775 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.775 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.775 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.775 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.775 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.775 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.775 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.775 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.776 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.776 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.776 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.776 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.776 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.776 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.776 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.776 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.776 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.776 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.776 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.776 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.777 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.777 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.777 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.777 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.777 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.777 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.777 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.777 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.777 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.777 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.777 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.777 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.777 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.778 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.778 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.778 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.778 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.778 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.778 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:50.778 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.778 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 5, 2024-09-14T03:47:50.778 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.778 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.778 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.778 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.778 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.778 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.779 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:50.779 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.779 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:50.779 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.779 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.779 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.779 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.779 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.779 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.779 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.660730+0000", 2024-09-14T03:47:50.779 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.880868+0000", 2024-09-14T03:47:50.779 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.880868+0000", 2024-09-14T03:47:50.779 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.881140+0000", 2024-09-14T03:47:50.780 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.660730+0000", 2024-09-14T03:47:50.780 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.780 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.780 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.780 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.780 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.660730+0000", 2024-09-14T03:47:50.780 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.660730+0000", 2024-09-14T03:47:50.780 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.660730+0000", 2024-09-14T03:47:50.780 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.780 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.780 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.781 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.660730+0000", 2024-09-14T03:47:50.781 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.660730+0000", 2024-09-14T03:47:50.781 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.781 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.781 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.781 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.781 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.781 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.781 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.781 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.781 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.782 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.782 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.782 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.782 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.782 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.13", 2024-09-14T03:47:50.782 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.782 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.782 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.782 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:50.782 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.782 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:29:05.729804+0000", 2024-09-14T03:47:50.782 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.782 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.783 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.783 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.783 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.783 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.783 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.783 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.783 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.783 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.783 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.783 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.783 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.783 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.783 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.783 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.784 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.784 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.784 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.784 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.784 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.784 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.784 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.784 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.784 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.784 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.784 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.784 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.784 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.785 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.785 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.785 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.785 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.785 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.785 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.785 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.785 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.785 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.785 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.785 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.785 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.785 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.786 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.786 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.786 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.786 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.786 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.786 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.786 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.786 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:50.786 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.786 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:50.786 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.787 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.787 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.787 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.787 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:50.787 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.787 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:50.787 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.787 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 6, 2024-09-14T03:47:50.787 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.787 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.788 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:50.788 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.788 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.788 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.788 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.544699+0000", 2024-09-14T03:47:50.788 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.729821+0000", 2024-09-14T03:47:50.788 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.729821+0000", 2024-09-14T03:47:50.788 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.730505+0000", 2024-09-14T03:47:50.788 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.544699+0000", 2024-09-14T03:47:50.788 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.788 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.788 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.788 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:50.788 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.544699+0000", 2024-09-14T03:47:50.789 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.544699+0000", 2024-09-14T03:47:50.789 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.544699+0000", 2024-09-14T03:47:50.789 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.789 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.789 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.789 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.544699+0000", 2024-09-14T03:47:50.789 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.544699+0000", 2024-09-14T03:47:50.789 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.789 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.789 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.789 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.789 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:50.789 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.790 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.790 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.790 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.790 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.790 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.790 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.790 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.790 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.11", 2024-09-14T03:47:50.790 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.790 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.790 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.790 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 23, 2024-09-14T03:47:50.790 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.790 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T10:17:44.091667+0000", 2024-09-14T03:47:50.791 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.791 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.791 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.791 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.791 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.791 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.791 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.791 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.791 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.791 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.791 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.791 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.791 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.791 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.791 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.792 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.792 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.792 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.792 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.792 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.792 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.792 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.792 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.792 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.792 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.792 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.792 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.792 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.792 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.793 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.793 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.793 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.793 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.793 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.793 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.793 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.793 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.793 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.793 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.794 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.794 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.794 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.794 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.794 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.794 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.794 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.794 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.794 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:50.794 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.794 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:50.794 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.794 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 6, 2024-09-14T03:47:50.794 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.795 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.795 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.795 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.795 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:50.795 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.795 INFO:tasks.workunit.client.0.smithi070.stdout: 5 2024-09-14T03:47:50.795 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.795 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 6, 2024-09-14T03:47:50.795 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.795 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.795 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.795 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.795 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.795 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.795 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.544408+0000", 2024-09-14T03:47:50.796 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.873714+0000", 2024-09-14T03:47:50.796 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.873714+0000", 2024-09-14T03:47:50.796 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.874477+0000", 2024-09-14T03:47:50.796 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.544408+0000", 2024-09-14T03:47:50.796 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.796 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.796 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.796 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.796 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.544408+0000", 2024-09-14T03:47:50.796 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.544408+0000", 2024-09-14T03:47:50.796 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.544408+0000", 2024-09-14T03:47:50.796 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.796 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.797 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.797 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.544408+0000", 2024-09-14T03:47:50.797 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.544408+0000", 2024-09-14T03:47:50.797 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.797 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.797 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.797 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.797 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.798 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.798 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.798 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.798 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.798 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.798 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.799 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.799 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.799 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.10", 2024-09-14T03:47:50.799 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.799 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.800 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.800 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:50.800 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.800 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:54:21.601768+0000", 2024-09-14T03:47:50.800 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.801 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.801 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.801 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.801 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.801 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.801 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.801 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.801 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.801 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.801 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.802 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.802 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.802 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.802 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.802 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.802 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.803 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.803 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.803 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.803 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.803 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.803 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.803 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.803 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.803 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.803 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.803 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.803 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.804 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.804 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.804 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.804 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.804 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.804 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.804 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.804 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.804 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.804 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.804 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.804 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.804 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.805 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.805 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.805 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.805 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.805 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.805 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:50.805 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.805 INFO:tasks.workunit.client.0.smithi070.stdout: 5 2024-09-14T03:47:50.805 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.805 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 6, 2024-09-14T03:47:50.805 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.805 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.805 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.806 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.806 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:50.806 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:50.806 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:50.806 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.806 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 2, 2024-09-14T03:47:50.806 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.806 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.806 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:50.806 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.806 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.806 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.806 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:15.135490+0000", 2024-09-14T03:47:50.806 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.726115+0000", 2024-09-14T03:47:50.806 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.726115+0000", 2024-09-14T03:47:50.807 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.726387+0000", 2024-09-14T03:47:50.807 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:15.135490+0000", 2024-09-14T03:47:50.807 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.807 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.807 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.807 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:50.807 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:15.135490+0000", 2024-09-14T03:47:50.807 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:15.135490+0000", 2024-09-14T03:47:50.807 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:15.135490+0000", 2024-09-14T03:47:50.807 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.808 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.808 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.808 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:15.135490+0000", 2024-09-14T03:47:50.808 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:15.135490+0000", 2024-09-14T03:47:50.808 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.808 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 8, 2024-09-14T03:47:50.808 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.808 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.808 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:50.808 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.808 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.808 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.809 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.809 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 8, 2024-09-14T03:47:50.809 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.809 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.809 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.809 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.10", 2024-09-14T03:47:50.809 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.809 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.809 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.809 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 31, 2024-09-14T03:47:50.809 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.809 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:18:04.176970+0000", 2024-09-14T03:47:50.809 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.809 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.810 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.810 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.810 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.810 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.810 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.810 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.810 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.810 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.810 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.810 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.810 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.810 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.810 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.810 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.810 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.811 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.811 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.811 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 12, 2024-09-14T03:47:50.811 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 4, 2024-09-14T03:47:50.811 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.811 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 4, 2024-09-14T03:47:50.811 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.811 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.811 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.811 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.811 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.811 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 4, 2024-09-14T03:47:50.811 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.811 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.811 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.812 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.812 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.812 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.812 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.812 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.812 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.812 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.812 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.812 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.812 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 12, 2024-09-14T03:47:50.813 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.813 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.813 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.813 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.813 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.813 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:50.813 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:50.813 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:50.814 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.814 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 2, 2024-09-14T03:47:50.814 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "58'8" 2024-09-14T03:47:50.814 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.814 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.814 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.814 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.814 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.814 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:50.814 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.814 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:50.814 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.814 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.814 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.815 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.815 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.815 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.815 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.660478+0000", 2024-09-14T03:47:50.815 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.878343+0000", 2024-09-14T03:47:50.815 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.878343+0000", 2024-09-14T03:47:50.815 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.878624+0000", 2024-09-14T03:47:50.815 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.660478+0000", 2024-09-14T03:47:50.815 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.815 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.815 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.815 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.815 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.660478+0000", 2024-09-14T03:47:50.815 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.660478+0000", 2024-09-14T03:47:50.816 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.660478+0000", 2024-09-14T03:47:50.816 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.816 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.816 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.816 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.660478+0000", 2024-09-14T03:47:50.816 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.660478+0000", 2024-09-14T03:47:50.816 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.816 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.816 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.816 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.816 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.816 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.816 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.817 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.817 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.817 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.817 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.817 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.817 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.817 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.11", 2024-09-14T03:47:50.817 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.817 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.817 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.817 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:50.817 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.817 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T10:44:05.205671+0000", 2024-09-14T03:47:50.818 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.818 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.818 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.818 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.818 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.818 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.818 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.818 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.818 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.818 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.818 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.818 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.818 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.818 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.819 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.819 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.819 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.819 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.819 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.819 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.819 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.819 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.819 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.819 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.819 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.819 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.819 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.819 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.819 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.820 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.820 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.820 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.820 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.820 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.820 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.820 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.820 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.820 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.820 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.820 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.820 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.820 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.820 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.821 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.821 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.821 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.821 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.821 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.821 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.821 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:50.821 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.821 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:50.821 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.821 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.821 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.821 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.822 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.822 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.822 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:50.822 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.822 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 4, 2024-09-14T03:47:50.822 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.822 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.822 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:50.822 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.822 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.822 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.822 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.458928+0000", 2024-09-14T03:47:50.822 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.728182+0000", 2024-09-14T03:47:50.822 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.728182+0000", 2024-09-14T03:47:50.823 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.728489+0000", 2024-09-14T03:47:50.823 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.458928+0000", 2024-09-14T03:47:50.823 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.823 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.823 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.823 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:50.823 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.458928+0000", 2024-09-14T03:47:50.823 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.458928+0000", 2024-09-14T03:47:50.823 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.458928+0000", 2024-09-14T03:47:50.823 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.823 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.824 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.824 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.458928+0000", 2024-09-14T03:47:50.824 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.458928+0000", 2024-09-14T03:47:50.824 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.824 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 1, 2024-09-14T03:47:50.824 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.824 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.824 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:50.824 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.825 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.825 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.825 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.825 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 1, 2024-09-14T03:47:50.825 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.825 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.825 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.825 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.f", 2024-09-14T03:47:50.825 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.825 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.825 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.826 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 23, 2024-09-14T03:47:50.826 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.826 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T09:15:56.310710+0000", 2024-09-14T03:47:50.826 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.826 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.826 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.826 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 22, 2024-09-14T03:47:50.826 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.826 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.826 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.826 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.826 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.826 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.827 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.827 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.827 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.827 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.827 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.827 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.827 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.827 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.827 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.827 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 3, 2024-09-14T03:47:50.827 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 1, 2024-09-14T03:47:50.827 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.827 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 1, 2024-09-14T03:47:50.828 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.828 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.828 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.828 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.828 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.828 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.828 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.828 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.828 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.828 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.828 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.828 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.828 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.828 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.829 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.829 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.829 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.829 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.829 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 1, 2024-09-14T03:47:50.829 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 1 2024-09-14T03:47:50.829 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.829 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.829 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.829 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.829 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.829 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.829 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:50.830 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.830 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 4, 2024-09-14T03:47:50.830 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "58'1" 2024-09-14T03:47:50.830 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.830 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.830 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.830 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.830 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.830 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:50.830 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.830 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:50.830 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.830 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.831 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.831 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.831 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.831 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.831 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.661108+0000", 2024-09-14T03:47:50.831 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.880483+0000", 2024-09-14T03:47:50.831 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.880483+0000", 2024-09-14T03:47:50.831 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.880754+0000", 2024-09-14T03:47:50.831 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.661108+0000", 2024-09-14T03:47:50.831 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.831 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.831 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.831 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.831 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.661108+0000", 2024-09-14T03:47:50.832 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.661108+0000", 2024-09-14T03:47:50.832 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.661108+0000", 2024-09-14T03:47:50.832 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.832 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.832 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.832 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.661108+0000", 2024-09-14T03:47:50.832 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.661108+0000", 2024-09-14T03:47:50.832 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.832 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.832 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.832 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.832 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.832 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.833 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.833 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.833 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.833 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.833 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.833 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.833 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.833 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.e", 2024-09-14T03:47:50.833 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.833 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.833 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.833 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:50.833 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.833 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:32:53.245007+0000", 2024-09-14T03:47:50.834 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.834 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.834 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.834 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.834 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.834 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.834 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.834 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.834 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.834 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.834 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.834 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.835 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.835 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.835 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.835 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.835 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.835 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.835 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.835 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.835 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.835 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.835 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.835 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.835 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.835 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.836 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.836 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.836 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.836 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.836 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.836 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.836 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.836 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.836 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.836 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.836 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.836 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.836 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.836 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.837 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.837 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.837 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.837 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.837 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.837 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.837 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.837 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.837 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.837 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:50.837 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.837 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:50.837 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.837 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.838 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.838 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.838 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:50.838 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.838 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:50.838 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.838 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 2, 2024-09-14T03:47:50.838 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.838 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.838 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:50.838 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.838 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.838 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.838 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:15.135841+0000", 2024-09-14T03:47:50.839 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.726090+0000", 2024-09-14T03:47:50.839 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.726090+0000", 2024-09-14T03:47:50.839 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.726366+0000", 2024-09-14T03:47:50.839 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:15.135841+0000", 2024-09-14T03:47:50.839 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.839 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.839 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.839 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:50.839 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:15.135841+0000", 2024-09-14T03:47:50.839 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:15.135841+0000", 2024-09-14T03:47:50.839 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:15.135841+0000", 2024-09-14T03:47:50.839 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.839 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.839 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.839 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:15.135841+0000", 2024-09-14T03:47:50.840 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:15.135841+0000", 2024-09-14T03:47:50.840 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.840 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 2, 2024-09-14T03:47:50.840 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.840 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.840 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:50.840 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.840 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.840 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.840 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.840 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 2, 2024-09-14T03:47:50.841 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.841 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.841 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.841 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.e", 2024-09-14T03:47:50.841 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.841 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.841 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.841 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 25, 2024-09-14T03:47:50.841 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.841 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:45:33.572820+0000", 2024-09-14T03:47:50.842 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.842 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.842 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.842 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.842 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.842 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.842 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.842 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.842 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.842 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.842 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.842 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.842 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.842 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.843 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.843 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.843 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.843 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.843 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.843 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 3, 2024-09-14T03:47:50.843 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 1, 2024-09-14T03:47:50.843 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.843 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 1, 2024-09-14T03:47:50.843 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.843 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.844 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.844 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.844 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.844 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 1, 2024-09-14T03:47:50.844 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.844 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.844 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.844 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.844 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.844 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.844 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.844 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.844 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.844 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.845 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.845 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.845 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 3, 2024-09-14T03:47:50.845 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.845 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.845 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.845 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.845 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.845 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:50.845 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.845 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:50.845 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.845 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 2, 2024-09-14T03:47:50.846 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "58'2" 2024-09-14T03:47:50.846 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.846 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.846 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.846 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.846 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.846 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:50.846 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.846 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:50.846 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.846 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.846 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.846 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.847 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.847 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.847 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.661089+0000", 2024-09-14T03:47:50.847 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.883448+0000", 2024-09-14T03:47:50.847 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.883448+0000", 2024-09-14T03:47:50.847 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.883704+0000", 2024-09-14T03:47:50.847 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.661089+0000", 2024-09-14T03:47:50.847 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.847 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.847 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.847 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.847 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.661089+0000", 2024-09-14T03:47:50.847 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.661089+0000", 2024-09-14T03:47:50.847 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.661089+0000", 2024-09-14T03:47:50.848 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.848 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.848 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.848 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.661089+0000", 2024-09-14T03:47:50.848 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.661089+0000", 2024-09-14T03:47:50.848 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.848 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.848 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.848 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.848 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.848 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.848 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.849 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.849 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.849 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.849 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.849 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.849 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.849 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.f", 2024-09-14T03:47:50.849 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.849 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.849 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.849 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:50.849 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.849 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:16:27.946770+0000", 2024-09-14T03:47:50.849 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.850 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.850 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.850 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.850 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.850 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.850 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.850 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.850 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.850 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.850 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.850 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.850 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.850 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.850 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.851 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.851 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.851 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.851 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.851 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.851 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.851 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.851 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.851 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.851 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.851 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.851 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.851 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.851 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.852 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.852 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.852 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.852 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.852 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.852 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.852 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.852 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.852 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.852 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.852 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.852 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.852 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.852 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.853 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.853 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.853 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.853 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.853 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.853 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.853 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:50.853 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.854 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:50.854 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.854 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.854 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.854 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.854 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:50.854 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.854 INFO:tasks.workunit.client.0.smithi070.stdout: 3 2024-09-14T03:47:50.854 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.854 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 1, 2024-09-14T03:47:50.854 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.855 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.855 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:50.855 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.855 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.855 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.855 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.458573+0000", 2024-09-14T03:47:50.855 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.732929+0000", 2024-09-14T03:47:50.855 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.732929+0000", 2024-09-14T03:47:50.855 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.733205+0000", 2024-09-14T03:47:50.855 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.458573+0000", 2024-09-14T03:47:50.855 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.855 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.855 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.856 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:50.856 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.458573+0000", 2024-09-14T03:47:50.856 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.458573+0000", 2024-09-14T03:47:50.856 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.458573+0000", 2024-09-14T03:47:50.856 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.856 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.856 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.856 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.458573+0000", 2024-09-14T03:47:50.856 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.458573+0000", 2024-09-14T03:47:50.856 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.857 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 1, 2024-09-14T03:47:50.857 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.857 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.857 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:50.857 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.857 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.858 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.858 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.858 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 1, 2024-09-14T03:47:50.858 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.858 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.859 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.859 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.d", 2024-09-14T03:47:50.859 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.859 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.859 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.859 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 24, 2024-09-14T03:47:50.860 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.860 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T15:31:13.660965+0000", 2024-09-14T03:47:50.860 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.860 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.860 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.861 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 70, 2024-09-14T03:47:50.861 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.861 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.861 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.861 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.861 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.862 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.862 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.862 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.862 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.862 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.862 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.862 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.862 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.862 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.863 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.863 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 3, 2024-09-14T03:47:50.863 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 1, 2024-09-14T03:47:50.863 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.863 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 1, 2024-09-14T03:47:50.863 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.863 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.863 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.863 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.863 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.863 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.863 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.863 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.863 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.864 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.864 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.864 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.864 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.864 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.864 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.864 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.864 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.864 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.864 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 1, 2024-09-14T03:47:50.864 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 1 2024-09-14T03:47:50.864 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.865 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.865 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.865 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.865 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:50.865 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.865 INFO:tasks.workunit.client.0.smithi070.stdout: 3 2024-09-14T03:47:50.865 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.865 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 1, 2024-09-14T03:47:50.865 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "58'1" 2024-09-14T03:47:50.865 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.865 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.865 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.865 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.866 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.866 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:50.866 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.866 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 5, 2024-09-14T03:47:50.866 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.866 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.866 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.866 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.866 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.866 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.866 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.461995+0000", 2024-09-14T03:47:50.866 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.884747+0000", 2024-09-14T03:47:50.867 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.884747+0000", 2024-09-14T03:47:50.867 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.886158+0000", 2024-09-14T03:47:50.867 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.461995+0000", 2024-09-14T03:47:50.867 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.867 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.867 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.867 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.867 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.461995+0000", 2024-09-14T03:47:50.867 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.461995+0000", 2024-09-14T03:47:50.867 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.461995+0000", 2024-09-14T03:47:50.867 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.867 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.867 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.868 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.461995+0000", 2024-09-14T03:47:50.868 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.461995+0000", 2024-09-14T03:47:50.868 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.868 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.868 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.868 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.868 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.868 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.868 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.868 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.868 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.868 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.868 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.869 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.869 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.869 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.c", 2024-09-14T03:47:50.869 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.869 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.869 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.869 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:50.869 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.869 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:49:56.265783+0000", 2024-09-14T03:47:50.869 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.869 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.869 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.869 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.870 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.870 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.870 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.870 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.870 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.870 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.870 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.870 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.870 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.871 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.871 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.871 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.871 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.871 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.871 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.871 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.871 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.871 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.871 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.871 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.872 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.872 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.872 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.872 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.872 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.872 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.872 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.872 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.872 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.872 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.872 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.873 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.873 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.873 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.873 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.873 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.873 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.873 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.873 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.873 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.873 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.874 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.874 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.874 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.874 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.874 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:50.874 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.874 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 5, 2024-09-14T03:47:50.874 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.874 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.874 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.874 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.875 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:50.875 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.875 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:50.875 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.875 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 2, 2024-09-14T03:47:50.875 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.875 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.875 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:50.875 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.875 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.875 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.875 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:15.135633+0000", 2024-09-14T03:47:50.875 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.732067+0000", 2024-09-14T03:47:50.876 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.732067+0000", 2024-09-14T03:47:50.876 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.732401+0000", 2024-09-14T03:47:50.876 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:15.135633+0000", 2024-09-14T03:47:50.876 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.876 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.876 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.876 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:50.876 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:15.135633+0000", 2024-09-14T03:47:50.876 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:15.135633+0000", 2024-09-14T03:47:50.876 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:15.135633+0000", 2024-09-14T03:47:50.876 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.876 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.877 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.877 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:15.135633+0000", 2024-09-14T03:47:50.877 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:15.135633+0000", 2024-09-14T03:47:50.877 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.877 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.877 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.877 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.877 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:50.877 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.877 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.877 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.877 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.877 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.878 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.878 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.878 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.878 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.c", 2024-09-14T03:47:50.878 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.878 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.878 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.878 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 23, 2024-09-14T03:47:50.878 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.878 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T09:34:00.566245+0000", 2024-09-14T03:47:50.879 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.879 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.879 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.879 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.879 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.879 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.879 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.879 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.879 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.879 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.879 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.879 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.879 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.880 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.880 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.880 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.880 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.880 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.880 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.881 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.881 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.881 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.881 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.881 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.881 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.881 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.881 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.882 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.882 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.882 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.882 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.882 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.882 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.882 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.883 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.883 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.883 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.883 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.883 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.883 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.883 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.883 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.883 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.883 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.883 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.884 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.884 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.884 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:50.884 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.884 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:50.884 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.884 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 2, 2024-09-14T03:47:50.884 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.884 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.884 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.884 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.884 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.885 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.885 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:50.885 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.885 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:50.885 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.885 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.885 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.885 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.885 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.885 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.885 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.660858+0000", 2024-09-14T03:47:50.885 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.881219+0000", 2024-09-14T03:47:50.885 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.881219+0000", 2024-09-14T03:47:50.886 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.881736+0000", 2024-09-14T03:47:50.886 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.660858+0000", 2024-09-14T03:47:50.886 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.886 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.886 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.886 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.886 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.660858+0000", 2024-09-14T03:47:50.886 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.660858+0000", 2024-09-14T03:47:50.886 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.660858+0000", 2024-09-14T03:47:50.886 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.886 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.886 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.886 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.660858+0000", 2024-09-14T03:47:50.887 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.660858+0000", 2024-09-14T03:47:50.887 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.887 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.887 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.887 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.887 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.887 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.887 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.887 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.887 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.887 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.887 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.887 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.887 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.888 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.d", 2024-09-14T03:47:50.888 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.888 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.888 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.888 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:50.888 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.888 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T09:24:23.342066+0000", 2024-09-14T03:47:50.888 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.888 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.888 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.888 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.888 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.888 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.889 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.889 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.889 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.889 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.889 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.889 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.889 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.889 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.889 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.889 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.889 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.889 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.889 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.890 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.890 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.890 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.890 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.890 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.890 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.890 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.890 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.890 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.890 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.890 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.890 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.890 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.890 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.891 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.891 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.891 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.891 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.891 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.891 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.891 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.891 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.891 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.891 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.891 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.891 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.891 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.891 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.892 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.892 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.892 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:50.892 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.892 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:50.892 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.892 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.892 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.892 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.892 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:50.893 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.893 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:50.893 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.893 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 6, 2024-09-14T03:47:50.893 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.893 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.893 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 62, 2024-09-14T03:47:50.893 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.893 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.893 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.893 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:20.503002+0000", 2024-09-14T03:47:50.893 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:45:09.755484+0000", 2024-09-14T03:47:50.893 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:45:09.755484+0000", 2024-09-14T03:47:50.893 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:45:14.545163+0000", 2024-09-14T03:47:50.894 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:20.503002+0000", 2024-09-14T03:47:50.894 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:50.894 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.894 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:50.894 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 63, 2024-09-14T03:47:50.894 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:20.503002+0000", 2024-09-14T03:47:50.894 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:20.503002+0000", 2024-09-14T03:47:50.894 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:20.503002+0000", 2024-09-14T03:47:50.894 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.894 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.895 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:50.895 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:20.503002+0000", 2024-09-14T03:47:50.895 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:20.503002+0000", 2024-09-14T03:47:50.895 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.895 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.895 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.895 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.895 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 62, 2024-09-14T03:47:50.895 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.895 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.895 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.896 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.896 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.896 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.896 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.896 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.896 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "4.5", 2024-09-14T03:47:50.896 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.896 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.896 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 68, 2024-09-14T03:47:50.896 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 19, 2024-09-14T03:47:50.896 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.896 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:18:49.100636+0000", 2024-09-14T03:47:50.896 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.000393397, 2024-09-14T03:47:50.897 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.897 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.897 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.897 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.897 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.897 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.897 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.897 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.897 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.897 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.897 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.897 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.897 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.897 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.898 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.898 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.898 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.898 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.898 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.898 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.898 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.898 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.898 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.898 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.898 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.898 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.898 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.899 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.899 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.899 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.899 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.899 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.899 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.899 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.899 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.899 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.899 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.899 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.899 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.899 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.900 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.900 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.900 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.900 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.900 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.900 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.900 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:50.900 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.900 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:50.900 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.900 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 6, 2024-09-14T03:47:50.900 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.900 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.900 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.901 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.901 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.901 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.901 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:50.901 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.901 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:50.901 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.901 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.901 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 20, 2024-09-14T03:47:50.901 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.901 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.901 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.902 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:47:47.131642+0000", 2024-09-14T03:47:50.902 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:36.141953+0000", 2024-09-14T03:47:50.902 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:36.141953+0000", 2024-09-14T03:47:50.902 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:36.147227+0000", 2024-09-14T03:47:50.902 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:47:47.131642+0000", 2024-09-14T03:47:50.902 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:41:39.458815+0000", 2024-09-14T03:47:50.902 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.902 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:41:39.458815+0000", 2024-09-14T03:47:50.902 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 51, 2024-09-14T03:47:50.902 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:47:47.131642+0000", 2024-09-14T03:47:50.902 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:47:47.131642+0000", 2024-09-14T03:47:50.902 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:47:47.131642+0000", 2024-09-14T03:47:50.902 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.902 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.903 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:41:39.458815+0000", 2024-09-14T03:47:50.903 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:47:47.131642+0000", 2024-09-14T03:47:50.903 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:47:47.131642+0000", 2024-09-14T03:47:50.903 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.903 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 95, 2024-09-14T03:47:50.903 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.903 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.903 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 50, 2024-09-14T03:47:50.903 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.903 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.903 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.903 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.903 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 95, 2024-09-14T03:47:50.903 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.904 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.904 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.904 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "1.0", 2024-09-14T03:47:50.904 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.904 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.904 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 68, 2024-09-14T03:47:50.904 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 116, 2024-09-14T03:47:50.904 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.904 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T05:38:43.056328+0000", 2024-09-14T03:47:50.904 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.904 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.904 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.905 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 590368, 2024-09-14T03:47:50.905 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.905 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 590368, 2024-09-14T03:47:50.905 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.905 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.905 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.905 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.905 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.905 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.905 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.905 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.905 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.905 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.905 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.906 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.906 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.906 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 6, 2024-09-14T03:47:50.906 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 2, 2024-09-14T03:47:50.906 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.906 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 2, 2024-09-14T03:47:50.906 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.906 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.906 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.906 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.906 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.906 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.906 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.906 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 2, 2024-09-14T03:47:50.907 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.907 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.907 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.907 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.907 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.907 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 272, 2024-09-14T03:47:50.907 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 479, 2024-09-14T03:47:50.907 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.907 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.907 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.907 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 145, 2024-09-14T03:47:50.907 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 1628 2024-09-14T03:47:50.907 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.907 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.908 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.908 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.908 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.908 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.908 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:50.908 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.908 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:50.908 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "68'95" 2024-09-14T03:47:50.908 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.908 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.908 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.908 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.909 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:50.909 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:50.909 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.909 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 5, 2024-09-14T03:47:50.909 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.909 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.909 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:50.909 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.909 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.909 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.909 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.461867+0000", 2024-09-14T03:47:50.909 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.733256+0000", 2024-09-14T03:47:50.909 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.733256+0000", 2024-09-14T03:47:50.910 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.733557+0000", 2024-09-14T03:47:50.910 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.461867+0000", 2024-09-14T03:47:50.910 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.910 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.910 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.910 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:50.910 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.461867+0000", 2024-09-14T03:47:50.910 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.461867+0000", 2024-09-14T03:47:50.910 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.461867+0000", 2024-09-14T03:47:50.910 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.910 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.910 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.910 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.461867+0000", 2024-09-14T03:47:50.910 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.461867+0000", 2024-09-14T03:47:50.911 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.911 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.911 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.911 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.911 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:50.911 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.911 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.911 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.911 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.911 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.911 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.911 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.911 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.911 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.3", 2024-09-14T03:47:50.912 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.912 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.912 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.912 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 23, 2024-09-14T03:47:50.912 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.912 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T10:09:12.987156+0000", 2024-09-14T03:47:50.912 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.912 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.912 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.912 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.913 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.913 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.913 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.913 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.913 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.913 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.913 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.913 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.913 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.913 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.913 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.913 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.913 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.913 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.914 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.914 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.914 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.914 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.914 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.914 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.914 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.914 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.914 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.914 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.914 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.914 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.914 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.915 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.915 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.915 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.915 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.915 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.915 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.915 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.915 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.915 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.915 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.915 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.915 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.916 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.916 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.916 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.916 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.916 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.916 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:50.916 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:50.916 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.916 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 5, 2024-09-14T03:47:50.916 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.916 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.916 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.916 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.917 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.917 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.917 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:50.917 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.917 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 3, 2024-09-14T03:47:50.917 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.917 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.917 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.917 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.917 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.917 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.917 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.857593+0000", 2024-09-14T03:47:50.917 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.875241+0000", 2024-09-14T03:47:50.918 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.875241+0000", 2024-09-14T03:47:50.918 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.875525+0000", 2024-09-14T03:47:50.918 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.857593+0000", 2024-09-14T03:47:50.918 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.918 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.918 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.918 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.918 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.857593+0000", 2024-09-14T03:47:50.918 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.857593+0000", 2024-09-14T03:47:50.918 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.857593+0000", 2024-09-14T03:47:50.918 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.919 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.919 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.919 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.857593+0000", 2024-09-14T03:47:50.919 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.857593+0000", 2024-09-14T03:47:50.919 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.919 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.919 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.919 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.919 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.919 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.919 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.919 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.919 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.919 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.920 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.920 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.920 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.920 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.2", 2024-09-14T03:47:50.920 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.920 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.920 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.920 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:50.920 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.920 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:34:47.346963+0000", 2024-09-14T03:47:50.920 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.920 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.920 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.920 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.921 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.921 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.921 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.921 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.921 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.921 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.921 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.921 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.921 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.921 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.921 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.921 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.921 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.922 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.922 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.922 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.922 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.922 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.922 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.922 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.922 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.922 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.922 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.922 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.922 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.922 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.922 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.922 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.923 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.923 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.923 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.923 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.923 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.923 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.923 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.923 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.923 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.923 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.923 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.923 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.924 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.924 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.924 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.924 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.924 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.924 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:50.924 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.924 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 3, 2024-09-14T03:47:50.924 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.924 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.924 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.924 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.924 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.925 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:50.925 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:50.925 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.925 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 0, 2024-09-14T03:47:50.925 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.925 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.925 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 62, 2024-09-14T03:47:50.925 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.925 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.925 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.925 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:20.503141+0000", 2024-09-14T03:47:50.925 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:45:09.711005+0000", 2024-09-14T03:47:50.926 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:45:09.711005+0000", 2024-09-14T03:47:50.926 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:45:14.684739+0000", 2024-09-14T03:47:50.926 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:20.503141+0000", 2024-09-14T03:47:50.926 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:50.926 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.926 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:50.926 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 63, 2024-09-14T03:47:50.926 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:20.503141+0000", 2024-09-14T03:47:50.926 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:20.503141+0000", 2024-09-14T03:47:50.926 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:20.503141+0000", 2024-09-14T03:47:50.926 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.926 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.927 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:50.927 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:20.503141+0000", 2024-09-14T03:47:50.927 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:20.503141+0000", 2024-09-14T03:47:50.927 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.927 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.927 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.927 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.927 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 62, 2024-09-14T03:47:50.927 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.927 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.927 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.928 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.928 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.928 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.928 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.928 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.928 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "4.6", 2024-09-14T03:47:50.928 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.928 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.928 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 68, 2024-09-14T03:47:50.928 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 19, 2024-09-14T03:47:50.928 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.928 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T09:20:35.312720+0000", 2024-09-14T03:47:50.929 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.00019552, 2024-09-14T03:47:50.929 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.929 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.929 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.929 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.929 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.929 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.929 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.929 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.929 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.930 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.930 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.930 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.930 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.930 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.930 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.930 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.930 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.930 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.930 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.930 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.930 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.930 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.930 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.931 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.931 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.931 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.931 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.931 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.931 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.931 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.931 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.931 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.931 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.931 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.931 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.931 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.931 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.931 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.932 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.932 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.932 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.932 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.932 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.932 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.932 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.932 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.932 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.932 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:50.932 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:50.932 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.932 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 0, 2024-09-14T03:47:50.933 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.933 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.933 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.933 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.933 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.933 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:50.933 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:50.933 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.933 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:50.933 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.933 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.933 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:50.933 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.933 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.934 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.934 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.660472+0000", 2024-09-14T03:47:50.934 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.731812+0000", 2024-09-14T03:47:50.934 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.731812+0000", 2024-09-14T03:47:50.934 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.732376+0000", 2024-09-14T03:47:50.934 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.660472+0000", 2024-09-14T03:47:50.934 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.934 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.934 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.934 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:50.934 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.660472+0000", 2024-09-14T03:47:50.934 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.660472+0000", 2024-09-14T03:47:50.934 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.660472+0000", 2024-09-14T03:47:50.934 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.934 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.935 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.935 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.660472+0000", 2024-09-14T03:47:50.935 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.660472+0000", 2024-09-14T03:47:50.935 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.935 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.935 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.935 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.935 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:50.935 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.935 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.935 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.935 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.935 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.935 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.936 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.936 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.936 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.0", 2024-09-14T03:47:50.936 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.936 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.936 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.936 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 24, 2024-09-14T03:47:50.936 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.936 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:35:26.262383+0000", 2024-09-14T03:47:50.936 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.936 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.936 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.936 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.936 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.937 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.937 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.937 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.937 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.937 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.937 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.937 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.937 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.937 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.937 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.937 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.937 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.937 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.938 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.938 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.938 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.938 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.938 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.938 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.938 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.938 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.938 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.938 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.938 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.938 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.938 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.939 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.939 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.939 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.939 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.939 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.939 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.939 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.939 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.939 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.939 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.939 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.939 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.939 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.939 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.940 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.940 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.940 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:50.940 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:50.940 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:50.940 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.940 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:50.940 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.940 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.940 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.940 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.940 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.940 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.941 INFO:tasks.workunit.client.0.smithi070.stdout: 3 2024-09-14T03:47:50.941 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.941 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 0, 2024-09-14T03:47:50.941 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.941 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.941 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.941 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.941 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.941 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.941 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.683969+0000", 2024-09-14T03:47:50.941 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.876225+0000", 2024-09-14T03:47:50.941 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.876225+0000", 2024-09-14T03:47:50.941 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.876511+0000", 2024-09-14T03:47:50.942 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.683969+0000", 2024-09-14T03:47:50.942 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.942 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.942 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.942 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.942 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.683969+0000", 2024-09-14T03:47:50.942 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.683969+0000", 2024-09-14T03:47:50.942 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.683969+0000", 2024-09-14T03:47:50.942 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.942 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.942 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.942 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.683969+0000", 2024-09-14T03:47:50.943 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.683969+0000", 2024-09-14T03:47:50.943 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.943 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.943 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.943 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.943 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.943 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.943 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.943 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.943 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.943 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.943 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.944 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.944 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.944 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.1", 2024-09-14T03:47:50.944 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.944 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.944 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.944 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:50.944 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.944 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T10:36:40.283622+0000", 2024-09-14T03:47:50.944 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.944 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.944 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.945 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.945 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.945 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.945 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.945 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.945 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.945 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.945 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.945 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.945 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.945 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.945 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.945 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.946 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.946 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.946 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.946 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.946 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.946 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.946 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.946 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.946 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.946 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.946 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.946 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.946 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.946 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.947 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.947 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.947 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.947 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.947 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.947 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.947 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.947 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.947 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.947 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.947 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.947 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.947 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.947 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.948 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.948 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.948 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.948 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:50.948 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.948 INFO:tasks.workunit.client.0.smithi070.stdout: 3 2024-09-14T03:47:50.948 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.948 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 0, 2024-09-14T03:47:50.948 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.948 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.948 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.948 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.948 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:50.949 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.949 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:50.949 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.949 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 1, 2024-09-14T03:47:50.949 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.949 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.949 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 62, 2024-09-14T03:47:50.949 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.949 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.949 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.949 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:20.464110+0000", 2024-09-14T03:47:50.949 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:45:09.879190+0000", 2024-09-14T03:47:50.949 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:45:09.879190+0000", 2024-09-14T03:47:50.950 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:45:14.458990+0000", 2024-09-14T03:47:50.950 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:20.464110+0000", 2024-09-14T03:47:50.950 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:50.950 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.950 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:50.950 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 63, 2024-09-14T03:47:50.950 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:20.464110+0000", 2024-09-14T03:47:50.950 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:20.464110+0000", 2024-09-14T03:47:50.950 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:20.464110+0000", 2024-09-14T03:47:50.950 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.950 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.950 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:50.950 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:20.464110+0000", 2024-09-14T03:47:50.950 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:20.464110+0000", 2024-09-14T03:47:50.951 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.951 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.951 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.951 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.951 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 62, 2024-09-14T03:47:50.951 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.951 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.951 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.951 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.951 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.951 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.951 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.951 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.952 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "4.7", 2024-09-14T03:47:50.952 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.952 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.952 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 68, 2024-09-14T03:47:50.952 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 19, 2024-09-14T03:47:50.952 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.952 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T07:23:17.328924+0000", 2024-09-14T03:47:50.952 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.000571641, 2024-09-14T03:47:50.952 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.952 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.952 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.952 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.952 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.953 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.953 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.953 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.953 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.953 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.953 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.953 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.953 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.953 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.953 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.953 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.953 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.953 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.954 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.954 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.954 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.954 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.954 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.954 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.954 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.954 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.954 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.954 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.955 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.955 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.955 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.955 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.955 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.955 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.955 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.955 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.955 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.955 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.955 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.955 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.955 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.955 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.955 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.956 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.956 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.956 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.956 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:50.956 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.956 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:50.956 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.956 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 1, 2024-09-14T03:47:50.956 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.956 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.956 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.956 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.956 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:50.957 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.957 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:50.957 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.957 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 2, 2024-09-14T03:47:50.957 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.957 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.957 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:50.957 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.957 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.957 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.957 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:15.135684+0000", 2024-09-14T03:47:50.957 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.726879+0000", 2024-09-14T03:47:50.957 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.726879+0000", 2024-09-14T03:47:50.958 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.727129+0000", 2024-09-14T03:47:50.958 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:15.135684+0000", 2024-09-14T03:47:50.958 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.958 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.958 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.958 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:50.958 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:15.135684+0000", 2024-09-14T03:47:50.958 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:15.135684+0000", 2024-09-14T03:47:50.958 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:15.135684+0000", 2024-09-14T03:47:50.958 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.958 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.958 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.958 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:15.135684+0000", 2024-09-14T03:47:50.958 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:15.135684+0000", 2024-09-14T03:47:50.959 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.959 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.959 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.959 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.959 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:50.959 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.959 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.959 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.959 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.959 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.959 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.959 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.959 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.959 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.1", 2024-09-14T03:47:50.960 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.960 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.960 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.960 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 23, 2024-09-14T03:47:50.960 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.960 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T09:54:41.748651+0000", 2024-09-14T03:47:50.960 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.960 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.960 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.961 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.961 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.961 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.961 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.961 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.961 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.961 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.961 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.961 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.961 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.961 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.961 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.962 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.962 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.962 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.962 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.962 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.962 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.962 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.962 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.963 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.963 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.963 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.963 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.963 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.963 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.963 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.963 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.963 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.963 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.963 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.964 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.964 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.964 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.964 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.964 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.964 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.964 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.964 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.964 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.964 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.964 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.964 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.964 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.965 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:50.965 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:50.965 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:50.965 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.965 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 2, 2024-09-14T03:47:50.965 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.965 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.965 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.965 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.965 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:50.965 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:50.965 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:50.965 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.966 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 1, 2024-09-14T03:47:50.966 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.966 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.966 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.966 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.966 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.966 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.966 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.457983+0000", 2024-09-14T03:47:50.966 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.870707+0000", 2024-09-14T03:47:50.966 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.870707+0000", 2024-09-14T03:47:50.966 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.871005+0000", 2024-09-14T03:47:50.966 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.457983+0000", 2024-09-14T03:47:50.967 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.967 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.967 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.967 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:50.967 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.457983+0000", 2024-09-14T03:47:50.967 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.457983+0000", 2024-09-14T03:47:50.967 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.457983+0000", 2024-09-14T03:47:50.967 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.967 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.967 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.967 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.457983+0000", 2024-09-14T03:47:50.968 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.457983+0000", 2024-09-14T03:47:50.968 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.968 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.968 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.968 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.968 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:50.968 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.968 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.968 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.968 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.968 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.968 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.968 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.969 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.969 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.0", 2024-09-14T03:47:50.969 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.969 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.969 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.969 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:50.969 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.969 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:30:22.758581+0000", 2024-09-14T03:47:50.969 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.969 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.969 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.969 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.969 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.970 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.970 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.970 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.970 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.970 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.970 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.970 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.970 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.970 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.970 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.970 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.970 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.970 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.970 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.971 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.971 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.971 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.971 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.971 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.971 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.971 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.971 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.971 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.971 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.971 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.972 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.972 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.972 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.972 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.972 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.972 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.972 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.972 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.972 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.972 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.972 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.972 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.972 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.973 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.973 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.973 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.973 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.973 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:50.973 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:50.973 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:50.973 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.973 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 1, 2024-09-14T03:47:50.973 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.973 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.973 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.973 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.974 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:50.974 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:50.974 INFO:tasks.workunit.client.0.smithi070.stdout: 3 2024-09-14T03:47:50.974 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.974 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 1, 2024-09-14T03:47:50.974 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.974 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.974 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 62, 2024-09-14T03:47:50.974 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.974 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.974 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.975 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:20.464062+0000", 2024-09-14T03:47:50.975 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:45:09.711634+0000", 2024-09-14T03:47:50.975 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:45:09.711634+0000", 2024-09-14T03:47:50.975 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:45:14.458757+0000", 2024-09-14T03:47:50.975 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:20.464062+0000", 2024-09-14T03:47:50.975 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:50.975 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.975 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:50.975 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 63, 2024-09-14T03:47:50.975 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:20.464062+0000", 2024-09-14T03:47:50.975 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:20.464062+0000", 2024-09-14T03:47:50.975 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:20.464062+0000", 2024-09-14T03:47:50.975 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.976 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.976 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:50.976 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:20.464062+0000", 2024-09-14T03:47:50.976 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:20.464062+0000", 2024-09-14T03:47:50.976 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.976 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.976 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.976 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.976 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 62, 2024-09-14T03:47:50.976 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.976 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.976 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.977 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.977 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.977 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.977 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.977 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.977 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "4.4", 2024-09-14T03:47:50.977 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.977 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.977 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 68, 2024-09-14T03:47:50.977 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 19, 2024-09-14T03:47:50.977 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.977 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:42:58.495982+0000", 2024-09-14T03:47:50.977 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.000547481, 2024-09-14T03:47:50.977 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.978 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.978 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.978 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.978 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.978 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.978 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.978 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.978 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.978 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.978 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.978 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.978 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.978 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.978 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.979 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.979 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.979 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.979 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.979 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.979 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.979 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.979 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.979 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.979 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.979 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.979 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.979 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.979 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.980 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.980 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.980 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.980 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.980 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.980 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.980 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.980 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.980 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.980 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.981 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.981 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.981 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.981 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.981 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.981 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.981 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.981 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:50.981 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:50.981 INFO:tasks.workunit.client.0.smithi070.stdout: 3 2024-09-14T03:47:50.981 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.981 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 1, 2024-09-14T03:47:50.981 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.982 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.982 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.982 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.982 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.982 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:50.982 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:50.982 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.982 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 5, 2024-09-14T03:47:50.982 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.982 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.982 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:50.983 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.983 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.983 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.983 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.461874+0000", 2024-09-14T03:47:50.983 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.732753+0000", 2024-09-14T03:47:50.983 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.732753+0000", 2024-09-14T03:47:50.983 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.733038+0000", 2024-09-14T03:47:50.983 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.461874+0000", 2024-09-14T03:47:50.983 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.983 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.983 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.983 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:50.983 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.461874+0000", 2024-09-14T03:47:50.983 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.461874+0000", 2024-09-14T03:47:50.984 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.461874+0000", 2024-09-14T03:47:50.984 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.984 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.984 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:50.984 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.461874+0000", 2024-09-14T03:47:50.984 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.461874+0000", 2024-09-14T03:47:50.984 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.984 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.984 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.984 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.984 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:50.984 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.984 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.985 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.985 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.985 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.985 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.985 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.985 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.985 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.2", 2024-09-14T03:47:50.985 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.985 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.985 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.985 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 23, 2024-09-14T03:47:50.985 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.985 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:40:49.754211+0000", 2024-09-14T03:47:50.986 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.986 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.986 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.986 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.986 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.986 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.986 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.986 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.986 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.986 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.986 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.986 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.986 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.986 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.986 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.987 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.987 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.987 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.987 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.987 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.987 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.987 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.987 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.987 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.987 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.987 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.987 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.988 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.988 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.988 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.988 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.988 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.988 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.989 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.989 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.989 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.989 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.989 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.989 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:50.989 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:50.989 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:50.989 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:50.989 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:50.989 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.990 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:50.990 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:50.990 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:50.990 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:50.990 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:50.990 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:50.990 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.991 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 5, 2024-09-14T03:47:50.991 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:50.991 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:50.991 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:50.991 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:50.991 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:50.992 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:50.992 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:50.992 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:50.992 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 4, 2024-09-14T03:47:50.992 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:50.992 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:50.993 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:50.993 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:50.993 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:50.993 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:50.993 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.458431+0000", 2024-09-14T03:47:50.993 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:26.429935+0000", 2024-09-14T03:47:50.993 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:26.429935+0000", 2024-09-14T03:47:50.993 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:26.430320+0000", 2024-09-14T03:47:50.993 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.458431+0000", 2024-09-14T03:47:50.993 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.993 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:50.994 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.994 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 60, 2024-09-14T03:47:50.994 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.458431+0000", 2024-09-14T03:47:50.994 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.458431+0000", 2024-09-14T03:47:50.994 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.458431+0000", 2024-09-14T03:47:50.994 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:50.995 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:50.995 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:50.995 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.458431+0000", 2024-09-14T03:47:50.995 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.458431+0000", 2024-09-14T03:47:50.995 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:50.995 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:50.995 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:50.995 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:50.995 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 59, 2024-09-14T03:47:50.995 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:50.995 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:50.996 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:50.996 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:50.996 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:50.996 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:50.996 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:50.996 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:50.996 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.3", 2024-09-14T03:47:50.996 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:50.996 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:50.996 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:50.996 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 30, 2024-09-14T03:47:50.996 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:50.996 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T15:12:35.577538+0000", 2024-09-14T03:47:50.997 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:50.997 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:50.997 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:50.997 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:50.997 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:50.997 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:50.997 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:50.997 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:50.997 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:50.997 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:50.997 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:50.997 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:50.997 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:50.998 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:50.998 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:50.998 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:50.998 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:50.998 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:50.998 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:50.998 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:50.998 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:50.998 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:50.998 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:50.998 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:50.998 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:50.998 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:50.998 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:50.999 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:50.999 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:50.999 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:50.999 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:50.999 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:50.999 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:50.999 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:50.999 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:50.999 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:50.999 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:50.999 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:50.999 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:51.000 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:51.000 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:51.000 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:51.000 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:51.000 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.000 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:51.001 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:51.001 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:51.001 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:51.001 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:51.001 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:51.001 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.001 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 4, 2024-09-14T03:47:51.001 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:51.001 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.001 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.001 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:51.001 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:51.001 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:51.001 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:51.002 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.002 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 1, 2024-09-14T03:47:51.002 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:51.002 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:51.002 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 62, 2024-09-14T03:47:51.002 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:51.002 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:51.002 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:51.002 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:20.464076+0000", 2024-09-14T03:47:51.002 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:45:09.879248+0000", 2024-09-14T03:47:51.003 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:45:09.879248+0000", 2024-09-14T03:47:51.004 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:45:14.458846+0000", 2024-09-14T03:47:51.004 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:20.464076+0000", 2024-09-14T03:47:51.004 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:51.004 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:51.004 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:51.004 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 63, 2024-09-14T03:47:51.004 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:20.464076+0000", 2024-09-14T03:47:51.004 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:20.464076+0000", 2024-09-14T03:47:51.004 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:20.464076+0000", 2024-09-14T03:47:51.004 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:51.004 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:51.004 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:51.005 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:20.464076+0000", 2024-09-14T03:47:51.005 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:20.464076+0000", 2024-09-14T03:47:51.005 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:51.005 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 2, 2024-09-14T03:47:51.005 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:51.005 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:51.005 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 62, 2024-09-14T03:47:51.005 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:51.005 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:51.005 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:51.005 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:51.005 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 2, 2024-09-14T03:47:51.006 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:51.006 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:51.006 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:51.006 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "4.2", 2024-09-14T03:47:51.006 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:51.006 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:51.006 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 68, 2024-09-14T03:47:51.006 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:51.006 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:51.006 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:02:23.634325+0000", 2024-09-14T03:47:51.006 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.000682786, 2024-09-14T03:47:51.006 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:51.006 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:51.006 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 19, 2024-09-14T03:47:51.007 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:51.007 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:51.007 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:51.007 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:51.007 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:51.007 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:51.007 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:51.007 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:51.007 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:51.007 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:51.007 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:51.007 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:51.007 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:51.008 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:51.008 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:51.008 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 3, 2024-09-14T03:47:51.008 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 1, 2024-09-14T03:47:51.008 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:51.008 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 1, 2024-09-14T03:47:51.008 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:51.008 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:51.008 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:51.009 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:51.009 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:51.009 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:51.009 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:51.009 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:51.009 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:51.009 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:51.009 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:51.009 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:51.009 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:51.009 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:51.009 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:51.009 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:51.010 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:51.010 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:51.010 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 2, 2024-09-14T03:47:51.010 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 2 2024-09-14T03:47:51.010 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.010 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:51.010 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:51.010 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:51.010 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:51.010 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:51.010 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:51.010 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.010 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 1, 2024-09-14T03:47:51.010 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "67'2" 2024-09-14T03:47:51.011 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.011 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.011 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:51.011 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:51.011 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:51.011 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:51.011 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.011 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 1, 2024-09-14T03:47:51.011 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:51.011 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:51.011 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:51.011 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:51.011 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:51.011 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:51.012 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.458338+0000", 2024-09-14T03:47:51.012 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.733357+0000", 2024-09-14T03:47:51.012 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.733357+0000", 2024-09-14T03:47:51.012 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.733621+0000", 2024-09-14T03:47:51.012 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.458338+0000", 2024-09-14T03:47:51.012 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.012 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:51.012 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.012 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:51.012 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.458338+0000", 2024-09-14T03:47:51.012 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.458338+0000", 2024-09-14T03:47:51.012 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.458338+0000", 2024-09-14T03:47:51.013 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:51.013 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:51.013 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.013 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.458338+0000", 2024-09-14T03:47:51.013 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.458338+0000", 2024-09-14T03:47:51.013 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:51.013 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:51.013 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:51.013 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:51.013 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:51.013 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:51.013 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:51.014 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:51.014 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:51.014 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:51.014 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:51.014 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:51.014 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:51.014 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.4", 2024-09-14T03:47:51.014 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:51.014 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:51.014 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:51.014 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 23, 2024-09-14T03:47:51.015 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:51.015 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T05:25:53.791926+0000", 2024-09-14T03:47:51.015 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:51.015 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:51.015 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:51.015 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:51.015 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:51.015 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:51.015 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:51.015 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:51.015 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:51.015 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:51.015 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:51.015 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:51.016 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:51.016 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:51.016 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:51.016 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:51.016 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:51.016 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:51.016 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:51.016 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:51.016 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:51.016 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:51.017 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:51.017 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:51.017 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:51.017 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:51.017 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:51.017 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:51.017 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:51.017 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:51.017 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:51.017 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:51.017 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:51.017 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:51.017 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:51.018 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:51.018 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:51.018 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:51.018 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:51.018 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:51.018 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:51.018 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:51.018 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:51.018 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.018 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:51.018 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:51.019 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:51.019 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:51.019 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:51.019 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:51.019 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.019 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 1, 2024-09-14T03:47:51.019 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:51.019 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.019 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.019 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:51.019 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:51.019 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:51.019 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:51.020 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.020 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 5, 2024-09-14T03:47:51.020 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:51.020 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:51.020 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:51.020 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:51.020 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:51.020 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:51.020 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.462075+0000", 2024-09-14T03:47:51.020 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.879925+0000", 2024-09-14T03:47:51.020 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.879925+0000", 2024-09-14T03:47:51.021 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.880200+0000", 2024-09-14T03:47:51.021 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.462075+0000", 2024-09-14T03:47:51.021 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.021 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:51.021 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.021 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:51.021 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.462075+0000", 2024-09-14T03:47:51.021 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.462075+0000", 2024-09-14T03:47:51.021 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.462075+0000", 2024-09-14T03:47:51.021 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:51.021 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:51.021 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.022 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.462075+0000", 2024-09-14T03:47:51.022 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.462075+0000", 2024-09-14T03:47:51.022 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:51.022 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:51.022 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:51.022 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:51.022 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:51.022 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:51.022 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:51.022 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:51.022 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:51.022 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:51.022 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:51.022 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:51.022 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:51.022 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.5", 2024-09-14T03:47:51.023 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:51.023 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:51.023 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:51.023 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:51.023 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:51.023 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T10:55:24.738215+0000", 2024-09-14T03:47:51.023 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:51.023 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:51.023 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:51.023 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:51.023 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:51.023 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:51.023 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:51.023 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:51.024 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:51.024 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:51.024 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:51.024 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:51.024 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:51.024 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:51.024 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:51.024 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:51.024 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:51.024 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:51.025 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:51.025 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:51.025 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:51.025 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:51.025 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:51.025 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:51.026 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:51.026 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:51.026 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:51.026 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:51.026 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:51.026 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:51.026 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:51.026 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:51.026 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:51.027 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:51.027 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:51.027 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:51.027 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:51.027 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:51.027 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:51.027 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:51.027 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:51.027 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:51.027 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:51.027 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.027 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:51.028 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:51.028 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:51.028 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:51.028 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:51.028 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:51.028 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.028 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 5, 2024-09-14T03:47:51.028 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:51.028 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.028 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.028 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:51.028 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:51.028 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:51.028 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:51.029 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.029 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 0, 2024-09-14T03:47:51.029 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:51.029 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:51.029 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 62, 2024-09-14T03:47:51.029 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:51.029 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:51.029 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:51.029 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:20.502304+0000", 2024-09-14T03:47:51.029 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:45:09.879229+0000", 2024-09-14T03:47:51.029 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:45:09.879229+0000", 2024-09-14T03:47:51.029 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:45:14.684595+0000", 2024-09-14T03:47:51.030 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:20.502304+0000", 2024-09-14T03:47:51.030 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:51.030 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:51.030 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:51.030 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 63, 2024-09-14T03:47:51.030 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:20.502304+0000", 2024-09-14T03:47:51.030 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:20.502304+0000", 2024-09-14T03:47:51.030 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:20.502304+0000", 2024-09-14T03:47:51.030 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:51.030 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:51.030 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:51.030 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:20.502304+0000", 2024-09-14T03:47:51.030 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:20.502304+0000", 2024-09-14T03:47:51.030 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:51.031 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 1, 2024-09-14T03:47:51.031 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:51.031 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:51.031 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 62, 2024-09-14T03:47:51.031 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:51.031 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:51.031 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:51.031 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:51.031 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 1, 2024-09-14T03:47:51.031 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:51.031 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:51.031 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:51.032 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "4.3", 2024-09-14T03:47:51.032 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:51.032 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:51.032 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 68, 2024-09-14T03:47:51.032 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:51.032 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:51.032 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T10:49:58.555610+0000", 2024-09-14T03:47:51.032 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.000406004, 2024-09-14T03:47:51.032 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:51.032 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:51.032 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:51.032 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:51.033 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:51.033 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:51.033 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:51.033 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:51.033 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:51.033 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:51.033 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:51.033 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:51.033 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:51.033 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:51.033 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:51.033 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:51.033 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:51.033 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:51.034 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 3, 2024-09-14T03:47:51.034 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 1, 2024-09-14T03:47:51.034 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:51.034 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 1, 2024-09-14T03:47:51.034 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:51.034 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:51.034 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:51.034 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:51.034 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:51.034 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:51.034 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:51.034 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:51.034 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:51.034 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:51.034 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:51.034 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:51.035 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:51.035 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:51.035 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:51.035 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:51.035 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:51.035 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:51.035 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:51.035 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:51.035 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.035 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:51.035 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:51.035 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:51.035 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:51.035 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:51.035 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:51.036 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.036 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 0, 2024-09-14T03:47:51.036 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "65'1" 2024-09-14T03:47:51.036 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.036 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.036 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:51.036 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:51.036 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:51.036 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:51.036 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.036 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:51.036 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:51.036 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:51.036 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:51.037 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:51.037 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:51.037 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:51.037 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.661082+0000", 2024-09-14T03:47:51.037 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.732868+0000", 2024-09-14T03:47:51.037 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.732868+0000", 2024-09-14T03:47:51.037 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.733230+0000", 2024-09-14T03:47:51.037 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.661082+0000", 2024-09-14T03:47:51.037 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.037 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:51.037 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.037 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:51.037 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.661082+0000", 2024-09-14T03:47:51.038 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.661082+0000", 2024-09-14T03:47:51.038 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.661082+0000", 2024-09-14T03:47:51.038 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:51.038 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:51.038 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.038 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.661082+0000", 2024-09-14T03:47:51.038 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.661082+0000", 2024-09-14T03:47:51.038 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:51.038 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:51.038 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:51.038 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:51.038 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:51.038 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:51.039 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:51.039 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:51.039 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:51.039 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:51.039 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:51.039 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:51.039 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:51.039 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.5", 2024-09-14T03:47:51.039 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:51.039 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:51.039 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:51.039 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 24, 2024-09-14T03:47:51.039 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:51.039 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:15:12.184851+0000", 2024-09-14T03:47:51.040 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:51.040 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:51.040 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:51.040 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:51.040 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:51.040 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:51.040 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:51.040 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:51.040 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:51.040 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:51.040 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:51.040 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:51.040 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:51.040 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:51.040 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:51.041 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:51.041 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:51.041 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:51.041 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:51.041 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:51.041 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:51.041 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:51.041 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:51.041 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:51.041 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:51.041 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:51.042 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:51.042 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:51.042 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:51.042 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:51.042 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:51.042 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:51.042 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:51.042 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:51.042 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:51.042 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:51.042 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:51.042 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:51.042 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:51.043 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:51.043 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:51.043 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:51.043 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:51.043 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.043 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:51.043 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:51.043 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:51.043 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:51.043 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:51.043 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:51.043 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.044 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:51.044 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:51.044 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.044 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.044 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:51.044 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:51.044 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:51.044 INFO:tasks.workunit.client.0.smithi070.stdout: 5 2024-09-14T03:47:51.044 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.045 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 1, 2024-09-14T03:47:51.045 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:51.045 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:51.045 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:51.045 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:51.045 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:51.045 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:51.045 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.458306+0000", 2024-09-14T03:47:51.045 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.877390+0000", 2024-09-14T03:47:51.045 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.877390+0000", 2024-09-14T03:47:51.045 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.878736+0000", 2024-09-14T03:47:51.045 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.458306+0000", 2024-09-14T03:47:51.046 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.046 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:51.046 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.046 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:51.046 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.458306+0000", 2024-09-14T03:47:51.046 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.458306+0000", 2024-09-14T03:47:51.046 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.458306+0000", 2024-09-14T03:47:51.046 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:51.046 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:51.046 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.046 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.458306+0000", 2024-09-14T03:47:51.046 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.458306+0000", 2024-09-14T03:47:51.046 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:51.046 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:51.047 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:51.047 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:51.047 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:51.047 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:51.047 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:51.047 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:51.047 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:51.047 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:51.047 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:51.047 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:51.047 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:51.047 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.4", 2024-09-14T03:47:51.048 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:51.048 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:51.048 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:51.048 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:51.048 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:51.048 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T09:17:42.812575+0000", 2024-09-14T03:47:51.049 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:51.049 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:51.049 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:51.049 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:51.049 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:51.050 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:51.050 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:51.050 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:51.050 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:51.050 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:51.050 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:51.051 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:51.051 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:51.051 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:51.051 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:51.051 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:51.052 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:51.052 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:51.052 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:51.052 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:51.052 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:51.052 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:51.053 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:51.053 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:51.053 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:51.053 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:51.053 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:51.054 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:51.054 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:51.054 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:51.054 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:51.054 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:51.054 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:51.055 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:51.055 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:51.055 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:51.055 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:51.055 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:51.056 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:51.056 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:51.056 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:51.056 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:51.056 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:51.056 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.057 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:51.057 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:51.057 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:51.057 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:51.057 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:51.057 INFO:tasks.workunit.client.0.smithi070.stdout: 5 2024-09-14T03:47:51.057 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.057 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 1, 2024-09-14T03:47:51.057 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:51.058 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.058 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.058 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:51.058 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:51.058 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:51.058 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:51.058 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.058 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 3, 2024-09-14T03:47:51.058 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:51.058 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:51.058 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 62, 2024-09-14T03:47:51.058 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:51.058 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:51.058 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:51.058 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:20.501909+0000", 2024-09-14T03:47:51.059 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:45:09.710318+0000", 2024-09-14T03:47:51.059 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:45:09.710318+0000", 2024-09-14T03:47:51.059 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:45:14.857657+0000", 2024-09-14T03:47:51.059 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:20.501909+0000", 2024-09-14T03:47:51.059 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:51.059 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:51.059 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:51.059 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 63, 2024-09-14T03:47:51.059 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:20.501909+0000", 2024-09-14T03:47:51.059 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:20.501909+0000", 2024-09-14T03:47:51.059 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:20.501909+0000", 2024-09-14T03:47:51.059 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:51.059 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:51.059 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:51.060 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:20.501909+0000", 2024-09-14T03:47:51.060 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:20.501909+0000", 2024-09-14T03:47:51.060 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:51.060 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:51.060 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:51.060 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:51.060 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 62, 2024-09-14T03:47:51.060 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:51.060 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:51.060 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:51.060 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:51.060 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:51.060 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:51.060 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:51.061 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:51.061 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "4.0", 2024-09-14T03:47:51.061 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:51.061 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:51.061 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 68, 2024-09-14T03:47:51.061 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 19, 2024-09-14T03:47:51.061 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:51.061 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:37:23.130384+0000", 2024-09-14T03:47:51.062 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.000315721, 2024-09-14T03:47:51.062 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:51.062 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:51.062 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:51.062 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:51.062 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:51.062 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:51.062 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:51.062 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:51.062 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:51.062 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:51.062 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:51.062 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:51.062 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:51.063 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:51.063 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:51.063 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:51.063 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:51.063 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:51.063 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:51.063 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:51.063 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:51.063 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:51.063 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:51.063 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:51.063 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:51.063 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:51.063 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:51.064 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:51.064 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:51.064 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:51.064 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:51.064 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:51.064 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:51.064 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:51.064 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:51.064 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:51.064 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:51.064 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:51.064 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:51.064 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:51.065 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:51.065 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:51.065 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.065 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:51.065 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:51.065 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:51.065 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:51.065 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:51.065 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:51.065 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.065 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 3, 2024-09-14T03:47:51.065 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:51.066 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.066 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.066 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:51.066 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:51.066 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:51.066 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:51.066 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.066 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 1, 2024-09-14T03:47:51.066 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:51.066 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:51.066 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:51.066 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:51.066 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:51.066 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:51.066 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.459070+0000", 2024-09-14T03:47:51.067 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.733457+0000", 2024-09-14T03:47:51.067 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.733457+0000", 2024-09-14T03:47:51.067 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.733688+0000", 2024-09-14T03:47:51.067 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.459070+0000", 2024-09-14T03:47:51.067 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.067 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:51.067 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.067 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:51.067 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.459070+0000", 2024-09-14T03:47:51.067 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.459070+0000", 2024-09-14T03:47:51.067 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.459070+0000", 2024-09-14T03:47:51.067 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:51.067 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:51.067 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.068 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.459070+0000", 2024-09-14T03:47:51.068 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.459070+0000", 2024-09-14T03:47:51.068 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:51.068 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 1, 2024-09-14T03:47:51.068 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:51.068 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:51.068 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:51.068 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:51.068 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:51.068 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:51.068 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:51.068 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 1, 2024-09-14T03:47:51.069 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:51.069 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:51.069 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:51.069 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.6", 2024-09-14T03:47:51.069 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:51.069 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:51.069 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:51.069 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 24, 2024-09-14T03:47:51.069 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:51.069 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:09:01.069988+0000", 2024-09-14T03:47:51.069 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:51.069 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:51.069 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:51.069 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:51.070 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:51.070 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:51.070 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:51.070 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:51.070 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:51.070 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:51.070 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:51.070 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:51.070 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:51.070 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:51.070 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:51.070 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:51.070 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:51.070 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:51.071 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:51.071 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 3, 2024-09-14T03:47:51.071 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 1, 2024-09-14T03:47:51.071 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:51.071 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 1, 2024-09-14T03:47:51.071 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:51.071 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:51.071 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:51.071 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:51.071 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:51.071 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 1, 2024-09-14T03:47:51.071 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:51.071 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:51.071 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:51.071 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:51.072 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:51.072 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:51.072 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:51.072 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:51.072 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:51.072 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:51.072 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:51.072 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:51.072 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 1, 2024-09-14T03:47:51.072 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:51.072 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.073 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:51.073 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:51.073 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:51.073 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:51.073 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:51.073 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:51.073 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.073 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 1, 2024-09-14T03:47:51.073 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "58'1" 2024-09-14T03:47:51.073 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.073 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.073 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:51.073 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:51.073 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:51.074 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:51.074 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.074 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 3, 2024-09-14T03:47:51.074 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:51.074 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:51.074 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:51.074 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:51.074 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:51.074 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:51.074 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.857916+0000", 2024-09-14T03:47:51.074 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.869780+0000", 2024-09-14T03:47:51.074 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.869780+0000", 2024-09-14T03:47:51.074 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.870045+0000", 2024-09-14T03:47:51.074 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.857916+0000", 2024-09-14T03:47:51.074 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.075 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:51.075 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.075 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:51.075 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.857916+0000", 2024-09-14T03:47:51.075 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.857916+0000", 2024-09-14T03:47:51.075 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.857916+0000", 2024-09-14T03:47:51.075 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:51.075 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:51.075 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.075 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.857916+0000", 2024-09-14T03:47:51.075 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.857916+0000", 2024-09-14T03:47:51.076 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:51.076 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:51.076 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:51.076 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:51.076 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:51.076 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:51.076 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:51.076 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:51.076 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:51.076 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:51.076 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:51.076 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:51.077 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:51.077 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.7", 2024-09-14T03:47:51.077 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:51.077 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:51.077 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:51.077 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:51.077 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:51.077 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T09:04:58.634704+0000", 2024-09-14T03:47:51.077 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:51.077 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:51.077 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:51.077 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:51.077 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:51.078 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:51.078 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:51.078 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:51.078 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:51.078 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:51.078 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:51.078 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:51.078 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:51.078 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:51.078 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:51.078 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:51.078 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:51.078 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:51.079 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:51.079 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:51.079 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:51.079 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:51.079 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:51.079 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:51.079 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:51.079 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:51.079 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:51.079 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:51.079 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:51.079 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:51.079 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:51.079 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:51.080 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:51.080 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:51.080 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:51.080 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:51.080 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:51.080 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:51.080 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:51.080 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:51.080 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:51.080 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:51.080 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:51.081 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.081 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:51.081 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:51.081 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:51.081 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:51.081 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:51.081 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:51.081 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.081 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 3, 2024-09-14T03:47:51.081 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:51.081 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.081 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.081 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:51.081 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:51.082 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:51.082 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:51.082 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.082 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 4, 2024-09-14T03:47:51.082 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:51.082 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:51.082 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 62, 2024-09-14T03:47:51.082 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:51.082 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:51.082 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:51.082 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:20.502067+0000", 2024-09-14T03:47:51.083 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:45:09.879281+0000", 2024-09-14T03:47:51.083 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:45:09.879281+0000", 2024-09-14T03:47:51.083 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:45:14.459017+0000", 2024-09-14T03:47:51.083 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:20.502067+0000", 2024-09-14T03:47:51.083 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:51.083 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:51.083 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:51.083 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 63, 2024-09-14T03:47:51.083 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:20.502067+0000", 2024-09-14T03:47:51.083 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:20.502067+0000", 2024-09-14T03:47:51.083 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:20.502067+0000", 2024-09-14T03:47:51.083 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:51.084 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:51.084 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:51.084 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:20.502067+0000", 2024-09-14T03:47:51.084 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:20.502067+0000", 2024-09-14T03:47:51.084 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:51.084 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:51.084 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:51.084 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:51.085 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 62, 2024-09-14T03:47:51.085 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:51.085 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:51.085 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:51.085 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:51.085 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:51.085 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:51.085 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:51.085 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:51.085 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "4.1", 2024-09-14T03:47:51.085 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:51.085 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:51.085 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 68, 2024-09-14T03:47:51.086 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 19, 2024-09-14T03:47:51.086 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:51.086 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:10:03.506895+0000", 2024-09-14T03:47:51.086 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.000827153, 2024-09-14T03:47:51.086 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:51.086 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:51.086 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:51.087 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:51.087 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:51.087 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:51.087 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:51.087 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:51.088 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:51.088 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:51.088 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:51.088 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:51.088 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:51.088 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:51.089 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:51.089 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:51.089 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:51.089 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:51.089 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:51.089 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:51.089 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:51.089 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:51.089 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:51.089 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:51.089 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:51.089 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:51.090 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:51.090 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:51.090 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:51.090 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:51.090 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:51.090 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:51.090 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:51.090 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:51.090 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:51.091 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:51.091 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:51.091 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:51.091 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:51.091 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:51.091 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:51.091 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:51.091 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.091 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:51.091 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:51.091 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:51.092 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:51.092 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:51.092 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:51.092 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.092 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 4, 2024-09-14T03:47:51.092 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:51.092 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.092 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.092 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:51.092 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:51.092 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:51.092 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:51.092 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.092 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 6, 2024-09-14T03:47:51.092 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:51.093 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:51.093 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:51.093 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:51.093 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:51.093 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:51.093 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.545051+0000", 2024-09-14T03:47:51.093 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.734833+0000", 2024-09-14T03:47:51.093 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.734833+0000", 2024-09-14T03:47:51.093 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.735364+0000", 2024-09-14T03:47:51.093 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.545051+0000", 2024-09-14T03:47:51.093 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.093 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:51.094 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.094 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:51.094 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.545051+0000", 2024-09-14T03:47:51.094 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.545051+0000", 2024-09-14T03:47:51.094 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.545051+0000", 2024-09-14T03:47:51.094 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:51.094 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:51.094 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.094 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.545051+0000", 2024-09-14T03:47:51.094 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.545051+0000", 2024-09-14T03:47:51.094 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:51.094 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 2, 2024-09-14T03:47:51.094 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:51.095 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:51.095 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:51.095 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:51.095 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:51.095 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:51.095 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:51.095 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 2, 2024-09-14T03:47:51.095 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:51.095 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:51.095 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:51.095 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.7", 2024-09-14T03:47:51.095 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:51.095 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:51.096 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:51.096 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 25, 2024-09-14T03:47:51.096 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:51.096 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T10:58:37.428713+0000", 2024-09-14T03:47:51.096 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:51.096 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:51.096 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:51.096 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:51.096 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:51.096 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:51.096 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:51.096 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:51.097 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:51.097 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:51.097 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:51.097 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:51.097 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:51.097 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:51.097 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:51.097 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:51.097 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:51.097 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:51.097 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:51.097 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 3, 2024-09-14T03:47:51.097 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 1, 2024-09-14T03:47:51.097 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:51.098 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 1, 2024-09-14T03:47:51.098 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:51.098 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:51.098 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:51.098 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:51.098 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:51.098 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 1, 2024-09-14T03:47:51.098 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:51.098 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:51.098 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:51.098 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:51.098 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:51.098 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:51.098 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:51.099 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:51.099 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:51.099 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:51.099 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:51.099 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:51.099 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 3, 2024-09-14T03:47:51.099 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:51.099 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.099 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:51.099 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:51.099 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:51.099 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:51.099 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:51.100 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:51.100 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.100 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 6, 2024-09-14T03:47:51.100 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "58'2" 2024-09-14T03:47:51.100 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.100 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.100 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:51.100 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:51.100 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:51.100 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:51.100 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.100 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 0, 2024-09-14T03:47:51.100 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:51.100 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:51.101 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:51.101 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:51.101 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:51.101 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:51.101 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.684514+0000", 2024-09-14T03:47:51.101 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.878316+0000", 2024-09-14T03:47:51.101 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.878316+0000", 2024-09-14T03:47:51.101 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.878573+0000", 2024-09-14T03:47:51.101 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.684514+0000", 2024-09-14T03:47:51.101 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.101 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:51.101 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.101 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:51.101 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.684514+0000", 2024-09-14T03:47:51.101 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.684514+0000", 2024-09-14T03:47:51.102 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.684514+0000", 2024-09-14T03:47:51.102 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:51.102 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:51.102 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.102 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.684514+0000", 2024-09-14T03:47:51.102 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.684514+0000", 2024-09-14T03:47:51.102 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:51.102 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:51.102 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:51.102 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:51.102 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:51.102 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:51.102 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:51.102 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:51.103 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:51.103 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:51.103 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:51.103 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:51.103 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:51.103 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.6", 2024-09-14T03:47:51.103 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:51.103 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:51.103 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:51.103 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:51.103 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:51.103 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:45:21.129163+0000", 2024-09-14T03:47:51.103 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:51.103 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:51.104 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:51.104 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:51.104 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:51.104 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:51.104 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:51.104 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:51.104 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:51.104 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:51.104 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:51.104 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:51.104 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:51.104 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:51.104 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:51.104 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:51.105 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:51.105 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:51.105 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:51.105 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:51.105 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:51.105 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:51.105 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:51.105 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:51.105 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:51.105 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:51.105 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:51.105 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:51.106 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:51.106 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:51.106 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:51.106 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:51.106 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:51.106 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:51.106 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:51.106 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:51.106 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:51.106 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:51.106 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:51.106 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:51.106 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:51.106 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:51.106 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:51.106 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.107 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:51.107 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:51.107 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:51.107 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:51.107 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:51.107 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:51.107 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.107 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 0, 2024-09-14T03:47:51.107 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:51.107 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.107 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.107 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:51.107 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:51.107 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:51.107 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:51.108 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.108 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:51.108 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:51.108 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:51.108 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:51.108 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:51.108 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:51.108 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:51.108 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.660416+0000", 2024-09-14T03:47:51.108 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.730538+0000", 2024-09-14T03:47:51.108 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.730538+0000", 2024-09-14T03:47:51.108 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.730843+0000", 2024-09-14T03:47:51.109 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.660416+0000", 2024-09-14T03:47:51.109 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.109 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:51.109 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.109 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:51.109 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.660416+0000", 2024-09-14T03:47:51.109 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.660416+0000", 2024-09-14T03:47:51.109 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.660416+0000", 2024-09-14T03:47:51.109 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:51.109 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:51.109 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.109 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.660416+0000", 2024-09-14T03:47:51.110 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.660416+0000", 2024-09-14T03:47:51.110 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:51.110 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:51.110 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:51.110 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:51.110 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:51.110 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:51.110 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:51.110 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:51.110 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:51.110 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:51.110 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:51.110 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:51.110 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:51.110 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.8", 2024-09-14T03:47:51.111 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:51.111 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:51.111 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:51.111 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 24, 2024-09-14T03:47:51.111 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:51.111 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T04:04:57.059345+0000", 2024-09-14T03:47:51.111 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:51.111 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:51.111 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:51.111 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:51.111 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:51.111 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:51.111 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:51.111 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:51.112 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:51.112 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:51.112 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:51.112 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:51.112 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:51.112 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:51.112 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:51.112 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:51.112 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:51.112 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:51.112 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:51.112 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:51.112 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:51.113 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:51.113 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:51.113 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:51.113 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:51.113 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:51.113 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:51.113 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:51.113 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:51.113 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:51.113 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:51.113 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:51.114 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:51.114 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:51.114 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:51.114 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:51.114 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:51.114 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:51.114 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:51.114 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:51.114 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:51.114 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:51.114 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:51.114 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.115 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:51.115 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:51.115 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:51.115 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:51.115 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:51.115 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:51.115 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.115 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:51.115 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:51.115 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.115 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.115 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:51.115 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:51.115 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:51.115 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:51.116 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.116 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 4, 2024-09-14T03:47:51.116 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:51.116 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:51.116 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:51.116 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:51.116 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:51.116 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:51.116 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.458548+0000", 2024-09-14T03:47:51.116 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.874049+0000", 2024-09-14T03:47:51.116 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.874049+0000", 2024-09-14T03:47:51.116 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.874355+0000", 2024-09-14T03:47:51.116 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.458548+0000", 2024-09-14T03:47:51.116 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.117 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:51.117 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.117 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:51.117 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.458548+0000", 2024-09-14T03:47:51.117 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.458548+0000", 2024-09-14T03:47:51.117 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.458548+0000", 2024-09-14T03:47:51.117 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:51.117 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:51.117 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.117 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.458548+0000", 2024-09-14T03:47:51.117 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.458548+0000", 2024-09-14T03:47:51.117 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:51.117 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:51.117 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:51.118 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:51.118 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:51.118 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:51.118 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:51.118 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:51.118 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:51.118 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:51.118 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:51.118 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:51.118 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:51.118 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.9", 2024-09-14T03:47:51.118 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:51.118 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:51.118 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:51.119 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:51.119 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:51.119 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:53:30.306175+0000", 2024-09-14T03:47:51.119 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:51.119 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:51.119 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:51.119 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:51.119 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:51.119 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:51.119 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:51.119 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:51.119 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:51.120 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:51.120 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:51.120 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:51.120 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:51.120 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:51.120 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:51.120 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:51.120 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:51.120 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:51.120 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:51.120 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:51.120 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:51.120 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:51.121 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:51.121 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:51.121 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:51.121 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:51.121 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:51.121 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:51.121 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:51.121 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:51.121 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:51.121 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:51.121 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:51.121 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:51.121 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:51.121 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:51.121 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:51.121 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:51.122 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:51.122 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:51.122 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:51.122 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:51.122 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:51.122 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.122 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:51.122 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:51.122 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:51.122 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:51.122 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:51.122 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:51.122 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.122 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 4, 2024-09-14T03:47:51.122 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:51.122 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.123 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.123 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:51.123 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:51.123 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:51.123 INFO:tasks.workunit.client.0.smithi070.stdout: 3 2024-09-14T03:47:51.123 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.123 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 1, 2024-09-14T03:47:51.123 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:51.123 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:51.123 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:51.123 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:51.123 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:51.123 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:51.123 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.458659+0000", 2024-09-14T03:47:51.124 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.727820+0000", 2024-09-14T03:47:51.124 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.727820+0000", 2024-09-14T03:47:51.124 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.728080+0000", 2024-09-14T03:47:51.124 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.458659+0000", 2024-09-14T03:47:51.124 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.124 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:51.124 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.124 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:51.124 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.458659+0000", 2024-09-14T03:47:51.124 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.458659+0000", 2024-09-14T03:47:51.124 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.458659+0000", 2024-09-14T03:47:51.124 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:51.124 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:51.125 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.125 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.458659+0000", 2024-09-14T03:47:51.125 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.458659+0000", 2024-09-14T03:47:51.125 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:51.125 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:51.125 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:51.125 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:51.125 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:51.125 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:51.125 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:51.126 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:51.126 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:51.126 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:51.126 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:51.126 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:51.126 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:51.126 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.9", 2024-09-14T03:47:51.126 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:51.126 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:51.126 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:51.126 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 23, 2024-09-14T03:47:51.126 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:51.126 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:37:58.166430+0000", 2024-09-14T03:47:51.126 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:51.126 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:51.127 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:51.127 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:51.127 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:51.127 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:51.127 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:51.127 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:51.127 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:51.127 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:51.127 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:51.127 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:51.127 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:51.127 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:51.128 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:51.128 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:51.128 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:51.128 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:51.128 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:51.128 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:51.128 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:51.128 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:51.128 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:51.128 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:51.129 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:51.129 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:51.129 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:51.129 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:51.129 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:51.129 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:51.129 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:51.129 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:51.129 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:51.129 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:51.129 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:51.129 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:51.129 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:51.129 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:51.129 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:51.130 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:51.130 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:51.130 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:51.130 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:51.130 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.130 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:51.130 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:51.130 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:51.130 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:51.130 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:51.130 INFO:tasks.workunit.client.0.smithi070.stdout: 3 2024-09-14T03:47:51.130 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.130 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 1, 2024-09-14T03:47:51.130 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:51.130 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.131 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.131 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:51.131 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:51.131 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:51.131 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:51.131 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.131 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 3, 2024-09-14T03:47:51.131 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:51.131 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:51.131 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:51.131 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:51.131 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:51.131 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:51.131 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.857492+0000", 2024-09-14T03:47:51.132 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.869065+0000", 2024-09-14T03:47:51.132 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.869065+0000", 2024-09-14T03:47:51.132 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.869311+0000", 2024-09-14T03:47:51.132 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.857492+0000", 2024-09-14T03:47:51.132 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.132 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:51.132 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.132 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:51.132 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.857492+0000", 2024-09-14T03:47:51.132 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.857492+0000", 2024-09-14T03:47:51.132 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.857492+0000", 2024-09-14T03:47:51.132 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:51.132 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:51.133 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.133 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.857492+0000", 2024-09-14T03:47:51.133 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.857492+0000", 2024-09-14T03:47:51.133 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:51.133 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:51.133 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:51.133 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:51.133 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:51.133 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:51.133 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:51.133 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:51.133 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:51.133 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:51.134 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:51.134 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:51.134 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:51.134 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.8", 2024-09-14T03:47:51.134 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:51.134 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:51.134 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:51.134 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:51.134 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:51.134 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T03:45:09.508019+0000", 2024-09-14T03:47:51.134 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:51.134 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:51.134 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:51.134 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:51.135 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:51.135 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:51.135 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:51.135 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:51.135 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:51.135 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:51.135 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:51.135 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:51.135 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:51.135 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:51.135 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:51.135 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:51.135 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:51.135 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:51.136 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:51.136 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:51.136 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:51.136 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:51.136 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:51.136 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:51.136 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:51.136 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:51.137 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:51.137 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:51.137 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:51.137 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:51.137 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:51.137 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:51.137 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:51.138 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:51.138 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:51.138 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:51.138 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:51.138 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:51.138 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:51.138 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:51.138 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:51.138 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:51.138 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:51.138 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.138 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:51.138 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:51.138 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:51.139 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:51.139 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:51.139 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:51.139 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.139 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 3, 2024-09-14T03:47:51.139 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:51.139 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.139 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.139 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:51.139 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:51.139 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:51.139 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:51.139 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.139 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 1, 2024-09-14T03:47:51.139 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:51.140 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:51.140 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:51.140 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:51.140 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:51.140 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:51.140 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.459148+0000", 2024-09-14T03:47:51.140 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.728647+0000", 2024-09-14T03:47:51.140 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.728647+0000", 2024-09-14T03:47:51.140 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.728931+0000", 2024-09-14T03:47:51.140 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.459148+0000", 2024-09-14T03:47:51.140 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.140 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:51.140 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.140 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:51.140 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.459148+0000", 2024-09-14T03:47:51.141 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.459148+0000", 2024-09-14T03:47:51.141 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.459148+0000", 2024-09-14T03:47:51.141 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:51.141 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:51.141 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.141 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.459148+0000", 2024-09-14T03:47:51.141 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.459148+0000", 2024-09-14T03:47:51.141 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:51.141 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:51.141 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:51.141 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:51.141 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:51.141 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:51.141 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:51.142 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:51.142 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:51.142 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:51.142 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:51.142 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:51.142 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:51.142 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.a", 2024-09-14T03:47:51.142 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:51.142 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:51.142 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:51.142 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 23, 2024-09-14T03:47:51.142 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:51.143 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:09:38.769523+0000", 2024-09-14T03:47:51.143 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:51.143 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:51.143 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:51.143 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:51.143 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:51.143 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:51.143 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:51.143 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:51.143 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:51.143 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:51.143 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:51.143 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:51.143 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:51.143 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:51.143 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:51.144 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:51.144 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:51.144 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:51.144 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:51.144 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:51.144 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:51.144 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:51.144 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:51.144 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:51.144 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:51.145 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:51.145 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:51.145 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:51.145 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:51.145 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:51.145 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:51.145 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:51.145 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:51.145 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:51.145 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:51.145 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:51.145 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:51.145 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:51.146 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:51.146 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:51.146 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:51.146 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:51.146 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:51.146 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.146 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:51.146 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:51.146 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:51.146 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:51.146 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:51.147 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:51.147 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.147 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 1, 2024-09-14T03:47:51.147 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:51.147 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.147 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.147 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:51.147 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:51.147 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:51.147 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:51.147 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.147 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 3, 2024-09-14T03:47:51.147 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:51.148 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:51.148 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:51.148 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:51.148 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:51.148 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:51.148 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.857689+0000", 2024-09-14T03:47:51.148 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:26.429880+0000", 2024-09-14T03:47:51.148 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:26.429880+0000", 2024-09-14T03:47:51.148 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:26.430230+0000", 2024-09-14T03:47:51.148 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.857689+0000", 2024-09-14T03:47:51.148 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.148 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:51.148 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.149 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 60, 2024-09-14T03:47:51.149 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.857689+0000", 2024-09-14T03:47:51.149 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.857689+0000", 2024-09-14T03:47:51.149 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.857689+0000", 2024-09-14T03:47:51.149 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:51.149 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:51.149 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.149 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.857689+0000", 2024-09-14T03:47:51.149 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.857689+0000", 2024-09-14T03:47:51.149 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:51.149 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:51.149 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:51.150 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:51.150 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 59, 2024-09-14T03:47:51.150 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:51.150 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:51.150 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:51.150 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:51.150 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:51.150 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:51.150 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:51.150 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:51.150 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.b", 2024-09-14T03:47:51.150 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:51.150 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:51.150 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:51.150 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 31, 2024-09-14T03:47:51.151 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:51.151 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:17:39.132572+0000", 2024-09-14T03:47:51.151 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:51.151 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:51.151 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:51.151 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:51.151 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:51.151 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:51.151 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:51.151 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:51.151 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:51.151 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:51.151 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:51.151 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:51.151 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:51.152 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:51.152 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:51.152 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:51.152 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:51.152 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:51.152 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:51.152 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:51.152 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:51.152 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:51.152 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:51.152 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:51.152 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:51.152 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:51.152 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:51.153 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:51.153 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:51.153 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:51.153 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:51.153 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:51.153 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:51.153 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:51.153 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:51.153 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:51.153 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:51.153 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:51.153 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:51.153 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:51.153 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:51.154 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:51.154 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:51.154 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.154 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:51.154 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:51.154 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:51.154 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:51.154 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:51.154 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:51.154 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.154 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 3, 2024-09-14T03:47:51.154 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:51.154 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.155 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.155 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:51.155 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:51.155 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:51.155 INFO:tasks.workunit.client.0.smithi070.stdout: 5 2024-09-14T03:47:51.155 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.155 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:51.155 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:51.155 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:51.155 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:51.155 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:51.155 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:51.155 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:51.155 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.660988+0000", 2024-09-14T03:47:51.156 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.736546+0000", 2024-09-14T03:47:51.156 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.736546+0000", 2024-09-14T03:47:51.156 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.737705+0000", 2024-09-14T03:47:51.156 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.660988+0000", 2024-09-14T03:47:51.156 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.156 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:51.156 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.156 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:51.156 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.660988+0000", 2024-09-14T03:47:51.156 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.660988+0000", 2024-09-14T03:47:51.156 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.660988+0000", 2024-09-14T03:47:51.156 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:51.156 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:51.156 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.157 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.660988+0000", 2024-09-14T03:47:51.157 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.660988+0000", 2024-09-14T03:47:51.157 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:51.157 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 2, 2024-09-14T03:47:51.157 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:51.157 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:51.157 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:51.157 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:51.157 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:51.157 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:51.157 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:51.157 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 2, 2024-09-14T03:47:51.157 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:51.158 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:51.158 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:51.158 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.b", 2024-09-14T03:47:51.158 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:51.158 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:51.158 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:51.158 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 26, 2024-09-14T03:47:51.158 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:51.158 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:27:12.542706+0000", 2024-09-14T03:47:51.158 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:51.158 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:51.158 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:51.158 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:51.159 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:51.159 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:51.159 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:51.159 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:51.159 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:51.159 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:51.159 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:51.159 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:51.159 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:51.159 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:51.159 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:51.159 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:51.159 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:51.159 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:51.160 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:51.160 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 3, 2024-09-14T03:47:51.160 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 1, 2024-09-14T03:47:51.160 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:51.160 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 1, 2024-09-14T03:47:51.160 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:51.160 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:51.160 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:51.160 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:51.160 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:51.160 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 1, 2024-09-14T03:47:51.160 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:51.160 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:51.161 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:51.161 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:51.161 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:51.161 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:51.161 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:51.161 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:51.161 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:51.161 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:51.161 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:51.161 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:51.162 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 1, 2024-09-14T03:47:51.162 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:51.162 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.162 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:51.162 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:51.162 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:51.162 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:51.162 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:51.162 INFO:tasks.workunit.client.0.smithi070.stdout: 5 2024-09-14T03:47:51.162 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.162 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:51.163 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "58'2" 2024-09-14T03:47:51.163 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.163 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.163 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:51.163 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:51.163 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:51.163 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:51.163 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.163 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 2, 2024-09-14T03:47:51.163 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:51.163 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:51.163 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:51.163 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:51.163 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:51.164 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:51.164 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:15.135501+0000", 2024-09-14T03:47:51.164 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:26.429364+0000", 2024-09-14T03:47:51.164 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:26.429364+0000", 2024-09-14T03:47:51.164 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:26.429678+0000", 2024-09-14T03:47:51.164 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:15.135501+0000", 2024-09-14T03:47:51.164 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.164 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:51.164 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.164 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 60, 2024-09-14T03:47:51.164 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:15.135501+0000", 2024-09-14T03:47:51.165 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:15.135501+0000", 2024-09-14T03:47:51.165 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:15.135501+0000", 2024-09-14T03:47:51.165 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:51.165 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:51.165 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.165 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:15.135501+0000", 2024-09-14T03:47:51.165 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:15.135501+0000", 2024-09-14T03:47:51.165 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:51.165 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:51.165 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:51.165 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:51.165 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 59, 2024-09-14T03:47:51.165 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:51.166 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:51.166 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:51.166 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:51.166 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:51.166 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:51.166 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:51.166 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:51.167 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.a", 2024-09-14T03:47:51.167 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:51.167 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:51.167 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:51.167 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 18, 2024-09-14T03:47:51.167 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:51.167 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:29:31.230108+0000", 2024-09-14T03:47:51.167 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:51.167 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:51.167 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:51.167 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:51.167 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:51.167 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:51.168 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:51.168 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:51.168 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:51.168 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:51.168 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:51.168 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:51.168 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:51.168 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:51.168 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:51.168 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:51.169 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:51.169 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:51.169 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:51.169 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:51.169 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:51.169 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:51.169 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:51.169 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:51.169 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:51.169 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:51.169 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:51.169 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:51.169 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:51.170 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:51.170 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:51.170 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:51.170 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:51.170 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:51.170 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:51.170 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:51.170 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:51.170 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:51.170 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:51.170 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:51.170 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:51.170 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:51.170 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:51.171 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.171 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:51.171 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:51.171 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:51.171 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:51.171 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:51.171 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:51.171 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.171 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 2, 2024-09-14T03:47:51.171 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:51.171 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.171 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.171 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:51.171 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:51.172 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:51.172 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:51.172 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.172 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 5, 2024-09-14T03:47:51.172 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:51.172 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:51.172 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:51.172 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:51.172 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:51.172 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:51.172 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.462051+0000", 2024-09-14T03:47:51.172 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.888257+0000", 2024-09-14T03:47:51.172 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.888257+0000", 2024-09-14T03:47:51.173 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.888591+0000", 2024-09-14T03:47:51.173 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.462051+0000", 2024-09-14T03:47:51.173 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.173 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:51.173 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.173 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:51.173 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.462051+0000", 2024-09-14T03:47:51.173 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.462051+0000", 2024-09-14T03:47:51.173 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.462051+0000", 2024-09-14T03:47:51.173 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:51.173 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:51.173 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.173 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.462051+0000", 2024-09-14T03:47:51.174 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.462051+0000", 2024-09-14T03:47:51.174 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:51.174 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:51.174 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:51.174 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:51.174 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:51.174 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:51.174 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:51.174 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:51.174 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:51.174 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:51.174 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:51.174 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:51.175 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:51.175 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.1d", 2024-09-14T03:47:51.175 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:51.175 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:51.175 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:51.175 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:51.175 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:51.175 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:24:40.992358+0000", 2024-09-14T03:47:51.175 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:51.175 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:51.175 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:51.175 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:51.176 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:51.176 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:51.176 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:51.176 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:51.176 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:51.176 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:51.176 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:51.176 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:51.176 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:51.176 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:51.176 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:51.176 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:51.177 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:51.177 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:51.177 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:51.177 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:51.177 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:51.177 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:51.178 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:51.178 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:51.178 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:51.178 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:51.178 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:51.178 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:51.179 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:51.179 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:51.179 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:51.179 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:51.179 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:51.179 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:51.179 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:51.180 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:51.180 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:51.180 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:51.180 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:51.180 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:51.180 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:51.181 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:51.181 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:51.181 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.181 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:51.181 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:51.181 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:51.182 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:51.182 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:51.182 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:51.182 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.182 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 5, 2024-09-14T03:47:51.182 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:51.182 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.183 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.183 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:51.183 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:51.183 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:51.183 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:51.183 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.183 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:51.184 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:51.184 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:51.184 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:51.184 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:51.184 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:51.184 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:51.184 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.660896+0000", 2024-09-14T03:47:51.185 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.734408+0000", 2024-09-14T03:47:51.185 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.734408+0000", 2024-09-14T03:47:51.185 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.734680+0000", 2024-09-14T03:47:51.185 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.660896+0000", 2024-09-14T03:47:51.185 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.186 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:51.186 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.186 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:51.186 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.660896+0000", 2024-09-14T03:47:51.186 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.660896+0000", 2024-09-14T03:47:51.186 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.660896+0000", 2024-09-14T03:47:51.186 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:51.187 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:51.187 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.187 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.660896+0000", 2024-09-14T03:47:51.187 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.660896+0000", 2024-09-14T03:47:51.187 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:51.187 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 2, 2024-09-14T03:47:51.188 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:51.188 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:51.188 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:51.188 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:51.188 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:51.188 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:51.189 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:51.189 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 2, 2024-09-14T03:47:51.189 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:51.189 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:51.189 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:51.189 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.1c", 2024-09-14T03:47:51.190 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:51.190 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:51.190 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:51.190 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 26, 2024-09-14T03:47:51.190 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:51.190 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:49:02.536404+0000", 2024-09-14T03:47:51.191 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:51.191 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:51.191 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:51.191 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:51.191 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:51.191 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:51.192 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:51.192 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:51.192 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:51.192 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:51.192 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:51.192 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:51.193 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:51.193 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:51.193 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:51.193 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:51.193 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:51.193 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:51.193 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:51.194 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 3, 2024-09-14T03:47:51.194 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 1, 2024-09-14T03:47:51.194 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:51.194 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 1, 2024-09-14T03:47:51.194 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:51.194 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:51.195 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:51.195 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:51.195 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:51.195 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 1, 2024-09-14T03:47:51.195 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:51.195 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:51.195 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:51.196 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:51.196 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:51.196 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:51.196 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:51.196 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:51.196 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:51.197 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:51.197 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:51.197 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:51.197 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 3, 2024-09-14T03:47:51.197 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:51.197 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.198 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:51.198 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:51.198 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:51.198 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:51.198 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:51.198 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:51.199 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.199 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:51.199 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "58'2" 2024-09-14T03:47:51.199 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.199 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.199 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:51.200 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:51.200 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:51.200 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:51.200 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.200 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 5, 2024-09-14T03:47:51.200 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:51.201 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:51.201 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:51.201 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:51.201 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:51.201 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:51.202 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.461880+0000", 2024-09-14T03:47:51.202 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.885110+0000", 2024-09-14T03:47:51.202 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.885110+0000", 2024-09-14T03:47:51.202 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.885572+0000", 2024-09-14T03:47:51.202 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.461880+0000", 2024-09-14T03:47:51.202 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.203 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:51.203 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.203 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:51.203 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.461880+0000", 2024-09-14T03:47:51.203 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.461880+0000", 2024-09-14T03:47:51.203 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.461880+0000", 2024-09-14T03:47:51.203 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:51.204 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:51.204 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.204 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.461880+0000", 2024-09-14T03:47:51.204 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.461880+0000", 2024-09-14T03:47:51.204 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:51.205 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:51.205 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:51.205 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:51.205 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:51.205 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:51.205 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:51.206 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:51.206 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:51.206 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:51.206 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:51.206 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:51.206 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:51.207 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.1c", 2024-09-14T03:47:51.207 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:51.207 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:51.207 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:51.207 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:51.207 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:51.208 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:16:10.439239+0000", 2024-09-14T03:47:51.208 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:51.208 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:51.208 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:51.208 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:51.208 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:51.209 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:51.209 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:51.209 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:51.209 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:51.209 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:51.209 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:51.210 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:51.210 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:51.210 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:51.210 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:51.210 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:51.210 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:51.211 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:51.211 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:51.211 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:51.211 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:51.211 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:51.211 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:51.212 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:51.212 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:51.212 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:51.212 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:51.212 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:51.212 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:51.213 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:51.213 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:51.213 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:51.213 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:51.213 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:51.213 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:51.214 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:51.214 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:51.214 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:51.214 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:51.214 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:51.214 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:51.215 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:51.215 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:51.215 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.215 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:51.215 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:51.215 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:51.216 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:51.216 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:51.216 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:51.216 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.216 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 5, 2024-09-14T03:47:51.216 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:51.217 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.217 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.217 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:51.217 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:51.217 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:51.217 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:51.217 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.218 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:51.218 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:51.218 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:51.218 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:51.218 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:51.218 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:51.218 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:51.219 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.660935+0000", 2024-09-14T03:47:51.219 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.730951+0000", 2024-09-14T03:47:51.219 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.730951+0000", 2024-09-14T03:47:51.219 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.731296+0000", 2024-09-14T03:47:51.219 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.660935+0000", 2024-09-14T03:47:51.219 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.219 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:51.219 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.219 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:51.219 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.660935+0000", 2024-09-14T03:47:51.219 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.660935+0000", 2024-09-14T03:47:51.220 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.660935+0000", 2024-09-14T03:47:51.220 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:51.220 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:51.220 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.220 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.660935+0000", 2024-09-14T03:47:51.220 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.660935+0000", 2024-09-14T03:47:51.220 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:51.220 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 2, 2024-09-14T03:47:51.220 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:51.220 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:51.220 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:51.220 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:51.221 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:51.221 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:51.221 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:51.221 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 2, 2024-09-14T03:47:51.221 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:51.221 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:51.221 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:51.221 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.1d", 2024-09-14T03:47:51.221 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:51.221 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:51.221 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:51.221 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 26, 2024-09-14T03:47:51.221 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:51.222 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:26:14.801414+0000", 2024-09-14T03:47:51.222 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:51.222 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:51.222 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:51.222 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:51.222 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:51.222 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:51.222 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:51.222 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:51.222 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:51.222 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:51.222 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:51.222 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:51.222 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:51.223 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:51.223 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:51.223 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:51.223 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:51.223 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:51.223 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:51.223 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 3, 2024-09-14T03:47:51.223 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 1, 2024-09-14T03:47:51.223 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:51.223 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 1, 2024-09-14T03:47:51.223 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:51.223 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:51.223 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:51.223 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:51.224 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:51.224 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 1, 2024-09-14T03:47:51.224 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:51.224 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:51.224 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:51.224 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:51.224 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:51.224 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:51.224 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:51.224 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:51.224 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:51.224 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:51.224 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:51.225 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:51.225 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 3, 2024-09-14T03:47:51.225 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:51.225 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.225 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:51.225 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:51.225 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:51.225 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:51.225 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:51.225 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:51.225 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.225 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:51.225 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "58'2" 2024-09-14T03:47:51.225 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.226 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.226 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:51.226 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:51.226 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:51.226 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:51.226 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.226 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 0, 2024-09-14T03:47:51.226 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:51.226 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:51.226 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:51.226 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:51.226 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:51.226 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:51.227 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.684378+0000", 2024-09-14T03:47:51.227 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.874668+0000", 2024-09-14T03:47:51.227 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.874668+0000", 2024-09-14T03:47:51.227 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.874935+0000", 2024-09-14T03:47:51.227 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.684378+0000", 2024-09-14T03:47:51.227 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.227 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:51.227 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.227 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:51.227 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.684378+0000", 2024-09-14T03:47:51.227 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.684378+0000", 2024-09-14T03:47:51.227 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.684378+0000", 2024-09-14T03:47:51.228 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:51.228 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:51.228 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.228 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.684378+0000", 2024-09-14T03:47:51.228 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.684378+0000", 2024-09-14T03:47:51.228 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:51.228 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:51.228 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:51.228 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:51.228 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:51.228 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:51.228 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:51.228 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:51.229 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:51.229 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:51.229 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:51.229 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:51.229 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:51.229 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.1f", 2024-09-14T03:47:51.229 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:51.229 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:51.229 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:51.229 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:51.229 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:51.229 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T15:24:31.404958+0000", 2024-09-14T03:47:51.229 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:51.230 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:51.230 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:51.230 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:51.230 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:51.230 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:51.230 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:51.230 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:51.230 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:51.230 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:51.230 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:51.230 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:51.230 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:51.230 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:51.230 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:51.231 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:51.231 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:51.231 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:51.231 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:51.231 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:51.231 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:51.231 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:51.231 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:51.231 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:51.231 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:51.231 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:51.231 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:51.231 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:51.231 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:51.232 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:51.232 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:51.232 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:51.232 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:51.232 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:51.232 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:51.232 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:51.232 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:51.232 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:51.232 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:51.232 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:51.232 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:51.232 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:51.233 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:51.233 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.233 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:51.233 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:51.233 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:51.233 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:51.233 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:51.233 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:51.233 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.233 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 0, 2024-09-14T03:47:51.233 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:51.233 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.233 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.234 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:51.234 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:51.234 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:51.234 INFO:tasks.workunit.client.0.smithi070.stdout: 5 2024-09-14T03:47:51.234 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.234 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 3, 2024-09-14T03:47:51.234 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:51.234 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:51.234 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:51.234 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:51.234 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:51.234 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:51.234 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.857877+0000", 2024-09-14T03:47:51.234 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.733960+0000", 2024-09-14T03:47:51.235 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.733960+0000", 2024-09-14T03:47:51.235 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.734206+0000", 2024-09-14T03:47:51.235 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.857877+0000", 2024-09-14T03:47:51.235 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.235 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:51.235 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.235 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:51.235 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.857877+0000", 2024-09-14T03:47:51.235 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.857877+0000", 2024-09-14T03:47:51.235 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.857877+0000", 2024-09-14T03:47:51.235 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:51.235 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:51.235 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.235 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.857877+0000", 2024-09-14T03:47:51.235 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.857877+0000", 2024-09-14T03:47:51.236 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:51.236 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 1, 2024-09-14T03:47:51.236 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:51.236 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:51.236 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:51.236 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:51.236 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:51.236 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:51.236 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:51.236 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 1, 2024-09-14T03:47:51.236 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:51.236 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:51.236 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:51.237 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.1e", 2024-09-14T03:47:51.237 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:51.237 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:51.237 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:51.237 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 24, 2024-09-14T03:47:51.237 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:51.237 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:36:44.073503+0000", 2024-09-14T03:47:51.237 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:51.237 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:51.237 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:51.237 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:51.238 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:51.238 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:51.238 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:51.238 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:51.238 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:51.238 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:51.238 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:51.238 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:51.238 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:51.238 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:51.238 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:51.239 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:51.239 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:51.239 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:51.239 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:51.239 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:51.239 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:51.239 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:51.239 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:51.239 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:51.239 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:51.239 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:51.239 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:51.239 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:51.240 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:51.240 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:51.240 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:51.240 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:51.240 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:51.240 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:51.240 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:51.240 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:51.240 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:51.240 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:51.240 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:51.240 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:51.241 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:51.241 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:51.241 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:51.241 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.241 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:51.241 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:51.241 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:51.241 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:51.241 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:51.241 INFO:tasks.workunit.client.0.smithi070.stdout: 5 2024-09-14T03:47:51.241 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.241 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 3, 2024-09-14T03:47:51.241 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "58'1" 2024-09-14T03:47:51.241 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.242 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.242 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:51.242 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:51.242 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:51.242 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:51.242 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.242 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 3, 2024-09-14T03:47:51.242 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:51.242 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:51.242 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:51.242 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:51.242 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:51.242 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:51.243 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.857768+0000", 2024-09-14T03:47:51.243 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.883265+0000", 2024-09-14T03:47:51.243 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.883265+0000", 2024-09-14T03:47:51.243 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.883550+0000", 2024-09-14T03:47:51.243 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.857768+0000", 2024-09-14T03:47:51.243 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.243 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:51.243 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.243 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:51.243 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.857768+0000", 2024-09-14T03:47:51.243 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.857768+0000", 2024-09-14T03:47:51.243 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.857768+0000", 2024-09-14T03:47:51.244 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:51.244 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:51.244 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:51.244 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.857768+0000", 2024-09-14T03:47:51.244 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.857768+0000", 2024-09-14T03:47:51.244 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:51.244 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:51.244 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:51.244 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:51.244 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:51.244 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:51.244 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:51.244 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:51.245 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:51.245 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:51.245 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:51.245 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:51.245 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:51.245 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.1e", 2024-09-14T03:47:51.245 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:51.245 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:51.245 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:51.245 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:51.246 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:51.246 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T03:52:04.512990+0000", 2024-09-14T03:47:51.246 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:51.246 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:51.246 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:51.246 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:51.246 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:51.246 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:51.246 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:51.246 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:51.246 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:51.246 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:51.246 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:51.246 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:51.247 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:51.247 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:51.247 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:51.247 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:51.247 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:51.247 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:51.247 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:51.247 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:51.247 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:51.247 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:51.247 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:51.247 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:51.247 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:51.247 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:51.248 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:51.248 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:51.248 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:51.248 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:51.248 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:51.248 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:51.248 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:51.248 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:51.248 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:51.248 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:51.248 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:51.248 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:51.248 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:51.248 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:51.249 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:51.249 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:51.249 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:51.249 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.249 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:51.249 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:51.249 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:51.249 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:51.249 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:51.249 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:51.249 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.249 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 3, 2024-09-14T03:47:51.249 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:51.249 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.250 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.250 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:51.250 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:51.250 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:51.250 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:51.250 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.250 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 0, 2024-09-14T03:47:51.250 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:51.250 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:51.250 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:51.250 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:51.250 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:51.250 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:51.251 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.684569+0000", 2024-09-14T03:47:51.251 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.731688+0000", 2024-09-14T03:47:51.251 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.731688+0000", 2024-09-14T03:47:51.251 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.731954+0000", 2024-09-14T03:47:51.251 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.684569+0000", 2024-09-14T03:47:51.251 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.251 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:51.251 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.251 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:51.251 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.684569+0000", 2024-09-14T03:47:51.251 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.684569+0000", 2024-09-14T03:47:51.251 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.684569+0000", 2024-09-14T03:47:51.251 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:51.251 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:51.252 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:51.252 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.684569+0000", 2024-09-14T03:47:51.252 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.684569+0000", 2024-09-14T03:47:51.252 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:51.252 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 4, 2024-09-14T03:47:51.252 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:51.252 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:51.252 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:51.252 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:51.252 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:51.252 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:51.252 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:51.252 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 4, 2024-09-14T03:47:51.253 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:51.253 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:51.253 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:51.253 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.1f", 2024-09-14T03:47:51.253 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:51.253 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:51.253 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:51.253 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 27, 2024-09-14T03:47:51.253 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:51.253 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:30:10.811609+0000", 2024-09-14T03:47:51.253 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:51.253 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:51.253 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:51.254 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 570, 2024-09-14T03:47:51.254 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:51.254 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:51.254 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:51.254 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:51.254 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:51.254 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:51.254 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:51.254 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:51.254 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:51.254 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:51.254 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:51.254 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:51.254 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:51.255 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:51.255 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:51.255 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 6, 2024-09-14T03:47:51.255 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 2, 2024-09-14T03:47:51.255 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:51.255 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 2, 2024-09-14T03:47:51.255 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:51.255 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:51.255 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:51.255 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:51.255 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:51.255 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 1, 2024-09-14T03:47:51.255 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:51.255 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:51.256 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:51.256 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:51.256 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:51.256 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:51.256 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:51.256 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:51.256 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:51.256 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:51.256 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:51.256 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:51.256 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 4, 2024-09-14T03:47:51.256 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 1 2024-09-14T03:47:51.256 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.257 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:51.257 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:51.257 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:51.257 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:51.257 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:51.257 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:51.257 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.257 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 0, 2024-09-14T03:47:51.257 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "58'4" 2024-09-14T03:47:51.257 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.257 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.257 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_per_pool": [ 2024-09-14T03:47:51.257 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.257 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": 24, 2024-09-14T03:47:51.258 INFO:tasks.workunit.client.0.smithi070.stdout: "application": [ 2024-09-14T03:47:51.258 INFO:tasks.workunit.client.0.smithi070.stdout: "rbd" 2024-09-14T03:47:51.258 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.258 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 3, 2024-09-14T03:47:51.258 INFO:tasks.workunit.client.0.smithi070.stdout: "num_pg": 8, 2024-09-14T03:47:51.258 INFO:tasks.workunit.client.0.smithi070.stdout: "num_store_stats": 8, 2024-09-14T03:47:51.258 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 3, 2024-09-14T03:47:51.258 INFO:tasks.workunit.client.0.smithi070.stdout: "poolid": 4, 2024-09-14T03:47:51.258 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:51.258 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 19, 2024-09-14T03:47:51.258 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:51.258 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:51.258 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:51.258 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:51.259 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:51.259 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:51.259 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:51.259 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:51.259 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:51.259 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:51.259 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:51.259 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:51.259 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:51.259 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:51.259 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:51.259 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 6, 2024-09-14T03:47:51.259 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 2, 2024-09-14T03:47:51.260 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:51.260 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 2, 2024-09-14T03:47:51.260 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:51.260 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:51.260 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:51.260 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:51.260 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:51.260 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:51.260 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:51.260 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:51.260 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:51.260 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:51.260 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:51.260 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:51.261 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:51.261 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:51.261 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:51.261 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:51.261 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:51.261 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:51.261 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 2, 2024-09-14T03:47:51.261 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 2 2024-09-14T03:47:51.261 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.261 INFO:tasks.workunit.client.0.smithi070.stdout: "store_stats": { 2024-09-14T03:47:51.261 INFO:tasks.workunit.client.0.smithi070.stdout: "allocated": 12288, 2024-09-14T03:47:51.261 INFO:tasks.workunit.client.0.smithi070.stdout: "available": 0, 2024-09-14T03:47:51.261 INFO:tasks.workunit.client.0.smithi070.stdout: "data_compressed": 0, 2024-09-14T03:47:51.262 INFO:tasks.workunit.client.0.smithi070.stdout: "data_compressed_allocated": 0, 2024-09-14T03:47:51.262 INFO:tasks.workunit.client.0.smithi070.stdout: "data_compressed_original": 0, 2024-09-14T03:47:51.262 INFO:tasks.workunit.client.0.smithi070.stdout: "data_stored": 57, 2024-09-14T03:47:51.262 INFO:tasks.workunit.client.0.smithi070.stdout: "internal_metadata": 0, 2024-09-14T03:47:51.262 INFO:tasks.workunit.client.0.smithi070.stdout: "internally_reserved": 0, 2024-09-14T03:47:51.262 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_allocated": 0, 2024-09-14T03:47:51.262 INFO:tasks.workunit.client.0.smithi070.stdout: "total": 0 2024-09-14T03:47:51.262 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.262 INFO:tasks.workunit.client.0.smithi070.stdout: "up": 24 2024-09-14T03:47:51.262 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.262 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.262 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": 192, 2024-09-14T03:47:51.263 INFO:tasks.workunit.client.0.smithi070.stdout: "application": [ 2024-09-14T03:47:51.263 INFO:tasks.workunit.client.0.smithi070.stdout: "cephfs" 2024-09-14T03:47:51.263 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.263 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:51.263 INFO:tasks.workunit.client.0.smithi070.stdout: "num_pg": 64, 2024-09-14T03:47:51.263 INFO:tasks.workunit.client.0.smithi070.stdout: "num_store_stats": 8, 2024-09-14T03:47:51.263 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:51.263 INFO:tasks.workunit.client.0.smithi070.stdout: "poolid": 3, 2024-09-14T03:47:51.263 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:51.263 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:51.263 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:51.263 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:51.263 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:51.263 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:51.264 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:51.264 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:51.264 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:51.264 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:51.264 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:51.264 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:51.264 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:51.264 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:51.264 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:51.264 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:51.264 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:51.264 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:51.264 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:51.265 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:51.265 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:51.265 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:51.265 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:51.265 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:51.265 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:51.265 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:51.265 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:51.265 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:51.265 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:51.265 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:51.265 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:51.265 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:51.265 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:51.266 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:51.266 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:51.266 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:51.266 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:51.266 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:51.266 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:51.266 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:51.266 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:51.266 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.266 INFO:tasks.workunit.client.0.smithi070.stdout: "store_stats": { 2024-09-14T03:47:51.266 INFO:tasks.workunit.client.0.smithi070.stdout: "allocated": 0, 2024-09-14T03:47:51.266 INFO:tasks.workunit.client.0.smithi070.stdout: "available": 0, 2024-09-14T03:47:51.266 INFO:tasks.workunit.client.0.smithi070.stdout: "data_compressed": 0, 2024-09-14T03:47:51.267 INFO:tasks.workunit.client.0.smithi070.stdout: "data_compressed_allocated": 0, 2024-09-14T03:47:51.267 INFO:tasks.workunit.client.0.smithi070.stdout: "data_compressed_original": 0, 2024-09-14T03:47:51.267 INFO:tasks.workunit.client.0.smithi070.stdout: "data_stored": 0, 2024-09-14T03:47:51.267 INFO:tasks.workunit.client.0.smithi070.stdout: "internal_metadata": 0, 2024-09-14T03:47:51.267 INFO:tasks.workunit.client.0.smithi070.stdout: "internally_reserved": 0, 2024-09-14T03:47:51.267 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_allocated": 0, 2024-09-14T03:47:51.267 INFO:tasks.workunit.client.0.smithi070.stdout: "total": 0 2024-09-14T03:47:51.267 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.267 INFO:tasks.workunit.client.0.smithi070.stdout: "up": 192 2024-09-14T03:47:51.267 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.267 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.268 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": 96, 2024-09-14T03:47:51.268 INFO:tasks.workunit.client.0.smithi070.stdout: "application": [ 2024-09-14T03:47:51.268 INFO:tasks.workunit.client.0.smithi070.stdout: "cephfs" 2024-09-14T03:47:51.268 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.268 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 45, 2024-09-14T03:47:51.268 INFO:tasks.workunit.client.0.smithi070.stdout: "num_pg": 32, 2024-09-14T03:47:51.268 INFO:tasks.workunit.client.0.smithi070.stdout: "num_store_stats": 8, 2024-09-14T03:47:51.268 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 45, 2024-09-14T03:47:51.268 INFO:tasks.workunit.client.0.smithi070.stdout: "poolid": 2, 2024-09-14T03:47:51.268 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:51.268 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 3410, 2024-09-14T03:47:51.268 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:51.268 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:51.268 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:51.269 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:51.269 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:51.269 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:51.269 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:51.269 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:51.269 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:51.269 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:51.269 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:51.269 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:51.269 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:51.270 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:51.270 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:51.270 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 66, 2024-09-14T03:47:51.270 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 22, 2024-09-14T03:47:51.270 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:51.270 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 22, 2024-09-14T03:47:51.270 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:51.270 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:51.270 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:51.271 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:51.271 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:51.271 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 14, 2024-09-14T03:47:51.271 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:51.271 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:51.271 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:51.271 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:51.271 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:51.271 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:51.271 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:51.271 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:51.272 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:51.272 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:51.272 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:51.272 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:51.272 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 51, 2024-09-14T03:47:51.272 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 19 2024-09-14T03:47:51.272 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.272 INFO:tasks.workunit.client.0.smithi070.stdout: "store_stats": { 2024-09-14T03:47:51.272 INFO:tasks.workunit.client.0.smithi070.stdout: "allocated": 98304, 2024-09-14T03:47:51.272 INFO:tasks.workunit.client.0.smithi070.stdout: "available": 0, 2024-09-14T03:47:51.272 INFO:tasks.workunit.client.0.smithi070.stdout: "data_compressed": 0, 2024-09-14T03:47:51.272 INFO:tasks.workunit.client.0.smithi070.stdout: "data_compressed_allocated": 0, 2024-09-14T03:47:51.272 INFO:tasks.workunit.client.0.smithi070.stdout: "data_compressed_original": 0, 2024-09-14T03:47:51.272 INFO:tasks.workunit.client.0.smithi070.stdout: "data_stored": 10230, 2024-09-14T03:47:51.273 INFO:tasks.workunit.client.0.smithi070.stdout: "internal_metadata": 0, 2024-09-14T03:47:51.273 INFO:tasks.workunit.client.0.smithi070.stdout: "internally_reserved": 0, 2024-09-14T03:47:51.273 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_allocated": 0, 2024-09-14T03:47:51.273 INFO:tasks.workunit.client.0.smithi070.stdout: "total": 0 2024-09-14T03:47:51.273 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.273 INFO:tasks.workunit.client.0.smithi070.stdout: "up": 96 2024-09-14T03:47:51.273 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.273 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.273 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": 3, 2024-09-14T03:47:51.273 INFO:tasks.workunit.client.0.smithi070.stdout: "application": [ 2024-09-14T03:47:51.273 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr" 2024-09-14T03:47:51.273 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.273 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 95, 2024-09-14T03:47:51.274 INFO:tasks.workunit.client.0.smithi070.stdout: "num_pg": 1, 2024-09-14T03:47:51.274 INFO:tasks.workunit.client.0.smithi070.stdout: "num_store_stats": 3, 2024-09-14T03:47:51.274 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 95, 2024-09-14T03:47:51.274 INFO:tasks.workunit.client.0.smithi070.stdout: "poolid": 1, 2024-09-14T03:47:51.274 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:51.274 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 590368, 2024-09-14T03:47:51.274 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:51.274 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 590368, 2024-09-14T03:47:51.274 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:51.274 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:51.274 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:51.274 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:51.274 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:51.274 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:51.275 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:51.275 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:51.275 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:51.275 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:51.275 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:51.275 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:51.275 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:51.275 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 6, 2024-09-14T03:47:51.275 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 2, 2024-09-14T03:47:51.275 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:51.275 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 2, 2024-09-14T03:47:51.276 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:51.276 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:51.276 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:51.276 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:51.276 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:51.276 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:51.276 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:51.276 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 2, 2024-09-14T03:47:51.276 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:51.276 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:51.276 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:51.276 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:51.276 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:51.277 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 272, 2024-09-14T03:47:51.277 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 479, 2024-09-14T03:47:51.277 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:51.277 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:51.277 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:51.277 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 145, 2024-09-14T03:47:51.277 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 1628 2024-09-14T03:47:51.277 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.277 INFO:tasks.workunit.client.0.smithi070.stdout: "store_stats": { 2024-09-14T03:47:51.277 INFO:tasks.workunit.client.0.smithi070.stdout: "allocated": 1781760, 2024-09-14T03:47:51.277 INFO:tasks.workunit.client.0.smithi070.stdout: "available": 0, 2024-09-14T03:47:51.277 INFO:tasks.workunit.client.0.smithi070.stdout: "data_compressed": 0, 2024-09-14T03:47:51.277 INFO:tasks.workunit.client.0.smithi070.stdout: "data_compressed_allocated": 0, 2024-09-14T03:47:51.277 INFO:tasks.workunit.client.0.smithi070.stdout: "data_compressed_original": 0, 2024-09-14T03:47:51.278 INFO:tasks.workunit.client.0.smithi070.stdout: "data_stored": 1771104, 2024-09-14T03:47:51.278 INFO:tasks.workunit.client.0.smithi070.stdout: "internal_metadata": 0, 2024-09-14T03:47:51.278 INFO:tasks.workunit.client.0.smithi070.stdout: "internally_reserved": 0, 2024-09-14T03:47:51.278 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_allocated": 0, 2024-09-14T03:47:51.278 INFO:tasks.workunit.client.0.smithi070.stdout: "total": 0 2024-09-14T03:47:51.278 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.278 INFO:tasks.workunit.client.0.smithi070.stdout: "up": 3 2024-09-14T03:47:51.278 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.278 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.278 INFO:tasks.workunit.client.0.smithi070.stdout: "usage": { 2024-09-14T03:47:51.278 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_num": 105, 2024-09-14T03:47:51.278 INFO:tasks.workunit.client.0.smithi070.stdout: "pools": 4, 2024-09-14T03:47:51.278 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_by_class": { 2024-09-14T03:47:51.278 INFO:tasks.workunit.client.0.smithi070.stdout: "ssd": { 2024-09-14T03:47:51.279 INFO:tasks.workunit.client.0.smithi070.stdout: "total_avail_bytes": 765516935168, 2024-09-14T03:47:51.279 INFO:tasks.workunit.client.0.smithi070.stdout: "total_bytes": 767960285184, 2024-09-14T03:47:51.279 INFO:tasks.workunit.client.0.smithi070.stdout: "total_used_bytes": 2443350016, 2024-09-14T03:47:51.279 INFO:tasks.workunit.client.0.smithi070.stdout: "total_used_raw_bytes": 2443350016, 2024-09-14T03:47:51.279 INFO:tasks.workunit.client.0.smithi070.stdout: "total_used_raw_ratio": 0.0031816097907721996 2024-09-14T03:47:51.279 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.279 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.279 INFO:tasks.workunit.client.0.smithi070.stdout: "total_avail_bytes": 765516935168, 2024-09-14T03:47:51.279 INFO:tasks.workunit.client.0.smithi070.stdout: "total_bytes": 767960285184, 2024-09-14T03:47:51.279 INFO:tasks.workunit.client.0.smithi070.stdout: "total_used_bytes": 2443350016 2024-09-14T03:47:51.279 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.279 INFO:tasks.workunit.client.0.smithi070.stdout:} 2024-09-14T03:47:51.279 INFO:tasks.workunit.client.0.smithi070.stdout:Device report is generated separately. To see it run 'ceph telemetry show-device'. 2024-09-14T03:47:51.280 INFO:tasks.workunit.client.0.smithi070.stderr:+ ceph telemetry show-device 2024-09-14T03:47:51.281 INFO:tasks.workunit.client.0.smithi070.stdout:{ 2024-09-14T03:47:51.281 INFO:tasks.workunit.client.0.smithi070.stdout: "1ad5ed40-724c-11ef-9ddf-0cc47a8fd3e7": { 2024-09-14T03:47:51.281 INFO:tasks.workunit.client.0.smithi070.stdout: "INTEL_SSDPED1D480GA_1ad66d74-724c-11ef-9ddf-0cc47a8fd3e7": { 2024-09-14T03:47:51.281 INFO:tasks.workunit.client.0.smithi070.stdout: "20240914-034143": { 2024-09-14T03:47:51.281 INFO:tasks.workunit.client.0.smithi070.stdout: "device": { 2024-09-14T03:47:51.281 INFO:tasks.workunit.client.0.smithi070.stdout: "info_name": "/dev/nvme0n1", 2024-09-14T03:47:51.282 INFO:tasks.workunit.client.0.smithi070.stdout: "name": "/dev/nvme0n1", 2024-09-14T03:47:51.282 INFO:tasks.workunit.client.0.smithi070.stdout: "protocol": "NVMe", 2024-09-14T03:47:51.282 INFO:tasks.workunit.client.0.smithi070.stdout: "type": "nvme" 2024-09-14T03:47:51.282 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.282 INFO:tasks.workunit.client.0.smithi070.stdout: "firmware_version": "E2010325", 2024-09-14T03:47:51.282 INFO:tasks.workunit.client.0.smithi070.stdout: "host_id": "1ad5ed40-724c-11ef-9ddf-0cc47a8fd3e7", 2024-09-14T03:47:51.282 INFO:tasks.workunit.client.0.smithi070.stdout: "json_format_version": [ 2024-09-14T03:47:51.282 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:51.282 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:51.282 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.282 INFO:tasks.workunit.client.0.smithi070.stdout: "local_time": { 2024-09-14T03:47:51.282 INFO:tasks.workunit.client.0.smithi070.stdout: "asctime": "Sat Sep 14 03:41:43 2024 UTC", 2024-09-14T03:47:51.282 INFO:tasks.workunit.client.0.smithi070.stdout: "time_t": 1726285303 2024-09-14T03:47:51.282 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.283 INFO:tasks.workunit.client.0.smithi070.stdout: "logical_block_size": 512, 2024-09-14T03:47:51.283 INFO:tasks.workunit.client.0.smithi070.stdout: "model_name": "INTEL SSDPED1D480GA", 2024-09-14T03:47:51.283 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_controller_id": 0, 2024-09-14T03:47:51.283 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-14T03:47:51.283 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_namespaces": [ 2024-09-14T03:47:51.283 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.283 INFO:tasks.workunit.client.0.smithi070.stdout: "capacity": { 2024-09-14T03:47:51.283 INFO:tasks.workunit.client.0.smithi070.stdout: "blocks": 937703088, 2024-09-14T03:47:51.283 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 480103981056 2024-09-14T03:47:51.283 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.283 INFO:tasks.workunit.client.0.smithi070.stdout: "formatted_lba_size": 512, 2024-09-14T03:47:51.284 INFO:tasks.workunit.client.0.smithi070.stdout: "id": 1, 2024-09-14T03:47:51.284 INFO:tasks.workunit.client.0.smithi070.stdout: "size": { 2024-09-14T03:47:51.284 INFO:tasks.workunit.client.0.smithi070.stdout: "blocks": 937703088, 2024-09-14T03:47:51.284 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 480103981056 2024-09-14T03:47:51.284 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.284 INFO:tasks.workunit.client.0.smithi070.stdout: "utilization": { 2024-09-14T03:47:51.284 INFO:tasks.workunit.client.0.smithi070.stdout: "blocks": 937703088, 2024-09-14T03:47:51.284 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 480103981056 2024-09-14T03:47:51.284 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.284 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.284 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.284 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_number_of_namespaces": 1, 2024-09-14T03:47:51.285 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_pci_vendor": { 2024-09-14T03:47:51.285 INFO:tasks.workunit.client.0.smithi070.stdout: "id": 32902, 2024-09-14T03:47:51.285 INFO:tasks.workunit.client.0.smithi070.stdout: "subsystem_id": 32902 2024-09-14T03:47:51.285 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.285 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_smart_health_information_add_log": { 2024-09-14T03:47:51.285 INFO:tasks.workunit.client.0.smithi070.stdout: "Device stats": { 2024-09-14T03:47:51.285 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_fw_downgrades": { 2024-09-14T03:47:51.285 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.285 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.285 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.285 INFO:tasks.workunit.client.0.smithi070.stdout: "crc_error_count": { 2024-09-14T03:47:51.285 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.285 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.285 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.286 INFO:tasks.workunit.client.0.smithi070.stdout: "end_to_end_error_detection_count": { 2024-09-14T03:47:51.286 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.286 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.286 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.286 INFO:tasks.workunit.client.0.smithi070.stdout: "erase_fail_count": { 2024-09-14T03:47:51.286 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.286 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.286 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.286 INFO:tasks.workunit.client.0.smithi070.stdout: "host_bytes_written": { 2024-09-14T03:47:51.286 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.286 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 13063793 2024-09-14T03:47:51.286 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.286 INFO:tasks.workunit.client.0.smithi070.stdout: "host_ctx_wear_used": { 2024-09-14T03:47:51.286 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.287 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.287 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.287 INFO:tasks.workunit.client.0.smithi070.stdout: "media_bytes_read": { 2024-09-14T03:47:51.287 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.287 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.287 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.287 INFO:tasks.workunit.client.0.smithi070.stdout: "nand_bytes_written": { 2024-09-14T03:47:51.287 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.287 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.287 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.287 INFO:tasks.workunit.client.0.smithi070.stdout: "perf_stat_indicator": { 2024-09-14T03:47:51.287 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.287 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.288 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.288 INFO:tasks.workunit.client.0.smithi070.stdout: "pll_lock_loss_count": { 2024-09-14T03:47:51.288 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.288 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.288 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.288 INFO:tasks.workunit.client.0.smithi070.stdout: "program_fail_count": { 2024-09-14T03:47:51.288 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.288 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.289 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.289 INFO:tasks.workunit.client.0.smithi070.stdout: "re_alloc_sectr_cnt": { 2024-09-14T03:47:51.289 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.289 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.289 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.289 INFO:tasks.workunit.client.0.smithi070.stdout: "retry_buffer_overflow_count": { 2024-09-14T03:47:51.289 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.289 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.289 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.289 INFO:tasks.workunit.client.0.smithi070.stdout: "soft_ecc_err_rate": { 2024-09-14T03:47:51.289 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.290 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.290 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.290 INFO:tasks.workunit.client.0.smithi070.stdout: "thermal_throttle_status": { 2024-09-14T03:47:51.290 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.290 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": { 2024-09-14T03:47:51.290 INFO:tasks.workunit.client.0.smithi070.stdout: "cnt": 0, 2024-09-14T03:47:51.290 INFO:tasks.workunit.client.0.smithi070.stdout: "pct": 0 2024-09-14T03:47:51.290 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.290 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.290 INFO:tasks.workunit.client.0.smithi070.stdout: "timed_workload_host_reads": { 2024-09-14T03:47:51.290 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.291 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 14 2024-09-14T03:47:51.291 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.291 INFO:tasks.workunit.client.0.smithi070.stdout: "timed_workload_media_wear": { 2024-09-14T03:47:51.291 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.291 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": "9.029297" 2024-09-14T03:47:51.291 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.291 INFO:tasks.workunit.client.0.smithi070.stdout: "timed_workload_timer": { 2024-09-14T03:47:51.291 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.291 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 1544484 2024-09-14T03:47:51.291 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.291 INFO:tasks.workunit.client.0.smithi070.stdout: "unexp_power_loss": { 2024-09-14T03:47:51.291 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.291 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.291 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.292 INFO:tasks.workunit.client.0.smithi070.stdout: "wear_leveling": { 2024-09-14T03:47:51.292 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.292 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": { 2024-09-14T03:47:51.292 INFO:tasks.workunit.client.0.smithi070.stdout: "avg": 0, 2024-09-14T03:47:51.292 INFO:tasks.workunit.client.0.smithi070.stdout: "max": 0, 2024-09-14T03:47:51.292 INFO:tasks.workunit.client.0.smithi070.stdout: "min": 0 2024-09-14T03:47:51.292 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.292 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.292 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.292 INFO:tasks.workunit.client.0.smithi070.stdout: "Intel Smart log": "nvme0n1" 2024-09-14T03:47:51.292 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.292 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_smart_health_information_log": { 2024-09-14T03:47:51.292 INFO:tasks.workunit.client.0.smithi070.stdout: "available_spare": 100, 2024-09-14T03:47:51.292 INFO:tasks.workunit.client.0.smithi070.stdout: "available_spare_threshold": 0, 2024-09-14T03:47:51.293 INFO:tasks.workunit.client.0.smithi070.stdout: "controller_busy_time": 5166, 2024-09-14T03:47:51.293 INFO:tasks.workunit.client.0.smithi070.stdout: "critical_warning": 0, 2024-09-14T03:47:51.293 INFO:tasks.workunit.client.0.smithi070.stdout: "data_units_read": 177890366, 2024-09-14T03:47:51.293 INFO:tasks.workunit.client.0.smithi070.stdout: "data_units_written": 856148757, 2024-09-14T03:47:51.293 INFO:tasks.workunit.client.0.smithi070.stdout: "host_reads": 3130935174, 2024-09-14T03:47:51.293 INFO:tasks.workunit.client.0.smithi070.stdout: "host_writes": 18536697060, 2024-09-14T03:47:51.293 INFO:tasks.workunit.client.0.smithi070.stdout: "media_errors": 0, 2024-09-14T03:47:51.293 INFO:tasks.workunit.client.0.smithi070.stdout: "num_err_log_entries": 0, 2024-09-14T03:47:51.293 INFO:tasks.workunit.client.0.smithi070.stdout: "percentage_used": 14, 2024-09-14T03:47:51.293 INFO:tasks.workunit.client.0.smithi070.stdout: "power_cycles": 35890, 2024-09-14T03:47:51.293 INFO:tasks.workunit.client.0.smithi070.stdout: "power_on_hours": 40704, 2024-09-14T03:47:51.293 INFO:tasks.workunit.client.0.smithi070.stdout: "temperature": 39, 2024-09-14T03:47:51.293 INFO:tasks.workunit.client.0.smithi070.stdout: "unsafe_shutdowns": 10931 2024-09-14T03:47:51.294 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.294 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_vendor": "intel", 2024-09-14T03:47:51.294 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_version": { 2024-09-14T03:47:51.294 INFO:tasks.workunit.client.0.smithi070.stdout: "string": "<1.2", 2024-09-14T03:47:51.294 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.294 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.294 INFO:tasks.workunit.client.0.smithi070.stdout: "power_cycle_count": 35890, 2024-09-14T03:47:51.294 INFO:tasks.workunit.client.0.smithi070.stdout: "power_on_time": { 2024-09-14T03:47:51.294 INFO:tasks.workunit.client.0.smithi070.stdout: "hours": 40704 2024-09-14T03:47:51.294 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.294 INFO:tasks.workunit.client.0.smithi070.stdout: "serial_number": "deleted", 2024-09-14T03:47:51.294 INFO:tasks.workunit.client.0.smithi070.stdout: "smart_status": { 2024-09-14T03:47:51.294 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme": { 2024-09-14T03:47:51.295 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.295 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.295 INFO:tasks.workunit.client.0.smithi070.stdout: "passed": true 2024-09-14T03:47:51.295 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.295 INFO:tasks.workunit.client.0.smithi070.stdout: "smartctl": { 2024-09-14T03:47:51.295 INFO:tasks.workunit.client.0.smithi070.stdout: "argv": [ 2024-09-14T03:47:51.295 INFO:tasks.workunit.client.0.smithi070.stdout: "smartctl", 2024-09-14T03:47:51.295 INFO:tasks.workunit.client.0.smithi070.stdout: "-x", 2024-09-14T03:47:51.296 INFO:tasks.workunit.client.0.smithi070.stdout: "--json=o", 2024-09-14T03:47:51.296 INFO:tasks.workunit.client.0.smithi070.stdout: "/dev/nvme0n1" 2024-09-14T03:47:51.296 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.296 INFO:tasks.workunit.client.0.smithi070.stdout: "build_info": "(local build)", 2024-09-14T03:47:51.296 INFO:tasks.workunit.client.0.smithi070.stdout: "exit_status": 0, 2024-09-14T03:47:51.296 INFO:tasks.workunit.client.0.smithi070.stdout: "output": [ 2024-09-14T03:47:51.296 INFO:tasks.workunit.client.0.smithi070.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-505.el9.x86_64] (local build)", 2024-09-14T03:47:51.296 INFO:tasks.workunit.client.0.smithi070.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-14T03:47:51.296 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:51.296 INFO:tasks.workunit.client.0.smithi070.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-14T03:47:51.297 INFO:tasks.workunit.client.0.smithi070.stdout: "Model Number: INTEL SSDPED1D480GA", 2024-09-14T03:47:51.297 INFO:tasks.workunit.client.0.smithi070.stdout: "Serial Number: deleted", 2024-09-14T03:47:51.297 INFO:tasks.workunit.client.0.smithi070.stdout: "Firmware Version: E2010325", 2024-09-14T03:47:51.297 INFO:tasks.workunit.client.0.smithi070.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-14T03:47:51.297 INFO:tasks.workunit.client.0.smithi070.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-14T03:47:51.297 INFO:tasks.workunit.client.0.smithi070.stdout: "Controller ID: 0", 2024-09-14T03:47:51.297 INFO:tasks.workunit.client.0.smithi070.stdout: "NVMe Version: <1.2", 2024-09-14T03:47:51.297 INFO:tasks.workunit.client.0.smithi070.stdout: "Number of Namespaces: 1", 2024-09-14T03:47:51.297 INFO:tasks.workunit.client.0.smithi070.stdout: "Namespace 1 Size/Capacity: 480,103,981,056 [480 GB]", 2024-09-14T03:47:51.297 INFO:tasks.workunit.client.0.smithi070.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-14T03:47:51.297 INFO:tasks.workunit.client.0.smithi070.stdout: "Local Time is: Sat Sep 14 03:41:43 2024 UTC", 2024-09-14T03:47:51.297 INFO:tasks.workunit.client.0.smithi070.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-14T03:47:51.298 INFO:tasks.workunit.client.0.smithi070.stdout: "Optional Admin Commands (0x0007): Security Format Frmw_DL", 2024-09-14T03:47:51.298 INFO:tasks.workunit.client.0.smithi070.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-14T03:47:51.298 INFO:tasks.workunit.client.0.smithi070.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-14T03:47:51.298 INFO:tasks.workunit.client.0.smithi070.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-14T03:47:51.298 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:51.298 INFO:tasks.workunit.client.0.smithi070.stdout: "Supported Power States", 2024-09-14T03:47:51.298 INFO:tasks.workunit.client.0.smithi070.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-14T03:47:51.298 INFO:tasks.workunit.client.0.smithi070.stdout: " 0 + 18.00W - - 0 0 0 0 0 0", 2024-09-14T03:47:51.298 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:51.298 INFO:tasks.workunit.client.0.smithi070.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-14T03:47:51.298 INFO:tasks.workunit.client.0.smithi070.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-14T03:47:51.298 INFO:tasks.workunit.client.0.smithi070.stdout: " 0 + 512 0 2", 2024-09-14T03:47:51.298 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:51.298 INFO:tasks.workunit.client.0.smithi070.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-14T03:47:51.299 INFO:tasks.workunit.client.0.smithi070.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-14T03:47:51.299 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:51.299 INFO:tasks.workunit.client.0.smithi070.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-14T03:47:51.299 INFO:tasks.workunit.client.0.smithi070.stdout: "Critical Warning: 0x00", 2024-09-14T03:47:51.299 INFO:tasks.workunit.client.0.smithi070.stdout: "Temperature: 39 Celsius", 2024-09-14T03:47:51.299 INFO:tasks.workunit.client.0.smithi070.stdout: "Available Spare: 100%", 2024-09-14T03:47:51.299 INFO:tasks.workunit.client.0.smithi070.stdout: "Available Spare Threshold: 0%", 2024-09-14T03:47:51.299 INFO:tasks.workunit.client.0.smithi070.stdout: "Percentage Used: 14%", 2024-09-14T03:47:51.299 INFO:tasks.workunit.client.0.smithi070.stdout: "Data Units Read: 177,890,366 [91.0 TB]", 2024-09-14T03:47:51.299 INFO:tasks.workunit.client.0.smithi070.stdout: "Data Units Written: 856,148,757 [438 TB]", 2024-09-14T03:47:51.299 INFO:tasks.workunit.client.0.smithi070.stdout: "Host Read Commands: 3,130,935,174", 2024-09-14T03:47:51.299 INFO:tasks.workunit.client.0.smithi070.stdout: "Host Write Commands: 18,536,697,060", 2024-09-14T03:47:51.300 INFO:tasks.workunit.client.0.smithi070.stdout: "Controller Busy Time: 5,166", 2024-09-14T03:47:51.300 INFO:tasks.workunit.client.0.smithi070.stdout: "Power Cycles: 35,890", 2024-09-14T03:47:51.300 INFO:tasks.workunit.client.0.smithi070.stdout: "Power On Hours: 40,704", 2024-09-14T03:47:51.300 INFO:tasks.workunit.client.0.smithi070.stdout: "Unsafe Shutdowns: 10,931", 2024-09-14T03:47:51.300 INFO:tasks.workunit.client.0.smithi070.stdout: "Media and Data Integrity Errors: 0", 2024-09-14T03:47:51.300 INFO:tasks.workunit.client.0.smithi070.stdout: "Error Information Log Entries: 0", 2024-09-14T03:47:51.300 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:51.300 INFO:tasks.workunit.client.0.smithi070.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-14T03:47:51.300 INFO:tasks.workunit.client.0.smithi070.stdout: "No Errors Logged", 2024-09-14T03:47:51.300 INFO:tasks.workunit.client.0.smithi070.stdout: "" 2024-09-14T03:47:51.300 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.300 INFO:tasks.workunit.client.0.smithi070.stdout: "platform_info": "x86_64-linux-5.14.0-505.el9.x86_64", 2024-09-14T03:47:51.300 INFO:tasks.workunit.client.0.smithi070.stdout: "svn_revision": "5155", 2024-09-14T03:47:51.301 INFO:tasks.workunit.client.0.smithi070.stdout: "version": [ 2024-09-14T03:47:51.301 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:51.301 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:51.301 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.301 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.301 INFO:tasks.workunit.client.0.smithi070.stdout: "temperature": { 2024-09-14T03:47:51.301 INFO:tasks.workunit.client.0.smithi070.stdout: "current": 39 2024-09-14T03:47:51.301 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.301 INFO:tasks.workunit.client.0.smithi070.stdout: "user_capacity": { 2024-09-14T03:47:51.301 INFO:tasks.workunit.client.0.smithi070.stdout: "blocks": 937703088, 2024-09-14T03:47:51.301 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 480103981056 2024-09-14T03:47:51.301 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.301 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.301 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.302 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.302 INFO:tasks.workunit.client.0.smithi070.stdout: "1ad766de-724c-11ef-9ddf-0cc47a8fd3e7": { 2024-09-14T03:47:51.302 INFO:tasks.workunit.client.0.smithi070.stdout: "INTEL_SSDPED1D480GA_1ad7d2f4-724c-11ef-9ddf-0cc47a8fd3e7": { 2024-09-14T03:47:51.302 INFO:tasks.workunit.client.0.smithi070.stdout: "20240914-034141": { 2024-09-14T03:47:51.302 INFO:tasks.workunit.client.0.smithi070.stdout: "device": { 2024-09-14T03:47:51.302 INFO:tasks.workunit.client.0.smithi070.stdout: "info_name": "/dev/nvme0n1", 2024-09-14T03:47:51.302 INFO:tasks.workunit.client.0.smithi070.stdout: "name": "/dev/nvme0n1", 2024-09-14T03:47:51.302 INFO:tasks.workunit.client.0.smithi070.stdout: "protocol": "NVMe", 2024-09-14T03:47:51.302 INFO:tasks.workunit.client.0.smithi070.stdout: "type": "nvme" 2024-09-14T03:47:51.302 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.302 INFO:tasks.workunit.client.0.smithi070.stdout: "firmware_version": "E2010325", 2024-09-14T03:47:51.302 INFO:tasks.workunit.client.0.smithi070.stdout: "host_id": "1ad766de-724c-11ef-9ddf-0cc47a8fd3e7", 2024-09-14T03:47:51.302 INFO:tasks.workunit.client.0.smithi070.stdout: "json_format_version": [ 2024-09-14T03:47:51.303 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:51.303 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:51.303 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.303 INFO:tasks.workunit.client.0.smithi070.stdout: "local_time": { 2024-09-14T03:47:51.303 INFO:tasks.workunit.client.0.smithi070.stdout: "asctime": "Sat Sep 14 03:41:40 2024 UTC", 2024-09-14T03:47:51.303 INFO:tasks.workunit.client.0.smithi070.stdout: "time_t": 1726285300 2024-09-14T03:47:51.303 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.303 INFO:tasks.workunit.client.0.smithi070.stdout: "logical_block_size": 512, 2024-09-14T03:47:51.303 INFO:tasks.workunit.client.0.smithi070.stdout: "model_name": "INTEL SSDPED1D480GA", 2024-09-14T03:47:51.303 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_controller_id": 0, 2024-09-14T03:47:51.303 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-14T03:47:51.303 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_namespaces": [ 2024-09-14T03:47:51.303 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.303 INFO:tasks.workunit.client.0.smithi070.stdout: "capacity": { 2024-09-14T03:47:51.304 INFO:tasks.workunit.client.0.smithi070.stdout: "blocks": 937703088, 2024-09-14T03:47:51.304 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 480103981056 2024-09-14T03:47:51.304 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.304 INFO:tasks.workunit.client.0.smithi070.stdout: "formatted_lba_size": 512, 2024-09-14T03:47:51.304 INFO:tasks.workunit.client.0.smithi070.stdout: "id": 1, 2024-09-14T03:47:51.304 INFO:tasks.workunit.client.0.smithi070.stdout: "size": { 2024-09-14T03:47:51.304 INFO:tasks.workunit.client.0.smithi070.stdout: "blocks": 937703088, 2024-09-14T03:47:51.304 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 480103981056 2024-09-14T03:47:51.304 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.304 INFO:tasks.workunit.client.0.smithi070.stdout: "utilization": { 2024-09-14T03:47:51.304 INFO:tasks.workunit.client.0.smithi070.stdout: "blocks": 937703088, 2024-09-14T03:47:51.304 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 480103981056 2024-09-14T03:47:51.304 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.305 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.305 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.305 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_number_of_namespaces": 1, 2024-09-14T03:47:51.305 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_pci_vendor": { 2024-09-14T03:47:51.305 INFO:tasks.workunit.client.0.smithi070.stdout: "id": 32902, 2024-09-14T03:47:51.305 INFO:tasks.workunit.client.0.smithi070.stdout: "subsystem_id": 32902 2024-09-14T03:47:51.305 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.305 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_smart_health_information_add_log": { 2024-09-14T03:47:51.305 INFO:tasks.workunit.client.0.smithi070.stdout: "Device stats": { 2024-09-14T03:47:51.305 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_fw_downgrades": { 2024-09-14T03:47:51.305 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.305 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.305 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.305 INFO:tasks.workunit.client.0.smithi070.stdout: "crc_error_count": { 2024-09-14T03:47:51.306 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.306 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.306 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.306 INFO:tasks.workunit.client.0.smithi070.stdout: "end_to_end_error_detection_count": { 2024-09-14T03:47:51.306 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.306 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.306 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.306 INFO:tasks.workunit.client.0.smithi070.stdout: "erase_fail_count": { 2024-09-14T03:47:51.306 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.306 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.306 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.306 INFO:tasks.workunit.client.0.smithi070.stdout: "host_bytes_written": { 2024-09-14T03:47:51.306 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.307 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 11224882 2024-09-14T03:47:51.307 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.307 INFO:tasks.workunit.client.0.smithi070.stdout: "host_ctx_wear_used": { 2024-09-14T03:47:51.307 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.307 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.307 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.307 INFO:tasks.workunit.client.0.smithi070.stdout: "media_bytes_read": { 2024-09-14T03:47:51.307 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.307 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.307 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.307 INFO:tasks.workunit.client.0.smithi070.stdout: "nand_bytes_written": { 2024-09-14T03:47:51.307 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.307 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.307 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.307 INFO:tasks.workunit.client.0.smithi070.stdout: "perf_stat_indicator": { 2024-09-14T03:47:51.308 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.308 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.308 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.308 INFO:tasks.workunit.client.0.smithi070.stdout: "pll_lock_loss_count": { 2024-09-14T03:47:51.308 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.308 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.308 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.308 INFO:tasks.workunit.client.0.smithi070.stdout: "program_fail_count": { 2024-09-14T03:47:51.308 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.308 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.308 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.308 INFO:tasks.workunit.client.0.smithi070.stdout: "re_alloc_sectr_cnt": { 2024-09-14T03:47:51.308 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.309 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.309 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.309 INFO:tasks.workunit.client.0.smithi070.stdout: "retry_buffer_overflow_count": { 2024-09-14T03:47:51.309 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.309 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.309 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.309 INFO:tasks.workunit.client.0.smithi070.stdout: "soft_ecc_err_rate": { 2024-09-14T03:47:51.309 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.309 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.309 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.309 INFO:tasks.workunit.client.0.smithi070.stdout: "thermal_throttle_status": { 2024-09-14T03:47:51.309 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.309 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": { 2024-09-14T03:47:51.309 INFO:tasks.workunit.client.0.smithi070.stdout: "cnt": 0, 2024-09-14T03:47:51.310 INFO:tasks.workunit.client.0.smithi070.stdout: "pct": 0 2024-09-14T03:47:51.310 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.310 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.310 INFO:tasks.workunit.client.0.smithi070.stdout: "timed_workload_host_reads": { 2024-09-14T03:47:51.310 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.310 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 14 2024-09-14T03:47:51.310 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.310 INFO:tasks.workunit.client.0.smithi070.stdout: "timed_workload_media_wear": { 2024-09-14T03:47:51.310 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.310 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": "0.000000" 2024-09-14T03:47:51.310 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.310 INFO:tasks.workunit.client.0.smithi070.stdout: "timed_workload_timer": { 2024-09-14T03:47:51.310 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.311 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 1493348 2024-09-14T03:47:51.311 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.311 INFO:tasks.workunit.client.0.smithi070.stdout: "unexp_power_loss": { 2024-09-14T03:47:51.311 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.311 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.311 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.311 INFO:tasks.workunit.client.0.smithi070.stdout: "wear_leveling": { 2024-09-14T03:47:51.311 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.311 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": { 2024-09-14T03:47:51.311 INFO:tasks.workunit.client.0.smithi070.stdout: "avg": 0, 2024-09-14T03:47:51.311 INFO:tasks.workunit.client.0.smithi070.stdout: "max": 0, 2024-09-14T03:47:51.311 INFO:tasks.workunit.client.0.smithi070.stdout: "min": 0 2024-09-14T03:47:51.311 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.311 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.312 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.312 INFO:tasks.workunit.client.0.smithi070.stdout: "Intel Smart log": "nvme0n1" 2024-09-14T03:47:51.312 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.312 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_smart_health_information_log": { 2024-09-14T03:47:51.312 INFO:tasks.workunit.client.0.smithi070.stdout: "available_spare": 100, 2024-09-14T03:47:51.312 INFO:tasks.workunit.client.0.smithi070.stdout: "available_spare_threshold": 0, 2024-09-14T03:47:51.312 INFO:tasks.workunit.client.0.smithi070.stdout: "controller_busy_time": 4368, 2024-09-14T03:47:51.312 INFO:tasks.workunit.client.0.smithi070.stdout: "critical_warning": 0, 2024-09-14T03:47:51.312 INFO:tasks.workunit.client.0.smithi070.stdout: "data_units_read": 155028165, 2024-09-14T03:47:51.312 INFO:tasks.workunit.client.0.smithi070.stdout: "data_units_written": 735633918, 2024-09-14T03:47:51.312 INFO:tasks.workunit.client.0.smithi070.stdout: "host_reads": 2607915323, 2024-09-14T03:47:51.312 INFO:tasks.workunit.client.0.smithi070.stdout: "host_writes": 16464089664, 2024-09-14T03:47:51.312 INFO:tasks.workunit.client.0.smithi070.stdout: "media_errors": 0, 2024-09-14T03:47:51.313 INFO:tasks.workunit.client.0.smithi070.stdout: "num_err_log_entries": 0, 2024-09-14T03:47:51.313 INFO:tasks.workunit.client.0.smithi070.stdout: "percentage_used": 0, 2024-09-14T03:47:51.313 INFO:tasks.workunit.client.0.smithi070.stdout: "power_cycles": 30550, 2024-09-14T03:47:51.313 INFO:tasks.workunit.client.0.smithi070.stdout: "power_on_hours": 37550, 2024-09-14T03:47:51.313 INFO:tasks.workunit.client.0.smithi070.stdout: "temperature": 34, 2024-09-14T03:47:51.313 INFO:tasks.workunit.client.0.smithi070.stdout: "unsafe_shutdowns": 9447 2024-09-14T03:47:51.313 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.313 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_vendor": "intel", 2024-09-14T03:47:51.313 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_version": { 2024-09-14T03:47:51.313 INFO:tasks.workunit.client.0.smithi070.stdout: "string": "<1.2", 2024-09-14T03:47:51.313 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.313 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.313 INFO:tasks.workunit.client.0.smithi070.stdout: "power_cycle_count": 30550, 2024-09-14T03:47:51.314 INFO:tasks.workunit.client.0.smithi070.stdout: "power_on_time": { 2024-09-14T03:47:51.314 INFO:tasks.workunit.client.0.smithi070.stdout: "hours": 37550 2024-09-14T03:47:51.314 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.314 INFO:tasks.workunit.client.0.smithi070.stdout: "serial_number": "deleted", 2024-09-14T03:47:51.314 INFO:tasks.workunit.client.0.smithi070.stdout: "smart_status": { 2024-09-14T03:47:51.314 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme": { 2024-09-14T03:47:51.314 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.314 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.314 INFO:tasks.workunit.client.0.smithi070.stdout: "passed": true 2024-09-14T03:47:51.314 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.314 INFO:tasks.workunit.client.0.smithi070.stdout: "smartctl": { 2024-09-14T03:47:51.314 INFO:tasks.workunit.client.0.smithi070.stdout: "argv": [ 2024-09-14T03:47:51.314 INFO:tasks.workunit.client.0.smithi070.stdout: "smartctl", 2024-09-14T03:47:51.314 INFO:tasks.workunit.client.0.smithi070.stdout: "-x", 2024-09-14T03:47:51.315 INFO:tasks.workunit.client.0.smithi070.stdout: "--json=o", 2024-09-14T03:47:51.315 INFO:tasks.workunit.client.0.smithi070.stdout: "/dev/nvme0n1" 2024-09-14T03:47:51.315 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.315 INFO:tasks.workunit.client.0.smithi070.stdout: "build_info": "(local build)", 2024-09-14T03:47:51.315 INFO:tasks.workunit.client.0.smithi070.stdout: "exit_status": 0, 2024-09-14T03:47:51.315 INFO:tasks.workunit.client.0.smithi070.stdout: "output": [ 2024-09-14T03:47:51.315 INFO:tasks.workunit.client.0.smithi070.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-505.el9.x86_64] (local build)", 2024-09-14T03:47:51.315 INFO:tasks.workunit.client.0.smithi070.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-14T03:47:51.315 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:51.315 INFO:tasks.workunit.client.0.smithi070.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-14T03:47:51.315 INFO:tasks.workunit.client.0.smithi070.stdout: "Model Number: INTEL SSDPED1D480GA", 2024-09-14T03:47:51.315 INFO:tasks.workunit.client.0.smithi070.stdout: "Serial Number: deleted", 2024-09-14T03:47:51.315 INFO:tasks.workunit.client.0.smithi070.stdout: "Firmware Version: E2010325", 2024-09-14T03:47:51.315 INFO:tasks.workunit.client.0.smithi070.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-14T03:47:51.316 INFO:tasks.workunit.client.0.smithi070.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-14T03:47:51.316 INFO:tasks.workunit.client.0.smithi070.stdout: "Controller ID: 0", 2024-09-14T03:47:51.316 INFO:tasks.workunit.client.0.smithi070.stdout: "NVMe Version: <1.2", 2024-09-14T03:47:51.316 INFO:tasks.workunit.client.0.smithi070.stdout: "Number of Namespaces: 1", 2024-09-14T03:47:51.316 INFO:tasks.workunit.client.0.smithi070.stdout: "Namespace 1 Size/Capacity: 480,103,981,056 [480 GB]", 2024-09-14T03:47:51.316 INFO:tasks.workunit.client.0.smithi070.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-14T03:47:51.316 INFO:tasks.workunit.client.0.smithi070.stdout: "Local Time is: Sat Sep 14 03:41:40 2024 UTC", 2024-09-14T03:47:51.316 INFO:tasks.workunit.client.0.smithi070.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-14T03:47:51.316 INFO:tasks.workunit.client.0.smithi070.stdout: "Optional Admin Commands (0x0007): Security Format Frmw_DL", 2024-09-14T03:47:51.316 INFO:tasks.workunit.client.0.smithi070.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-14T03:47:51.316 INFO:tasks.workunit.client.0.smithi070.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-14T03:47:51.316 INFO:tasks.workunit.client.0.smithi070.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-14T03:47:51.316 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:51.317 INFO:tasks.workunit.client.0.smithi070.stdout: "Supported Power States", 2024-09-14T03:47:51.317 INFO:tasks.workunit.client.0.smithi070.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-14T03:47:51.317 INFO:tasks.workunit.client.0.smithi070.stdout: " 0 + 18.00W - - 0 0 0 0 0 0", 2024-09-14T03:47:51.317 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:51.317 INFO:tasks.workunit.client.0.smithi070.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-14T03:47:51.317 INFO:tasks.workunit.client.0.smithi070.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-14T03:47:51.317 INFO:tasks.workunit.client.0.smithi070.stdout: " 0 + 512 0 2", 2024-09-14T03:47:51.317 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:51.317 INFO:tasks.workunit.client.0.smithi070.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-14T03:47:51.317 INFO:tasks.workunit.client.0.smithi070.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-14T03:47:51.317 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:51.317 INFO:tasks.workunit.client.0.smithi070.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-14T03:47:51.317 INFO:tasks.workunit.client.0.smithi070.stdout: "Critical Warning: 0x00", 2024-09-14T03:47:51.317 INFO:tasks.workunit.client.0.smithi070.stdout: "Temperature: 34 Celsius", 2024-09-14T03:47:51.318 INFO:tasks.workunit.client.0.smithi070.stdout: "Available Spare: 100%", 2024-09-14T03:47:51.318 INFO:tasks.workunit.client.0.smithi070.stdout: "Available Spare Threshold: 0%", 2024-09-14T03:47:51.318 INFO:tasks.workunit.client.0.smithi070.stdout: "Percentage Used: 0%", 2024-09-14T03:47:51.318 INFO:tasks.workunit.client.0.smithi070.stdout: "Data Units Read: 155,028,165 [79.3 TB]", 2024-09-14T03:47:51.318 INFO:tasks.workunit.client.0.smithi070.stdout: "Data Units Written: 735,633,918 [376 TB]", 2024-09-14T03:47:51.318 INFO:tasks.workunit.client.0.smithi070.stdout: "Host Read Commands: 2,607,915,323", 2024-09-14T03:47:51.318 INFO:tasks.workunit.client.0.smithi070.stdout: "Host Write Commands: 16,464,089,664", 2024-09-14T03:47:51.318 INFO:tasks.workunit.client.0.smithi070.stdout: "Controller Busy Time: 4,368", 2024-09-14T03:47:51.318 INFO:tasks.workunit.client.0.smithi070.stdout: "Power Cycles: 30,550", 2024-09-14T03:47:51.318 INFO:tasks.workunit.client.0.smithi070.stdout: "Power On Hours: 37,550", 2024-09-14T03:47:51.318 INFO:tasks.workunit.client.0.smithi070.stdout: "Unsafe Shutdowns: 9,447", 2024-09-14T03:47:51.318 INFO:tasks.workunit.client.0.smithi070.stdout: "Media and Data Integrity Errors: 0", 2024-09-14T03:47:51.318 INFO:tasks.workunit.client.0.smithi070.stdout: "Error Information Log Entries: 0", 2024-09-14T03:47:51.318 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:51.319 INFO:tasks.workunit.client.0.smithi070.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-14T03:47:51.319 INFO:tasks.workunit.client.0.smithi070.stdout: "No Errors Logged", 2024-09-14T03:47:51.319 INFO:tasks.workunit.client.0.smithi070.stdout: "" 2024-09-14T03:47:51.319 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.319 INFO:tasks.workunit.client.0.smithi070.stdout: "platform_info": "x86_64-linux-5.14.0-505.el9.x86_64", 2024-09-14T03:47:51.319 INFO:tasks.workunit.client.0.smithi070.stdout: "svn_revision": "5155", 2024-09-14T03:47:51.319 INFO:tasks.workunit.client.0.smithi070.stdout: "version": [ 2024-09-14T03:47:51.319 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:51.319 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:51.319 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.319 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.319 INFO:tasks.workunit.client.0.smithi070.stdout: "temperature": { 2024-09-14T03:47:51.320 INFO:tasks.workunit.client.0.smithi070.stdout: "current": 34 2024-09-14T03:47:51.320 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.320 INFO:tasks.workunit.client.0.smithi070.stdout: "user_capacity": { 2024-09-14T03:47:51.320 INFO:tasks.workunit.client.0.smithi070.stdout: "blocks": 937703088, 2024-09-14T03:47:51.320 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 480103981056 2024-09-14T03:47:51.320 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.321 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.321 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.321 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.321 INFO:tasks.workunit.client.0.smithi070.stdout:} 2024-09-14T03:47:51.321 INFO:tasks.workunit.client.0.smithi070.stderr:+ ceph telemetry show-all 2024-09-14T03:47:51.322 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:51.322 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:51.322 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: pgmap v91: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-14T03:47:51.322 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='client.24770 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:51.322 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:51.322 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:51.322 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.322 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.322 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.322 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.323 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.323 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.323 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.323 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.323 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.323 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:51.323 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:51.323 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.323 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.323 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.323 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:51.323 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:51.323 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.324 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.324 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.324 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:51.324 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:51.324 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:51.324 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:51.324 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: pgmap v91: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-14T03:47:51.324 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='client.24770 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:51.324 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:51.324 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:51.325 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.325 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.325 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.325 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.325 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.325 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.325 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.325 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.325 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.325 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:51.325 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:51.325 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.325 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.326 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.326 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:51.326 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:51.326 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.326 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.326 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.326 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:51.326 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:51.326 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.326 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.326 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.326 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:51.326 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:51.327 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.327 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.327 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.327 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.327 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.327 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.327 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.327 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.327 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.327 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:51.327 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:51.327 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.327 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.327 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.328 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:51.328 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:51.328 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.328 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.328 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.328 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:51.328 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:51.328 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.328 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.328 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:49 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.328 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='client.24770 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:51.328 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='client.24788 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:51.329 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='client.24788 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:51.329 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='client.14886 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:51.329 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:51.329 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:51.329 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.329 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.329 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.329 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.329 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.329 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.329 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.329 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.329 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.330 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:51.330 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:51.330 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.330 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.330 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.330 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:51.330 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:51.330 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.330 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.330 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.330 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:51.330 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:51.330 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.331 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.331 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.331 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:51.331 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:51.331 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.331 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.331 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.331 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.331 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.331 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.331 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.331 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.331 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.332 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:51.332 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:51.332 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.332 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.332 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.332 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:51.332 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:51.332 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.332 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.332 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.332 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:51.332 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:51.332 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.333 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.333 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:50 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.333 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:51.333 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:51.333 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: pgmap v91: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-14T03:47:51.333 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='client.24770 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:51.333 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:51.333 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:51.334 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.334 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.334 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.334 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.334 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.334 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.334 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.334 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.334 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.334 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:51.334 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:51.334 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.334 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.335 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.335 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:51.335 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:51.335 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.335 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.335 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.335 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:51.335 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:51.335 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.335 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.335 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.335 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:51.335 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:51.335 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.336 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.336 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.336 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.336 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.336 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.336 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.336 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.336 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.336 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:51.336 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:51.336 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.336 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.336 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.337 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:51.337 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:51.337 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.337 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.337 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.337 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:51.337 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:51.337 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.337 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.337 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.338 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.338 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.338 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.338 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:51.338 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:51.338 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.338 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.338 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.338 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.339 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.339 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.339 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.339 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.339 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.339 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:51.339 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:51.339 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.339 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.339 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.339 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:51.339 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:51.340 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.340 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.340 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.340 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:51.340 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:51.340 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:51.340 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:51.340 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:49 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:51.347 INFO:tasks.workunit.client.0.smithi070.stdout:{ 2024-09-14T03:47:51.348 INFO:tasks.workunit.client.0.smithi070.stdout: "device_report": { 2024-09-14T03:47:51.348 INFO:tasks.workunit.client.0.smithi070.stdout: "1ad5ed40-724c-11ef-9ddf-0cc47a8fd3e7": { 2024-09-14T03:47:51.348 INFO:tasks.workunit.client.0.smithi070.stdout: "INTEL_SSDPED1D480GA_1ad66d74-724c-11ef-9ddf-0cc47a8fd3e7": { 2024-09-14T03:47:51.348 INFO:tasks.workunit.client.0.smithi070.stdout: "20240914-034143": { 2024-09-14T03:47:51.348 INFO:tasks.workunit.client.0.smithi070.stdout: "device": { 2024-09-14T03:47:51.348 INFO:tasks.workunit.client.0.smithi070.stdout: "info_name": "/dev/nvme0n1", 2024-09-14T03:47:51.348 INFO:tasks.workunit.client.0.smithi070.stdout: "name": "/dev/nvme0n1", 2024-09-14T03:47:51.348 INFO:tasks.workunit.client.0.smithi070.stdout: "protocol": "NVMe", 2024-09-14T03:47:51.348 INFO:tasks.workunit.client.0.smithi070.stdout: "type": "nvme" 2024-09-14T03:47:51.348 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.348 INFO:tasks.workunit.client.0.smithi070.stdout: "firmware_version": "E2010325", 2024-09-14T03:47:51.348 INFO:tasks.workunit.client.0.smithi070.stdout: "host_id": "1ad5ed40-724c-11ef-9ddf-0cc47a8fd3e7", 2024-09-14T03:47:51.349 INFO:tasks.workunit.client.0.smithi070.stdout: "json_format_version": [ 2024-09-14T03:47:51.349 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:51.349 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:51.349 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.349 INFO:tasks.workunit.client.0.smithi070.stdout: "local_time": { 2024-09-14T03:47:51.349 INFO:tasks.workunit.client.0.smithi070.stdout: "asctime": "Sat Sep 14 03:41:43 2024 UTC", 2024-09-14T03:47:51.349 INFO:tasks.workunit.client.0.smithi070.stdout: "time_t": 1726285303 2024-09-14T03:47:51.349 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.349 INFO:tasks.workunit.client.0.smithi070.stdout: "logical_block_size": 512, 2024-09-14T03:47:51.349 INFO:tasks.workunit.client.0.smithi070.stdout: "model_name": "INTEL SSDPED1D480GA", 2024-09-14T03:47:51.349 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_controller_id": 0, 2024-09-14T03:47:51.349 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-14T03:47:51.349 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_namespaces": [ 2024-09-14T03:47:51.349 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.350 INFO:tasks.workunit.client.0.smithi070.stdout: "capacity": { 2024-09-14T03:47:51.350 INFO:tasks.workunit.client.0.smithi070.stdout: "blocks": 937703088, 2024-09-14T03:47:51.350 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 480103981056 2024-09-14T03:47:51.350 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.350 INFO:tasks.workunit.client.0.smithi070.stdout: "formatted_lba_size": 512, 2024-09-14T03:47:51.350 INFO:tasks.workunit.client.0.smithi070.stdout: "id": 1, 2024-09-14T03:47:51.350 INFO:tasks.workunit.client.0.smithi070.stdout: "size": { 2024-09-14T03:47:51.350 INFO:tasks.workunit.client.0.smithi070.stdout: "blocks": 937703088, 2024-09-14T03:47:51.350 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 480103981056 2024-09-14T03:47:51.350 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.350 INFO:tasks.workunit.client.0.smithi070.stdout: "utilization": { 2024-09-14T03:47:51.350 INFO:tasks.workunit.client.0.smithi070.stdout: "blocks": 937703088, 2024-09-14T03:47:51.350 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 480103981056 2024-09-14T03:47:51.351 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.351 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.351 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.351 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_number_of_namespaces": 1, 2024-09-14T03:47:51.351 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_pci_vendor": { 2024-09-14T03:47:51.351 INFO:tasks.workunit.client.0.smithi070.stdout: "id": 32902, 2024-09-14T03:47:51.351 INFO:tasks.workunit.client.0.smithi070.stdout: "subsystem_id": 32902 2024-09-14T03:47:51.351 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.351 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_smart_health_information_add_log": { 2024-09-14T03:47:51.351 INFO:tasks.workunit.client.0.smithi070.stdout: "Device stats": { 2024-09-14T03:47:51.351 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_fw_downgrades": { 2024-09-14T03:47:51.351 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.351 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.351 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.352 INFO:tasks.workunit.client.0.smithi070.stdout: "crc_error_count": { 2024-09-14T03:47:51.352 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.352 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.352 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.352 INFO:tasks.workunit.client.0.smithi070.stdout: "end_to_end_error_detection_count": { 2024-09-14T03:47:51.352 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.352 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.352 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.352 INFO:tasks.workunit.client.0.smithi070.stdout: "erase_fail_count": { 2024-09-14T03:47:51.352 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.352 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.352 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.353 INFO:tasks.workunit.client.0.smithi070.stdout: "host_bytes_written": { 2024-09-14T03:47:51.353 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.353 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 13063793 2024-09-14T03:47:51.353 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.353 INFO:tasks.workunit.client.0.smithi070.stdout: "host_ctx_wear_used": { 2024-09-14T03:47:51.353 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.353 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.353 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.353 INFO:tasks.workunit.client.0.smithi070.stdout: "media_bytes_read": { 2024-09-14T03:47:51.353 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.353 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.353 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.353 INFO:tasks.workunit.client.0.smithi070.stdout: "nand_bytes_written": { 2024-09-14T03:47:51.354 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.354 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.354 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.354 INFO:tasks.workunit.client.0.smithi070.stdout: "perf_stat_indicator": { 2024-09-14T03:47:51.354 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.354 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.354 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.354 INFO:tasks.workunit.client.0.smithi070.stdout: "pll_lock_loss_count": { 2024-09-14T03:47:51.354 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.354 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.354 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.354 INFO:tasks.workunit.client.0.smithi070.stdout: "program_fail_count": { 2024-09-14T03:47:51.354 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.354 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.355 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.355 INFO:tasks.workunit.client.0.smithi070.stdout: "re_alloc_sectr_cnt": { 2024-09-14T03:47:51.355 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.355 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.355 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.355 INFO:tasks.workunit.client.0.smithi070.stdout: "retry_buffer_overflow_count": { 2024-09-14T03:47:51.355 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.355 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.355 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.355 INFO:tasks.workunit.client.0.smithi070.stdout: "soft_ecc_err_rate": { 2024-09-14T03:47:51.355 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.355 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.355 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.355 INFO:tasks.workunit.client.0.smithi070.stdout: "thermal_throttle_status": { 2024-09-14T03:47:51.356 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.356 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": { 2024-09-14T03:47:51.356 INFO:tasks.workunit.client.0.smithi070.stdout: "cnt": 0, 2024-09-14T03:47:51.356 INFO:tasks.workunit.client.0.smithi070.stdout: "pct": 0 2024-09-14T03:47:51.356 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.356 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.356 INFO:tasks.workunit.client.0.smithi070.stdout: "timed_workload_host_reads": { 2024-09-14T03:47:51.356 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.356 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 14 2024-09-14T03:47:51.356 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.356 INFO:tasks.workunit.client.0.smithi070.stdout: "timed_workload_media_wear": { 2024-09-14T03:47:51.356 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.356 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": "9.029297" 2024-09-14T03:47:51.357 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.357 INFO:tasks.workunit.client.0.smithi070.stdout: "timed_workload_timer": { 2024-09-14T03:47:51.357 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.357 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 1544484 2024-09-14T03:47:51.357 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.357 INFO:tasks.workunit.client.0.smithi070.stdout: "unexp_power_loss": { 2024-09-14T03:47:51.357 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.357 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.357 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.357 INFO:tasks.workunit.client.0.smithi070.stdout: "wear_leveling": { 2024-09-14T03:47:51.357 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.357 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": { 2024-09-14T03:47:51.357 INFO:tasks.workunit.client.0.smithi070.stdout: "avg": 0, 2024-09-14T03:47:51.358 INFO:tasks.workunit.client.0.smithi070.stdout: "max": 0, 2024-09-14T03:47:51.358 INFO:tasks.workunit.client.0.smithi070.stdout: "min": 0 2024-09-14T03:47:51.358 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.358 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.358 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.358 INFO:tasks.workunit.client.0.smithi070.stdout: "Intel Smart log": "nvme0n1" 2024-09-14T03:47:51.358 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.358 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_smart_health_information_log": { 2024-09-14T03:47:51.358 INFO:tasks.workunit.client.0.smithi070.stdout: "available_spare": 100, 2024-09-14T03:47:51.358 INFO:tasks.workunit.client.0.smithi070.stdout: "available_spare_threshold": 0, 2024-09-14T03:47:51.358 INFO:tasks.workunit.client.0.smithi070.stdout: "controller_busy_time": 5166, 2024-09-14T03:47:51.358 INFO:tasks.workunit.client.0.smithi070.stdout: "critical_warning": 0, 2024-09-14T03:47:51.358 INFO:tasks.workunit.client.0.smithi070.stdout: "data_units_read": 177890366, 2024-09-14T03:47:51.358 INFO:tasks.workunit.client.0.smithi070.stdout: "data_units_written": 856148757, 2024-09-14T03:47:51.359 INFO:tasks.workunit.client.0.smithi070.stdout: "host_reads": 3130935174, 2024-09-14T03:47:51.359 INFO:tasks.workunit.client.0.smithi070.stdout: "host_writes": 18536697060, 2024-09-14T03:47:51.359 INFO:tasks.workunit.client.0.smithi070.stdout: "media_errors": 0, 2024-09-14T03:47:51.359 INFO:tasks.workunit.client.0.smithi070.stdout: "num_err_log_entries": 0, 2024-09-14T03:47:51.359 INFO:tasks.workunit.client.0.smithi070.stdout: "percentage_used": 14, 2024-09-14T03:47:51.359 INFO:tasks.workunit.client.0.smithi070.stdout: "power_cycles": 35890, 2024-09-14T03:47:51.359 INFO:tasks.workunit.client.0.smithi070.stdout: "power_on_hours": 40704, 2024-09-14T03:47:51.359 INFO:tasks.workunit.client.0.smithi070.stdout: "temperature": 39, 2024-09-14T03:47:51.359 INFO:tasks.workunit.client.0.smithi070.stdout: "unsafe_shutdowns": 10931 2024-09-14T03:47:51.359 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.359 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_vendor": "intel", 2024-09-14T03:47:51.359 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_version": { 2024-09-14T03:47:51.360 INFO:tasks.workunit.client.0.smithi070.stdout: "string": "<1.2", 2024-09-14T03:47:51.360 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.360 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.360 INFO:tasks.workunit.client.0.smithi070.stdout: "power_cycle_count": 35890, 2024-09-14T03:47:51.360 INFO:tasks.workunit.client.0.smithi070.stdout: "power_on_time": { 2024-09-14T03:47:51.360 INFO:tasks.workunit.client.0.smithi070.stdout: "hours": 40704 2024-09-14T03:47:51.360 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.360 INFO:tasks.workunit.client.0.smithi070.stdout: "serial_number": "deleted", 2024-09-14T03:47:51.360 INFO:tasks.workunit.client.0.smithi070.stdout: "smart_status": { 2024-09-14T03:47:51.361 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme": { 2024-09-14T03:47:51.361 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.361 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.361 INFO:tasks.workunit.client.0.smithi070.stdout: "passed": true 2024-09-14T03:47:51.361 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.361 INFO:tasks.workunit.client.0.smithi070.stdout: "smartctl": { 2024-09-14T03:47:51.361 INFO:tasks.workunit.client.0.smithi070.stdout: "argv": [ 2024-09-14T03:47:51.361 INFO:tasks.workunit.client.0.smithi070.stdout: "smartctl", 2024-09-14T03:47:51.361 INFO:tasks.workunit.client.0.smithi070.stdout: "-x", 2024-09-14T03:47:51.361 INFO:tasks.workunit.client.0.smithi070.stdout: "--json=o", 2024-09-14T03:47:51.361 INFO:tasks.workunit.client.0.smithi070.stdout: "/dev/nvme0n1" 2024-09-14T03:47:51.362 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.362 INFO:tasks.workunit.client.0.smithi070.stdout: "build_info": "(local build)", 2024-09-14T03:47:51.362 INFO:tasks.workunit.client.0.smithi070.stdout: "exit_status": 0, 2024-09-14T03:47:51.362 INFO:tasks.workunit.client.0.smithi070.stdout: "output": [ 2024-09-14T03:47:51.362 INFO:tasks.workunit.client.0.smithi070.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-505.el9.x86_64] (local build)", 2024-09-14T03:47:51.362 INFO:tasks.workunit.client.0.smithi070.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-14T03:47:51.362 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:51.362 INFO:tasks.workunit.client.0.smithi070.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-14T03:47:51.362 INFO:tasks.workunit.client.0.smithi070.stdout: "Model Number: INTEL SSDPED1D480GA", 2024-09-14T03:47:51.362 INFO:tasks.workunit.client.0.smithi070.stdout: "Serial Number: deleted", 2024-09-14T03:47:51.363 INFO:tasks.workunit.client.0.smithi070.stdout: "Firmware Version: E2010325", 2024-09-14T03:47:51.363 INFO:tasks.workunit.client.0.smithi070.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-14T03:47:51.363 INFO:tasks.workunit.client.0.smithi070.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-14T03:47:51.363 INFO:tasks.workunit.client.0.smithi070.stdout: "Controller ID: 0", 2024-09-14T03:47:51.363 INFO:tasks.workunit.client.0.smithi070.stdout: "NVMe Version: <1.2", 2024-09-14T03:47:51.363 INFO:tasks.workunit.client.0.smithi070.stdout: "Number of Namespaces: 1", 2024-09-14T03:47:51.363 INFO:tasks.workunit.client.0.smithi070.stdout: "Namespace 1 Size/Capacity: 480,103,981,056 [480 GB]", 2024-09-14T03:47:51.363 INFO:tasks.workunit.client.0.smithi070.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-14T03:47:51.363 INFO:tasks.workunit.client.0.smithi070.stdout: "Local Time is: Sat Sep 14 03:41:43 2024 UTC", 2024-09-14T03:47:51.363 INFO:tasks.workunit.client.0.smithi070.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-14T03:47:51.363 INFO:tasks.workunit.client.0.smithi070.stdout: "Optional Admin Commands (0x0007): Security Format Frmw_DL", 2024-09-14T03:47:51.364 INFO:tasks.workunit.client.0.smithi070.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-14T03:47:51.364 INFO:tasks.workunit.client.0.smithi070.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-14T03:47:51.364 INFO:tasks.workunit.client.0.smithi070.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-14T03:47:51.364 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:51.364 INFO:tasks.workunit.client.0.smithi070.stdout: "Supported Power States", 2024-09-14T03:47:51.364 INFO:tasks.workunit.client.0.smithi070.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-14T03:47:51.364 INFO:tasks.workunit.client.0.smithi070.stdout: " 0 + 18.00W - - 0 0 0 0 0 0", 2024-09-14T03:47:51.364 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:51.364 INFO:tasks.workunit.client.0.smithi070.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-14T03:47:51.364 INFO:tasks.workunit.client.0.smithi070.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-14T03:47:51.365 INFO:tasks.workunit.client.0.smithi070.stdout: " 0 + 512 0 2", 2024-09-14T03:47:51.365 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:51.365 INFO:tasks.workunit.client.0.smithi070.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-14T03:47:51.365 INFO:tasks.workunit.client.0.smithi070.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-14T03:47:51.365 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:51.365 INFO:tasks.workunit.client.0.smithi070.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-14T03:47:51.365 INFO:tasks.workunit.client.0.smithi070.stdout: "Critical Warning: 0x00", 2024-09-14T03:47:51.365 INFO:tasks.workunit.client.0.smithi070.stdout: "Temperature: 39 Celsius", 2024-09-14T03:47:51.365 INFO:tasks.workunit.client.0.smithi070.stdout: "Available Spare: 100%", 2024-09-14T03:47:51.365 INFO:tasks.workunit.client.0.smithi070.stdout: "Available Spare Threshold: 0%", 2024-09-14T03:47:51.365 INFO:tasks.workunit.client.0.smithi070.stdout: "Percentage Used: 14%", 2024-09-14T03:47:51.365 INFO:tasks.workunit.client.0.smithi070.stdout: "Data Units Read: 177,890,366 [91.0 TB]", 2024-09-14T03:47:51.366 INFO:tasks.workunit.client.0.smithi070.stdout: "Data Units Written: 856,148,757 [438 TB]", 2024-09-14T03:47:51.366 INFO:tasks.workunit.client.0.smithi070.stdout: "Host Read Commands: 3,130,935,174", 2024-09-14T03:47:51.366 INFO:tasks.workunit.client.0.smithi070.stdout: "Host Write Commands: 18,536,697,060", 2024-09-14T03:47:51.366 INFO:tasks.workunit.client.0.smithi070.stdout: "Controller Busy Time: 5,166", 2024-09-14T03:47:51.366 INFO:tasks.workunit.client.0.smithi070.stdout: "Power Cycles: 35,890", 2024-09-14T03:47:51.366 INFO:tasks.workunit.client.0.smithi070.stdout: "Power On Hours: 40,704", 2024-09-14T03:47:51.366 INFO:tasks.workunit.client.0.smithi070.stdout: "Unsafe Shutdowns: 10,931", 2024-09-14T03:47:51.366 INFO:tasks.workunit.client.0.smithi070.stdout: "Media and Data Integrity Errors: 0", 2024-09-14T03:47:51.366 INFO:tasks.workunit.client.0.smithi070.stdout: "Error Information Log Entries: 0", 2024-09-14T03:47:51.366 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:51.366 INFO:tasks.workunit.client.0.smithi070.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-14T03:47:51.367 INFO:tasks.workunit.client.0.smithi070.stdout: "No Errors Logged", 2024-09-14T03:47:51.367 INFO:tasks.workunit.client.0.smithi070.stdout: "" 2024-09-14T03:47:51.367 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.367 INFO:tasks.workunit.client.0.smithi070.stdout: "platform_info": "x86_64-linux-5.14.0-505.el9.x86_64", 2024-09-14T03:47:51.367 INFO:tasks.workunit.client.0.smithi070.stdout: "svn_revision": "5155", 2024-09-14T03:47:51.367 INFO:tasks.workunit.client.0.smithi070.stdout: "version": [ 2024-09-14T03:47:51.367 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:51.367 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:51.367 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.367 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.367 INFO:tasks.workunit.client.0.smithi070.stdout: "temperature": { 2024-09-14T03:47:51.367 INFO:tasks.workunit.client.0.smithi070.stdout: "current": 39 2024-09-14T03:47:51.368 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.368 INFO:tasks.workunit.client.0.smithi070.stdout: "user_capacity": { 2024-09-14T03:47:51.368 INFO:tasks.workunit.client.0.smithi070.stdout: "blocks": 937703088, 2024-09-14T03:47:51.368 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 480103981056 2024-09-14T03:47:51.368 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.368 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.368 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.368 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.368 INFO:tasks.workunit.client.0.smithi070.stdout: "1ad766de-724c-11ef-9ddf-0cc47a8fd3e7": { 2024-09-14T03:47:51.368 INFO:tasks.workunit.client.0.smithi070.stdout: "INTEL_SSDPED1D480GA_1ad7d2f4-724c-11ef-9ddf-0cc47a8fd3e7": { 2024-09-14T03:47:51.368 INFO:tasks.workunit.client.0.smithi070.stdout: "20240914-034141": { 2024-09-14T03:47:51.369 INFO:tasks.workunit.client.0.smithi070.stdout: "device": { 2024-09-14T03:47:51.369 INFO:tasks.workunit.client.0.smithi070.stdout: "info_name": "/dev/nvme0n1", 2024-09-14T03:47:51.369 INFO:tasks.workunit.client.0.smithi070.stdout: "name": "/dev/nvme0n1", 2024-09-14T03:47:51.369 INFO:tasks.workunit.client.0.smithi070.stdout: "protocol": "NVMe", 2024-09-14T03:47:51.369 INFO:tasks.workunit.client.0.smithi070.stdout: "type": "nvme" 2024-09-14T03:47:51.369 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.369 INFO:tasks.workunit.client.0.smithi070.stdout: "firmware_version": "E2010325", 2024-09-14T03:47:51.369 INFO:tasks.workunit.client.0.smithi070.stdout: "host_id": "1ad766de-724c-11ef-9ddf-0cc47a8fd3e7", 2024-09-14T03:47:51.369 INFO:tasks.workunit.client.0.smithi070.stdout: "json_format_version": [ 2024-09-14T03:47:51.369 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:51.370 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:51.370 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.370 INFO:tasks.workunit.client.0.smithi070.stdout: "local_time": { 2024-09-14T03:47:51.370 INFO:tasks.workunit.client.0.smithi070.stdout: "asctime": "Sat Sep 14 03:41:40 2024 UTC", 2024-09-14T03:47:51.370 INFO:tasks.workunit.client.0.smithi070.stdout: "time_t": 1726285300 2024-09-14T03:47:51.370 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.370 INFO:tasks.workunit.client.0.smithi070.stdout: "logical_block_size": 512, 2024-09-14T03:47:51.370 INFO:tasks.workunit.client.0.smithi070.stdout: "model_name": "INTEL SSDPED1D480GA", 2024-09-14T03:47:51.370 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_controller_id": 0, 2024-09-14T03:47:51.370 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-14T03:47:51.370 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_namespaces": [ 2024-09-14T03:47:51.370 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.371 INFO:tasks.workunit.client.0.smithi070.stdout: "capacity": { 2024-09-14T03:47:51.371 INFO:tasks.workunit.client.0.smithi070.stdout: "blocks": 937703088, 2024-09-14T03:47:51.371 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 480103981056 2024-09-14T03:47:51.371 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.371 INFO:tasks.workunit.client.0.smithi070.stdout: "formatted_lba_size": 512, 2024-09-14T03:47:51.371 INFO:tasks.workunit.client.0.smithi070.stdout: "id": 1, 2024-09-14T03:47:51.371 INFO:tasks.workunit.client.0.smithi070.stdout: "size": { 2024-09-14T03:47:51.371 INFO:tasks.workunit.client.0.smithi070.stdout: "blocks": 937703088, 2024-09-14T03:47:51.371 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 480103981056 2024-09-14T03:47:51.371 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.371 INFO:tasks.workunit.client.0.smithi070.stdout: "utilization": { 2024-09-14T03:47:51.371 INFO:tasks.workunit.client.0.smithi070.stdout: "blocks": 937703088, 2024-09-14T03:47:51.372 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 480103981056 2024-09-14T03:47:51.372 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.372 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.372 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.372 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_number_of_namespaces": 1, 2024-09-14T03:47:51.372 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_pci_vendor": { 2024-09-14T03:47:51.372 INFO:tasks.workunit.client.0.smithi070.stdout: "id": 32902, 2024-09-14T03:47:51.372 INFO:tasks.workunit.client.0.smithi070.stdout: "subsystem_id": 32902 2024-09-14T03:47:51.372 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.372 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_smart_health_information_add_log": { 2024-09-14T03:47:51.372 INFO:tasks.workunit.client.0.smithi070.stdout: "Device stats": { 2024-09-14T03:47:51.372 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_fw_downgrades": { 2024-09-14T03:47:51.373 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.373 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.373 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.373 INFO:tasks.workunit.client.0.smithi070.stdout: "crc_error_count": { 2024-09-14T03:47:51.373 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.373 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.373 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.373 INFO:tasks.workunit.client.0.smithi070.stdout: "end_to_end_error_detection_count": { 2024-09-14T03:47:51.373 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.373 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.373 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.374 INFO:tasks.workunit.client.0.smithi070.stdout: "erase_fail_count": { 2024-09-14T03:47:51.374 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.374 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.374 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.374 INFO:tasks.workunit.client.0.smithi070.stdout: "host_bytes_written": { 2024-09-14T03:47:51.374 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.374 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 11224882 2024-09-14T03:47:51.374 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.374 INFO:tasks.workunit.client.0.smithi070.stdout: "host_ctx_wear_used": { 2024-09-14T03:47:51.374 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.374 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.374 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.375 INFO:tasks.workunit.client.0.smithi070.stdout: "media_bytes_read": { 2024-09-14T03:47:51.375 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.375 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.375 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.375 INFO:tasks.workunit.client.0.smithi070.stdout: "nand_bytes_written": { 2024-09-14T03:47:51.375 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.375 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.375 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.375 INFO:tasks.workunit.client.0.smithi070.stdout: "perf_stat_indicator": { 2024-09-14T03:47:51.375 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.375 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.375 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.376 INFO:tasks.workunit.client.0.smithi070.stdout: "pll_lock_loss_count": { 2024-09-14T03:47:51.376 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.376 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.376 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.376 INFO:tasks.workunit.client.0.smithi070.stdout: "program_fail_count": { 2024-09-14T03:47:51.376 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.376 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.376 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.376 INFO:tasks.workunit.client.0.smithi070.stdout: "re_alloc_sectr_cnt": { 2024-09-14T03:47:51.376 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.376 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.376 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.377 INFO:tasks.workunit.client.0.smithi070.stdout: "retry_buffer_overflow_count": { 2024-09-14T03:47:51.377 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.377 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.377 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.377 INFO:tasks.workunit.client.0.smithi070.stdout: "soft_ecc_err_rate": { 2024-09-14T03:47:51.377 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.377 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.377 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.377 INFO:tasks.workunit.client.0.smithi070.stdout: "thermal_throttle_status": { 2024-09-14T03:47:51.377 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.378 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": { 2024-09-14T03:47:51.378 INFO:tasks.workunit.client.0.smithi070.stdout: "cnt": 0, 2024-09-14T03:47:51.378 INFO:tasks.workunit.client.0.smithi070.stdout: "pct": 0 2024-09-14T03:47:51.378 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.378 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.378 INFO:tasks.workunit.client.0.smithi070.stdout: "timed_workload_host_reads": { 2024-09-14T03:47:51.378 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.378 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 14 2024-09-14T03:47:51.378 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.378 INFO:tasks.workunit.client.0.smithi070.stdout: "timed_workload_media_wear": { 2024-09-14T03:47:51.378 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.379 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": "0.000000" 2024-09-14T03:47:51.379 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.379 INFO:tasks.workunit.client.0.smithi070.stdout: "timed_workload_timer": { 2024-09-14T03:47:51.379 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 100, 2024-09-14T03:47:51.379 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 1493348 2024-09-14T03:47:51.379 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.379 INFO:tasks.workunit.client.0.smithi070.stdout: "unexp_power_loss": { 2024-09-14T03:47:51.379 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.379 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": 0 2024-09-14T03:47:51.379 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.379 INFO:tasks.workunit.client.0.smithi070.stdout: "wear_leveling": { 2024-09-14T03:47:51.379 INFO:tasks.workunit.client.0.smithi070.stdout: "normalized": 0, 2024-09-14T03:47:51.379 INFO:tasks.workunit.client.0.smithi070.stdout: "raw": { 2024-09-14T03:47:51.380 INFO:tasks.workunit.client.0.smithi070.stdout: "avg": 0, 2024-09-14T03:47:51.380 INFO:tasks.workunit.client.0.smithi070.stdout: "max": 0, 2024-09-14T03:47:51.380 INFO:tasks.workunit.client.0.smithi070.stdout: "min": 0 2024-09-14T03:47:51.380 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.380 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.380 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.380 INFO:tasks.workunit.client.0.smithi070.stdout: "Intel Smart log": "nvme0n1" 2024-09-14T03:47:51.380 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.380 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_smart_health_information_log": { 2024-09-14T03:47:51.380 INFO:tasks.workunit.client.0.smithi070.stdout: "available_spare": 100, 2024-09-14T03:47:51.380 INFO:tasks.workunit.client.0.smithi070.stdout: "available_spare_threshold": 0, 2024-09-14T03:47:51.380 INFO:tasks.workunit.client.0.smithi070.stdout: "controller_busy_time": 4368, 2024-09-14T03:47:51.381 INFO:tasks.workunit.client.0.smithi070.stdout: "critical_warning": 0, 2024-09-14T03:47:51.381 INFO:tasks.workunit.client.0.smithi070.stdout: "data_units_read": 155028165, 2024-09-14T03:47:51.381 INFO:tasks.workunit.client.0.smithi070.stdout: "data_units_written": 735633918, 2024-09-14T03:47:51.381 INFO:tasks.workunit.client.0.smithi070.stdout: "host_reads": 2607915323, 2024-09-14T03:47:51.381 INFO:tasks.workunit.client.0.smithi070.stdout: "host_writes": 16464089664, 2024-09-14T03:47:51.381 INFO:tasks.workunit.client.0.smithi070.stdout: "media_errors": 0, 2024-09-14T03:47:51.381 INFO:tasks.workunit.client.0.smithi070.stdout: "num_err_log_entries": 0, 2024-09-14T03:47:51.381 INFO:tasks.workunit.client.0.smithi070.stdout: "percentage_used": 0, 2024-09-14T03:47:51.381 INFO:tasks.workunit.client.0.smithi070.stdout: "power_cycles": 30550, 2024-09-14T03:47:51.381 INFO:tasks.workunit.client.0.smithi070.stdout: "power_on_hours": 37550, 2024-09-14T03:47:51.381 INFO:tasks.workunit.client.0.smithi070.stdout: "temperature": 34, 2024-09-14T03:47:51.382 INFO:tasks.workunit.client.0.smithi070.stdout: "unsafe_shutdowns": 9447 2024-09-14T03:47:51.382 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.382 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_vendor": "intel", 2024-09-14T03:47:51.382 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme_version": { 2024-09-14T03:47:51.382 INFO:tasks.workunit.client.0.smithi070.stdout: "string": "<1.2", 2024-09-14T03:47:51.382 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.382 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.382 INFO:tasks.workunit.client.0.smithi070.stdout: "power_cycle_count": 30550, 2024-09-14T03:47:51.382 INFO:tasks.workunit.client.0.smithi070.stdout: "power_on_time": { 2024-09-14T03:47:51.382 INFO:tasks.workunit.client.0.smithi070.stdout: "hours": 37550 2024-09-14T03:47:51.382 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.383 INFO:tasks.workunit.client.0.smithi070.stdout: "serial_number": "deleted", 2024-09-14T03:47:51.383 INFO:tasks.workunit.client.0.smithi070.stdout: "smart_status": { 2024-09-14T03:47:51.383 INFO:tasks.workunit.client.0.smithi070.stdout: "nvme": { 2024-09-14T03:47:51.383 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.383 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.383 INFO:tasks.workunit.client.0.smithi070.stdout: "passed": true 2024-09-14T03:47:51.383 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.383 INFO:tasks.workunit.client.0.smithi070.stdout: "smartctl": { 2024-09-14T03:47:51.383 INFO:tasks.workunit.client.0.smithi070.stdout: "argv": [ 2024-09-14T03:47:51.383 INFO:tasks.workunit.client.0.smithi070.stdout: "smartctl", 2024-09-14T03:47:51.383 INFO:tasks.workunit.client.0.smithi070.stdout: "-x", 2024-09-14T03:47:51.384 INFO:tasks.workunit.client.0.smithi070.stdout: "--json=o", 2024-09-14T03:47:51.384 INFO:tasks.workunit.client.0.smithi070.stdout: "/dev/nvme0n1" 2024-09-14T03:47:51.384 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.384 INFO:tasks.workunit.client.0.smithi070.stdout: "build_info": "(local build)", 2024-09-14T03:47:51.384 INFO:tasks.workunit.client.0.smithi070.stdout: "exit_status": 0, 2024-09-14T03:47:51.384 INFO:tasks.workunit.client.0.smithi070.stdout: "output": [ 2024-09-14T03:47:51.384 INFO:tasks.workunit.client.0.smithi070.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-505.el9.x86_64] (local build)", 2024-09-14T03:47:51.384 INFO:tasks.workunit.client.0.smithi070.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-14T03:47:51.384 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:51.384 INFO:tasks.workunit.client.0.smithi070.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-14T03:47:51.384 INFO:tasks.workunit.client.0.smithi070.stdout: "Model Number: INTEL SSDPED1D480GA", 2024-09-14T03:47:51.384 INFO:tasks.workunit.client.0.smithi070.stdout: "Serial Number: deleted", 2024-09-14T03:47:51.385 INFO:tasks.workunit.client.0.smithi070.stdout: "Firmware Version: E2010325", 2024-09-14T03:47:51.385 INFO:tasks.workunit.client.0.smithi070.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-14T03:47:51.385 INFO:tasks.workunit.client.0.smithi070.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-14T03:47:51.385 INFO:tasks.workunit.client.0.smithi070.stdout: "Controller ID: 0", 2024-09-14T03:47:51.385 INFO:tasks.workunit.client.0.smithi070.stdout: "NVMe Version: <1.2", 2024-09-14T03:47:51.385 INFO:tasks.workunit.client.0.smithi070.stdout: "Number of Namespaces: 1", 2024-09-14T03:47:51.385 INFO:tasks.workunit.client.0.smithi070.stdout: "Namespace 1 Size/Capacity: 480,103,981,056 [480 GB]", 2024-09-14T03:47:51.385 INFO:tasks.workunit.client.0.smithi070.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-14T03:47:51.385 INFO:tasks.workunit.client.0.smithi070.stdout: "Local Time is: Sat Sep 14 03:41:40 2024 UTC", 2024-09-14T03:47:51.385 INFO:tasks.workunit.client.0.smithi070.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-14T03:47:51.385 INFO:tasks.workunit.client.0.smithi070.stdout: "Optional Admin Commands (0x0007): Security Format Frmw_DL", 2024-09-14T03:47:51.386 INFO:tasks.workunit.client.0.smithi070.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-14T03:47:51.386 INFO:tasks.workunit.client.0.smithi070.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-14T03:47:51.386 INFO:tasks.workunit.client.0.smithi070.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-14T03:47:51.386 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:51.386 INFO:tasks.workunit.client.0.smithi070.stdout: "Supported Power States", 2024-09-14T03:47:51.386 INFO:tasks.workunit.client.0.smithi070.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-14T03:47:51.386 INFO:tasks.workunit.client.0.smithi070.stdout: " 0 + 18.00W - - 0 0 0 0 0 0", 2024-09-14T03:47:51.386 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:51.386 INFO:tasks.workunit.client.0.smithi070.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-14T03:47:51.386 INFO:tasks.workunit.client.0.smithi070.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-14T03:47:51.386 INFO:tasks.workunit.client.0.smithi070.stdout: " 0 + 512 0 2", 2024-09-14T03:47:51.386 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:51.386 INFO:tasks.workunit.client.0.smithi070.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-14T03:47:51.387 INFO:tasks.workunit.client.0.smithi070.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-14T03:47:51.387 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:51.387 INFO:tasks.workunit.client.0.smithi070.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-14T03:47:51.387 INFO:tasks.workunit.client.0.smithi070.stdout: "Critical Warning: 0x00", 2024-09-14T03:47:51.387 INFO:tasks.workunit.client.0.smithi070.stdout: "Temperature: 34 Celsius", 2024-09-14T03:47:51.387 INFO:tasks.workunit.client.0.smithi070.stdout: "Available Spare: 100%", 2024-09-14T03:47:51.387 INFO:tasks.workunit.client.0.smithi070.stdout: "Available Spare Threshold: 0%", 2024-09-14T03:47:51.387 INFO:tasks.workunit.client.0.smithi070.stdout: "Percentage Used: 0%", 2024-09-14T03:47:51.387 INFO:tasks.workunit.client.0.smithi070.stdout: "Data Units Read: 155,028,165 [79.3 TB]", 2024-09-14T03:47:51.387 INFO:tasks.workunit.client.0.smithi070.stdout: "Data Units Written: 735,633,918 [376 TB]", 2024-09-14T03:47:51.387 INFO:tasks.workunit.client.0.smithi070.stdout: "Host Read Commands: 2,607,915,323", 2024-09-14T03:47:51.387 INFO:tasks.workunit.client.0.smithi070.stdout: "Host Write Commands: 16,464,089,664", 2024-09-14T03:47:51.388 INFO:tasks.workunit.client.0.smithi070.stdout: "Controller Busy Time: 4,368", 2024-09-14T03:47:51.388 INFO:tasks.workunit.client.0.smithi070.stdout: "Power Cycles: 30,550", 2024-09-14T03:47:51.388 INFO:tasks.workunit.client.0.smithi070.stdout: "Power On Hours: 37,550", 2024-09-14T03:47:51.388 INFO:tasks.workunit.client.0.smithi070.stdout: "Unsafe Shutdowns: 9,447", 2024-09-14T03:47:51.388 INFO:tasks.workunit.client.0.smithi070.stdout: "Media and Data Integrity Errors: 0", 2024-09-14T03:47:51.388 INFO:tasks.workunit.client.0.smithi070.stdout: "Error Information Log Entries: 0", 2024-09-14T03:47:51.388 INFO:tasks.workunit.client.0.smithi070.stdout: "", 2024-09-14T03:47:51.388 INFO:tasks.workunit.client.0.smithi070.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-14T03:47:51.388 INFO:tasks.workunit.client.0.smithi070.stdout: "No Errors Logged", 2024-09-14T03:47:51.388 INFO:tasks.workunit.client.0.smithi070.stdout: "" 2024-09-14T03:47:51.388 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.388 INFO:tasks.workunit.client.0.smithi070.stdout: "platform_info": "x86_64-linux-5.14.0-505.el9.x86_64", 2024-09-14T03:47:51.389 INFO:tasks.workunit.client.0.smithi070.stdout: "svn_revision": "5155", 2024-09-14T03:47:51.389 INFO:tasks.workunit.client.0.smithi070.stdout: "version": [ 2024-09-14T03:47:51.389 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:51.389 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:51.389 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.389 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.389 INFO:tasks.workunit.client.0.smithi070.stdout: "temperature": { 2024-09-14T03:47:51.389 INFO:tasks.workunit.client.0.smithi070.stdout: "current": 34 2024-09-14T03:47:51.389 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.389 INFO:tasks.workunit.client.0.smithi070.stdout: "user_capacity": { 2024-09-14T03:47:51.389 INFO:tasks.workunit.client.0.smithi070.stdout: "blocks": 937703088, 2024-09-14T03:47:51.390 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 480103981056 2024-09-14T03:47:51.390 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.390 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.390 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.390 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.390 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.390 INFO:tasks.workunit.client.0.smithi070.stdout: "report": { 2024-09-14T03:47:51.390 INFO:tasks.workunit.client.0.smithi070.stdout: "balancer": { 2024-09-14T03:47:51.390 INFO:tasks.workunit.client.0.smithi070.stdout: "active": true, 2024-09-14T03:47:51.390 INFO:tasks.workunit.client.0.smithi070.stdout: "mode": "" 2024-09-14T03:47:51.390 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.391 INFO:tasks.workunit.client.0.smithi070.stdout: "channels": [ 2024-09-14T03:47:51.391 INFO:tasks.workunit.client.0.smithi070.stdout: "basic", 2024-09-14T03:47:51.391 INFO:tasks.workunit.client.0.smithi070.stdout: "crash", 2024-09-14T03:47:51.391 INFO:tasks.workunit.client.0.smithi070.stdout: "device", 2024-09-14T03:47:51.391 INFO:tasks.workunit.client.0.smithi070.stdout: "ident", 2024-09-14T03:47:51.391 INFO:tasks.workunit.client.0.smithi070.stdout: "perf" 2024-09-14T03:47:51.391 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.391 INFO:tasks.workunit.client.0.smithi070.stdout: "channels_available": [ 2024-09-14T03:47:51.391 INFO:tasks.workunit.client.0.smithi070.stdout: "basic", 2024-09-14T03:47:51.391 INFO:tasks.workunit.client.0.smithi070.stdout: "ident", 2024-09-14T03:47:51.391 INFO:tasks.workunit.client.0.smithi070.stdout: "crash", 2024-09-14T03:47:51.391 INFO:tasks.workunit.client.0.smithi070.stdout: "device", 2024-09-14T03:47:51.392 INFO:tasks.workunit.client.0.smithi070.stdout: "perf" 2024-09-14T03:47:51.392 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.392 INFO:tasks.workunit.client.0.smithi070.stdout: "collections_available": [ 2024-09-14T03:47:51.392 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_base", 2024-09-14T03:47:51.392 INFO:tasks.workunit.client.0.smithi070.stdout: "device_base", 2024-09-14T03:47:51.392 INFO:tasks.workunit.client.0.smithi070.stdout: "crash_base", 2024-09-14T03:47:51.392 INFO:tasks.workunit.client.0.smithi070.stdout: "ident_base", 2024-09-14T03:47:51.392 INFO:tasks.workunit.client.0.smithi070.stdout: "perf_perf", 2024-09-14T03:47:51.392 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_mds_metadata", 2024-09-14T03:47:51.392 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_pool_usage", 2024-09-14T03:47:51.393 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_usage_by_class", 2024-09-14T03:47:51.393 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_rook_v01", 2024-09-14T03:47:51.393 INFO:tasks.workunit.client.0.smithi070.stdout: "perf_memory_metrics", 2024-09-14T03:47:51.393 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_pool_options_bluestore" 2024-09-14T03:47:51.393 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.393 INFO:tasks.workunit.client.0.smithi070.stdout: "collections_opted_in": [ 2024-09-14T03:47:51.393 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_base", 2024-09-14T03:47:51.393 INFO:tasks.workunit.client.0.smithi070.stdout: "device_base", 2024-09-14T03:47:51.393 INFO:tasks.workunit.client.0.smithi070.stdout: "crash_base", 2024-09-14T03:47:51.393 INFO:tasks.workunit.client.0.smithi070.stdout: "ident_base", 2024-09-14T03:47:51.393 INFO:tasks.workunit.client.0.smithi070.stdout: "perf_perf", 2024-09-14T03:47:51.394 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_mds_metadata", 2024-09-14T03:47:51.394 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_pool_usage", 2024-09-14T03:47:51.394 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_usage_by_class", 2024-09-14T03:47:51.394 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_rook_v01", 2024-09-14T03:47:51.394 INFO:tasks.workunit.client.0.smithi070.stdout: "perf_memory_metrics", 2024-09-14T03:47:51.394 INFO:tasks.workunit.client.0.smithi070.stdout: "basic_pool_options_bluestore" 2024-09-14T03:47:51.394 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.394 INFO:tasks.workunit.client.0.smithi070.stdout: "config": { 2024-09-14T03:47:51.394 INFO:tasks.workunit.client.0.smithi070.stdout: "active_changed": [ 2024-09-14T03:47:51.394 INFO:tasks.workunit.client.0.smithi070.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-14T03:47:51.394 INFO:tasks.workunit.client.0.smithi070.stdout: "auth_debug", 2024-09-14T03:47:51.395 INFO:tasks.workunit.client.0.smithi070.stdout: "auth_mon_ticket_ttl", 2024-09-14T03:47:51.395 INFO:tasks.workunit.client.0.smithi070.stdout: "auth_service_ticket_ttl", 2024-09-14T03:47:51.395 INFO:tasks.workunit.client.0.smithi070.stdout: "bdev_debug_aio", 2024-09-14T03:47:51.395 INFO:tasks.workunit.client.0.smithi070.stdout: "client_check_pool_perm", 2024-09-14T03:47:51.395 INFO:tasks.workunit.client.0.smithi070.stdout: "client_mount_gid", 2024-09-14T03:47:51.395 INFO:tasks.workunit.client.0.smithi070.stdout: "client_mount_uid", 2024-09-14T03:47:51.395 INFO:tasks.workunit.client.0.smithi070.stdout: "client_quota", 2024-09-14T03:47:51.395 INFO:tasks.workunit.client.0.smithi070.stdout: "container_image", 2024-09-14T03:47:51.395 INFO:tasks.workunit.client.0.smithi070.stdout: "daemonize", 2024-09-14T03:47:51.395 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_asserts_on_shutdown", 2024-09-14T03:47:51.395 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_mgr", 2024-09-14T03:47:51.395 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_mon", 2024-09-14T03:47:51.396 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_ms", 2024-09-14T03:47:51.396 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_osd", 2024-09-14T03:47:51.396 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_paxos", 2024-09-14T03:47:51.396 INFO:tasks.workunit.client.0.smithi070.stdout: "keyring", 2024-09-14T03:47:51.396 INFO:tasks.workunit.client.0.smithi070.stdout: "leveldb_block_size", 2024-09-14T03:47:51.396 INFO:tasks.workunit.client.0.smithi070.stdout: "leveldb_cache_size", 2024-09-14T03:47:51.396 INFO:tasks.workunit.client.0.smithi070.stdout: "leveldb_compression", 2024-09-14T03:47:51.396 INFO:tasks.workunit.client.0.smithi070.stdout: "leveldb_log", 2024-09-14T03:47:51.396 INFO:tasks.workunit.client.0.smithi070.stdout: "leveldb_write_buffer_size", 2024-09-14T03:47:51.396 INFO:tasks.workunit.client.0.smithi070.stdout: "log_to_file", 2024-09-14T03:47:51.396 INFO:tasks.workunit.client.0.smithi070.stdout: "log_to_journald", 2024-09-14T03:47:51.397 INFO:tasks.workunit.client.0.smithi070.stdout: "log_to_stderr", 2024-09-14T03:47:51.397 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_allow_pool_delete", 2024-09-14T03:47:51.397 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_clock_drift_allowed", 2024-09-14T03:47:51.397 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_cluster_log_to_file", 2024-09-14T03:47:51.397 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_cluster_log_to_journald", 2024-09-14T03:47:51.397 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_cluster_log_to_stderr", 2024-09-14T03:47:51.397 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_data_avail_warn", 2024-09-14T03:47:51.397 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_election_default_strategy", 2024-09-14T03:47:51.397 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_host", 2024-09-14T03:47:51.397 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_max_pg_per_osd", 2024-09-14T03:47:51.397 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_mgr_mkfs_grace", 2024-09-14T03:47:51.397 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_osd_prime_pg_temp", 2024-09-14T03:47:51.398 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_osd_reporter_subtree_level", 2024-09-14T03:47:51.398 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_pg_warn_max_object_skew", 2024-09-14T03:47:51.398 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-14T03:47:51.398 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-14T03:47:51.398 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-14T03:47:51.398 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-14T03:47:51.398 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-14T03:47:51.398 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-14T03:47:51.398 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-14T03:47:51.398 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-14T03:47:51.398 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_too_few_osds", 2024-09-14T03:47:51.398 INFO:tasks.workunit.client.0.smithi070.stdout: "ms_die_on_bug", 2024-09-14T03:47:51.399 INFO:tasks.workunit.client.0.smithi070.stdout: "ms_die_on_old_message", 2024-09-14T03:47:51.399 INFO:tasks.workunit.client.0.smithi070.stdout: "no_config_file", 2024-09-14T03:47:51.399 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_class_default_list", 2024-09-14T03:47:51.399 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_class_load_list", 2024-09-14T03:47:51.399 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_misdirected_ops", 2024-09-14T03:47:51.399 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_op_order", 2024-09-14T03:47:51.399 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_pg_log_writeout", 2024-09-14T03:47:51.399 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_shutdown", 2024-09-14T03:47:51.399 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_verify_cached_snaps", 2024-09-14T03:47:51.399 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_verify_missing_on_start", 2024-09-14T03:47:51.399 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_verify_stray_on_activate", 2024-09-14T03:47:51.399 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-14T03:47:51.400 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_profile", 2024-09-14T03:47:51.400 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_background_best_effort_lim", 2024-09-14T03:47:51.400 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_background_best_effort_res", 2024-09-14T03:47:51.400 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_background_best_effort_wgt", 2024-09-14T03:47:51.400 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_background_recovery_lim", 2024-09-14T03:47:51.400 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_background_recovery_res", 2024-09-14T03:47:51.400 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_background_recovery_wgt", 2024-09-14T03:47:51.400 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_client_lim", 2024-09-14T03:47:51.400 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_client_res", 2024-09-14T03:47:51.400 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_scheduler_client_wgt", 2024-09-14T03:47:51.400 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_memory_target", 2024-09-14T03:47:51.401 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_memory_target_autotune", 2024-09-14T03:47:51.401 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_op_queue", 2024-09-14T03:47:51.401 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_op_queue_cut_off", 2024-09-14T03:47:51.401 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pool_default_erasure_code_profile", 2024-09-14T03:47:51.401 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-14T03:47:51.401 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_recover_clone_overlap", 2024-09-14T03:47:51.401 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_recovery_max_chunk", 2024-09-14T03:47:51.401 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_scrub_load_threshold", 2024-09-14T03:47:51.401 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_scrub_max_interval", 2024-09-14T03:47:51.401 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_shutdown_pgref_assert", 2024-09-14T03:47:51.401 INFO:tasks.workunit.client.0.smithi070.stdout: "public_addr", 2024-09-14T03:47:51.401 INFO:tasks.workunit.client.0.smithi070.stdout: "public_addrv", 2024-09-14T03:47:51.402 INFO:tasks.workunit.client.0.smithi070.stdout: "public_network", 2024-09-14T03:47:51.402 INFO:tasks.workunit.client.0.smithi070.stdout: "rbd_default_features", 2024-09-14T03:47:51.402 INFO:tasks.workunit.client.0.smithi070.stdout: "rbd_qos_exclude_ops", 2024-09-14T03:47:51.402 INFO:tasks.workunit.client.0.smithi070.stdout: "setgroup", 2024-09-14T03:47:51.402 INFO:tasks.workunit.client.0.smithi070.stdout: "setuser" 2024-09-14T03:47:51.402 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.402 INFO:tasks.workunit.client.0.smithi070.stdout: "cluster_changed": [ 2024-09-14T03:47:51.402 INFO:tasks.workunit.client.0.smithi070.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-14T03:47:51.402 INFO:tasks.workunit.client.0.smithi070.stdout: "auth_debug", 2024-09-14T03:47:51.402 INFO:tasks.workunit.client.0.smithi070.stdout: "auth_mon_ticket_ttl", 2024-09-14T03:47:51.402 INFO:tasks.workunit.client.0.smithi070.stdout: "auth_service_ticket_ttl", 2024-09-14T03:47:51.403 INFO:tasks.workunit.client.0.smithi070.stdout: "bdev_debug_aio", 2024-09-14T03:47:51.403 INFO:tasks.workunit.client.0.smithi070.stdout: "container_image", 2024-09-14T03:47:51.403 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_asserts_on_shutdown", 2024-09-14T03:47:51.403 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_mgr", 2024-09-14T03:47:51.403 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_mon", 2024-09-14T03:47:51.403 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_ms", 2024-09-14T03:47:51.403 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_osd", 2024-09-14T03:47:51.403 INFO:tasks.workunit.client.0.smithi070.stdout: "debug_paxos", 2024-09-14T03:47:51.403 INFO:tasks.workunit.client.0.smithi070.stdout: "log_to_file", 2024-09-14T03:47:51.403 INFO:tasks.workunit.client.0.smithi070.stdout: "log_to_journald", 2024-09-14T03:47:51.403 INFO:tasks.workunit.client.0.smithi070.stdout: "log_to_stderr", 2024-09-14T03:47:51.403 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_join_fs", 2024-09-14T03:47:51.403 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/cephadm/allow_ptrace", 2024-09-14T03:47:51.404 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/cephadm/container_init", 2024-09-14T03:47:51.404 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/cephadm/migration_current", 2024-09-14T03:47:51.404 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/cephadm/use_repo_digest", 2024-09-14T03:47:51.404 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/dashboard/ALERTMANAGER_API_HOST", 2024-09-14T03:47:51.404 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/dashboard/GRAFANA_API_SSL_VERIFY", 2024-09-14T03:47:51.404 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/dashboard/GRAFANA_API_URL", 2024-09-14T03:47:51.404 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/dashboard/PROMETHEUS_API_HOST", 2024-09-14T03:47:51.404 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/dashboard/ssl_server_port", 2024-09-14T03:47:51.404 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/orchestrator/orchestrator", 2024-09-14T03:47:51.404 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/telemetry/channel_ident", 2024-09-14T03:47:51.404 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/telemetry/channel_perf", 2024-09-14T03:47:51.404 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/telemetry/description", 2024-09-14T03:47:51.405 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/telemetry/enabled", 2024-09-14T03:47:51.405 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr/telemetry/organization", 2024-09-14T03:47:51.405 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_allow_pool_delete", 2024-09-14T03:47:51.405 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_clock_drift_allowed", 2024-09-14T03:47:51.405 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_cluster_log_to_file", 2024-09-14T03:47:51.405 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_data_avail_warn", 2024-09-14T03:47:51.405 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_election_default_strategy", 2024-09-14T03:47:51.405 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_max_pg_per_osd", 2024-09-14T03:47:51.405 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_mgr_mkfs_grace", 2024-09-14T03:47:51.405 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_osd_prime_pg_temp", 2024-09-14T03:47:51.405 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_osd_reporter_subtree_level", 2024-09-14T03:47:51.405 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_pg_warn_max_object_skew", 2024-09-14T03:47:51.406 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-14T03:47:51.406 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-14T03:47:51.406 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-14T03:47:51.406 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-14T03:47:51.406 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-14T03:47:51.406 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-14T03:47:51.406 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-14T03:47:51.406 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-14T03:47:51.406 INFO:tasks.workunit.client.0.smithi070.stdout: "mon_warn_on_too_few_osds", 2024-09-14T03:47:51.406 INFO:tasks.workunit.client.0.smithi070.stdout: "ms_die_on_bug", 2024-09-14T03:47:51.406 INFO:tasks.workunit.client.0.smithi070.stdout: "ms_die_on_old_message", 2024-09-14T03:47:51.406 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_class_default_list", 2024-09-14T03:47:51.407 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_class_load_list", 2024-09-14T03:47:51.407 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_misdirected_ops", 2024-09-14T03:47:51.407 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_op_order", 2024-09-14T03:47:51.407 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_pg_log_writeout", 2024-09-14T03:47:51.407 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_shutdown", 2024-09-14T03:47:51.407 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_verify_cached_snaps", 2024-09-14T03:47:51.407 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_verify_missing_on_start", 2024-09-14T03:47:51.407 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_debug_verify_stray_on_activate", 2024-09-14T03:47:51.407 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-14T03:47:51.407 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mclock_profile", 2024-09-14T03:47:51.407 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_memory_target", 2024-09-14T03:47:51.407 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_memory_target_autotune", 2024-09-14T03:47:51.407 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_op_queue", 2024-09-14T03:47:51.408 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_op_queue_cut_off", 2024-09-14T03:47:51.408 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pool_default_erasure_code_profile", 2024-09-14T03:47:51.408 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-14T03:47:51.408 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_recover_clone_overlap", 2024-09-14T03:47:51.408 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_recovery_max_chunk", 2024-09-14T03:47:51.408 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_scrub_load_threshold", 2024-09-14T03:47:51.408 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_scrub_max_interval", 2024-09-14T03:47:51.408 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_shutdown_pgref_assert", 2024-09-14T03:47:51.408 INFO:tasks.workunit.client.0.smithi070.stdout: "public_network", 2024-09-14T03:47:51.408 INFO:tasks.workunit.client.0.smithi070.stdout: "rgw_cache_enabled", 2024-09-14T03:47:51.408 INFO:tasks.workunit.client.0.smithi070.stdout: "rgw_enable_ops_log", 2024-09-14T03:47:51.409 INFO:tasks.workunit.client.0.smithi070.stdout: "rgw_enable_usage_log" 2024-09-14T03:47:51.409 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.409 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.409 INFO:tasks.workunit.client.0.smithi070.stdout: "contact": null, 2024-09-14T03:47:51.409 INFO:tasks.workunit.client.0.smithi070.stdout: "crashes": [], 2024-09-14T03:47:51.409 INFO:tasks.workunit.client.0.smithi070.stdout: "created": "2024-09-14T03:38:26.272229Z", 2024-09-14T03:47:51.409 INFO:tasks.workunit.client.0.smithi070.stdout: "crush": { 2024-09-14T03:47:51.409 INFO:tasks.workunit.client.0.smithi070.stdout: "bucket_algs": { 2024-09-14T03:47:51.409 INFO:tasks.workunit.client.0.smithi070.stdout: "straw2": 3 2024-09-14T03:47:51.409 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.410 INFO:tasks.workunit.client.0.smithi070.stdout: "bucket_sizes": { 2024-09-14T03:47:51.410 INFO:tasks.workunit.client.0.smithi070.stdout: "2": 1, 2024-09-14T03:47:51.410 INFO:tasks.workunit.client.0.smithi070.stdout: "4": 2 2024-09-14T03:47:51.410 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.410 INFO:tasks.workunit.client.0.smithi070.stdout: "bucket_types": { 2024-09-14T03:47:51.410 INFO:tasks.workunit.client.0.smithi070.stdout: "1": 2, 2024-09-14T03:47:51.410 INFO:tasks.workunit.client.0.smithi070.stdout: "11": 1 2024-09-14T03:47:51.410 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.410 INFO:tasks.workunit.client.0.smithi070.stdout: "compat_weight_set": false, 2024-09-14T03:47:51.410 INFO:tasks.workunit.client.0.smithi070.stdout: "device_classes": [ 2024-09-14T03:47:51.410 INFO:tasks.workunit.client.0.smithi070.stdout: 8 2024-09-14T03:47:51.411 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.411 INFO:tasks.workunit.client.0.smithi070.stdout: "num_buckets": 6, 2024-09-14T03:47:51.411 INFO:tasks.workunit.client.0.smithi070.stdout: "num_devices": 8, 2024-09-14T03:47:51.411 INFO:tasks.workunit.client.0.smithi070.stdout: "num_rules": 1, 2024-09-14T03:47:51.411 INFO:tasks.workunit.client.0.smithi070.stdout: "num_types": 12, 2024-09-14T03:47:51.411 INFO:tasks.workunit.client.0.smithi070.stdout: "num_weight_sets": 0, 2024-09-14T03:47:51.411 INFO:tasks.workunit.client.0.smithi070.stdout: "tunables": { 2024-09-14T03:47:51.411 INFO:tasks.workunit.client.0.smithi070.stdout: "allowed_bucket_algs": 54, 2024-09-14T03:47:51.411 INFO:tasks.workunit.client.0.smithi070.stdout: "choose_local_fallback_tries": 0, 2024-09-14T03:47:51.411 INFO:tasks.workunit.client.0.smithi070.stdout: "choose_local_tries": 0, 2024-09-14T03:47:51.411 INFO:tasks.workunit.client.0.smithi070.stdout: "choose_total_tries": 50, 2024-09-14T03:47:51.412 INFO:tasks.workunit.client.0.smithi070.stdout: "chooseleaf_descend_once": 1, 2024-09-14T03:47:51.412 INFO:tasks.workunit.client.0.smithi070.stdout: "chooseleaf_stable": 1, 2024-09-14T03:47:51.412 INFO:tasks.workunit.client.0.smithi070.stdout: "chooseleaf_vary_r": 1, 2024-09-14T03:47:51.412 INFO:tasks.workunit.client.0.smithi070.stdout: "has_v2_rules": 0, 2024-09-14T03:47:51.412 INFO:tasks.workunit.client.0.smithi070.stdout: "has_v3_rules": 0, 2024-09-14T03:47:51.412 INFO:tasks.workunit.client.0.smithi070.stdout: "has_v4_buckets": 1, 2024-09-14T03:47:51.412 INFO:tasks.workunit.client.0.smithi070.stdout: "has_v5_rules": 0, 2024-09-14T03:47:51.412 INFO:tasks.workunit.client.0.smithi070.stdout: "legacy_tunables": 0, 2024-09-14T03:47:51.412 INFO:tasks.workunit.client.0.smithi070.stdout: "minimum_required_version": "jewel", 2024-09-14T03:47:51.412 INFO:tasks.workunit.client.0.smithi070.stdout: "optimal_tunables": 1, 2024-09-14T03:47:51.412 INFO:tasks.workunit.client.0.smithi070.stdout: "profile": "jewel", 2024-09-14T03:47:51.413 INFO:tasks.workunit.client.0.smithi070.stdout: "require_feature_tunables": 1, 2024-09-14T03:47:51.413 INFO:tasks.workunit.client.0.smithi070.stdout: "require_feature_tunables2": 1, 2024-09-14T03:47:51.413 INFO:tasks.workunit.client.0.smithi070.stdout: "require_feature_tunables3": 1, 2024-09-14T03:47:51.413 INFO:tasks.workunit.client.0.smithi070.stdout: "require_feature_tunables5": 1, 2024-09-14T03:47:51.413 INFO:tasks.workunit.client.0.smithi070.stdout: "straw_calc_version": 1 2024-09-14T03:47:51.413 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.413 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.413 INFO:tasks.workunit.client.0.smithi070.stdout: "description": "upgrade test cluster", 2024-09-14T03:47:51.413 INFO:tasks.workunit.client.0.smithi070.stdout: "fs": { 2024-09-14T03:47:51.416 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:51.416 INFO:tasks.workunit.client.0.smithi070.stdout: "feature_flags": { 2024-09-14T03:47:51.416 INFO:tasks.workunit.client.0.smithi070.stdout: "enable_multiple": true, 2024-09-14T03:47:51.417 INFO:tasks.workunit.client.0.smithi070.stdout: "ever_enabled_multiple": true 2024-09-14T03:47:51.417 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.417 INFO:tasks.workunit.client.0.smithi070.stdout: "filesystems": [ 2024-09-14T03:47:51.417 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.417 INFO:tasks.workunit.client.0.smithi070.stdout: "approx_ctime": "2024-09", 2024-09-14T03:47:51.417 INFO:tasks.workunit.client.0.smithi070.stdout: "balancer_enabled": false, 2024-09-14T03:47:51.417 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.417 INFO:tasks.workunit.client.0.smithi070.stdout: "cached_caps": 0, 2024-09-14T03:47:51.417 INFO:tasks.workunit.client.0.smithi070.stdout: "cached_dns": 10, 2024-09-14T03:47:51.417 INFO:tasks.workunit.client.0.smithi070.stdout: "cached_inos": 13, 2024-09-14T03:47:51.417 INFO:tasks.workunit.client.0.smithi070.stdout: "cached_subtrees": 2, 2024-09-14T03:47:51.417 INFO:tasks.workunit.client.0.smithi070.stdout: "ever_allowed_features": 0, 2024-09-14T03:47:51.418 INFO:tasks.workunit.client.0.smithi070.stdout: "explicitly_allowed_features": 0, 2024-09-14T03:47:51.418 INFO:tasks.workunit.client.0.smithi070.stdout: "files": 0, 2024-09-14T03:47:51.418 INFO:tasks.workunit.client.0.smithi070.stdout: "max_mds": 1, 2024-09-14T03:47:51.418 INFO:tasks.workunit.client.0.smithi070.stdout: "num_data_pools": 1, 2024-09-14T03:47:51.418 INFO:tasks.workunit.client.0.smithi070.stdout: "num_in": 1, 2024-09-14T03:47:51.418 INFO:tasks.workunit.client.0.smithi070.stdout: "num_mds": 1, 2024-09-14T03:47:51.418 INFO:tasks.workunit.client.0.smithi070.stdout: "num_sessions": 0, 2024-09-14T03:47:51.418 INFO:tasks.workunit.client.0.smithi070.stdout: "num_standby_replay": 0, 2024-09-14T03:47:51.418 INFO:tasks.workunit.client.0.smithi070.stdout: "num_up": 1, 2024-09-14T03:47:51.418 INFO:tasks.workunit.client.0.smithi070.stdout: "snaps": 0, 2024-09-14T03:47:51.418 INFO:tasks.workunit.client.0.smithi070.stdout: "standby_count_wanted": 0 2024-09-14T03:47:51.418 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.419 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.419 INFO:tasks.workunit.client.0.smithi070.stdout: "num_standby_mds": 1, 2024-09-14T03:47:51.419 INFO:tasks.workunit.client.0.smithi070.stdout: "total_num_mds": 2 2024-09-14T03:47:51.419 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.419 INFO:tasks.workunit.client.0.smithi070.stdout: "heap_stats": { 2024-09-14T03:47:51.419 INFO:tasks.workunit.client.0.smithi070.stdout: "mds": { 2024-09-14T03:47:51.419 INFO:tasks.workunit.client.0.smithi070.stdout: "mds.a8e17c6c2bb7d35df7944161574528c366e6a388": { 2024-09-14T03:47:51.419 INFO:tasks.workunit.client.0.smithi070.stdout: "actual_memory_used": 20594688, 2024-09-14T03:47:51.419 INFO:tasks.workunit.client.0.smithi070.stdout: "central_cache_freelist": 388872, 2024-09-14T03:47:51.419 INFO:tasks.workunit.client.0.smithi070.stdout: "malloc_metadata": 2621440, 2024-09-14T03:47:51.419 INFO:tasks.workunit.client.0.smithi070.stdout: "page_heap_freelist": 0, 2024-09-14T03:47:51.419 INFO:tasks.workunit.client.0.smithi070.stdout: "released_to_os": 589824, 2024-09-14T03:47:51.420 INFO:tasks.workunit.client.0.smithi070.stdout: "spans_in_use": 433, 2024-09-14T03:47:51.420 INFO:tasks.workunit.client.0.smithi070.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:47:51.420 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_cache_freelists": 1308624, 2024-09-14T03:47:51.420 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_heaps_in_use": 20, 2024-09-14T03:47:51.420 INFO:tasks.workunit.client.0.smithi070.stdout: "transfer_cache_freelist": 1171968, 2024-09-14T03:47:51.420 INFO:tasks.workunit.client.0.smithi070.stdout: "use_by_application": 15103784, 2024-09-14T03:47:51.420 INFO:tasks.workunit.client.0.smithi070.stdout: "virtual_address_space_used": 21184512 2024-09-14T03:47:51.420 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.420 INFO:tasks.workunit.client.0.smithi070.stdout: "mds.b1d2edfb86a8f225d9283baf06ff7497eda48d2d": { 2024-09-14T03:47:51.420 INFO:tasks.workunit.client.0.smithi070.stdout: "actual_memory_used": 20135936, 2024-09-14T03:47:51.420 INFO:tasks.workunit.client.0.smithi070.stdout: "central_cache_freelist": 351992, 2024-09-14T03:47:51.420 INFO:tasks.workunit.client.0.smithi070.stdout: "malloc_metadata": 2621440, 2024-09-14T03:47:51.421 INFO:tasks.workunit.client.0.smithi070.stdout: "page_heap_freelist": 794624, 2024-09-14T03:47:51.421 INFO:tasks.workunit.client.0.smithi070.stdout: "released_to_os": 0, 2024-09-14T03:47:51.421 INFO:tasks.workunit.client.0.smithi070.stdout: "spans_in_use": 353, 2024-09-14T03:47:51.421 INFO:tasks.workunit.client.0.smithi070.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:47:51.421 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_cache_freelists": 772696, 2024-09-14T03:47:51.421 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_heaps_in_use": 12, 2024-09-14T03:47:51.421 INFO:tasks.workunit.client.0.smithi070.stdout: "transfer_cache_freelist": 910848, 2024-09-14T03:47:51.421 INFO:tasks.workunit.client.0.smithi070.stdout: "use_by_application": 14684336, 2024-09-14T03:47:51.421 INFO:tasks.workunit.client.0.smithi070.stdout: "virtual_address_space_used": 20135936 2024-09-14T03:47:51.421 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.421 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.421 INFO:tasks.workunit.client.0.smithi070.stdout: "mon": { 2024-09-14T03:47:51.422 INFO:tasks.workunit.client.0.smithi070.stdout: "mon.a8e17c6c2bb7d35df7944161574528c366e6a388": { 2024-09-14T03:47:51.422 INFO:tasks.workunit.client.0.smithi070.stdout: "actual_memory_used": 45850624, 2024-09-14T03:47:51.422 INFO:tasks.workunit.client.0.smithi070.stdout: "central_cache_freelist": 1368376, 2024-09-14T03:47:51.422 INFO:tasks.workunit.client.0.smithi070.stdout: "malloc_metadata": 2621440, 2024-09-14T03:47:51.422 INFO:tasks.workunit.client.0.smithi070.stdout: "page_heap_freelist": 0, 2024-09-14T03:47:51.422 INFO:tasks.workunit.client.0.smithi070.stdout: "released_to_os": 6807552, 2024-09-14T03:47:51.422 INFO:tasks.workunit.client.0.smithi070.stdout: "spans_in_use": 1894, 2024-09-14T03:47:51.422 INFO:tasks.workunit.client.0.smithi070.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:47:51.422 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_cache_freelists": 11124864, 2024-09-14T03:47:51.422 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_heaps_in_use": 23, 2024-09-14T03:47:51.422 INFO:tasks.workunit.client.0.smithi070.stdout: "transfer_cache_freelist": 3050496, 2024-09-14T03:47:51.422 INFO:tasks.workunit.client.0.smithi070.stdout: "use_by_application": 27685448, 2024-09-14T03:47:51.423 INFO:tasks.workunit.client.0.smithi070.stdout: "virtual_address_space_used": 52658176 2024-09-14T03:47:51.423 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.423 INFO:tasks.workunit.client.0.smithi070.stdout: "mon.b1d2edfb86a8f225d9283baf06ff7497eda48d2d": { 2024-09-14T03:47:51.423 INFO:tasks.workunit.client.0.smithi070.stdout: "actual_memory_used": 36331520, 2024-09-14T03:47:51.423 INFO:tasks.workunit.client.0.smithi070.stdout: "central_cache_freelist": 980448, 2024-09-14T03:47:51.423 INFO:tasks.workunit.client.0.smithi070.stdout: "malloc_metadata": 2621440, 2024-09-14T03:47:51.423 INFO:tasks.workunit.client.0.smithi070.stdout: "page_heap_freelist": 0, 2024-09-14T03:47:51.423 INFO:tasks.workunit.client.0.smithi070.stdout: "released_to_os": 15368192, 2024-09-14T03:47:51.423 INFO:tasks.workunit.client.0.smithi070.stdout: "spans_in_use": 1142, 2024-09-14T03:47:51.423 INFO:tasks.workunit.client.0.smithi070.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:47:51.423 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_cache_freelists": 7203416, 2024-09-14T03:47:51.423 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_heaps_in_use": 21, 2024-09-14T03:47:51.424 INFO:tasks.workunit.client.0.smithi070.stdout: "transfer_cache_freelist": 1773568, 2024-09-14T03:47:51.424 INFO:tasks.workunit.client.0.smithi070.stdout: "use_by_application": 23752648, 2024-09-14T03:47:51.424 INFO:tasks.workunit.client.0.smithi070.stdout: "virtual_address_space_used": 51699712 2024-09-14T03:47:51.424 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.424 INFO:tasks.workunit.client.0.smithi070.stdout: "mon.bb10d63b3f8e26c4a0c5da991c12ff03df4a20b0": { 2024-09-14T03:47:51.424 INFO:tasks.workunit.client.0.smithi070.stdout: "actual_memory_used": 37257216, 2024-09-14T03:47:51.424 INFO:tasks.workunit.client.0.smithi070.stdout: "central_cache_freelist": 1255848, 2024-09-14T03:47:51.424 INFO:tasks.workunit.client.0.smithi070.stdout: "malloc_metadata": 2621440, 2024-09-14T03:47:51.424 INFO:tasks.workunit.client.0.smithi070.stdout: "page_heap_freelist": 548864, 2024-09-14T03:47:51.424 INFO:tasks.workunit.client.0.smithi070.stdout: "released_to_os": 13393920, 2024-09-14T03:47:51.424 INFO:tasks.workunit.client.0.smithi070.stdout: "spans_in_use": 1052, 2024-09-14T03:47:51.424 INFO:tasks.workunit.client.0.smithi070.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:47:51.425 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_cache_freelists": 6874528, 2024-09-14T03:47:51.425 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_heaps_in_use": 21, 2024-09-14T03:47:51.425 INFO:tasks.workunit.client.0.smithi070.stdout: "transfer_cache_freelist": 2264832, 2024-09-14T03:47:51.425 INFO:tasks.workunit.client.0.smithi070.stdout: "use_by_application": 23691704, 2024-09-14T03:47:51.425 INFO:tasks.workunit.client.0.smithi070.stdout: "virtual_address_space_used": 50651136 2024-09-14T03:47:51.425 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.425 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.425 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:51.425 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.0": { 2024-09-14T03:47:51.425 INFO:tasks.workunit.client.0.smithi070.stdout: "actual_memory_used": 273907712, 2024-09-14T03:47:51.425 INFO:tasks.workunit.client.0.smithi070.stdout: "central_cache_freelist": 736792, 2024-09-14T03:47:51.425 INFO:tasks.workunit.client.0.smithi070.stdout: "malloc_metadata": 2752512, 2024-09-14T03:47:51.426 INFO:tasks.workunit.client.0.smithi070.stdout: "page_heap_freelist": 0, 2024-09-14T03:47:51.426 INFO:tasks.workunit.client.0.smithi070.stdout: "released_to_os": 868352, 2024-09-14T03:47:51.426 INFO:tasks.workunit.client.0.smithi070.stdout: "spans_in_use": 1379, 2024-09-14T03:47:51.426 INFO:tasks.workunit.client.0.smithi070.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:47:51.426 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_cache_freelists": 5224352, 2024-09-14T03:47:51.426 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_heaps_in_use": 45, 2024-09-14T03:47:51.426 INFO:tasks.workunit.client.0.smithi070.stdout: "transfer_cache_freelist": 2694656, 2024-09-14T03:47:51.426 INFO:tasks.workunit.client.0.smithi070.stdout: "use_by_application": 262499400, 2024-09-14T03:47:51.426 INFO:tasks.workunit.client.0.smithi070.stdout: "virtual_address_space_used": 274776064 2024-09-14T03:47:51.426 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.426 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.1": { 2024-09-14T03:47:51.426 INFO:tasks.workunit.client.0.smithi070.stdout: "actual_memory_used": 273268736, 2024-09-14T03:47:51.427 INFO:tasks.workunit.client.0.smithi070.stdout: "central_cache_freelist": 725568, 2024-09-14T03:47:51.427 INFO:tasks.workunit.client.0.smithi070.stdout: "malloc_metadata": 2752512, 2024-09-14T03:47:51.427 INFO:tasks.workunit.client.0.smithi070.stdout: "page_heap_freelist": 0, 2024-09-14T03:47:51.427 INFO:tasks.workunit.client.0.smithi070.stdout: "released_to_os": 393216, 2024-09-14T03:47:51.427 INFO:tasks.workunit.client.0.smithi070.stdout: "spans_in_use": 1343, 2024-09-14T03:47:51.427 INFO:tasks.workunit.client.0.smithi070.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:47:51.427 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_cache_freelists": 5457448, 2024-09-14T03:47:51.427 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_heaps_in_use": 46, 2024-09-14T03:47:51.427 INFO:tasks.workunit.client.0.smithi070.stdout: "transfer_cache_freelist": 2205184, 2024-09-14T03:47:51.427 INFO:tasks.workunit.client.0.smithi070.stdout: "use_by_application": 262128024, 2024-09-14T03:47:51.427 INFO:tasks.workunit.client.0.smithi070.stdout: "virtual_address_space_used": 273661952 2024-09-14T03:47:51.427 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.427 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.2": { 2024-09-14T03:47:51.428 INFO:tasks.workunit.client.0.smithi070.stdout: "actual_memory_used": 272056320, 2024-09-14T03:47:51.428 INFO:tasks.workunit.client.0.smithi070.stdout: "central_cache_freelist": 764024, 2024-09-14T03:47:51.428 INFO:tasks.workunit.client.0.smithi070.stdout: "malloc_metadata": 2752512, 2024-09-14T03:47:51.428 INFO:tasks.workunit.client.0.smithi070.stdout: "page_heap_freelist": 32768, 2024-09-14T03:47:51.428 INFO:tasks.workunit.client.0.smithi070.stdout: "released_to_os": 557056, 2024-09-14T03:47:51.428 INFO:tasks.workunit.client.0.smithi070.stdout: "spans_in_use": 1273, 2024-09-14T03:47:51.428 INFO:tasks.workunit.client.0.smithi070.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:47:51.428 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_cache_freelists": 4956992, 2024-09-14T03:47:51.428 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_heaps_in_use": 47, 2024-09-14T03:47:51.428 INFO:tasks.workunit.client.0.smithi070.stdout: "transfer_cache_freelist": 2395392, 2024-09-14T03:47:51.428 INFO:tasks.workunit.client.0.smithi070.stdout: "use_by_application": 261154632, 2024-09-14T03:47:51.428 INFO:tasks.workunit.client.0.smithi070.stdout: "virtual_address_space_used": 272613376 2024-09-14T03:47:51.429 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.429 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.3": { 2024-09-14T03:47:51.429 INFO:tasks.workunit.client.0.smithi070.stdout: "actual_memory_used": 273817600, 2024-09-14T03:47:51.429 INFO:tasks.workunit.client.0.smithi070.stdout: "central_cache_freelist": 623584, 2024-09-14T03:47:51.429 INFO:tasks.workunit.client.0.smithi070.stdout: "malloc_metadata": 4849664, 2024-09-14T03:47:51.429 INFO:tasks.workunit.client.0.smithi070.stdout: "page_heap_freelist": 8192, 2024-09-14T03:47:51.429 INFO:tasks.workunit.client.0.smithi070.stdout: "released_to_os": 958464, 2024-09-14T03:47:51.429 INFO:tasks.workunit.client.0.smithi070.stdout: "spans_in_use": 1248, 2024-09-14T03:47:51.429 INFO:tasks.workunit.client.0.smithi070.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:47:51.430 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_cache_freelists": 4726040, 2024-09-14T03:47:51.430 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_heaps_in_use": 46, 2024-09-14T03:47:51.430 INFO:tasks.workunit.client.0.smithi070.stdout: "transfer_cache_freelist": 1794304, 2024-09-14T03:47:51.430 INFO:tasks.workunit.client.0.smithi070.stdout: "use_by_application": 261815816, 2024-09-14T03:47:51.430 INFO:tasks.workunit.client.0.smithi070.stdout: "virtual_address_space_used": 274776064 2024-09-14T03:47:51.430 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.430 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.4": { 2024-09-14T03:47:51.430 INFO:tasks.workunit.client.0.smithi070.stdout: "actual_memory_used": 272695296, 2024-09-14T03:47:51.430 INFO:tasks.workunit.client.0.smithi070.stdout: "central_cache_freelist": 842216, 2024-09-14T03:47:51.430 INFO:tasks.workunit.client.0.smithi070.stdout: "malloc_metadata": 2752512, 2024-09-14T03:47:51.430 INFO:tasks.workunit.client.0.smithi070.stdout: "page_heap_freelist": 0, 2024-09-14T03:47:51.431 INFO:tasks.workunit.client.0.smithi070.stdout: "released_to_os": 1032192, 2024-09-14T03:47:51.431 INFO:tasks.workunit.client.0.smithi070.stdout: "spans_in_use": 1261, 2024-09-14T03:47:51.431 INFO:tasks.workunit.client.0.smithi070.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:47:51.431 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_cache_freelists": 4617992, 2024-09-14T03:47:51.431 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_heaps_in_use": 47, 2024-09-14T03:47:51.431 INFO:tasks.workunit.client.0.smithi070.stdout: "transfer_cache_freelist": 2138880, 2024-09-14T03:47:51.431 INFO:tasks.workunit.client.0.smithi070.stdout: "use_by_application": 262343696, 2024-09-14T03:47:51.431 INFO:tasks.workunit.client.0.smithi070.stdout: "virtual_address_space_used": 273727488 2024-09-14T03:47:51.431 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.431 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.5": { 2024-09-14T03:47:51.431 INFO:tasks.workunit.client.0.smithi070.stdout: "actual_memory_used": 272441344, 2024-09-14T03:47:51.432 INFO:tasks.workunit.client.0.smithi070.stdout: "central_cache_freelist": 707560, 2024-09-14T03:47:51.432 INFO:tasks.workunit.client.0.smithi070.stdout: "malloc_metadata": 2752512, 2024-09-14T03:47:51.432 INFO:tasks.workunit.client.0.smithi070.stdout: "page_heap_freelist": 0, 2024-09-14T03:47:51.432 INFO:tasks.workunit.client.0.smithi070.stdout: "released_to_os": 237568, 2024-09-14T03:47:51.432 INFO:tasks.workunit.client.0.smithi070.stdout: "spans_in_use": 1257, 2024-09-14T03:47:51.432 INFO:tasks.workunit.client.0.smithi070.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:47:51.432 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_cache_freelists": 4594240, 2024-09-14T03:47:51.432 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_heaps_in_use": 46, 2024-09-14T03:47:51.432 INFO:tasks.workunit.client.0.smithi070.stdout: "transfer_cache_freelist": 2042112, 2024-09-14T03:47:51.432 INFO:tasks.workunit.client.0.smithi070.stdout: "use_by_application": 262344920, 2024-09-14T03:47:51.432 INFO:tasks.workunit.client.0.smithi070.stdout: "virtual_address_space_used": 272678912 2024-09-14T03:47:51.433 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.433 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.6": { 2024-09-14T03:47:51.433 INFO:tasks.workunit.client.0.smithi070.stdout: "actual_memory_used": 272293888, 2024-09-14T03:47:51.433 INFO:tasks.workunit.client.0.smithi070.stdout: "central_cache_freelist": 1140040, 2024-09-14T03:47:51.433 INFO:tasks.workunit.client.0.smithi070.stdout: "malloc_metadata": 2752512, 2024-09-14T03:47:51.433 INFO:tasks.workunit.client.0.smithi070.stdout: "page_heap_freelist": 0, 2024-09-14T03:47:51.433 INFO:tasks.workunit.client.0.smithi070.stdout: "released_to_os": 1433600, 2024-09-14T03:47:51.433 INFO:tasks.workunit.client.0.smithi070.stdout: "spans_in_use": 1268, 2024-09-14T03:47:51.433 INFO:tasks.workunit.client.0.smithi070.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:47:51.433 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_cache_freelists": 4629088, 2024-09-14T03:47:51.433 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_heaps_in_use": 45, 2024-09-14T03:47:51.433 INFO:tasks.workunit.client.0.smithi070.stdout: "transfer_cache_freelist": 2132224, 2024-09-14T03:47:51.434 INFO:tasks.workunit.client.0.smithi070.stdout: "use_by_application": 261640024, 2024-09-14T03:47:51.434 INFO:tasks.workunit.client.0.smithi070.stdout: "virtual_address_space_used": 273727488 2024-09-14T03:47:51.434 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.434 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.7": { 2024-09-14T03:47:51.434 INFO:tasks.workunit.client.0.smithi070.stdout: "actual_memory_used": 273113088, 2024-09-14T03:47:51.434 INFO:tasks.workunit.client.0.smithi070.stdout: "central_cache_freelist": 1285480, 2024-09-14T03:47:51.434 INFO:tasks.workunit.client.0.smithi070.stdout: "malloc_metadata": 2752512, 2024-09-14T03:47:51.434 INFO:tasks.workunit.client.0.smithi070.stdout: "page_heap_freelist": 0, 2024-09-14T03:47:51.434 INFO:tasks.workunit.client.0.smithi070.stdout: "released_to_os": 1597440, 2024-09-14T03:47:51.434 INFO:tasks.workunit.client.0.smithi070.stdout: "spans_in_use": 1339, 2024-09-14T03:47:51.434 INFO:tasks.workunit.client.0.smithi070.stdout: "tcmalloc_page_size": 8192, 2024-09-14T03:47:51.435 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_cache_freelists": 4837624, 2024-09-14T03:47:51.435 INFO:tasks.workunit.client.0.smithi070.stdout: "thread_heaps_in_use": 45, 2024-09-14T03:47:51.435 INFO:tasks.workunit.client.0.smithi070.stdout: "transfer_cache_freelist": 2105088, 2024-09-14T03:47:51.435 INFO:tasks.workunit.client.0.smithi070.stdout: "use_by_application": 262132384, 2024-09-14T03:47:51.435 INFO:tasks.workunit.client.0.smithi070.stdout: "virtual_address_space_used": 274710528 2024-09-14T03:47:51.435 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.435 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.435 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.435 INFO:tasks.workunit.client.0.smithi070.stdout: "hosts": { 2024-09-14T03:47:51.435 INFO:tasks.workunit.client.0.smithi070.stdout: "num": 2, 2024-09-14T03:47:51.436 INFO:tasks.workunit.client.0.smithi070.stdout: "num_with_mds": 2, 2024-09-14T03:47:51.436 INFO:tasks.workunit.client.0.smithi070.stdout: "num_with_mgr": 2, 2024-09-14T03:47:51.436 INFO:tasks.workunit.client.0.smithi070.stdout: "num_with_mon": 2, 2024-09-14T03:47:51.436 INFO:tasks.workunit.client.0.smithi070.stdout: "num_with_osd": 2 2024-09-14T03:47:51.436 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.436 INFO:tasks.workunit.client.0.smithi070.stdout: "io_rate": { 2024-09-14T03:47:51.436 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_stats_delta": { 2024-09-14T03:47:51.436 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": 0, 2024-09-14T03:47:51.436 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:51.436 INFO:tasks.workunit.client.0.smithi070.stdout: "num_store_stats": 0, 2024-09-14T03:47:51.436 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:51.437 INFO:tasks.workunit.client.0.smithi070.stdout: "stamp_delta": "12.002341", 2024-09-14T03:47:51.437 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:51.437 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:51.437 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:51.437 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:51.437 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:51.437 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:51.437 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:51.437 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:51.437 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:51.437 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:51.437 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:51.438 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:51.438 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:51.438 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:51.438 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:51.438 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:51.438 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:51.438 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:51.438 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:51.438 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:51.438 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:51.438 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:51.438 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:51.439 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:51.439 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:51.439 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:51.439 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:51.439 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:51.439 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:51.439 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:51.439 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:51.439 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:51.439 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:51.439 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:51.440 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 146, 2024-09-14T03:47:51.440 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 248, 2024-09-14T03:47:51.440 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:51.440 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:51.440 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:51.440 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 32, 2024-09-14T03:47:51.440 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:51.440 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.440 INFO:tasks.workunit.client.0.smithi070.stdout: "store_stats": { 2024-09-14T03:47:51.440 INFO:tasks.workunit.client.0.smithi070.stdout: "allocated": 0, 2024-09-14T03:47:51.440 INFO:tasks.workunit.client.0.smithi070.stdout: "available": 0, 2024-09-14T03:47:51.440 INFO:tasks.workunit.client.0.smithi070.stdout: "data_compressed": 0, 2024-09-14T03:47:51.441 INFO:tasks.workunit.client.0.smithi070.stdout: "data_compressed_allocated": 0, 2024-09-14T03:47:51.441 INFO:tasks.workunit.client.0.smithi070.stdout: "data_compressed_original": 0, 2024-09-14T03:47:51.441 INFO:tasks.workunit.client.0.smithi070.stdout: "data_stored": 0, 2024-09-14T03:47:51.441 INFO:tasks.workunit.client.0.smithi070.stdout: "internal_metadata": 0, 2024-09-14T03:47:51.441 INFO:tasks.workunit.client.0.smithi070.stdout: "internally_reserved": 0, 2024-09-14T03:47:51.441 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_allocated": 0, 2024-09-14T03:47:51.441 INFO:tasks.workunit.client.0.smithi070.stdout: "total": 0 2024-09-14T03:47:51.441 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.441 INFO:tasks.workunit.client.0.smithi070.stdout: "up": 0 2024-09-14T03:47:51.441 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.441 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.442 INFO:tasks.workunit.client.0.smithi070.stdout: "leaderboard": false, 2024-09-14T03:47:51.442 INFO:tasks.workunit.client.0.smithi070.stdout: "license": "sharing-1-0", 2024-09-14T03:47:51.442 INFO:tasks.workunit.client.0.smithi070.stdout: "mempool": { 2024-09-14T03:47:51.442 INFO:tasks.workunit.client.0.smithi070.stdout: "mds": { 2024-09-14T03:47:51.442 INFO:tasks.workunit.client.0.smithi070.stdout: "mds.a8e17c6c2bb7d35df7944161574528c366e6a388": { 2024-09-14T03:47:51.442 INFO:tasks.workunit.client.0.smithi070.stdout: "bloom_filter": { 2024-09-14T03:47:51.442 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.442 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.442 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.442 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:51.442 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.442 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.442 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.443 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_reader": { 2024-09-14T03:47:51.443 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.443 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.443 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.443 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_writer": { 2024-09-14T03:47:51.443 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.443 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.443 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.443 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Blob": { 2024-09-14T03:47:51.443 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.444 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.444 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.444 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Buffer": { 2024-09-14T03:47:51.444 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.444 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.444 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.444 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Extent": { 2024-09-14T03:47:51.444 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.444 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.444 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.444 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_SharedBlob": { 2024-09-14T03:47:51.444 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.445 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.445 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.445 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_alloc": { 2024-09-14T03:47:51.445 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.445 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.445 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.445 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_data": { 2024-09-14T03:47:51.445 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.445 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.445 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.445 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_meta": { 2024-09-14T03:47:51.445 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.446 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.446 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.446 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_onode": { 2024-09-14T03:47:51.446 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.446 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.446 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.446 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_other": { 2024-09-14T03:47:51.446 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.446 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.446 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.446 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_fsck": { 2024-09-14T03:47:51.447 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.447 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.447 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.447 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_inline_bl": { 2024-09-14T03:47:51.447 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.447 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.447 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.447 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_txc": { 2024-09-14T03:47:51.447 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.447 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.448 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.448 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing": { 2024-09-14T03:47:51.448 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.448 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.448 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.448 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing_deferred": { 2024-09-14T03:47:51.448 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.448 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.448 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.448 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_anon": { 2024-09-14T03:47:51.448 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 222471, 2024-09-14T03:47:51.448 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 72 2024-09-14T03:47:51.449 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.449 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_meta": { 2024-09-14T03:47:51.449 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.449 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.449 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.449 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_co": { 2024-09-14T03:47:51.449 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 50488, 2024-09-14T03:47:51.449 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 208 2024-09-14T03:47:51.449 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.449 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:51.449 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.449 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.450 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.450 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mapbl": { 2024-09-14T03:47:51.450 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.450 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.450 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.450 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pglog": { 2024-09-14T03:47:51.450 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.450 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.450 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.450 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap": { 2024-09-14T03:47:51.450 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 8048, 2024-09-14T03:47:51.451 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 138 2024-09-14T03:47:51.451 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.451 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap_mapping": { 2024-09-14T03:47:51.451 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.451 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.451 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.451 INFO:tasks.workunit.client.0.smithi070.stdout: "pgmap": { 2024-09-14T03:47:51.451 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.451 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.451 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.451 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_1": { 2024-09-14T03:47:51.451 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.451 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.452 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.452 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_2": { 2024-09-14T03:47:51.452 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.452 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.452 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.452 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.452 INFO:tasks.workunit.client.0.smithi070.stdout: "mds.b1d2edfb86a8f225d9283baf06ff7497eda48d2d": { 2024-09-14T03:47:51.452 INFO:tasks.workunit.client.0.smithi070.stdout: "bloom_filter": { 2024-09-14T03:47:51.452 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.452 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.452 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.452 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:51.453 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.453 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.453 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.453 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_reader": { 2024-09-14T03:47:51.453 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.453 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.453 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.453 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_writer": { 2024-09-14T03:47:51.453 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.453 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.453 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.454 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Blob": { 2024-09-14T03:47:51.454 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.454 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.454 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.454 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Buffer": { 2024-09-14T03:47:51.454 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.454 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.454 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.454 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Extent": { 2024-09-14T03:47:51.454 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.454 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.454 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.455 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_SharedBlob": { 2024-09-14T03:47:51.455 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.455 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.455 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.455 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_alloc": { 2024-09-14T03:47:51.455 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.455 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.455 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.455 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_data": { 2024-09-14T03:47:51.455 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.455 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.455 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.456 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_meta": { 2024-09-14T03:47:51.456 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.456 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.456 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.456 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_onode": { 2024-09-14T03:47:51.456 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.456 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.456 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.456 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_other": { 2024-09-14T03:47:51.456 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.456 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.457 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.457 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_fsck": { 2024-09-14T03:47:51.457 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.457 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.457 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.457 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_inline_bl": { 2024-09-14T03:47:51.457 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.457 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.457 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.457 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_txc": { 2024-09-14T03:47:51.457 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.457 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.458 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.458 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing": { 2024-09-14T03:47:51.458 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.458 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.458 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.458 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing_deferred": { 2024-09-14T03:47:51.458 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.458 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.458 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.458 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_anon": { 2024-09-14T03:47:51.458 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 149057, 2024-09-14T03:47:51.458 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 32 2024-09-14T03:47:51.459 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.459 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_meta": { 2024-09-14T03:47:51.459 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.459 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.459 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.459 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_co": { 2024-09-14T03:47:51.459 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 856, 2024-09-14T03:47:51.459 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 2 2024-09-14T03:47:51.459 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.459 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:51.459 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.459 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.460 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.460 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mapbl": { 2024-09-14T03:47:51.460 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.460 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.460 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.460 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pglog": { 2024-09-14T03:47:51.460 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.460 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.460 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.460 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap": { 2024-09-14T03:47:51.460 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.461 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.461 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.461 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap_mapping": { 2024-09-14T03:47:51.461 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.461 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.461 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.461 INFO:tasks.workunit.client.0.smithi070.stdout: "pgmap": { 2024-09-14T03:47:51.461 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.461 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.461 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.461 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_1": { 2024-09-14T03:47:51.461 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.462 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.462 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.462 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_2": { 2024-09-14T03:47:51.462 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.462 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.462 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.462 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.462 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.462 INFO:tasks.workunit.client.0.smithi070.stdout: "mon": { 2024-09-14T03:47:51.462 INFO:tasks.workunit.client.0.smithi070.stdout: "mon.a8e17c6c2bb7d35df7944161574528c366e6a388": { 2024-09-14T03:47:51.462 INFO:tasks.workunit.client.0.smithi070.stdout: "bloom_filter": { 2024-09-14T03:47:51.462 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.463 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.463 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.463 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:51.463 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.463 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.463 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.463 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_reader": { 2024-09-14T03:47:51.463 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.463 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.463 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.463 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_writer": { 2024-09-14T03:47:51.463 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.464 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.464 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.464 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Blob": { 2024-09-14T03:47:51.464 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.464 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.464 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.464 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Buffer": { 2024-09-14T03:47:51.464 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.464 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.464 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.464 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Extent": { 2024-09-14T03:47:51.464 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.465 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.465 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.465 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_SharedBlob": { 2024-09-14T03:47:51.465 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.465 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.465 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.465 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_alloc": { 2024-09-14T03:47:51.465 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.465 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.465 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.465 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_data": { 2024-09-14T03:47:51.465 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.466 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.466 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.466 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_meta": { 2024-09-14T03:47:51.466 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.466 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.466 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.466 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_onode": { 2024-09-14T03:47:51.466 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.466 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.466 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.466 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_other": { 2024-09-14T03:47:51.466 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.467 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.467 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.467 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_fsck": { 2024-09-14T03:47:51.467 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.467 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.467 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.467 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_inline_bl": { 2024-09-14T03:47:51.467 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.467 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.467 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.467 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_txc": { 2024-09-14T03:47:51.467 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.468 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.468 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.468 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing": { 2024-09-14T03:47:51.468 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.468 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.468 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.468 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing_deferred": { 2024-09-14T03:47:51.468 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.468 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.468 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.469 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_anon": { 2024-09-14T03:47:51.469 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 1129782, 2024-09-14T03:47:51.469 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 335 2024-09-14T03:47:51.469 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.469 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_meta": { 2024-09-14T03:47:51.469 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 440, 2024-09-14T03:47:51.469 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 5 2024-09-14T03:47:51.469 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.469 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_co": { 2024-09-14T03:47:51.469 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.469 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.469 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.470 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:51.470 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.470 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.470 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.470 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mapbl": { 2024-09-14T03:47:51.470 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.470 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.470 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.470 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pglog": { 2024-09-14T03:47:51.470 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.470 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.471 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.471 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap": { 2024-09-14T03:47:51.471 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 6696, 2024-09-14T03:47:51.471 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 137 2024-09-14T03:47:51.471 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.471 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap_mapping": { 2024-09-14T03:47:51.471 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 12744, 2024-09-14T03:47:51.471 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 1546 2024-09-14T03:47:51.471 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.471 INFO:tasks.workunit.client.0.smithi070.stdout: "pgmap": { 2024-09-14T03:47:51.471 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 10624, 2024-09-14T03:47:51.471 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 206 2024-09-14T03:47:51.472 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.472 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_1": { 2024-09-14T03:47:51.472 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.472 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.472 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.472 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_2": { 2024-09-14T03:47:51.472 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.472 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.472 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.472 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.472 INFO:tasks.workunit.client.0.smithi070.stdout: "mon.b1d2edfb86a8f225d9283baf06ff7497eda48d2d": { 2024-09-14T03:47:51.472 INFO:tasks.workunit.client.0.smithi070.stdout: "bloom_filter": { 2024-09-14T03:47:51.473 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.473 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.473 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.473 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:51.473 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.473 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.473 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.473 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_reader": { 2024-09-14T03:47:51.473 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.473 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.473 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.474 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_writer": { 2024-09-14T03:47:51.474 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.474 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.474 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.474 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Blob": { 2024-09-14T03:47:51.474 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.474 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.474 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.474 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Buffer": { 2024-09-14T03:47:51.474 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.474 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.474 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.474 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Extent": { 2024-09-14T03:47:51.475 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.475 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.475 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.475 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_SharedBlob": { 2024-09-14T03:47:51.475 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.475 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.475 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.475 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_alloc": { 2024-09-14T03:47:51.475 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.475 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.475 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.475 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_data": { 2024-09-14T03:47:51.475 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.476 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.476 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.476 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_meta": { 2024-09-14T03:47:51.476 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.476 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.476 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.476 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_onode": { 2024-09-14T03:47:51.476 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.476 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.476 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.476 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_other": { 2024-09-14T03:47:51.477 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.477 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.477 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.477 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_fsck": { 2024-09-14T03:47:51.477 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.477 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.477 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.477 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_inline_bl": { 2024-09-14T03:47:51.477 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.477 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.477 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.477 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_txc": { 2024-09-14T03:47:51.478 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.478 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.478 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.478 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing": { 2024-09-14T03:47:51.478 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.478 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.478 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.478 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing_deferred": { 2024-09-14T03:47:51.478 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.478 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.478 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.478 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_anon": { 2024-09-14T03:47:51.479 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 1095766, 2024-09-14T03:47:51.479 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 345 2024-09-14T03:47:51.479 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.479 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_meta": { 2024-09-14T03:47:51.479 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.479 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.479 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.479 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_co": { 2024-09-14T03:47:51.479 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.479 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.479 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.480 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:51.480 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.480 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.480 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.480 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mapbl": { 2024-09-14T03:47:51.480 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.480 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.480 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.480 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pglog": { 2024-09-14T03:47:51.480 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.480 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.480 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.481 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap": { 2024-09-14T03:47:51.481 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 6696, 2024-09-14T03:47:51.481 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 137 2024-09-14T03:47:51.481 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.481 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap_mapping": { 2024-09-14T03:47:51.481 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 12744, 2024-09-14T03:47:51.481 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 1546 2024-09-14T03:47:51.481 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.481 INFO:tasks.workunit.client.0.smithi070.stdout: "pgmap": { 2024-09-14T03:47:51.481 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 5312, 2024-09-14T03:47:51.481 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 103 2024-09-14T03:47:51.481 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.481 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_1": { 2024-09-14T03:47:51.482 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.482 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.482 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.482 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_2": { 2024-09-14T03:47:51.482 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.482 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.482 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.482 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.482 INFO:tasks.workunit.client.0.smithi070.stdout: "mon.bb10d63b3f8e26c4a0c5da991c12ff03df4a20b0": { 2024-09-14T03:47:51.482 INFO:tasks.workunit.client.0.smithi070.stdout: "bloom_filter": { 2024-09-14T03:47:51.482 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.482 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.483 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.483 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:51.483 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.483 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.483 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.483 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_reader": { 2024-09-14T03:47:51.483 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.483 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.483 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.483 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_writer": { 2024-09-14T03:47:51.483 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.483 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.483 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.484 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Blob": { 2024-09-14T03:47:51.484 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.484 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.484 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.484 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Buffer": { 2024-09-14T03:47:51.484 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.484 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.484 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.484 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Extent": { 2024-09-14T03:47:51.484 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.484 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.485 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.485 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_SharedBlob": { 2024-09-14T03:47:51.485 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.485 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.485 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.485 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_alloc": { 2024-09-14T03:47:51.485 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.485 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.485 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.485 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_data": { 2024-09-14T03:47:51.485 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.485 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.486 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.486 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_meta": { 2024-09-14T03:47:51.486 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.486 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.486 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.486 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_onode": { 2024-09-14T03:47:51.486 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.486 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.486 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.486 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_other": { 2024-09-14T03:47:51.486 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.486 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.486 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.486 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_fsck": { 2024-09-14T03:47:51.487 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.487 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.487 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.487 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_inline_bl": { 2024-09-14T03:47:51.487 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.487 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.487 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.487 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_txc": { 2024-09-14T03:47:51.487 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.487 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.487 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.487 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing": { 2024-09-14T03:47:51.488 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.488 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.488 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.488 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing_deferred": { 2024-09-14T03:47:51.488 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.488 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.488 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.488 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_anon": { 2024-09-14T03:47:51.488 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 1099878, 2024-09-14T03:47:51.488 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 353 2024-09-14T03:47:51.488 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.489 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_meta": { 2024-09-14T03:47:51.489 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.489 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.489 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.489 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_co": { 2024-09-14T03:47:51.489 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.489 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.489 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.489 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:51.489 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.489 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.489 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.490 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mapbl": { 2024-09-14T03:47:51.490 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.490 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.490 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.490 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pglog": { 2024-09-14T03:47:51.490 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.490 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.490 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.490 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap": { 2024-09-14T03:47:51.490 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 6696, 2024-09-14T03:47:51.490 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 137 2024-09-14T03:47:51.490 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.490 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap_mapping": { 2024-09-14T03:47:51.491 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 12744, 2024-09-14T03:47:51.491 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 1546 2024-09-14T03:47:51.491 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.491 INFO:tasks.workunit.client.0.smithi070.stdout: "pgmap": { 2024-09-14T03:47:51.491 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 5312, 2024-09-14T03:47:51.491 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 103 2024-09-14T03:47:51.491 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.491 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_1": { 2024-09-14T03:47:51.491 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.491 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.491 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.491 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_2": { 2024-09-14T03:47:51.491 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.492 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.492 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.492 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.492 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.492 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:51.492 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.0": { 2024-09-14T03:47:51.492 INFO:tasks.workunit.client.0.smithi070.stdout: "bloom_filter": { 2024-09-14T03:47:51.492 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.492 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.492 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.492 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:51.492 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 6304, 2024-09-14T03:47:51.493 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 115 2024-09-14T03:47:51.493 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.493 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_reader": { 2024-09-14T03:47:51.493 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 16896, 2024-09-14T03:47:51.493 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 8 2024-09-14T03:47:51.493 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.493 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_writer": { 2024-09-14T03:47:51.493 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 576, 2024-09-14T03:47:51.493 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3 2024-09-14T03:47:51.493 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.493 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Blob": { 2024-09-14T03:47:51.493 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 20496, 2024-09-14T03:47:51.494 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 183 2024-09-14T03:47:51.494 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.494 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Buffer": { 2024-09-14T03:47:51.494 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 224, 2024-09-14T03:47:51.494 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 2 2024-09-14T03:47:51.494 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.494 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Extent": { 2024-09-14T03:47:51.494 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 8784, 2024-09-14T03:47:51.494 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 183 2024-09-14T03:47:51.494 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.494 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_SharedBlob": { 2024-09-14T03:47:51.494 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 20496, 2024-09-14T03:47:51.494 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 183 2024-09-14T03:47:51.495 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.495 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_alloc": { 2024-09-14T03:47:51.495 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 5280, 2024-09-14T03:47:51.495 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 66 2024-09-14T03:47:51.495 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.495 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_data": { 2024-09-14T03:47:51.495 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 8192, 2024-09-14T03:47:51.495 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 2 2024-09-14T03:47:51.495 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.495 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_meta": { 2024-09-14T03:47:51.495 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 54475, 2024-09-14T03:47:51.495 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 9882 2024-09-14T03:47:51.495 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.496 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_onode": { 2024-09-14T03:47:51.496 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 135320, 2024-09-14T03:47:51.496 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 199 2024-09-14T03:47:51.496 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.496 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_other": { 2024-09-14T03:47:51.496 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 11104, 2024-09-14T03:47:51.496 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 711 2024-09-14T03:47:51.496 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.496 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_fsck": { 2024-09-14T03:47:51.496 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.496 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.496 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.497 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_inline_bl": { 2024-09-14T03:47:51.497 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 31588, 2024-09-14T03:47:51.497 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 199 2024-09-14T03:47:51.497 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.497 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_txc": { 2024-09-14T03:47:51.497 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.497 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.497 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.497 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing": { 2024-09-14T03:47:51.497 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 399331, 2024-09-14T03:47:51.497 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 87 2024-09-14T03:47:51.497 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.497 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing_deferred": { 2024-09-14T03:47:51.498 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.498 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.498 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.498 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_anon": { 2024-09-14T03:47:51.498 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 2433372, 2024-09-14T03:47:51.498 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 193 2024-09-14T03:47:51.498 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.498 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_meta": { 2024-09-14T03:47:51.498 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 5632, 2024-09-14T03:47:51.498 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 64 2024-09-14T03:47:51.498 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.498 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_co": { 2024-09-14T03:47:51.498 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.499 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.499 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.499 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:51.499 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 521280, 2024-09-14T03:47:51.499 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 45 2024-09-14T03:47:51.499 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.499 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mapbl": { 2024-09-14T03:47:51.499 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.499 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.499 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.499 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pglog": { 2024-09-14T03:47:51.500 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 76544, 2024-09-14T03:47:51.500 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 152 2024-09-14T03:47:51.500 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.500 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap": { 2024-09-14T03:47:51.500 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 235720, 2024-09-14T03:47:51.500 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3529 2024-09-14T03:47:51.500 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.500 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap_mapping": { 2024-09-14T03:47:51.500 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.500 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.500 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.500 INFO:tasks.workunit.client.0.smithi070.stdout: "pgmap": { 2024-09-14T03:47:51.501 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.501 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.501 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.501 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_1": { 2024-09-14T03:47:51.501 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.501 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.501 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.501 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_2": { 2024-09-14T03:47:51.501 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.501 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.501 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.501 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.502 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.1": { 2024-09-14T03:47:51.502 INFO:tasks.workunit.client.0.smithi070.stdout: "bloom_filter": { 2024-09-14T03:47:51.502 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.502 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.502 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.502 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:51.502 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 6304, 2024-09-14T03:47:51.502 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 115 2024-09-14T03:47:51.502 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.502 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_reader": { 2024-09-14T03:47:51.502 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 16896, 2024-09-14T03:47:51.502 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 8 2024-09-14T03:47:51.502 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.503 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_writer": { 2024-09-14T03:47:51.503 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 576, 2024-09-14T03:47:51.503 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3 2024-09-14T03:47:51.503 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.503 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Blob": { 2024-09-14T03:47:51.503 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 19040, 2024-09-14T03:47:51.503 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 170 2024-09-14T03:47:51.503 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.503 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Buffer": { 2024-09-14T03:47:51.503 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 896, 2024-09-14T03:47:51.503 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 8 2024-09-14T03:47:51.503 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.504 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Extent": { 2024-09-14T03:47:51.504 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 8160, 2024-09-14T03:47:51.504 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 170 2024-09-14T03:47:51.504 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.504 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_SharedBlob": { 2024-09-14T03:47:51.504 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 19040, 2024-09-14T03:47:51.504 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 170 2024-09-14T03:47:51.504 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.504 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_alloc": { 2024-09-14T03:47:51.504 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 4560, 2024-09-14T03:47:51.504 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 57 2024-09-14T03:47:51.504 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.505 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_data": { 2024-09-14T03:47:51.505 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 12288, 2024-09-14T03:47:51.505 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3 2024-09-14T03:47:51.505 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.505 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_meta": { 2024-09-14T03:47:51.505 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 48337, 2024-09-14T03:47:51.505 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 8825 2024-09-14T03:47:51.505 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.505 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_onode": { 2024-09-14T03:47:51.505 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 125120, 2024-09-14T03:47:51.505 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 184 2024-09-14T03:47:51.505 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.505 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_other": { 2024-09-14T03:47:51.506 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 9152, 2024-09-14T03:47:51.506 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 535 2024-09-14T03:47:51.506 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.506 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_fsck": { 2024-09-14T03:47:51.506 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.506 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.506 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.506 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_inline_bl": { 2024-09-14T03:47:51.506 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 27312, 2024-09-14T03:47:51.506 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 184 2024-09-14T03:47:51.506 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.506 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_txc": { 2024-09-14T03:47:51.507 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 736, 2024-09-14T03:47:51.507 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 1 2024-09-14T03:47:51.507 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.507 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing": { 2024-09-14T03:47:51.507 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 427120, 2024-09-14T03:47:51.507 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 94 2024-09-14T03:47:51.507 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.507 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing_deferred": { 2024-09-14T03:47:51.507 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 12096, 2024-09-14T03:47:51.507 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3 2024-09-14T03:47:51.507 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.507 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_anon": { 2024-09-14T03:47:51.508 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 2411092, 2024-09-14T03:47:51.508 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 181 2024-09-14T03:47:51.508 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.508 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_meta": { 2024-09-14T03:47:51.508 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 4928, 2024-09-14T03:47:51.508 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 56 2024-09-14T03:47:51.509 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.509 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_co": { 2024-09-14T03:47:51.509 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.509 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.509 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.509 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:51.509 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 440192, 2024-09-14T03:47:51.509 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 38 2024-09-14T03:47:51.509 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.509 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mapbl": { 2024-09-14T03:47:51.509 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.509 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.510 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.510 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pglog": { 2024-09-14T03:47:51.510 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 8528, 2024-09-14T03:47:51.510 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 22 2024-09-14T03:47:51.510 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.510 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap": { 2024-09-14T03:47:51.510 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 235208, 2024-09-14T03:47:51.510 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3505 2024-09-14T03:47:51.510 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.510 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap_mapping": { 2024-09-14T03:47:51.510 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.511 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.511 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.511 INFO:tasks.workunit.client.0.smithi070.stdout: "pgmap": { 2024-09-14T03:47:51.511 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.511 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.511 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.511 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_1": { 2024-09-14T03:47:51.511 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.511 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.511 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.511 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_2": { 2024-09-14T03:47:51.511 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.512 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.512 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.512 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.512 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.2": { 2024-09-14T03:47:51.512 INFO:tasks.workunit.client.0.smithi070.stdout: "bloom_filter": { 2024-09-14T03:47:51.512 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.512 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.512 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.512 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:51.512 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 6304, 2024-09-14T03:47:51.512 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 115 2024-09-14T03:47:51.513 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.513 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_reader": { 2024-09-14T03:47:51.513 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 16896, 2024-09-14T03:47:51.513 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 8 2024-09-14T03:47:51.513 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.513 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_writer": { 2024-09-14T03:47:51.513 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 576, 2024-09-14T03:47:51.513 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3 2024-09-14T03:47:51.513 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.513 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Blob": { 2024-09-14T03:47:51.513 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 18928, 2024-09-14T03:47:51.514 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 169 2024-09-14T03:47:51.514 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.514 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Buffer": { 2024-09-14T03:47:51.514 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 784, 2024-09-14T03:47:51.514 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 7 2024-09-14T03:47:51.514 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.514 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Extent": { 2024-09-14T03:47:51.514 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 8112, 2024-09-14T03:47:51.514 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 169 2024-09-14T03:47:51.514 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.514 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_SharedBlob": { 2024-09-14T03:47:51.514 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 18928, 2024-09-14T03:47:51.515 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 169 2024-09-14T03:47:51.515 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.515 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_alloc": { 2024-09-14T03:47:51.515 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 4160, 2024-09-14T03:47:51.515 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 52 2024-09-14T03:47:51.515 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.515 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_data": { 2024-09-14T03:47:51.515 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 8192, 2024-09-14T03:47:51.515 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 2 2024-09-14T03:47:51.515 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.515 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_meta": { 2024-09-14T03:47:51.516 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 45727, 2024-09-14T03:47:51.516 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 8489 2024-09-14T03:47:51.516 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.516 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_onode": { 2024-09-14T03:47:51.516 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 120360, 2024-09-14T03:47:51.516 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 177 2024-09-14T03:47:51.516 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.516 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_other": { 2024-09-14T03:47:51.516 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 8348, 2024-09-14T03:47:51.516 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 526 2024-09-14T03:47:51.516 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.516 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_fsck": { 2024-09-14T03:47:51.517 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.517 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.517 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.517 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_inline_bl": { 2024-09-14T03:47:51.517 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 25688, 2024-09-14T03:47:51.517 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 177 2024-09-14T03:47:51.517 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.517 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_txc": { 2024-09-14T03:47:51.517 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 736, 2024-09-14T03:47:51.517 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 1 2024-09-14T03:47:51.517 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.518 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing": { 2024-09-14T03:47:51.518 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 329388, 2024-09-14T03:47:51.518 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 70 2024-09-14T03:47:51.518 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.518 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing_deferred": { 2024-09-14T03:47:51.518 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 12096, 2024-09-14T03:47:51.518 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3 2024-09-14T03:47:51.518 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.518 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_anon": { 2024-09-14T03:47:51.518 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 2422832, 2024-09-14T03:47:51.518 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 191 2024-09-14T03:47:51.518 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.519 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_meta": { 2024-09-14T03:47:51.519 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 4312, 2024-09-14T03:47:51.519 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 49 2024-09-14T03:47:51.519 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.519 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_co": { 2024-09-14T03:47:51.519 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.519 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.519 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.519 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:51.519 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 359104, 2024-09-14T03:47:51.519 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 31 2024-09-14T03:47:51.519 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.520 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mapbl": { 2024-09-14T03:47:51.520 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.520 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.520 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.520 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pglog": { 2024-09-14T03:47:51.520 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 8320, 2024-09-14T03:47:51.520 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 24 2024-09-14T03:47:51.520 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.520 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap": { 2024-09-14T03:47:51.520 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 235720, 2024-09-14T03:47:51.520 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3529 2024-09-14T03:47:51.520 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.521 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap_mapping": { 2024-09-14T03:47:51.521 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.521 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.521 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.521 INFO:tasks.workunit.client.0.smithi070.stdout: "pgmap": { 2024-09-14T03:47:51.521 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.521 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.521 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.521 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_1": { 2024-09-14T03:47:51.521 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.521 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.521 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.522 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_2": { 2024-09-14T03:47:51.522 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.522 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.522 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.522 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.522 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.3": { 2024-09-14T03:47:51.522 INFO:tasks.workunit.client.0.smithi070.stdout: "bloom_filter": { 2024-09-14T03:47:51.522 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.522 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.522 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.522 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:51.523 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 6304, 2024-09-14T03:47:51.523 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 115 2024-09-14T03:47:51.523 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.523 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_reader": { 2024-09-14T03:47:51.523 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 16896, 2024-09-14T03:47:51.523 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 8 2024-09-14T03:47:51.523 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.523 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_writer": { 2024-09-14T03:47:51.523 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 576, 2024-09-14T03:47:51.523 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3 2024-09-14T03:47:51.523 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.523 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Blob": { 2024-09-14T03:47:51.524 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 19488, 2024-09-14T03:47:51.524 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 174 2024-09-14T03:47:51.524 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.524 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Buffer": { 2024-09-14T03:47:51.524 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 224, 2024-09-14T03:47:51.524 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 2 2024-09-14T03:47:51.524 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.524 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Extent": { 2024-09-14T03:47:51.524 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 8352, 2024-09-14T03:47:51.524 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 174 2024-09-14T03:47:51.524 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.524 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_SharedBlob": { 2024-09-14T03:47:51.525 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 19488, 2024-09-14T03:47:51.525 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 174 2024-09-14T03:47:51.525 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.525 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_alloc": { 2024-09-14T03:47:51.525 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 3760, 2024-09-14T03:47:51.525 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 47 2024-09-14T03:47:51.525 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.525 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_data": { 2024-09-14T03:47:51.525 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.525 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.525 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.525 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_meta": { 2024-09-14T03:47:51.526 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 47391, 2024-09-14T03:47:51.526 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 8817 2024-09-14T03:47:51.526 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.526 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_onode": { 2024-09-14T03:47:51.526 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 125120, 2024-09-14T03:47:51.526 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 184 2024-09-14T03:47:51.526 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.526 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_other": { 2024-09-14T03:47:51.526 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 9152, 2024-09-14T03:47:51.526 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 544 2024-09-14T03:47:51.526 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.526 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_fsck": { 2024-09-14T03:47:51.527 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.527 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.527 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.527 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_inline_bl": { 2024-09-14T03:47:51.527 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 26640, 2024-09-14T03:47:51.527 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 184 2024-09-14T03:47:51.527 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.527 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_txc": { 2024-09-14T03:47:51.527 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 2208, 2024-09-14T03:47:51.527 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3 2024-09-14T03:47:51.527 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.527 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing": { 2024-09-14T03:47:51.528 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 276833, 2024-09-14T03:47:51.528 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 57 2024-09-14T03:47:51.528 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.528 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing_deferred": { 2024-09-14T03:47:51.528 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 30637, 2024-09-14T03:47:51.528 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 9 2024-09-14T03:47:51.528 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.528 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_anon": { 2024-09-14T03:47:51.528 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 2413964, 2024-09-14T03:47:51.528 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 185 2024-09-14T03:47:51.528 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.528 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_meta": { 2024-09-14T03:47:51.529 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 4752, 2024-09-14T03:47:51.529 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 54 2024-09-14T03:47:51.529 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.529 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_co": { 2024-09-14T03:47:51.529 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.529 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.529 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.529 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:51.529 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 428608, 2024-09-14T03:47:51.529 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 37 2024-09-14T03:47:51.530 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.530 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mapbl": { 2024-09-14T03:47:51.530 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.530 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.530 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.530 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pglog": { 2024-09-14T03:47:51.530 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 10400, 2024-09-14T03:47:51.530 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 20 2024-09-14T03:47:51.530 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.530 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap": { 2024-09-14T03:47:51.530 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 235720, 2024-09-14T03:47:51.530 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3529 2024-09-14T03:47:51.531 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.531 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap_mapping": { 2024-09-14T03:47:51.531 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.531 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.531 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.531 INFO:tasks.workunit.client.0.smithi070.stdout: "pgmap": { 2024-09-14T03:47:51.531 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.531 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.531 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.531 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_1": { 2024-09-14T03:47:51.531 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.532 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.532 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.532 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_2": { 2024-09-14T03:47:51.532 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.532 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.532 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.532 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.532 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.4": { 2024-09-14T03:47:51.532 INFO:tasks.workunit.client.0.smithi070.stdout: "bloom_filter": { 2024-09-14T03:47:51.532 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.532 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.532 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.533 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:51.533 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 6304, 2024-09-14T03:47:51.533 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 115 2024-09-14T03:47:51.533 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.533 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_reader": { 2024-09-14T03:47:51.533 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 16896, 2024-09-14T03:47:51.533 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 8 2024-09-14T03:47:51.533 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.533 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_writer": { 2024-09-14T03:47:51.533 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 576, 2024-09-14T03:47:51.533 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3 2024-09-14T03:47:51.534 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.534 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Blob": { 2024-09-14T03:47:51.534 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 19600, 2024-09-14T03:47:51.534 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 175 2024-09-14T03:47:51.534 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.534 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Buffer": { 2024-09-14T03:47:51.534 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 560, 2024-09-14T03:47:51.534 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 5 2024-09-14T03:47:51.534 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.534 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Extent": { 2024-09-14T03:47:51.534 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 8400, 2024-09-14T03:47:51.534 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 175 2024-09-14T03:47:51.535 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.535 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_SharedBlob": { 2024-09-14T03:47:51.535 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 19600, 2024-09-14T03:47:51.535 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 175 2024-09-14T03:47:51.535 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.535 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_alloc": { 2024-09-14T03:47:51.535 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 3280, 2024-09-14T03:47:51.535 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 41 2024-09-14T03:47:51.535 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.535 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_data": { 2024-09-14T03:47:51.535 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.535 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.536 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.536 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_meta": { 2024-09-14T03:47:51.536 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 48760, 2024-09-14T03:47:51.536 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 9014 2024-09-14T03:47:51.536 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.536 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_onode": { 2024-09-14T03:47:51.536 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 127840, 2024-09-14T03:47:51.536 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 188 2024-09-14T03:47:51.536 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.536 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_other": { 2024-09-14T03:47:51.536 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 9364, 2024-09-14T03:47:51.537 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 546 2024-09-14T03:47:51.537 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.537 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_fsck": { 2024-09-14T03:47:51.537 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.537 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.537 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.537 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_inline_bl": { 2024-09-14T03:47:51.537 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 27496, 2024-09-14T03:47:51.537 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 188 2024-09-14T03:47:51.537 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.537 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_txc": { 2024-09-14T03:47:51.537 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 736, 2024-09-14T03:47:51.538 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 1 2024-09-14T03:47:51.538 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.538 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing": { 2024-09-14T03:47:51.538 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 901365, 2024-09-14T03:47:51.538 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 67 2024-09-14T03:47:51.538 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.538 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing_deferred": { 2024-09-14T03:47:51.538 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 12096, 2024-09-14T03:47:51.538 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3 2024-09-14T03:47:51.538 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.538 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_anon": { 2024-09-14T03:47:51.538 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 2405394, 2024-09-14T03:47:51.538 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 170 2024-09-14T03:47:51.539 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.539 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_meta": { 2024-09-14T03:47:51.539 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 4752, 2024-09-14T03:47:51.539 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 54 2024-09-14T03:47:51.539 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.539 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_co": { 2024-09-14T03:47:51.539 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.539 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.539 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.539 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:51.539 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 451776, 2024-09-14T03:47:51.540 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 39 2024-09-14T03:47:51.540 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.540 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mapbl": { 2024-09-14T03:47:51.540 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.540 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.540 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.540 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pglog": { 2024-09-14T03:47:51.540 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 8944, 2024-09-14T03:47:51.540 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 18 2024-09-14T03:47:51.540 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.540 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap": { 2024-09-14T03:47:51.541 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 235464, 2024-09-14T03:47:51.541 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3517 2024-09-14T03:47:51.541 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.541 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap_mapping": { 2024-09-14T03:47:51.541 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.541 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.541 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.541 INFO:tasks.workunit.client.0.smithi070.stdout: "pgmap": { 2024-09-14T03:47:51.541 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.541 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.541 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.541 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_1": { 2024-09-14T03:47:51.542 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.542 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.542 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.542 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_2": { 2024-09-14T03:47:51.542 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.542 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.542 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.542 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.542 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.5": { 2024-09-14T03:47:51.542 INFO:tasks.workunit.client.0.smithi070.stdout: "bloom_filter": { 2024-09-14T03:47:51.542 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.542 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.543 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.543 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:51.543 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 6304, 2024-09-14T03:47:51.543 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 115 2024-09-14T03:47:51.543 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.543 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_reader": { 2024-09-14T03:47:51.543 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 16896, 2024-09-14T03:47:51.543 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 8 2024-09-14T03:47:51.543 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.543 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_writer": { 2024-09-14T03:47:51.543 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 576, 2024-09-14T03:47:51.544 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3 2024-09-14T03:47:51.544 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.544 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Blob": { 2024-09-14T03:47:51.544 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 19488, 2024-09-14T03:47:51.544 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 174 2024-09-14T03:47:51.544 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.544 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Buffer": { 2024-09-14T03:47:51.544 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 224, 2024-09-14T03:47:51.544 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 2 2024-09-14T03:47:51.544 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.544 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Extent": { 2024-09-14T03:47:51.544 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 8352, 2024-09-14T03:47:51.545 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 174 2024-09-14T03:47:51.545 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.545 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_SharedBlob": { 2024-09-14T03:47:51.545 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 19488, 2024-09-14T03:47:51.545 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 174 2024-09-14T03:47:51.545 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.545 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_alloc": { 2024-09-14T03:47:51.545 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 3040, 2024-09-14T03:47:51.545 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 38 2024-09-14T03:47:51.545 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.545 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_data": { 2024-09-14T03:47:51.545 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.546 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.546 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.546 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_meta": { 2024-09-14T03:47:51.546 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 49727, 2024-09-14T03:47:51.546 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 9103 2024-09-14T03:47:51.546 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.546 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_onode": { 2024-09-14T03:47:51.546 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 129200, 2024-09-14T03:47:51.546 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 190 2024-09-14T03:47:51.546 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.546 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_other": { 2024-09-14T03:47:51.546 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 9600, 2024-09-14T03:47:51.547 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 548 2024-09-14T03:47:51.547 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.547 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_fsck": { 2024-09-14T03:47:51.547 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.547 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.547 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.547 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_inline_bl": { 2024-09-14T03:47:51.547 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 28176, 2024-09-14T03:47:51.547 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 190 2024-09-14T03:47:51.547 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.547 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_txc": { 2024-09-14T03:47:51.547 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 2208, 2024-09-14T03:47:51.548 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3 2024-09-14T03:47:51.548 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.548 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing": { 2024-09-14T03:47:51.548 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 846007, 2024-09-14T03:47:51.548 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 53 2024-09-14T03:47:51.548 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.548 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing_deferred": { 2024-09-14T03:47:51.548 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 30637, 2024-09-14T03:47:51.548 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 9 2024-09-14T03:47:51.548 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.548 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_anon": { 2024-09-14T03:47:51.548 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 2403941, 2024-09-14T03:47:51.549 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 169 2024-09-14T03:47:51.549 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.549 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_meta": { 2024-09-14T03:47:51.549 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 5456, 2024-09-14T03:47:51.549 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 62 2024-09-14T03:47:51.549 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.549 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_co": { 2024-09-14T03:47:51.549 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.549 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.549 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.550 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:51.550 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 474944, 2024-09-14T03:47:51.550 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 41 2024-09-14T03:47:51.550 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.550 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mapbl": { 2024-09-14T03:47:51.552 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.552 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.552 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.552 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pglog": { 2024-09-14T03:47:51.552 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 13000, 2024-09-14T03:47:51.552 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 25 2024-09-14T03:47:51.552 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.552 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap": { 2024-09-14T03:47:51.552 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 235720, 2024-09-14T03:47:51.552 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3529 2024-09-14T03:47:51.552 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.553 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap_mapping": { 2024-09-14T03:47:51.553 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.553 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.553 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.553 INFO:tasks.workunit.client.0.smithi070.stdout: "pgmap": { 2024-09-14T03:47:51.553 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.553 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.553 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.553 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_1": { 2024-09-14T03:47:51.553 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.553 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.553 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.554 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_2": { 2024-09-14T03:47:51.554 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.554 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.554 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.554 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.554 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.6": { 2024-09-14T03:47:51.554 INFO:tasks.workunit.client.0.smithi070.stdout: "bloom_filter": { 2024-09-14T03:47:51.554 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.554 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.554 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.554 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:51.554 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 6304, 2024-09-14T03:47:51.555 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 115 2024-09-14T03:47:51.555 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.555 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_reader": { 2024-09-14T03:47:51.555 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 16896, 2024-09-14T03:47:51.555 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 8 2024-09-14T03:47:51.555 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.555 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_writer": { 2024-09-14T03:47:51.555 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 576, 2024-09-14T03:47:51.555 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3 2024-09-14T03:47:51.555 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.555 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Blob": { 2024-09-14T03:47:51.555 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 20384, 2024-09-14T03:47:51.556 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 182 2024-09-14T03:47:51.556 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.556 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Buffer": { 2024-09-14T03:47:51.556 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 224, 2024-09-14T03:47:51.556 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 2 2024-09-14T03:47:51.556 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.556 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Extent": { 2024-09-14T03:47:51.556 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 8736, 2024-09-14T03:47:51.556 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 182 2024-09-14T03:47:51.556 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.556 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_SharedBlob": { 2024-09-14T03:47:51.556 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 20384, 2024-09-14T03:47:51.557 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 182 2024-09-14T03:47:51.557 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.557 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_alloc": { 2024-09-14T03:47:51.557 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 2560, 2024-09-14T03:47:51.557 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 32 2024-09-14T03:47:51.557 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.557 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_data": { 2024-09-14T03:47:51.557 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 14723, 2024-09-14T03:47:51.557 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3 2024-09-14T03:47:51.557 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.557 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_meta": { 2024-09-14T03:47:51.557 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 48913, 2024-09-14T03:47:51.558 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 9250 2024-09-14T03:47:51.558 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.558 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_onode": { 2024-09-14T03:47:51.558 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 126480, 2024-09-14T03:47:51.558 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 186 2024-09-14T03:47:51.558 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.558 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_other": { 2024-09-14T03:47:51.558 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 10492, 2024-09-14T03:47:51.558 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 702 2024-09-14T03:47:51.558 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.558 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_fsck": { 2024-09-14T03:47:51.558 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.559 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.559 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.559 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_inline_bl": { 2024-09-14T03:47:51.559 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 28348, 2024-09-14T03:47:51.559 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 186 2024-09-14T03:47:51.559 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.559 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_txc": { 2024-09-14T03:47:51.559 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.559 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.559 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.559 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing": { 2024-09-14T03:47:51.559 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 210427, 2024-09-14T03:47:51.560 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 42 2024-09-14T03:47:51.560 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.560 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing_deferred": { 2024-09-14T03:47:51.560 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.560 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.560 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.560 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_anon": { 2024-09-14T03:47:51.560 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 2410728, 2024-09-14T03:47:51.560 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 177 2024-09-14T03:47:51.560 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.560 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_meta": { 2024-09-14T03:47:51.560 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 4576, 2024-09-14T03:47:51.561 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 52 2024-09-14T03:47:51.561 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.561 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_co": { 2024-09-14T03:47:51.561 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.561 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.561 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.561 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:51.561 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 440192, 2024-09-14T03:47:51.561 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 38 2024-09-14T03:47:51.561 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.561 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mapbl": { 2024-09-14T03:47:51.562 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.562 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.562 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.562 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pglog": { 2024-09-14T03:47:51.562 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 75712, 2024-09-14T03:47:51.562 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 156 2024-09-14T03:47:51.562 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.562 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap": { 2024-09-14T03:47:51.562 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 235208, 2024-09-14T03:47:51.562 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3505 2024-09-14T03:47:51.562 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.562 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap_mapping": { 2024-09-14T03:47:51.563 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.563 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.563 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.563 INFO:tasks.workunit.client.0.smithi070.stdout: "pgmap": { 2024-09-14T03:47:51.563 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.563 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.563 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.563 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_1": { 2024-09-14T03:47:51.563 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.563 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.563 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.563 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_2": { 2024-09-14T03:47:51.564 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.564 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.564 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.564 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.564 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.7": { 2024-09-14T03:47:51.564 INFO:tasks.workunit.client.0.smithi070.stdout: "bloom_filter": { 2024-09-14T03:47:51.564 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.564 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.564 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.564 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:51.564 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 6304, 2024-09-14T03:47:51.564 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 115 2024-09-14T03:47:51.565 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.565 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_reader": { 2024-09-14T03:47:51.565 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 16896, 2024-09-14T03:47:51.565 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 8 2024-09-14T03:47:51.565 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.565 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs_file_writer": { 2024-09-14T03:47:51.565 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 576, 2024-09-14T03:47:51.565 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3 2024-09-14T03:47:51.565 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.565 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Blob": { 2024-09-14T03:47:51.565 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 20384, 2024-09-14T03:47:51.565 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 182 2024-09-14T03:47:51.566 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.566 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Buffer": { 2024-09-14T03:47:51.566 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 1232, 2024-09-14T03:47:51.566 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 11 2024-09-14T03:47:51.566 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.566 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_Extent": { 2024-09-14T03:47:51.566 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 8736, 2024-09-14T03:47:51.566 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 182 2024-09-14T03:47:51.566 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.566 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_SharedBlob": { 2024-09-14T03:47:51.566 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 20384, 2024-09-14T03:47:51.566 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 182 2024-09-14T03:47:51.567 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.567 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_alloc": { 2024-09-14T03:47:51.567 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 2240, 2024-09-14T03:47:51.567 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 28 2024-09-14T03:47:51.567 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.567 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_data": { 2024-09-14T03:47:51.567 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 293251, 2024-09-14T03:47:51.567 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 12 2024-09-14T03:47:51.567 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.567 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_meta": { 2024-09-14T03:47:51.567 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 53184, 2024-09-14T03:47:51.567 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 9789 2024-09-14T03:47:51.568 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.568 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_onode": { 2024-09-14T03:47:51.568 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 133960, 2024-09-14T03:47:51.568 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 197 2024-09-14T03:47:51.568 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.568 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_cache_other": { 2024-09-14T03:47:51.568 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 11388, 2024-09-14T03:47:51.568 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 710 2024-09-14T03:47:51.568 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.568 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_fsck": { 2024-09-14T03:47:51.568 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.569 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.569 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.569 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_inline_bl": { 2024-09-14T03:47:51.569 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 31164, 2024-09-14T03:47:51.569 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 197 2024-09-14T03:47:51.569 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.569 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_txc": { 2024-09-14T03:47:51.569 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.569 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.569 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.569 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing": { 2024-09-14T03:47:51.569 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 181403, 2024-09-14T03:47:51.570 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 35 2024-09-14T03:47:51.570 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.570 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore_writing_deferred": { 2024-09-14T03:47:51.570 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.570 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.570 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.570 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_anon": { 2024-09-14T03:47:51.570 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 2430620, 2024-09-14T03:47:51.570 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 217 2024-09-14T03:47:51.570 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.570 INFO:tasks.workunit.client.0.smithi070.stdout: "buffer_meta": { 2024-09-14T03:47:51.570 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 6336, 2024-09-14T03:47:51.571 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 72 2024-09-14T03:47:51.571 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.571 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_co": { 2024-09-14T03:47:51.571 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.571 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.571 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.571 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:51.571 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 532864, 2024-09-14T03:47:51.571 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 46 2024-09-14T03:47:51.571 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.572 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_mapbl": { 2024-09-14T03:47:51.572 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.572 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.572 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.572 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_pglog": { 2024-09-14T03:47:51.572 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 79560, 2024-09-14T03:47:51.572 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 197 2024-09-14T03:47:51.572 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.572 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap": { 2024-09-14T03:47:51.572 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 235208, 2024-09-14T03:47:51.572 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 3505 2024-09-14T03:47:51.573 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.573 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap_mapping": { 2024-09-14T03:47:51.573 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.573 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.573 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.573 INFO:tasks.workunit.client.0.smithi070.stdout: "pgmap": { 2024-09-14T03:47:51.573 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.573 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.573 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.573 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_1": { 2024-09-14T03:47:51.573 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.573 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.574 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.574 INFO:tasks.workunit.client.0.smithi070.stdout: "unittest_2": { 2024-09-14T03:47:51.574 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes": 0, 2024-09-14T03:47:51.574 INFO:tasks.workunit.client.0.smithi070.stdout: "items": 0 2024-09-14T03:47:51.574 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.574 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.574 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.574 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.574 INFO:tasks.workunit.client.0.smithi070.stdout: "metadata": { 2024-09-14T03:47:51.574 INFO:tasks.workunit.client.0.smithi070.stdout: "mds": { 2024-09-14T03:47:51.574 INFO:tasks.workunit.client.0.smithi070.stdout: "arch": { 2024-09-14T03:47:51.575 INFO:tasks.workunit.client.0.smithi070.stdout: "x86_64": 2 2024-09-14T03:47:51.575 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.575 INFO:tasks.workunit.client.0.smithi070.stdout: "ceph_version": { 2024-09-14T03:47:51.575 INFO:tasks.workunit.client.0.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:47:51.575 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.575 INFO:tasks.workunit.client.0.smithi070.stdout: "cpu": { 2024-09-14T03:47:51.575 INFO:tasks.workunit.client.0.smithi070.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 2 2024-09-14T03:47:51.575 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.575 INFO:tasks.workunit.client.0.smithi070.stdout: "distro": { 2024-09-14T03:47:51.575 INFO:tasks.workunit.client.0.smithi070.stdout: "centos": 2 2024-09-14T03:47:51.575 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.575 INFO:tasks.workunit.client.0.smithi070.stdout: "distro_description": { 2024-09-14T03:47:51.576 INFO:tasks.workunit.client.0.smithi070.stdout: "CentOS Stream 9": 2 2024-09-14T03:47:51.576 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.576 INFO:tasks.workunit.client.0.smithi070.stdout: "kernel_description": { 2024-09-14T03:47:51.576 INFO:tasks.workunit.client.0.smithi070.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 2 2024-09-14T03:47:51.576 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.576 INFO:tasks.workunit.client.0.smithi070.stdout: "kernel_version": { 2024-09-14T03:47:51.576 INFO:tasks.workunit.client.0.smithi070.stdout: "5.14.0-505.el9.x86_64": 2 2024-09-14T03:47:51.576 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.576 INFO:tasks.workunit.client.0.smithi070.stdout: "os": { 2024-09-14T03:47:51.576 INFO:tasks.workunit.client.0.smithi070.stdout: "Linux": 2 2024-09-14T03:47:51.576 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.577 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.577 INFO:tasks.workunit.client.0.smithi070.stdout: "mon": { 2024-09-14T03:47:51.577 INFO:tasks.workunit.client.0.smithi070.stdout: "arch": { 2024-09-14T03:47:51.577 INFO:tasks.workunit.client.0.smithi070.stdout: "x86_64": 3 2024-09-14T03:47:51.577 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.577 INFO:tasks.workunit.client.0.smithi070.stdout: "ceph_version": { 2024-09-14T03:47:51.577 INFO:tasks.workunit.client.0.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 3 2024-09-14T03:47:51.577 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.577 INFO:tasks.workunit.client.0.smithi070.stdout: "cpu": { 2024-09-14T03:47:51.577 INFO:tasks.workunit.client.0.smithi070.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 3 2024-09-14T03:47:51.577 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.577 INFO:tasks.workunit.client.0.smithi070.stdout: "distro": { 2024-09-14T03:47:51.578 INFO:tasks.workunit.client.0.smithi070.stdout: "centos": 3 2024-09-14T03:47:51.578 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.578 INFO:tasks.workunit.client.0.smithi070.stdout: "distro_description": { 2024-09-14T03:47:51.578 INFO:tasks.workunit.client.0.smithi070.stdout: "CentOS Stream 9": 3 2024-09-14T03:47:51.578 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.578 INFO:tasks.workunit.client.0.smithi070.stdout: "kernel_description": { 2024-09-14T03:47:51.578 INFO:tasks.workunit.client.0.smithi070.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 3 2024-09-14T03:47:51.578 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.578 INFO:tasks.workunit.client.0.smithi070.stdout: "kernel_version": { 2024-09-14T03:47:51.578 INFO:tasks.workunit.client.0.smithi070.stdout: "5.14.0-505.el9.x86_64": 3 2024-09-14T03:47:51.578 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.578 INFO:tasks.workunit.client.0.smithi070.stdout: "os": { 2024-09-14T03:47:51.579 INFO:tasks.workunit.client.0.smithi070.stdout: "Linux": 3 2024-09-14T03:47:51.579 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.579 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.579 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:51.579 INFO:tasks.workunit.client.0.smithi070.stdout: "arch": { 2024-09-14T03:47:51.579 INFO:tasks.workunit.client.0.smithi070.stdout: "x86_64": 8 2024-09-14T03:47:51.579 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.579 INFO:tasks.workunit.client.0.smithi070.stdout: "ceph_version": { 2024-09-14T03:47:51.579 INFO:tasks.workunit.client.0.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 8 2024-09-14T03:47:51.579 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.579 INFO:tasks.workunit.client.0.smithi070.stdout: "cpu": { 2024-09-14T03:47:51.580 INFO:tasks.workunit.client.0.smithi070.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 8 2024-09-14T03:47:51.580 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.580 INFO:tasks.workunit.client.0.smithi070.stdout: "distro": { 2024-09-14T03:47:51.580 INFO:tasks.workunit.client.0.smithi070.stdout: "centos": 8 2024-09-14T03:47:51.580 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.580 INFO:tasks.workunit.client.0.smithi070.stdout: "distro_description": { 2024-09-14T03:47:51.580 INFO:tasks.workunit.client.0.smithi070.stdout: "CentOS Stream 9": 8 2024-09-14T03:47:51.580 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.580 INFO:tasks.workunit.client.0.smithi070.stdout: "kernel_description": { 2024-09-14T03:47:51.580 INFO:tasks.workunit.client.0.smithi070.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024": 8 2024-09-14T03:47:51.580 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.580 INFO:tasks.workunit.client.0.smithi070.stdout: "kernel_version": { 2024-09-14T03:47:51.581 INFO:tasks.workunit.client.0.smithi070.stdout: "5.14.0-505.el9.x86_64": 8 2024-09-14T03:47:51.581 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.581 INFO:tasks.workunit.client.0.smithi070.stdout: "os": { 2024-09-14T03:47:51.581 INFO:tasks.workunit.client.0.smithi070.stdout: "Linux": 8 2024-09-14T03:47:51.581 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.581 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_objectstore": { 2024-09-14T03:47:51.581 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore": 8 2024-09-14T03:47:51.581 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.581 INFO:tasks.workunit.client.0.smithi070.stdout: "rotational": { 2024-09-14T03:47:51.581 INFO:tasks.workunit.client.0.smithi070.stdout: "0": 8 2024-09-14T03:47:51.581 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.581 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.581 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.582 INFO:tasks.workunit.client.0.smithi070.stdout: "mon": { 2024-09-14T03:47:51.582 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 3, 2024-09-14T03:47:51.582 INFO:tasks.workunit.client.0.smithi070.stdout: "features": { 2024-09-14T03:47:51.582 INFO:tasks.workunit.client.0.smithi070.stdout: "optional": [], 2024-09-14T03:47:51.582 INFO:tasks.workunit.client.0.smithi070.stdout: "persistent": [ 2024-09-14T03:47:51.582 INFO:tasks.workunit.client.0.smithi070.stdout: "kraken", 2024-09-14T03:47:51.582 INFO:tasks.workunit.client.0.smithi070.stdout: "luminous", 2024-09-14T03:47:51.582 INFO:tasks.workunit.client.0.smithi070.stdout: "mimic", 2024-09-14T03:47:51.582 INFO:tasks.workunit.client.0.smithi070.stdout: "osdmap-prune", 2024-09-14T03:47:51.582 INFO:tasks.workunit.client.0.smithi070.stdout: "nautilus", 2024-09-14T03:47:51.582 INFO:tasks.workunit.client.0.smithi070.stdout: "octopus", 2024-09-14T03:47:51.582 INFO:tasks.workunit.client.0.smithi070.stdout: "pacific", 2024-09-14T03:47:51.583 INFO:tasks.workunit.client.0.smithi070.stdout: "elector-pinging", 2024-09-14T03:47:51.583 INFO:tasks.workunit.client.0.smithi070.stdout: "quincy" 2024-09-14T03:47:51.583 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.583 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.583 INFO:tasks.workunit.client.0.smithi070.stdout: "ipv4_addr_mons": 6, 2024-09-14T03:47:51.583 INFO:tasks.workunit.client.0.smithi070.stdout: "ipv6_addr_mons": 0, 2024-09-14T03:47:51.583 INFO:tasks.workunit.client.0.smithi070.stdout: "min_mon_release": 17, 2024-09-14T03:47:51.583 INFO:tasks.workunit.client.0.smithi070.stdout: "v1_addr_mons": 3, 2024-09-14T03:47:51.583 INFO:tasks.workunit.client.0.smithi070.stdout: "v2_addr_mons": 3 2024-09-14T03:47:51.583 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.583 INFO:tasks.workunit.client.0.smithi070.stdout: "organization": "ceph-qa", 2024-09-14T03:47:51.583 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:51.583 INFO:tasks.workunit.client.0.smithi070.stdout: "cluster_network": false, 2024-09-14T03:47:51.584 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 8, 2024-09-14T03:47:51.584 INFO:tasks.workunit.client.0.smithi070.stdout: "require_min_compat_client": "luminous", 2024-09-14T03:47:51.584 INFO:tasks.workunit.client.0.smithi070.stdout: "require_osd_release": "quincy" 2024-09-14T03:47:51.584 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.584 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_perf_histograms": [ 2024-09-14T03:47:51.584 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.584 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:47:51.584 INFO:tasks.workunit.client.0.smithi070.stdout: "axes": [ 2024-09-14T03:47:51.584 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.584 INFO:tasks.workunit.client.0.smithi070.stdout: "buckets": 32, 2024-09-14T03:47:51.584 INFO:tasks.workunit.client.0.smithi070.stdout: "min": 0, 2024-09-14T03:47:51.584 INFO:tasks.workunit.client.0.smithi070.stdout: "name": "Latency (usec)", 2024-09-14T03:47:51.585 INFO:tasks.workunit.client.0.smithi070.stdout: "quant_size": 100000, 2024-09-14T03:47:51.585 INFO:tasks.workunit.client.0.smithi070.stdout: "ranges": [ 2024-09-14T03:47:51.585 INFO:tasks.workunit.client.0.smithi070.stdout: "[None, -1]", 2024-09-14T03:47:51.585 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 99999]", 2024-09-14T03:47:51.585 INFO:tasks.workunit.client.0.smithi070.stdout: "[100000, 199999]", 2024-09-14T03:47:51.585 INFO:tasks.workunit.client.0.smithi070.stdout: "[200000, 399999]", 2024-09-14T03:47:51.585 INFO:tasks.workunit.client.0.smithi070.stdout: "[400000, 799999]", 2024-09-14T03:47:51.585 INFO:tasks.workunit.client.0.smithi070.stdout: "[800000, 1599999]", 2024-09-14T03:47:51.585 INFO:tasks.workunit.client.0.smithi070.stdout: "[1600000, 3199999]", 2024-09-14T03:47:51.585 INFO:tasks.workunit.client.0.smithi070.stdout: "[3200000, 6399999]", 2024-09-14T03:47:51.585 INFO:tasks.workunit.client.0.smithi070.stdout: "[6400000, 12799999]", 2024-09-14T03:47:51.585 INFO:tasks.workunit.client.0.smithi070.stdout: "[12800000, 25599999]", 2024-09-14T03:47:51.586 INFO:tasks.workunit.client.0.smithi070.stdout: "[25600000, 51199999]", 2024-09-14T03:47:51.586 INFO:tasks.workunit.client.0.smithi070.stdout: "[51200000, 102399999]", 2024-09-14T03:47:51.586 INFO:tasks.workunit.client.0.smithi070.stdout: "[102400000, 204799999]", 2024-09-14T03:47:51.586 INFO:tasks.workunit.client.0.smithi070.stdout: "[204800000, 409599999]", 2024-09-14T03:47:51.586 INFO:tasks.workunit.client.0.smithi070.stdout: "[409600000, 819199999]", 2024-09-14T03:47:51.586 INFO:tasks.workunit.client.0.smithi070.stdout: "[819200000, 1638399999]", 2024-09-14T03:47:51.586 INFO:tasks.workunit.client.0.smithi070.stdout: "[1638400000, 3276799999]", 2024-09-14T03:47:51.586 INFO:tasks.workunit.client.0.smithi070.stdout: "[3276800000, 6553599999]", 2024-09-14T03:47:51.586 INFO:tasks.workunit.client.0.smithi070.stdout: "[6553600000, 13107199999]", 2024-09-14T03:47:51.586 INFO:tasks.workunit.client.0.smithi070.stdout: "[13107200000, 26214399999]", 2024-09-14T03:47:51.586 INFO:tasks.workunit.client.0.smithi070.stdout: "[26214400000, 52428799999]", 2024-09-14T03:47:51.586 INFO:tasks.workunit.client.0.smithi070.stdout: "[52428800000, 104857599999]", 2024-09-14T03:47:51.587 INFO:tasks.workunit.client.0.smithi070.stdout: "[104857600000, 209715199999]", 2024-09-14T03:47:51.587 INFO:tasks.workunit.client.0.smithi070.stdout: "[209715200000, 419430399999]", 2024-09-14T03:47:51.587 INFO:tasks.workunit.client.0.smithi070.stdout: "[419430400000, 838860799999]", 2024-09-14T03:47:51.587 INFO:tasks.workunit.client.0.smithi070.stdout: "[838860800000, 1677721599999]", 2024-09-14T03:47:51.587 INFO:tasks.workunit.client.0.smithi070.stdout: "[1677721600000, 3355443199999]", 2024-09-14T03:47:51.587 INFO:tasks.workunit.client.0.smithi070.stdout: "[3355443200000, 6710886399999]", 2024-09-14T03:47:51.587 INFO:tasks.workunit.client.0.smithi070.stdout: "[6710886400000, 13421772799999]", 2024-09-14T03:47:51.587 INFO:tasks.workunit.client.0.smithi070.stdout: "[13421772800000, 26843545599999]", 2024-09-14T03:47:51.587 INFO:tasks.workunit.client.0.smithi070.stdout: "[26843545600000, 53687091199999]", 2024-09-14T03:47:51.587 INFO:tasks.workunit.client.0.smithi070.stdout: "[53687091200000, None]" 2024-09-14T03:47:51.587 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.587 INFO:tasks.workunit.client.0.smithi070.stdout: "scale_type": "log2" 2024-09-14T03:47:51.588 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.588 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.588 INFO:tasks.workunit.client.0.smithi070.stdout: "buckets": 32, 2024-09-14T03:47:51.588 INFO:tasks.workunit.client.0.smithi070.stdout: "min": 0, 2024-09-14T03:47:51.588 INFO:tasks.workunit.client.0.smithi070.stdout: "name": "Request size (bytes)", 2024-09-14T03:47:51.588 INFO:tasks.workunit.client.0.smithi070.stdout: "quant_size": 512, 2024-09-14T03:47:51.588 INFO:tasks.workunit.client.0.smithi070.stdout: "ranges": [ 2024-09-14T03:47:51.588 INFO:tasks.workunit.client.0.smithi070.stdout: "[None, -1]", 2024-09-14T03:47:51.588 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 511]", 2024-09-14T03:47:51.588 INFO:tasks.workunit.client.0.smithi070.stdout: "[512, 1023]", 2024-09-14T03:47:51.588 INFO:tasks.workunit.client.0.smithi070.stdout: "[1024, 2047]", 2024-09-14T03:47:51.588 INFO:tasks.workunit.client.0.smithi070.stdout: "[2048, 4095]", 2024-09-14T03:47:51.589 INFO:tasks.workunit.client.0.smithi070.stdout: "[4096, 8191]", 2024-09-14T03:47:51.589 INFO:tasks.workunit.client.0.smithi070.stdout: "[8192, 16383]", 2024-09-14T03:47:51.589 INFO:tasks.workunit.client.0.smithi070.stdout: "[16384, 32767]", 2024-09-14T03:47:51.589 INFO:tasks.workunit.client.0.smithi070.stdout: "[32768, 65535]", 2024-09-14T03:47:51.589 INFO:tasks.workunit.client.0.smithi070.stdout: "[65536, 131071]", 2024-09-14T03:47:51.589 INFO:tasks.workunit.client.0.smithi070.stdout: "[131072, 262143]", 2024-09-14T03:47:51.589 INFO:tasks.workunit.client.0.smithi070.stdout: "[262144, 524287]", 2024-09-14T03:47:51.589 INFO:tasks.workunit.client.0.smithi070.stdout: "[524288, 1048575]", 2024-09-14T03:47:51.589 INFO:tasks.workunit.client.0.smithi070.stdout: "[1048576, 2097151]", 2024-09-14T03:47:51.589 INFO:tasks.workunit.client.0.smithi070.stdout: "[2097152, 4194303]", 2024-09-14T03:47:51.589 INFO:tasks.workunit.client.0.smithi070.stdout: "[4194304, 8388607]", 2024-09-14T03:47:51.589 INFO:tasks.workunit.client.0.smithi070.stdout: "[8388608, 16777215]", 2024-09-14T03:47:51.590 INFO:tasks.workunit.client.0.smithi070.stdout: "[16777216, 33554431]", 2024-09-14T03:47:51.590 INFO:tasks.workunit.client.0.smithi070.stdout: "[33554432, 67108863]", 2024-09-14T03:47:51.590 INFO:tasks.workunit.client.0.smithi070.stdout: "[67108864, 134217727]", 2024-09-14T03:47:51.590 INFO:tasks.workunit.client.0.smithi070.stdout: "[134217728, 268435455]", 2024-09-14T03:47:51.590 INFO:tasks.workunit.client.0.smithi070.stdout: "[268435456, 536870911]", 2024-09-14T03:47:51.590 INFO:tasks.workunit.client.0.smithi070.stdout: "[536870912, 1073741823]", 2024-09-14T03:47:51.590 INFO:tasks.workunit.client.0.smithi070.stdout: "[1073741824, 2147483647]", 2024-09-14T03:47:51.590 INFO:tasks.workunit.client.0.smithi070.stdout: "[2147483648, 4294967295]", 2024-09-14T03:47:51.590 INFO:tasks.workunit.client.0.smithi070.stdout: "[4294967296, 8589934591]", 2024-09-14T03:47:51.590 INFO:tasks.workunit.client.0.smithi070.stdout: "[8589934592, 17179869183]", 2024-09-14T03:47:51.591 INFO:tasks.workunit.client.0.smithi070.stdout: "[17179869184, 34359738367]", 2024-09-14T03:47:51.591 INFO:tasks.workunit.client.0.smithi070.stdout: "[34359738368, 68719476735]", 2024-09-14T03:47:51.591 INFO:tasks.workunit.client.0.smithi070.stdout: "[68719476736, 137438953471]", 2024-09-14T03:47:51.591 INFO:tasks.workunit.client.0.smithi070.stdout: "[137438953472, 274877906943]", 2024-09-14T03:47:51.591 INFO:tasks.workunit.client.0.smithi070.stdout: "[274877906944, None]" 2024-09-14T03:47:51.591 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.591 INFO:tasks.workunit.client.0.smithi070.stdout: "scale_type": "log2" 2024-09-14T03:47:51.591 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.591 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.591 INFO:tasks.workunit.client.0.smithi070.stdout: "osds": [ 2024-09-14T03:47:51.591 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.591 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 0, 2024-09-14T03:47:51.592 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:51.592 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.592 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.592 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.592 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.592 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.592 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.592 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.592 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.592 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.592 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.592 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.593 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.593 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.593 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.593 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.593 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.593 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.593 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.593 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.593 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.593 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.593 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.593 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.594 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.594 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.594 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.594 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.594 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.594 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.594 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.594 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.594 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:51.594 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.594 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.594 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.594 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 1, 2024-09-14T03:47:51.595 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:51.595 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.595 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.595 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.595 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.595 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.595 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.595 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.595 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.595 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.595 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.595 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.596 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.596 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.596 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.596 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.596 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.596 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.596 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.596 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.596 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.596 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.596 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.597 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.597 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.597 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.597 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.597 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.597 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.597 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.597 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.597 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.597 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:51.597 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.597 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.598 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.598 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 2, 2024-09-14T03:47:51.598 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:51.598 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.598 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.598 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.598 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.598 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.598 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.598 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.598 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.598 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.599 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.599 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.599 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.599 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.599 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.599 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.599 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.599 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.599 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.599 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.599 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.599 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.600 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.600 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.600 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.600 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.600 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.600 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.600 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.600 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.600 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.600 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.600 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:51.601 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.601 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.601 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.601 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 3, 2024-09-14T03:47:51.601 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:51.601 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.601 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.601 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.601 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.601 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.601 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.601 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.601 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.602 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.602 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.602 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.602 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.602 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.602 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.602 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.602 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.602 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.602 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.602 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.602 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.603 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.603 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.603 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.603 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.603 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.603 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.603 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.603 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.603 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.603 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.603 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.603 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:51.603 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.604 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.604 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.604 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 4, 2024-09-14T03:47:51.604 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:51.604 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.604 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.604 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.604 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.604 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.604 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.604 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.604 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.605 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.605 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.605 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.605 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.605 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.605 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.605 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.605 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.605 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.605 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.605 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.606 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.606 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.606 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.606 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.606 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.606 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.606 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.606 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.606 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.606 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.606 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.607 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.607 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:51.607 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.607 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.607 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.607 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 5, 2024-09-14T03:47:51.607 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:51.607 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.607 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.607 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.607 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.607 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.608 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.608 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.608 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.608 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.608 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.608 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.608 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.608 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.608 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.608 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.609 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.609 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.609 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.609 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.609 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.609 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.609 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.609 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.609 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.609 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.609 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.609 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.610 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.610 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.610 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.610 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.610 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:51.610 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.610 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.610 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.610 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 6, 2024-09-14T03:47:51.610 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:51.610 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.610 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.611 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.611 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.611 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.611 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.611 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.611 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.611 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.611 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.611 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.611 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.611 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.611 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.612 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.612 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.612 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.612 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.612 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.612 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.612 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.612 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.612 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.612 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.612 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.612 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.613 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.613 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.613 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.613 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.613 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.613 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:51.613 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.613 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.613 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.613 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 7, 2024-09-14T03:47:51.613 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:51.613 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.614 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.614 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 103, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.614 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.614 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.614 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.614 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.614 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.614 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.614 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.614 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.614 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.614 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.615 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.615 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.615 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.615 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.615 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.615 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.615 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.615 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.615 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.615 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.615 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.615 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.616 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.616 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.616 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.616 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.616 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.616 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.616 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:51.616 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.616 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.616 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.616 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.616 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:47:51.617 INFO:tasks.workunit.client.0.smithi070.stdout: "axes": [ 2024-09-14T03:47:51.617 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.617 INFO:tasks.workunit.client.0.smithi070.stdout: "buckets": 32, 2024-09-14T03:47:51.617 INFO:tasks.workunit.client.0.smithi070.stdout: "min": 0, 2024-09-14T03:47:51.617 INFO:tasks.workunit.client.0.smithi070.stdout: "name": "Latency (usec)", 2024-09-14T03:47:51.617 INFO:tasks.workunit.client.0.smithi070.stdout: "quant_size": 100000, 2024-09-14T03:47:51.617 INFO:tasks.workunit.client.0.smithi070.stdout: "ranges": [ 2024-09-14T03:47:51.617 INFO:tasks.workunit.client.0.smithi070.stdout: "[None, -1]", 2024-09-14T03:47:51.617 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 99999]", 2024-09-14T03:47:51.617 INFO:tasks.workunit.client.0.smithi070.stdout: "[100000, 199999]", 2024-09-14T03:47:51.617 INFO:tasks.workunit.client.0.smithi070.stdout: "[200000, 399999]", 2024-09-14T03:47:51.617 INFO:tasks.workunit.client.0.smithi070.stdout: "[400000, 799999]", 2024-09-14T03:47:51.618 INFO:tasks.workunit.client.0.smithi070.stdout: "[800000, 1599999]", 2024-09-14T03:47:51.618 INFO:tasks.workunit.client.0.smithi070.stdout: "[1600000, 3199999]", 2024-09-14T03:47:51.618 INFO:tasks.workunit.client.0.smithi070.stdout: "[3200000, 6399999]", 2024-09-14T03:47:51.618 INFO:tasks.workunit.client.0.smithi070.stdout: "[6400000, 12799999]", 2024-09-14T03:47:51.618 INFO:tasks.workunit.client.0.smithi070.stdout: "[12800000, 25599999]", 2024-09-14T03:47:51.618 INFO:tasks.workunit.client.0.smithi070.stdout: "[25600000, 51199999]", 2024-09-14T03:47:51.618 INFO:tasks.workunit.client.0.smithi070.stdout: "[51200000, 102399999]", 2024-09-14T03:47:51.618 INFO:tasks.workunit.client.0.smithi070.stdout: "[102400000, 204799999]", 2024-09-14T03:47:51.618 INFO:tasks.workunit.client.0.smithi070.stdout: "[204800000, 409599999]", 2024-09-14T03:47:51.618 INFO:tasks.workunit.client.0.smithi070.stdout: "[409600000, 819199999]", 2024-09-14T03:47:51.618 INFO:tasks.workunit.client.0.smithi070.stdout: "[819200000, 1638399999]", 2024-09-14T03:47:51.619 INFO:tasks.workunit.client.0.smithi070.stdout: "[1638400000, 3276799999]", 2024-09-14T03:47:51.619 INFO:tasks.workunit.client.0.smithi070.stdout: "[3276800000, 6553599999]", 2024-09-14T03:47:51.619 INFO:tasks.workunit.client.0.smithi070.stdout: "[6553600000, 13107199999]", 2024-09-14T03:47:51.619 INFO:tasks.workunit.client.0.smithi070.stdout: "[13107200000, 26214399999]", 2024-09-14T03:47:51.619 INFO:tasks.workunit.client.0.smithi070.stdout: "[26214400000, 52428799999]", 2024-09-14T03:47:51.619 INFO:tasks.workunit.client.0.smithi070.stdout: "[52428800000, 104857599999]", 2024-09-14T03:47:51.619 INFO:tasks.workunit.client.0.smithi070.stdout: "[104857600000, 209715199999]", 2024-09-14T03:47:51.619 INFO:tasks.workunit.client.0.smithi070.stdout: "[209715200000, 419430399999]", 2024-09-14T03:47:51.619 INFO:tasks.workunit.client.0.smithi070.stdout: "[419430400000, 838860799999]", 2024-09-14T03:47:51.619 INFO:tasks.workunit.client.0.smithi070.stdout: "[838860800000, 1677721599999]", 2024-09-14T03:47:51.619 INFO:tasks.workunit.client.0.smithi070.stdout: "[1677721600000, 3355443199999]", 2024-09-14T03:47:51.619 INFO:tasks.workunit.client.0.smithi070.stdout: "[3355443200000, 6710886399999]", 2024-09-14T03:47:51.620 INFO:tasks.workunit.client.0.smithi070.stdout: "[6710886400000, 13421772799999]", 2024-09-14T03:47:51.620 INFO:tasks.workunit.client.0.smithi070.stdout: "[13421772800000, 26843545599999]", 2024-09-14T03:47:51.620 INFO:tasks.workunit.client.0.smithi070.stdout: "[26843545600000, 53687091199999]", 2024-09-14T03:47:51.620 INFO:tasks.workunit.client.0.smithi070.stdout: "[53687091200000, None]" 2024-09-14T03:47:51.620 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.620 INFO:tasks.workunit.client.0.smithi070.stdout: "scale_type": "log2" 2024-09-14T03:47:51.620 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.620 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.620 INFO:tasks.workunit.client.0.smithi070.stdout: "buckets": 32, 2024-09-14T03:47:51.620 INFO:tasks.workunit.client.0.smithi070.stdout: "min": 0, 2024-09-14T03:47:51.620 INFO:tasks.workunit.client.0.smithi070.stdout: "name": "Request size (bytes)", 2024-09-14T03:47:51.620 INFO:tasks.workunit.client.0.smithi070.stdout: "quant_size": 512, 2024-09-14T03:47:51.621 INFO:tasks.workunit.client.0.smithi070.stdout: "ranges": [ 2024-09-14T03:47:51.621 INFO:tasks.workunit.client.0.smithi070.stdout: "[None, -1]", 2024-09-14T03:47:51.621 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 511]", 2024-09-14T03:47:51.621 INFO:tasks.workunit.client.0.smithi070.stdout: "[512, 1023]", 2024-09-14T03:47:51.621 INFO:tasks.workunit.client.0.smithi070.stdout: "[1024, 2047]", 2024-09-14T03:47:51.621 INFO:tasks.workunit.client.0.smithi070.stdout: "[2048, 4095]", 2024-09-14T03:47:51.621 INFO:tasks.workunit.client.0.smithi070.stdout: "[4096, 8191]", 2024-09-14T03:47:51.621 INFO:tasks.workunit.client.0.smithi070.stdout: "[8192, 16383]", 2024-09-14T03:47:51.621 INFO:tasks.workunit.client.0.smithi070.stdout: "[16384, 32767]", 2024-09-14T03:47:51.621 INFO:tasks.workunit.client.0.smithi070.stdout: "[32768, 65535]", 2024-09-14T03:47:51.621 INFO:tasks.workunit.client.0.smithi070.stdout: "[65536, 131071]", 2024-09-14T03:47:51.621 INFO:tasks.workunit.client.0.smithi070.stdout: "[131072, 262143]", 2024-09-14T03:47:51.621 INFO:tasks.workunit.client.0.smithi070.stdout: "[262144, 524287]", 2024-09-14T03:47:51.622 INFO:tasks.workunit.client.0.smithi070.stdout: "[524288, 1048575]", 2024-09-14T03:47:51.622 INFO:tasks.workunit.client.0.smithi070.stdout: "[1048576, 2097151]", 2024-09-14T03:47:51.622 INFO:tasks.workunit.client.0.smithi070.stdout: "[2097152, 4194303]", 2024-09-14T03:47:51.622 INFO:tasks.workunit.client.0.smithi070.stdout: "[4194304, 8388607]", 2024-09-14T03:47:51.622 INFO:tasks.workunit.client.0.smithi070.stdout: "[8388608, 16777215]", 2024-09-14T03:47:51.622 INFO:tasks.workunit.client.0.smithi070.stdout: "[16777216, 33554431]", 2024-09-14T03:47:51.622 INFO:tasks.workunit.client.0.smithi070.stdout: "[33554432, 67108863]", 2024-09-14T03:47:51.622 INFO:tasks.workunit.client.0.smithi070.stdout: "[67108864, 134217727]", 2024-09-14T03:47:51.622 INFO:tasks.workunit.client.0.smithi070.stdout: "[134217728, 268435455]", 2024-09-14T03:47:51.622 INFO:tasks.workunit.client.0.smithi070.stdout: "[268435456, 536870911]", 2024-09-14T03:47:51.622 INFO:tasks.workunit.client.0.smithi070.stdout: "[536870912, 1073741823]", 2024-09-14T03:47:51.622 INFO:tasks.workunit.client.0.smithi070.stdout: "[1073741824, 2147483647]", 2024-09-14T03:47:51.623 INFO:tasks.workunit.client.0.smithi070.stdout: "[2147483648, 4294967295]", 2024-09-14T03:47:51.623 INFO:tasks.workunit.client.0.smithi070.stdout: "[4294967296, 8589934591]", 2024-09-14T03:47:51.623 INFO:tasks.workunit.client.0.smithi070.stdout: "[8589934592, 17179869183]", 2024-09-14T03:47:51.623 INFO:tasks.workunit.client.0.smithi070.stdout: "[17179869184, 34359738367]", 2024-09-14T03:47:51.623 INFO:tasks.workunit.client.0.smithi070.stdout: "[34359738368, 68719476735]", 2024-09-14T03:47:51.623 INFO:tasks.workunit.client.0.smithi070.stdout: "[68719476736, 137438953471]", 2024-09-14T03:47:51.623 INFO:tasks.workunit.client.0.smithi070.stdout: "[137438953472, 274877906943]", 2024-09-14T03:47:51.623 INFO:tasks.workunit.client.0.smithi070.stdout: "[274877906944, None]" 2024-09-14T03:47:51.623 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.623 INFO:tasks.workunit.client.0.smithi070.stdout: "scale_type": "log2" 2024-09-14T03:47:51.623 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.623 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.624 INFO:tasks.workunit.client.0.smithi070.stdout: "osds": [ 2024-09-14T03:47:51.624 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.624 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 0, 2024-09-14T03:47:51.624 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:51.624 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.624 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.624 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.624 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.624 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.624 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.625 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.625 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.625 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.625 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.625 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.625 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.625 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.625 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.625 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.625 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.625 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.625 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.625 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.626 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.626 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.626 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.626 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.626 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.626 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.626 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.626 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.626 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.626 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.626 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.626 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.627 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:51.627 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.627 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.627 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.627 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 1, 2024-09-14T03:47:51.627 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:51.627 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.627 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.627 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.627 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.627 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.628 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.628 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.628 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.628 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.628 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.628 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.628 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.628 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.628 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.628 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.628 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.629 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.629 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.629 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.629 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.629 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.629 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.629 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.629 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.629 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.629 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.629 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.629 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.630 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.630 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.630 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.630 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:51.630 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.630 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.630 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.630 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 2, 2024-09-14T03:47:51.630 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:51.630 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.630 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.631 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.631 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.631 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.631 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.631 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.631 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.631 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.631 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.631 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.631 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.631 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.631 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.632 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.632 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.632 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.632 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.632 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.632 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.632 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.632 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.632 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.632 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.632 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.633 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.633 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.633 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.633 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.633 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.633 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.633 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:51.633 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.633 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.633 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.633 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 3, 2024-09-14T03:47:51.633 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:51.634 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.634 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.634 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.634 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.634 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.634 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.634 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.634 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.634 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.634 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.634 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.634 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.634 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.635 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.635 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.635 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.635 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.635 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.635 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.635 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.635 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.635 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.635 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.635 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.636 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.636 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.636 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.636 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.636 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.636 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.636 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.636 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:51.636 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.636 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.636 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.637 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 4, 2024-09-14T03:47:51.637 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:51.637 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.637 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.637 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.637 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.637 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.637 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.637 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.637 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.637 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.637 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.638 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.638 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.638 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.638 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.638 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.638 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.638 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.638 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.638 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.638 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.638 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.638 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.639 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.639 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.639 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.639 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.639 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.639 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.639 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.639 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.639 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.639 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:51.639 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.639 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.640 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.640 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 5, 2024-09-14T03:47:51.640 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:51.640 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.640 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.640 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.640 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.640 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.640 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.640 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.640 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.641 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.641 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.641 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.641 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.641 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.641 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.641 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.641 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.641 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.641 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.641 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.641 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.642 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.642 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.642 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.642 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.642 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.642 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.642 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.642 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.642 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.642 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.642 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.643 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:51.643 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.643 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.643 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.643 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 6, 2024-09-14T03:47:51.643 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:51.643 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.643 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.643 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.643 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.643 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.643 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.644 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.644 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.644 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.644 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.644 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.644 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.644 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.644 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.644 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.644 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.644 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.645 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.645 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.645 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.645 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.645 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.645 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.645 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.645 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.645 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.645 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.645 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.645 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.646 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.646 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.646 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:51.646 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.646 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.646 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.646 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 7, 2024-09-14T03:47:51.646 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:51.646 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.646 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.646 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.646 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.647 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.647 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.647 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.647 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.647 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.647 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.647 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.647 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.647 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.647 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.647 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.648 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.648 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.648 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.648 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.648 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.648 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.648 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.648 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.648 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.648 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.649 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.649 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.649 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.649 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.649 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.649 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.649 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:51.649 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.649 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.649 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.649 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.650 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:47:51.650 INFO:tasks.workunit.client.0.smithi070.stdout: "axes": [ 2024-09-14T03:47:51.650 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.650 INFO:tasks.workunit.client.0.smithi070.stdout: "buckets": 32, 2024-09-14T03:47:51.650 INFO:tasks.workunit.client.0.smithi070.stdout: "min": 0, 2024-09-14T03:47:51.650 INFO:tasks.workunit.client.0.smithi070.stdout: "name": "Latency (usec)", 2024-09-14T03:47:51.650 INFO:tasks.workunit.client.0.smithi070.stdout: "quant_size": 100000, 2024-09-14T03:47:51.650 INFO:tasks.workunit.client.0.smithi070.stdout: "ranges": [ 2024-09-14T03:47:51.650 INFO:tasks.workunit.client.0.smithi070.stdout: "[None, -1]", 2024-09-14T03:47:51.650 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 99999]", 2024-09-14T03:47:51.650 INFO:tasks.workunit.client.0.smithi070.stdout: "[100000, 199999]", 2024-09-14T03:47:51.650 INFO:tasks.workunit.client.0.smithi070.stdout: "[200000, 399999]", 2024-09-14T03:47:51.651 INFO:tasks.workunit.client.0.smithi070.stdout: "[400000, 799999]", 2024-09-14T03:47:51.651 INFO:tasks.workunit.client.0.smithi070.stdout: "[800000, 1599999]", 2024-09-14T03:47:51.651 INFO:tasks.workunit.client.0.smithi070.stdout: "[1600000, 3199999]", 2024-09-14T03:47:51.651 INFO:tasks.workunit.client.0.smithi070.stdout: "[3200000, 6399999]", 2024-09-14T03:47:51.651 INFO:tasks.workunit.client.0.smithi070.stdout: "[6400000, 12799999]", 2024-09-14T03:47:51.651 INFO:tasks.workunit.client.0.smithi070.stdout: "[12800000, 25599999]", 2024-09-14T03:47:51.651 INFO:tasks.workunit.client.0.smithi070.stdout: "[25600000, 51199999]", 2024-09-14T03:47:51.651 INFO:tasks.workunit.client.0.smithi070.stdout: "[51200000, 102399999]", 2024-09-14T03:47:51.651 INFO:tasks.workunit.client.0.smithi070.stdout: "[102400000, 204799999]", 2024-09-14T03:47:51.651 INFO:tasks.workunit.client.0.smithi070.stdout: "[204800000, 409599999]", 2024-09-14T03:47:51.651 INFO:tasks.workunit.client.0.smithi070.stdout: "[409600000, 819199999]", 2024-09-14T03:47:51.651 INFO:tasks.workunit.client.0.smithi070.stdout: "[819200000, 1638399999]", 2024-09-14T03:47:51.652 INFO:tasks.workunit.client.0.smithi070.stdout: "[1638400000, 3276799999]", 2024-09-14T03:47:51.652 INFO:tasks.workunit.client.0.smithi070.stdout: "[3276800000, 6553599999]", 2024-09-14T03:47:51.652 INFO:tasks.workunit.client.0.smithi070.stdout: "[6553600000, 13107199999]", 2024-09-14T03:47:51.652 INFO:tasks.workunit.client.0.smithi070.stdout: "[13107200000, 26214399999]", 2024-09-14T03:47:51.652 INFO:tasks.workunit.client.0.smithi070.stdout: "[26214400000, 52428799999]", 2024-09-14T03:47:51.652 INFO:tasks.workunit.client.0.smithi070.stdout: "[52428800000, 104857599999]", 2024-09-14T03:47:51.652 INFO:tasks.workunit.client.0.smithi070.stdout: "[104857600000, 209715199999]", 2024-09-14T03:47:51.652 INFO:tasks.workunit.client.0.smithi070.stdout: "[209715200000, 419430399999]", 2024-09-14T03:47:51.652 INFO:tasks.workunit.client.0.smithi070.stdout: "[419430400000, 838860799999]", 2024-09-14T03:47:51.652 INFO:tasks.workunit.client.0.smithi070.stdout: "[838860800000, 1677721599999]", 2024-09-14T03:47:51.652 INFO:tasks.workunit.client.0.smithi070.stdout: "[1677721600000, 3355443199999]", 2024-09-14T03:47:51.652 INFO:tasks.workunit.client.0.smithi070.stdout: "[3355443200000, 6710886399999]", 2024-09-14T03:47:51.652 INFO:tasks.workunit.client.0.smithi070.stdout: "[6710886400000, 13421772799999]", 2024-09-14T03:47:51.653 INFO:tasks.workunit.client.0.smithi070.stdout: "[13421772800000, 26843545599999]", 2024-09-14T03:47:51.653 INFO:tasks.workunit.client.0.smithi070.stdout: "[26843545600000, 53687091199999]", 2024-09-14T03:47:51.653 INFO:tasks.workunit.client.0.smithi070.stdout: "[53687091200000, None]" 2024-09-14T03:47:51.653 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.653 INFO:tasks.workunit.client.0.smithi070.stdout: "scale_type": "log2" 2024-09-14T03:47:51.653 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.653 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.653 INFO:tasks.workunit.client.0.smithi070.stdout: "buckets": 32, 2024-09-14T03:47:51.653 INFO:tasks.workunit.client.0.smithi070.stdout: "min": 0, 2024-09-14T03:47:51.653 INFO:tasks.workunit.client.0.smithi070.stdout: "name": "Request size (bytes)", 2024-09-14T03:47:51.653 INFO:tasks.workunit.client.0.smithi070.stdout: "quant_size": 512, 2024-09-14T03:47:51.653 INFO:tasks.workunit.client.0.smithi070.stdout: "ranges": [ 2024-09-14T03:47:51.654 INFO:tasks.workunit.client.0.smithi070.stdout: "[None, -1]", 2024-09-14T03:47:51.654 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 511]", 2024-09-14T03:47:51.654 INFO:tasks.workunit.client.0.smithi070.stdout: "[512, 1023]", 2024-09-14T03:47:51.654 INFO:tasks.workunit.client.0.smithi070.stdout: "[1024, 2047]", 2024-09-14T03:47:51.654 INFO:tasks.workunit.client.0.smithi070.stdout: "[2048, 4095]", 2024-09-14T03:47:51.654 INFO:tasks.workunit.client.0.smithi070.stdout: "[4096, 8191]", 2024-09-14T03:47:51.654 INFO:tasks.workunit.client.0.smithi070.stdout: "[8192, 16383]", 2024-09-14T03:47:51.654 INFO:tasks.workunit.client.0.smithi070.stdout: "[16384, 32767]", 2024-09-14T03:47:51.654 INFO:tasks.workunit.client.0.smithi070.stdout: "[32768, 65535]", 2024-09-14T03:47:51.654 INFO:tasks.workunit.client.0.smithi070.stdout: "[65536, 131071]", 2024-09-14T03:47:51.654 INFO:tasks.workunit.client.0.smithi070.stdout: "[131072, 262143]", 2024-09-14T03:47:51.654 INFO:tasks.workunit.client.0.smithi070.stdout: "[262144, 524287]", 2024-09-14T03:47:51.654 INFO:tasks.workunit.client.0.smithi070.stdout: "[524288, 1048575]", 2024-09-14T03:47:51.655 INFO:tasks.workunit.client.0.smithi070.stdout: "[1048576, 2097151]", 2024-09-14T03:47:51.655 INFO:tasks.workunit.client.0.smithi070.stdout: "[2097152, 4194303]", 2024-09-14T03:47:51.655 INFO:tasks.workunit.client.0.smithi070.stdout: "[4194304, 8388607]", 2024-09-14T03:47:51.655 INFO:tasks.workunit.client.0.smithi070.stdout: "[8388608, 16777215]", 2024-09-14T03:47:51.655 INFO:tasks.workunit.client.0.smithi070.stdout: "[16777216, 33554431]", 2024-09-14T03:47:51.655 INFO:tasks.workunit.client.0.smithi070.stdout: "[33554432, 67108863]", 2024-09-14T03:47:51.655 INFO:tasks.workunit.client.0.smithi070.stdout: "[67108864, 134217727]", 2024-09-14T03:47:51.655 INFO:tasks.workunit.client.0.smithi070.stdout: "[134217728, 268435455]", 2024-09-14T03:47:51.655 INFO:tasks.workunit.client.0.smithi070.stdout: "[268435456, 536870911]", 2024-09-14T03:47:51.655 INFO:tasks.workunit.client.0.smithi070.stdout: "[536870912, 1073741823]", 2024-09-14T03:47:51.655 INFO:tasks.workunit.client.0.smithi070.stdout: "[1073741824, 2147483647]", 2024-09-14T03:47:51.656 INFO:tasks.workunit.client.0.smithi070.stdout: "[2147483648, 4294967295]", 2024-09-14T03:47:51.656 INFO:tasks.workunit.client.0.smithi070.stdout: "[4294967296, 8589934591]", 2024-09-14T03:47:51.656 INFO:tasks.workunit.client.0.smithi070.stdout: "[8589934592, 17179869183]", 2024-09-14T03:47:51.656 INFO:tasks.workunit.client.0.smithi070.stdout: "[17179869184, 34359738367]", 2024-09-14T03:47:51.656 INFO:tasks.workunit.client.0.smithi070.stdout: "[34359738368, 68719476735]", 2024-09-14T03:47:51.656 INFO:tasks.workunit.client.0.smithi070.stdout: "[68719476736, 137438953471]", 2024-09-14T03:47:51.656 INFO:tasks.workunit.client.0.smithi070.stdout: "[137438953472, 274877906943]", 2024-09-14T03:47:51.656 INFO:tasks.workunit.client.0.smithi070.stdout: "[274877906944, None]" 2024-09-14T03:47:51.656 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.656 INFO:tasks.workunit.client.0.smithi070.stdout: "scale_type": "log2" 2024-09-14T03:47:51.656 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.656 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.657 INFO:tasks.workunit.client.0.smithi070.stdout: "osds": [ 2024-09-14T03:47:51.657 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.657 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 0, 2024-09-14T03:47:51.657 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:51.657 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.657 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.657 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.657 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.657 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.657 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.657 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.657 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.658 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.658 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.658 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.658 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.658 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.658 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.658 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.658 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.658 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.658 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.658 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.659 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.659 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.659 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.659 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.659 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.659 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.659 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.659 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.659 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.659 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.660 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.660 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.660 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:51.660 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.660 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.660 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.660 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 1, 2024-09-14T03:47:51.660 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:51.660 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.660 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.660 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.660 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.661 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.661 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.661 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.661 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.661 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.661 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.661 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.661 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.661 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.661 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.662 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.662 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.662 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.662 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.662 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.662 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.662 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.662 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.662 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.662 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.662 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.662 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.663 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.663 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.663 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.663 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.663 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.663 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:51.663 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.663 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.663 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.663 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 2, 2024-09-14T03:47:51.663 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:51.664 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.664 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.664 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.664 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.664 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.664 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.664 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.664 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.664 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.664 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.664 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.665 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.665 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.665 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.665 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.665 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.665 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.665 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.665 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.665 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.665 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.665 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.666 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.666 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.666 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.666 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.666 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.666 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.666 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.666 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.666 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.666 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:51.666 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.666 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.666 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.667 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 3, 2024-09-14T03:47:51.667 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:51.667 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.667 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.667 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.667 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.667 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.667 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.667 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.667 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.667 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.667 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.668 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.668 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.668 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.668 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.668 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.668 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.668 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.668 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.668 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.668 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.668 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.669 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.669 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.669 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.669 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.669 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.669 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.669 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.669 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.669 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.669 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.669 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:51.670 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.670 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.670 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.670 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 4, 2024-09-14T03:47:51.670 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:51.670 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.670 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.670 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.670 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.670 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.670 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.671 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.671 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.671 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.671 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.671 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.671 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.671 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.671 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.671 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.671 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.671 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.671 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.672 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.672 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.672 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.672 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.672 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.672 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.672 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.672 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.672 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.672 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.672 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.672 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.673 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.673 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:51.673 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.673 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.673 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.673 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 5, 2024-09-14T03:47:51.673 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:51.673 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.673 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.673 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.673 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.673 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.674 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.674 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.674 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.674 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.674 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.674 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.674 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.674 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.674 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.674 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.674 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.674 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.675 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.675 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.675 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.675 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.675 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.675 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.675 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.675 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.675 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.675 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.675 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.675 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.676 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.676 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.676 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:51.676 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.676 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.676 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.676 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 6, 2024-09-14T03:47:51.676 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:51.676 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.676 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.676 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.677 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.677 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.677 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.677 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.677 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.677 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.677 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.677 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.677 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.677 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.677 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.677 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.678 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.678 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.678 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.678 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.678 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.678 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.678 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.678 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.678 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.678 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.678 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.679 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.679 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.679 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.679 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.679 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.679 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:51.679 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.679 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.679 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.679 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 7, 2024-09-14T03:47:51.679 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:51.680 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.680 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.680 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.680 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.680 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.680 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.680 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.680 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.680 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.680 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.680 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.680 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.681 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.681 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.681 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.681 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.681 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.681 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.681 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.681 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.681 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.681 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.681 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.681 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.681 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.682 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.682 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.682 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.682 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.682 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.682 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.682 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:51.682 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.682 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.682 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.682 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.683 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:47:51.683 INFO:tasks.workunit.client.0.smithi070.stdout: "axes": [ 2024-09-14T03:47:51.683 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.683 INFO:tasks.workunit.client.0.smithi070.stdout: "buckets": 32, 2024-09-14T03:47:51.683 INFO:tasks.workunit.client.0.smithi070.stdout: "min": 0, 2024-09-14T03:47:51.683 INFO:tasks.workunit.client.0.smithi070.stdout: "name": "Latency (usec)", 2024-09-14T03:47:51.683 INFO:tasks.workunit.client.0.smithi070.stdout: "quant_size": 100000, 2024-09-14T03:47:51.683 INFO:tasks.workunit.client.0.smithi070.stdout: "ranges": [ 2024-09-14T03:47:51.683 INFO:tasks.workunit.client.0.smithi070.stdout: "[None, -1]", 2024-09-14T03:47:51.683 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 99999]", 2024-09-14T03:47:51.683 INFO:tasks.workunit.client.0.smithi070.stdout: "[100000, 199999]", 2024-09-14T03:47:51.683 INFO:tasks.workunit.client.0.smithi070.stdout: "[200000, 399999]", 2024-09-14T03:47:51.683 INFO:tasks.workunit.client.0.smithi070.stdout: "[400000, 799999]", 2024-09-14T03:47:51.684 INFO:tasks.workunit.client.0.smithi070.stdout: "[800000, 1599999]", 2024-09-14T03:47:51.684 INFO:tasks.workunit.client.0.smithi070.stdout: "[1600000, 3199999]", 2024-09-14T03:47:51.684 INFO:tasks.workunit.client.0.smithi070.stdout: "[3200000, 6399999]", 2024-09-14T03:47:51.684 INFO:tasks.workunit.client.0.smithi070.stdout: "[6400000, 12799999]", 2024-09-14T03:47:51.684 INFO:tasks.workunit.client.0.smithi070.stdout: "[12800000, 25599999]", 2024-09-14T03:47:51.684 INFO:tasks.workunit.client.0.smithi070.stdout: "[25600000, 51199999]", 2024-09-14T03:47:51.684 INFO:tasks.workunit.client.0.smithi070.stdout: "[51200000, 102399999]", 2024-09-14T03:47:51.684 INFO:tasks.workunit.client.0.smithi070.stdout: "[102400000, 204799999]", 2024-09-14T03:47:51.684 INFO:tasks.workunit.client.0.smithi070.stdout: "[204800000, 409599999]", 2024-09-14T03:47:51.684 INFO:tasks.workunit.client.0.smithi070.stdout: "[409600000, 819199999]", 2024-09-14T03:47:51.684 INFO:tasks.workunit.client.0.smithi070.stdout: "[819200000, 1638399999]", 2024-09-14T03:47:51.685 INFO:tasks.workunit.client.0.smithi070.stdout: "[1638400000, 3276799999]", 2024-09-14T03:47:51.685 INFO:tasks.workunit.client.0.smithi070.stdout: "[3276800000, 6553599999]", 2024-09-14T03:47:51.685 INFO:tasks.workunit.client.0.smithi070.stdout: "[6553600000, 13107199999]", 2024-09-14T03:47:51.685 INFO:tasks.workunit.client.0.smithi070.stdout: "[13107200000, 26214399999]", 2024-09-14T03:47:51.685 INFO:tasks.workunit.client.0.smithi070.stdout: "[26214400000, 52428799999]", 2024-09-14T03:47:51.685 INFO:tasks.workunit.client.0.smithi070.stdout: "[52428800000, 104857599999]", 2024-09-14T03:47:51.685 INFO:tasks.workunit.client.0.smithi070.stdout: "[104857600000, 209715199999]", 2024-09-14T03:47:51.685 INFO:tasks.workunit.client.0.smithi070.stdout: "[209715200000, 419430399999]", 2024-09-14T03:47:51.685 INFO:tasks.workunit.client.0.smithi070.stdout: "[419430400000, 838860799999]", 2024-09-14T03:47:51.685 INFO:tasks.workunit.client.0.smithi070.stdout: "[838860800000, 1677721599999]", 2024-09-14T03:47:51.685 INFO:tasks.workunit.client.0.smithi070.stdout: "[1677721600000, 3355443199999]", 2024-09-14T03:47:51.686 INFO:tasks.workunit.client.0.smithi070.stdout: "[3355443200000, 6710886399999]", 2024-09-14T03:47:51.686 INFO:tasks.workunit.client.0.smithi070.stdout: "[6710886400000, 13421772799999]", 2024-09-14T03:47:51.686 INFO:tasks.workunit.client.0.smithi070.stdout: "[13421772800000, 26843545599999]", 2024-09-14T03:47:51.686 INFO:tasks.workunit.client.0.smithi070.stdout: "[26843545600000, 53687091199999]", 2024-09-14T03:47:51.686 INFO:tasks.workunit.client.0.smithi070.stdout: "[53687091200000, None]" 2024-09-14T03:47:51.686 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.686 INFO:tasks.workunit.client.0.smithi070.stdout: "scale_type": "log2" 2024-09-14T03:47:51.686 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.686 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.686 INFO:tasks.workunit.client.0.smithi070.stdout: "buckets": 32, 2024-09-14T03:47:51.686 INFO:tasks.workunit.client.0.smithi070.stdout: "min": 0, 2024-09-14T03:47:51.686 INFO:tasks.workunit.client.0.smithi070.stdout: "name": "Request size (bytes)", 2024-09-14T03:47:51.687 INFO:tasks.workunit.client.0.smithi070.stdout: "quant_size": 512, 2024-09-14T03:47:51.687 INFO:tasks.workunit.client.0.smithi070.stdout: "ranges": [ 2024-09-14T03:47:51.687 INFO:tasks.workunit.client.0.smithi070.stdout: "[None, -1]", 2024-09-14T03:47:51.687 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 511]", 2024-09-14T03:47:51.687 INFO:tasks.workunit.client.0.smithi070.stdout: "[512, 1023]", 2024-09-14T03:47:51.687 INFO:tasks.workunit.client.0.smithi070.stdout: "[1024, 2047]", 2024-09-14T03:47:51.687 INFO:tasks.workunit.client.0.smithi070.stdout: "[2048, 4095]", 2024-09-14T03:47:51.687 INFO:tasks.workunit.client.0.smithi070.stdout: "[4096, 8191]", 2024-09-14T03:47:51.687 INFO:tasks.workunit.client.0.smithi070.stdout: "[8192, 16383]", 2024-09-14T03:47:51.687 INFO:tasks.workunit.client.0.smithi070.stdout: "[16384, 32767]", 2024-09-14T03:47:51.687 INFO:tasks.workunit.client.0.smithi070.stdout: "[32768, 65535]", 2024-09-14T03:47:51.687 INFO:tasks.workunit.client.0.smithi070.stdout: "[65536, 131071]", 2024-09-14T03:47:51.688 INFO:tasks.workunit.client.0.smithi070.stdout: "[131072, 262143]", 2024-09-14T03:47:51.688 INFO:tasks.workunit.client.0.smithi070.stdout: "[262144, 524287]", 2024-09-14T03:47:51.688 INFO:tasks.workunit.client.0.smithi070.stdout: "[524288, 1048575]", 2024-09-14T03:47:51.688 INFO:tasks.workunit.client.0.smithi070.stdout: "[1048576, 2097151]", 2024-09-14T03:47:51.688 INFO:tasks.workunit.client.0.smithi070.stdout: "[2097152, 4194303]", 2024-09-14T03:47:51.688 INFO:tasks.workunit.client.0.smithi070.stdout: "[4194304, 8388607]", 2024-09-14T03:47:51.688 INFO:tasks.workunit.client.0.smithi070.stdout: "[8388608, 16777215]", 2024-09-14T03:47:51.688 INFO:tasks.workunit.client.0.smithi070.stdout: "[16777216, 33554431]", 2024-09-14T03:47:51.688 INFO:tasks.workunit.client.0.smithi070.stdout: "[33554432, 67108863]", 2024-09-14T03:47:51.688 INFO:tasks.workunit.client.0.smithi070.stdout: "[67108864, 134217727]", 2024-09-14T03:47:51.688 INFO:tasks.workunit.client.0.smithi070.stdout: "[134217728, 268435455]", 2024-09-14T03:47:51.689 INFO:tasks.workunit.client.0.smithi070.stdout: "[268435456, 536870911]", 2024-09-14T03:47:51.689 INFO:tasks.workunit.client.0.smithi070.stdout: "[536870912, 1073741823]", 2024-09-14T03:47:51.689 INFO:tasks.workunit.client.0.smithi070.stdout: "[1073741824, 2147483647]", 2024-09-14T03:47:51.689 INFO:tasks.workunit.client.0.smithi070.stdout: "[2147483648, 4294967295]", 2024-09-14T03:47:51.689 INFO:tasks.workunit.client.0.smithi070.stdout: "[4294967296, 8589934591]", 2024-09-14T03:47:51.689 INFO:tasks.workunit.client.0.smithi070.stdout: "[8589934592, 17179869183]", 2024-09-14T03:47:51.689 INFO:tasks.workunit.client.0.smithi070.stdout: "[17179869184, 34359738367]", 2024-09-14T03:47:51.689 INFO:tasks.workunit.client.0.smithi070.stdout: "[34359738368, 68719476735]", 2024-09-14T03:47:51.689 INFO:tasks.workunit.client.0.smithi070.stdout: "[68719476736, 137438953471]", 2024-09-14T03:47:51.689 INFO:tasks.workunit.client.0.smithi070.stdout: "[137438953472, 274877906943]", 2024-09-14T03:47:51.689 INFO:tasks.workunit.client.0.smithi070.stdout: "[274877906944, None]" 2024-09-14T03:47:51.690 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.690 INFO:tasks.workunit.client.0.smithi070.stdout: "scale_type": "log2" 2024-09-14T03:47:51.690 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.690 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.690 INFO:tasks.workunit.client.0.smithi070.stdout: "osds": [ 2024-09-14T03:47:51.690 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.690 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 0, 2024-09-14T03:47:51.690 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:51.690 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.690 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.690 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.691 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.691 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.691 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.691 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.691 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.691 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.691 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.691 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.691 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.691 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.691 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.691 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.692 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.692 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.692 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.692 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.692 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.692 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.692 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.692 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.692 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.692 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.692 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.693 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.693 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.693 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.693 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.693 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.693 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:51.693 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.693 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.693 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.693 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 1, 2024-09-14T03:47:51.693 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:51.693 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.694 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.694 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.694 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.694 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.694 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 4, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.694 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 6, 2, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.694 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 11, 1, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.694 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 9, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.694 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.694 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.694 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.694 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.694 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.695 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.695 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.695 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.695 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.695 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.695 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.695 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.695 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.695 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.695 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.695 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.695 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.696 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.696 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.696 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.696 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.696 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.696 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:51.696 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.696 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.696 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.696 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 2, 2024-09-14T03:47:51.696 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:51.696 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.697 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.697 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.697 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.697 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.697 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.697 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.697 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.697 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.697 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.697 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.697 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.697 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.698 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.698 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.698 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.698 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.698 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.698 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.698 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.698 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.698 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.698 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.698 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.698 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.699 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.699 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.699 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.699 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.699 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.699 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.699 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:51.699 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.699 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.699 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.700 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 3, 2024-09-14T03:47:51.700 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:51.700 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.700 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.700 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.700 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.700 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.700 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.700 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.700 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.700 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.701 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.701 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.701 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.701 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.701 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.701 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.701 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.701 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.701 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.701 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.701 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.701 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.702 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.702 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.702 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.702 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.702 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.702 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.702 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.702 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.702 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.702 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.702 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:51.702 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.702 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.702 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.703 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 4, 2024-09-14T03:47:51.703 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:51.703 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.703 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.703 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.703 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.703 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.703 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.703 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.703 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.703 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.704 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.704 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.704 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.704 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.704 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.704 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.704 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.704 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.704 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.704 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.704 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.704 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.704 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.705 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.705 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.705 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.705 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.705 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.705 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.705 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.705 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.705 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.705 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:51.705 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.705 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.706 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.706 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 5, 2024-09-14T03:47:51.706 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:51.706 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.706 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.706 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.706 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.706 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.706 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.706 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.706 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.706 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.706 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.707 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.707 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.707 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.707 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.707 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.707 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.707 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.707 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.707 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.707 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.707 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.707 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.707 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.708 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.708 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.708 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.708 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.708 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.708 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.708 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.708 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.708 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:51.708 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.708 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.708 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.709 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 6, 2024-09-14T03:47:51.709 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:51.709 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.709 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.709 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.709 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.709 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.709 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.709 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.709 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.710 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.710 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.710 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.710 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.710 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.710 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.710 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.710 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.710 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.710 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.710 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.710 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.711 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.711 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.711 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.711 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.711 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.711 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.711 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.711 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.711 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.711 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.711 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.711 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:51.712 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.712 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.712 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:51.712 INFO:tasks.workunit.client.0.smithi070.stdout: "osd_id": 7, 2024-09-14T03:47:51.712 INFO:tasks.workunit.client.0.smithi070.stdout: "values": [ 2024-09-14T03:47:51.712 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.712 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.712 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.712 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.712 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.712 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.713 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.713 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.713 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.713 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.713 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.713 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.713 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.713 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.713 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.713 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.713 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.713 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.714 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.714 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.714 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.714 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.714 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.714 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.714 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.714 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.714 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.714 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.714 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.715 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.715 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-14T03:47:51.715 INFO:tasks.workunit.client.0.smithi070.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-14T03:47:51.715 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.715 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.715 INFO:tasks.workunit.client.0.smithi070.stdout: ] 2024-09-14T03:47:51.715 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.715 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.715 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:51.715 INFO:tasks.workunit.client.0.smithi070.stdout: "perf_counters": { 2024-09-14T03:47:51.715 INFO:tasks.workunit.client.0.smithi070.stdout: "mds.a8e17c6c2bb7d35df7944161574528c366e6a388": { 2024-09-14T03:47:51.715 INFO:tasks.workunit.client.0.smithi070.stdout: "mds": { 2024-09-14T03:47:51.716 INFO:tasks.workunit.client.0.smithi070.stdout: "caps": { 2024-09-14T03:47:51.716 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.716 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.716 INFO:tasks.workunit.client.0.smithi070.stdout: "ceph_cap_op_flush_ack": { 2024-09-14T03:47:51.716 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.716 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.716 INFO:tasks.workunit.client.0.smithi070.stdout: "ceph_cap_op_flushsnap_ack": { 2024-09-14T03:47:51.716 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.716 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.716 INFO:tasks.workunit.client.0.smithi070.stdout: "ceph_cap_op_grant": { 2024-09-14T03:47:51.716 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.716 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.716 INFO:tasks.workunit.client.0.smithi070.stdout: "ceph_cap_op_revoke": { 2024-09-14T03:47:51.717 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.717 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.717 INFO:tasks.workunit.client.0.smithi070.stdout: "ceph_cap_op_trunc": { 2024-09-14T03:47:51.717 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.717 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.717 INFO:tasks.workunit.client.0.smithi070.stdout: "dir_commit": { 2024-09-14T03:47:51.717 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12 2024-09-14T03:47:51.717 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.717 INFO:tasks.workunit.client.0.smithi070.stdout: "dir_fetch": { 2024-09-14T03:47:51.717 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.718 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.718 INFO:tasks.workunit.client.0.smithi070.stdout: "dir_merge": { 2024-09-14T03:47:51.718 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.718 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.718 INFO:tasks.workunit.client.0.smithi070.stdout: "dir_split": { 2024-09-14T03:47:51.718 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.718 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.718 INFO:tasks.workunit.client.0.smithi070.stdout: "exported_inodes": { 2024-09-14T03:47:51.718 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.718 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.718 INFO:tasks.workunit.client.0.smithi070.stdout: "forward": { 2024-09-14T03:47:51.719 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.719 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.719 INFO:tasks.workunit.client.0.smithi070.stdout: "handle_client_cap_release": { 2024-09-14T03:47:51.719 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.719 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.719 INFO:tasks.workunit.client.0.smithi070.stdout: "handle_client_caps": { 2024-09-14T03:47:51.719 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.719 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.719 INFO:tasks.workunit.client.0.smithi070.stdout: "handle_client_caps_dirty": { 2024-09-14T03:47:51.719 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.719 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.720 INFO:tasks.workunit.client.0.smithi070.stdout: "handle_inode_file_caps": { 2024-09-14T03:47:51.720 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.720 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.720 INFO:tasks.workunit.client.0.smithi070.stdout: "imported_inodes": { 2024-09-14T03:47:51.720 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.720 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.720 INFO:tasks.workunit.client.0.smithi070.stdout: "inodes": { 2024-09-14T03:47:51.720 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 10 2024-09-14T03:47:51.720 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.720 INFO:tasks.workunit.client.0.smithi070.stdout: "inodes_expired": { 2024-09-14T03:47:51.720 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.720 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.721 INFO:tasks.workunit.client.0.smithi070.stdout: "inodes_pinned": { 2024-09-14T03:47:51.721 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 10 2024-09-14T03:47:51.721 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.721 INFO:tasks.workunit.client.0.smithi070.stdout: "inodes_with_caps": { 2024-09-14T03:47:51.721 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.721 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.721 INFO:tasks.workunit.client.0.smithi070.stdout: "load_cent": { 2024-09-14T03:47:51.721 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.721 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.721 INFO:tasks.workunit.client.0.smithi070.stdout: "openino_dir_fetch": { 2024-09-14T03:47:51.721 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.721 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.722 INFO:tasks.workunit.client.0.smithi070.stdout: "process_request_cap_release": { 2024-09-14T03:47:51.722 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.722 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.722 INFO:tasks.workunit.client.0.smithi070.stdout: "reply_latency": { 2024-09-14T03:47:51.722 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:51.722 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 125024 2024-09-14T03:47:51.722 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.722 INFO:tasks.workunit.client.0.smithi070.stdout: "request": { 2024-09-14T03:47:51.722 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:51.722 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.722 INFO:tasks.workunit.client.0.smithi070.stdout: "root_rbytes": { 2024-09-14T03:47:51.722 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.723 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.723 INFO:tasks.workunit.client.0.smithi070.stdout: "root_rfiles": { 2024-09-14T03:47:51.723 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.723 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.723 INFO:tasks.workunit.client.0.smithi070.stdout: "root_rsnaps": { 2024-09-14T03:47:51.723 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.723 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.723 INFO:tasks.workunit.client.0.smithi070.stdout: "slow_reply": { 2024-09-14T03:47:51.723 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.723 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.723 INFO:tasks.workunit.client.0.smithi070.stdout: "subtrees": { 2024-09-14T03:47:51.723 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:51.724 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.724 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.724 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_cache": { 2024-09-14T03:47:51.724 INFO:tasks.workunit.client.0.smithi070.stdout: "ireq_enqueue_scrub": { 2024-09-14T03:47:51.724 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.724 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.724 INFO:tasks.workunit.client.0.smithi070.stdout: "ireq_exportdir": { 2024-09-14T03:47:51.724 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.724 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.724 INFO:tasks.workunit.client.0.smithi070.stdout: "ireq_flush": { 2024-09-14T03:47:51.724 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.724 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.725 INFO:tasks.workunit.client.0.smithi070.stdout: "ireq_fragmentdir": { 2024-09-14T03:47:51.725 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.725 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.725 INFO:tasks.workunit.client.0.smithi070.stdout: "ireq_fragstats": { 2024-09-14T03:47:51.725 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.725 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.725 INFO:tasks.workunit.client.0.smithi070.stdout: "ireq_inodestats": { 2024-09-14T03:47:51.725 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.725 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.725 INFO:tasks.workunit.client.0.smithi070.stdout: "num_recovering_enqueued": { 2024-09-14T03:47:51.725 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.725 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.725 INFO:tasks.workunit.client.0.smithi070.stdout: "num_recovering_prioritized": { 2024-09-14T03:47:51.726 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.726 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.726 INFO:tasks.workunit.client.0.smithi070.stdout: "num_recovering_processing": { 2024-09-14T03:47:51.726 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.726 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.726 INFO:tasks.workunit.client.0.smithi070.stdout: "num_strays": { 2024-09-14T03:47:51.726 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.726 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.726 INFO:tasks.workunit.client.0.smithi070.stdout: "num_strays_delayed": { 2024-09-14T03:47:51.726 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.726 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.726 INFO:tasks.workunit.client.0.smithi070.stdout: "num_strays_enqueuing": { 2024-09-14T03:47:51.726 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.727 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.727 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_completed": { 2024-09-14T03:47:51.727 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.727 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.727 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_started": { 2024-09-14T03:47:51.727 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.727 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.727 INFO:tasks.workunit.client.0.smithi070.stdout: "strays_created": { 2024-09-14T03:47:51.727 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.727 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.727 INFO:tasks.workunit.client.0.smithi070.stdout: "strays_enqueued": { 2024-09-14T03:47:51.727 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.728 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.728 INFO:tasks.workunit.client.0.smithi070.stdout: "strays_migrated": { 2024-09-14T03:47:51.728 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.728 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.728 INFO:tasks.workunit.client.0.smithi070.stdout: "strays_reintegrated": { 2024-09-14T03:47:51.728 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.728 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.728 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.728 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_log": { 2024-09-14T03:47:51.728 INFO:tasks.workunit.client.0.smithi070.stdout: "ev": { 2024-09-14T03:47:51.728 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5 2024-09-14T03:47:51.728 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.729 INFO:tasks.workunit.client.0.smithi070.stdout: "evadd": { 2024-09-14T03:47:51.729 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5 2024-09-14T03:47:51.729 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.729 INFO:tasks.workunit.client.0.smithi070.stdout: "evex": { 2024-09-14T03:47:51.729 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.729 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.729 INFO:tasks.workunit.client.0.smithi070.stdout: "evexd": { 2024-09-14T03:47:51.729 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.729 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.729 INFO:tasks.workunit.client.0.smithi070.stdout: "evexg": { 2024-09-14T03:47:51.729 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.730 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.730 INFO:tasks.workunit.client.0.smithi070.stdout: "evtrm": { 2024-09-14T03:47:51.730 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.730 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.730 INFO:tasks.workunit.client.0.smithi070.stdout: "jlat": { 2024-09-14T03:47:51.730 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 5, 2024-09-14T03:47:51.730 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12472377 2024-09-14T03:47:51.730 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.730 INFO:tasks.workunit.client.0.smithi070.stdout: "replayed": { 2024-09-14T03:47:51.730 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.730 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.731 INFO:tasks.workunit.client.0.smithi070.stdout: "seg": { 2024-09-14T03:47:51.731 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1 2024-09-14T03:47:51.731 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.731 INFO:tasks.workunit.client.0.smithi070.stdout: "segadd": { 2024-09-14T03:47:51.731 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1 2024-09-14T03:47:51.731 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.731 INFO:tasks.workunit.client.0.smithi070.stdout: "segex": { 2024-09-14T03:47:51.731 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.731 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.731 INFO:tasks.workunit.client.0.smithi070.stdout: "segexd": { 2024-09-14T03:47:51.731 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.732 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.732 INFO:tasks.workunit.client.0.smithi070.stdout: "segexg": { 2024-09-14T03:47:51.732 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.732 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.732 INFO:tasks.workunit.client.0.smithi070.stdout: "segtrm": { 2024-09-14T03:47:51.732 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.732 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.732 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.732 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_mem": { 2024-09-14T03:47:51.732 INFO:tasks.workunit.client.0.smithi070.stdout: "cap": { 2024-09-14T03:47:51.732 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.733 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.733 INFO:tasks.workunit.client.0.smithi070.stdout: "cap+": { 2024-09-14T03:47:51.733 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:51.733 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.733 INFO:tasks.workunit.client.0.smithi070.stdout: "cap-": { 2024-09-14T03:47:51.733 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:51.733 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.733 INFO:tasks.workunit.client.0.smithi070.stdout: "dir": { 2024-09-14T03:47:51.733 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12 2024-09-14T03:47:51.733 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.733 INFO:tasks.workunit.client.0.smithi070.stdout: "dir+": { 2024-09-14T03:47:51.733 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12 2024-09-14T03:47:51.734 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.734 INFO:tasks.workunit.client.0.smithi070.stdout: "dir-": { 2024-09-14T03:47:51.734 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.734 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.734 INFO:tasks.workunit.client.0.smithi070.stdout: "dn": { 2024-09-14T03:47:51.734 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 10 2024-09-14T03:47:51.734 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.734 INFO:tasks.workunit.client.0.smithi070.stdout: "dn+": { 2024-09-14T03:47:51.734 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 10 2024-09-14T03:47:51.734 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.734 INFO:tasks.workunit.client.0.smithi070.stdout: "dn-": { 2024-09-14T03:47:51.734 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.735 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.735 INFO:tasks.workunit.client.0.smithi070.stdout: "heap": { 2024-09-14T03:47:51.735 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 207068 2024-09-14T03:47:51.735 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.735 INFO:tasks.workunit.client.0.smithi070.stdout: "ino": { 2024-09-14T03:47:51.735 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 13 2024-09-14T03:47:51.735 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.735 INFO:tasks.workunit.client.0.smithi070.stdout: "ino+": { 2024-09-14T03:47:51.735 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 13 2024-09-14T03:47:51.735 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.735 INFO:tasks.workunit.client.0.smithi070.stdout: "ino-": { 2024-09-14T03:47:51.736 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.736 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.736 INFO:tasks.workunit.client.0.smithi070.stdout: "rss": { 2024-09-14T03:47:51.736 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 38060 2024-09-14T03:47:51.736 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.736 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.736 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_server": { 2024-09-14T03:47:51.736 INFO:tasks.workunit.client.0.smithi070.stdout: "cap_acquisition_throttle": { 2024-09-14T03:47:51.736 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.736 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.736 INFO:tasks.workunit.client.0.smithi070.stdout: "cap_revoke_eviction": { 2024-09-14T03:47:51.737 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.737 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.737 INFO:tasks.workunit.client.0.smithi070.stdout: "handle_client_request": { 2024-09-14T03:47:51.737 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:51.737 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.737 INFO:tasks.workunit.client.0.smithi070.stdout: "handle_client_session": { 2024-09-14T03:47:51.737 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 16 2024-09-14T03:47:51.737 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.737 INFO:tasks.workunit.client.0.smithi070.stdout: "handle_peer_request": { 2024-09-14T03:47:51.737 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.737 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.737 INFO:tasks.workunit.client.0.smithi070.stdout: "req_create_latency": { 2024-09-14T03:47:51.737 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.738 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.738 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.738 INFO:tasks.workunit.client.0.smithi070.stdout: "req_getattr_latency": { 2024-09-14T03:47:51.738 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:51.738 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 125024 2024-09-14T03:47:51.738 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.738 INFO:tasks.workunit.client.0.smithi070.stdout: "req_getfilelock_latency": { 2024-09-14T03:47:51.738 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.738 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.738 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.738 INFO:tasks.workunit.client.0.smithi070.stdout: "req_getvxattr_latency": { 2024-09-14T03:47:51.738 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.739 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.739 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.739 INFO:tasks.workunit.client.0.smithi070.stdout: "req_link_latency": { 2024-09-14T03:47:51.739 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.739 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.739 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.739 INFO:tasks.workunit.client.0.smithi070.stdout: "req_lookup_latency": { 2024-09-14T03:47:51.739 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.739 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.739 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.739 INFO:tasks.workunit.client.0.smithi070.stdout: "req_lookuphash_latency": { 2024-09-14T03:47:51.740 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.740 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.740 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.740 INFO:tasks.workunit.client.0.smithi070.stdout: "req_lookupino_latency": { 2024-09-14T03:47:51.740 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.740 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.740 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.740 INFO:tasks.workunit.client.0.smithi070.stdout: "req_lookupname_latency": { 2024-09-14T03:47:51.740 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.740 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.740 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.741 INFO:tasks.workunit.client.0.smithi070.stdout: "req_lookupparent_latency": { 2024-09-14T03:47:51.741 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.741 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.741 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.741 INFO:tasks.workunit.client.0.smithi070.stdout: "req_lookupsnap_latency": { 2024-09-14T03:47:51.741 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.741 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.741 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.741 INFO:tasks.workunit.client.0.smithi070.stdout: "req_lssnap_latency": { 2024-09-14T03:47:51.741 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.741 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.742 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.742 INFO:tasks.workunit.client.0.smithi070.stdout: "req_mkdir_latency": { 2024-09-14T03:47:51.742 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.742 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.742 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.742 INFO:tasks.workunit.client.0.smithi070.stdout: "req_mknod_latency": { 2024-09-14T03:47:51.742 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.742 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.742 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.742 INFO:tasks.workunit.client.0.smithi070.stdout: "req_mksnap_latency": { 2024-09-14T03:47:51.742 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.743 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.743 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.743 INFO:tasks.workunit.client.0.smithi070.stdout: "req_open_latency": { 2024-09-14T03:47:51.743 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.743 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.743 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.743 INFO:tasks.workunit.client.0.smithi070.stdout: "req_readdir_latency": { 2024-09-14T03:47:51.743 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.743 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.743 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.743 INFO:tasks.workunit.client.0.smithi070.stdout: "req_rename_latency": { 2024-09-14T03:47:51.743 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.744 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.744 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.744 INFO:tasks.workunit.client.0.smithi070.stdout: "req_renamesnap_latency": { 2024-09-14T03:47:51.744 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.744 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.744 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.744 INFO:tasks.workunit.client.0.smithi070.stdout: "req_rmdir_latency": { 2024-09-14T03:47:51.744 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.744 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.744 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.744 INFO:tasks.workunit.client.0.smithi070.stdout: "req_rmsnap_latency": { 2024-09-14T03:47:51.745 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.745 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.745 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.745 INFO:tasks.workunit.client.0.smithi070.stdout: "req_rmxattr_latency": { 2024-09-14T03:47:51.745 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.745 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.745 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.745 INFO:tasks.workunit.client.0.smithi070.stdout: "req_setattr_latency": { 2024-09-14T03:47:51.745 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.745 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.745 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.746 INFO:tasks.workunit.client.0.smithi070.stdout: "req_setdirlayout_latency": { 2024-09-14T03:47:51.746 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.746 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.746 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.746 INFO:tasks.workunit.client.0.smithi070.stdout: "req_setfilelock_latency": { 2024-09-14T03:47:51.746 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.746 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.746 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.746 INFO:tasks.workunit.client.0.smithi070.stdout: "req_setlayout_latency": { 2024-09-14T03:47:51.746 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.746 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.746 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.747 INFO:tasks.workunit.client.0.smithi070.stdout: "req_setxattr_latency": { 2024-09-14T03:47:51.747 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.747 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.747 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.747 INFO:tasks.workunit.client.0.smithi070.stdout: "req_symlink_latency": { 2024-09-14T03:47:51.747 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.747 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.747 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.747 INFO:tasks.workunit.client.0.smithi070.stdout: "req_unlink_latency": { 2024-09-14T03:47:51.747 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.747 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.748 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.748 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.748 INFO:tasks.workunit.client.0.smithi070.stdout: "mds_sessions": { 2024-09-14T03:47:51.748 INFO:tasks.workunit.client.0.smithi070.stdout: "average_load": { 2024-09-14T03:47:51.748 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1 2024-09-14T03:47:51.748 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.748 INFO:tasks.workunit.client.0.smithi070.stdout: "avg_session_uptime": { 2024-09-14T03:47:51.748 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 89 2024-09-14T03:47:51.748 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.748 INFO:tasks.workunit.client.0.smithi070.stdout: "mdthresh_evicted": { 2024-09-14T03:47:51.749 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.749 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.749 INFO:tasks.workunit.client.0.smithi070.stdout: "session_add": { 2024-09-14T03:47:51.749 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:51.749 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.749 INFO:tasks.workunit.client.0.smithi070.stdout: "session_count": { 2024-09-14T03:47:51.749 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.749 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.749 INFO:tasks.workunit.client.0.smithi070.stdout: "session_remove": { 2024-09-14T03:47:51.749 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:51.749 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.750 INFO:tasks.workunit.client.0.smithi070.stdout: "sessions_open": { 2024-09-14T03:47:51.750 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.750 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.750 INFO:tasks.workunit.client.0.smithi070.stdout: "sessions_stale": { 2024-09-14T03:47:51.750 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.750 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.750 INFO:tasks.workunit.client.0.smithi070.stdout: "total_load": { 2024-09-14T03:47:51.750 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:51.750 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.750 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.750 INFO:tasks.workunit.client.0.smithi070.stdout: "objecter": { 2024-09-14T03:47:51.751 INFO:tasks.workunit.client.0.smithi070.stdout: "op_active": { 2024-09-14T03:47:51.751 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.751 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.751 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r": { 2024-09-14T03:47:51.751 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.751 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.751 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rmw": { 2024-09-14T03:47:51.751 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.751 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.751 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w": { 2024-09-14T03:47:51.752 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 45 2024-09-14T03:47:51.752 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.752 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.752 INFO:tasks.workunit.client.0.smithi070.stdout: "purge_queue": { 2024-09-14T03:47:51.752 INFO:tasks.workunit.client.0.smithi070.stdout: "pq_executed": { 2024-09-14T03:47:51.752 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.752 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.752 INFO:tasks.workunit.client.0.smithi070.stdout: "pq_executing": { 2024-09-14T03:47:51.752 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.752 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.752 INFO:tasks.workunit.client.0.smithi070.stdout: "pq_executing_high_water": { 2024-09-14T03:47:51.752 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.753 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.753 INFO:tasks.workunit.client.0.smithi070.stdout: "pq_executing_ops": { 2024-09-14T03:47:51.753 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.753 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.753 INFO:tasks.workunit.client.0.smithi070.stdout: "pq_executing_ops_high_water": { 2024-09-14T03:47:51.753 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.753 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.753 INFO:tasks.workunit.client.0.smithi070.stdout: "pq_item_in_journal": { 2024-09-14T03:47:51.753 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.753 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.753 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.753 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.754 INFO:tasks.workunit.client.0.smithi070.stdout: "mon.a8e17c6c2bb7d35df7944161574528c366e6a388": { 2024-09-14T03:47:51.754 INFO:tasks.workunit.client.0.smithi070.stdout: "mon": { 2024-09-14T03:47:51.754 INFO:tasks.workunit.client.0.smithi070.stdout: "election_call": { 2024-09-14T03:47:51.754 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:51.754 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.754 INFO:tasks.workunit.client.0.smithi070.stdout: "election_lose": { 2024-09-14T03:47:51.754 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.754 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.754 INFO:tasks.workunit.client.0.smithi070.stdout: "election_win": { 2024-09-14T03:47:51.754 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3 2024-09-14T03:47:51.754 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.755 INFO:tasks.workunit.client.0.smithi070.stdout: "num_elections": { 2024-09-14T03:47:51.755 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 9 2024-09-14T03:47:51.755 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.755 INFO:tasks.workunit.client.0.smithi070.stdout: "num_sessions": { 2024-09-14T03:47:51.755 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 8 2024-09-14T03:47:51.755 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.755 INFO:tasks.workunit.client.0.smithi070.stdout: "session_add": { 2024-09-14T03:47:51.755 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3 2024-09-14T03:47:51.755 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.755 INFO:tasks.workunit.client.0.smithi070.stdout: "session_rm": { 2024-09-14T03:47:51.755 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 179 2024-09-14T03:47:51.756 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.756 INFO:tasks.workunit.client.0.smithi070.stdout: "session_trim": { 2024-09-14T03:47:51.756 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.756 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.756 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.756 INFO:tasks.workunit.client.0.smithi070.stdout: "paxos": { 2024-09-14T03:47:51.756 INFO:tasks.workunit.client.0.smithi070.stdout: "accept_timeout": { 2024-09-14T03:47:51.756 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.756 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.756 INFO:tasks.workunit.client.0.smithi070.stdout: "begin": { 2024-09-14T03:47:51.756 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 962 2024-09-14T03:47:51.756 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.757 INFO:tasks.workunit.client.0.smithi070.stdout: "begin_bytes": { 2024-09-14T03:47:51.757 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 962, 2024-09-14T03:47:51.757 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5106207 2024-09-14T03:47:51.757 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.757 INFO:tasks.workunit.client.0.smithi070.stdout: "begin_keys": { 2024-09-14T03:47:51.757 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 962, 2024-09-14T03:47:51.757 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2886 2024-09-14T03:47:51.757 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.757 INFO:tasks.workunit.client.0.smithi070.stdout: "begin_latency": { 2024-09-14T03:47:51.757 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 962, 2024-09-14T03:47:51.757 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 131996113 2024-09-14T03:47:51.757 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.758 INFO:tasks.workunit.client.0.smithi070.stdout: "collect": { 2024-09-14T03:47:51.758 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.758 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.758 INFO:tasks.workunit.client.0.smithi070.stdout: "collect_bytes": { 2024-09-14T03:47:51.758 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.758 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.758 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.758 INFO:tasks.workunit.client.0.smithi070.stdout: "collect_keys": { 2024-09-14T03:47:51.758 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.758 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.758 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.759 INFO:tasks.workunit.client.0.smithi070.stdout: "collect_latency": { 2024-09-14T03:47:51.759 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.759 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.759 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.759 INFO:tasks.workunit.client.0.smithi070.stdout: "collect_timeout": { 2024-09-14T03:47:51.759 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.759 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.759 INFO:tasks.workunit.client.0.smithi070.stdout: "collect_uncommitted": { 2024-09-14T03:47:51.759 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.759 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.759 INFO:tasks.workunit.client.0.smithi070.stdout: "commit": { 2024-09-14T03:47:51.759 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 962 2024-09-14T03:47:51.760 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.760 INFO:tasks.workunit.client.0.smithi070.stdout: "commit_bytes": { 2024-09-14T03:47:51.760 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 962, 2024-09-14T03:47:51.760 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5037001 2024-09-14T03:47:51.760 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.760 INFO:tasks.workunit.client.0.smithi070.stdout: "commit_keys": { 2024-09-14T03:47:51.760 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 962, 2024-09-14T03:47:51.760 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5480 2024-09-14T03:47:51.760 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.760 INFO:tasks.workunit.client.0.smithi070.stdout: "commit_latency": { 2024-09-14T03:47:51.760 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 962, 2024-09-14T03:47:51.760 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 131023916 2024-09-14T03:47:51.761 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.761 INFO:tasks.workunit.client.0.smithi070.stdout: "lease_ack_timeout": { 2024-09-14T03:47:51.761 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.761 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.761 INFO:tasks.workunit.client.0.smithi070.stdout: "lease_timeout": { 2024-09-14T03:47:51.761 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.761 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.761 INFO:tasks.workunit.client.0.smithi070.stdout: "new_pn": { 2024-09-14T03:47:51.761 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:51.761 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.761 INFO:tasks.workunit.client.0.smithi070.stdout: "new_pn_latency": { 2024-09-14T03:47:51.762 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:51.762 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.762 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.762 INFO:tasks.workunit.client.0.smithi070.stdout: "refresh": { 2024-09-14T03:47:51.762 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 964 2024-09-14T03:47:51.762 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.762 INFO:tasks.workunit.client.0.smithi070.stdout: "refresh_latency": { 2024-09-14T03:47:51.762 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 964, 2024-09-14T03:47:51.762 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2283932853 2024-09-14T03:47:51.762 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.762 INFO:tasks.workunit.client.0.smithi070.stdout: "restart": { 2024-09-14T03:47:51.762 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12 2024-09-14T03:47:51.763 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.763 INFO:tasks.workunit.client.0.smithi070.stdout: "share_state": { 2024-09-14T03:47:51.763 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3 2024-09-14T03:47:51.763 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.763 INFO:tasks.workunit.client.0.smithi070.stdout: "share_state_bytes": { 2024-09-14T03:47:51.763 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 3, 2024-09-14T03:47:51.763 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1627 2024-09-14T03:47:51.763 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.763 INFO:tasks.workunit.client.0.smithi070.stdout: "share_state_keys": { 2024-09-14T03:47:51.763 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 3, 2024-09-14T03:47:51.763 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3 2024-09-14T03:47:51.764 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.764 INFO:tasks.workunit.client.0.smithi070.stdout: "start_leader": { 2024-09-14T03:47:51.764 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3 2024-09-14T03:47:51.764 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.764 INFO:tasks.workunit.client.0.smithi070.stdout: "start_peon": { 2024-09-14T03:47:51.764 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.764 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.764 INFO:tasks.workunit.client.0.smithi070.stdout: "store_state": { 2024-09-14T03:47:51.764 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.764 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.764 INFO:tasks.workunit.client.0.smithi070.stdout: "store_state_bytes": { 2024-09-14T03:47:51.764 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.765 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.765 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.765 INFO:tasks.workunit.client.0.smithi070.stdout: "store_state_keys": { 2024-09-14T03:47:51.765 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.765 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.765 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.765 INFO:tasks.workunit.client.0.smithi070.stdout: "store_state_latency": { 2024-09-14T03:47:51.765 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.765 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.765 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.765 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.765 INFO:tasks.workunit.client.0.smithi070.stdout: "prioritycache": { 2024-09-14T03:47:51.766 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_bytes": { 2024-09-14T03:47:51.766 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1020054731 2024-09-14T03:47:51.766 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.766 INFO:tasks.workunit.client.0.smithi070.stdout: "heap_bytes": { 2024-09-14T03:47:51.766 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 50036736 2024-09-14T03:47:51.766 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.766 INFO:tasks.workunit.client.0.smithi070.stdout: "mapped_bytes": { 2024-09-14T03:47:51.766 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 41148416 2024-09-14T03:47:51.766 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.766 INFO:tasks.workunit.client.0.smithi070.stdout: "target_bytes": { 2024-09-14T03:47:51.766 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2147483648 2024-09-14T03:47:51.766 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.767 INFO:tasks.workunit.client.0.smithi070.stdout: "unmapped_bytes": { 2024-09-14T03:47:51.767 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 8888320 2024-09-14T03:47:51.767 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.767 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.767 INFO:tasks.workunit.client.0.smithi070.stdout: "prioritycache:full": { 2024-09-14T03:47:51.767 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:51.767 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 348127232 2024-09-14T03:47:51.767 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.767 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:51.767 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.767 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.767 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:51.768 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.768 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.768 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:51.768 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 277021771 2024-09-14T03:47:51.768 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.768 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:51.768 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 273495 2024-09-14T03:47:51.768 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.768 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:51.768 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.768 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.769 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:51.769 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.769 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.769 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:51.769 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.769 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.769 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:51.769 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.769 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.769 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:51.769 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.769 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.770 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:51.770 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.770 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.770 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:51.770 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.770 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.770 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:51.770 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.770 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.770 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:51.770 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 70831966 2024-09-14T03:47:51.770 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.770 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.771 INFO:tasks.workunit.client.0.smithi070.stdout: "prioritycache:inc": { 2024-09-14T03:47:51.771 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:51.771 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 348127232 2024-09-14T03:47:51.771 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.771 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:51.771 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.771 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.771 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:51.771 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.771 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.771 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:51.771 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 277021771 2024-09-14T03:47:51.772 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.772 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:51.772 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 38144 2024-09-14T03:47:51.772 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.772 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:51.772 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.772 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.772 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:51.772 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.772 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.772 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:51.772 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.773 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.773 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:51.773 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.773 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.773 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:51.773 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.773 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.773 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:51.773 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.773 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.773 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:51.773 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.774 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.774 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:51.774 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.774 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.774 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:51.774 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71067317 2024-09-14T03:47:51.774 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.774 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.774 INFO:tasks.workunit.client.0.smithi070.stdout: "prioritycache:kv": { 2024-09-14T03:47:51.774 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:51.774 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 322961408 2024-09-14T03:47:51.774 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.775 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:51.775 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:51.775 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.775 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:51.775 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.775 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.775 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:51.775 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 184681181 2024-09-14T03:47:51.775 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.775 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:51.775 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.775 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.776 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:51.776 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.776 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.776 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:51.776 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.776 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.776 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:51.776 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.776 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.776 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:51.776 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.777 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.777 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:51.777 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.777 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.777 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:51.777 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.777 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.777 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:51.777 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.777 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.777 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:51.777 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.778 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.778 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:51.778 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71171363 2024-09-14T03:47:51.778 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.778 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.778 INFO:tasks.workunit.client.0.smithi070.stdout: "rocksdb": { 2024-09-14T03:47:51.778 INFO:tasks.workunit.client.0.smithi070.stdout: "compact": { 2024-09-14T03:47:51.778 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.778 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.778 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_completed": { 2024-09-14T03:47:51.778 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1 2024-09-14T03:47:51.778 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.779 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lasted": { 2024-09-14T03:47:51.779 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 25282850 2024-09-14T03:47:51.779 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.779 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_queue_len": { 2024-09-14T03:47:51.779 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.779 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.779 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_queue_merge": { 2024-09-14T03:47:51.779 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.779 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.779 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_running": { 2024-09-14T03:47:51.779 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.779 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.780 INFO:tasks.workunit.client.0.smithi070.stdout: "get_latency": { 2024-09-14T03:47:51.780 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 43311, 2024-09-14T03:47:51.780 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 163882893 2024-09-14T03:47:51.780 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.780 INFO:tasks.workunit.client.0.smithi070.stdout: "rocksdb_write_delay_time": { 2024-09-14T03:47:51.780 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.780 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.780 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.780 INFO:tasks.workunit.client.0.smithi070.stdout: "rocksdb_write_memtable_time": { 2024-09-14T03:47:51.780 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.780 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.781 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.781 INFO:tasks.workunit.client.0.smithi070.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-14T03:47:51.781 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.781 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.781 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.781 INFO:tasks.workunit.client.0.smithi070.stdout: "rocksdb_write_wal_time": { 2024-09-14T03:47:51.781 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.781 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.781 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.781 INFO:tasks.workunit.client.0.smithi070.stdout: "submit_latency": { 2024-09-14T03:47:51.781 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.781 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.782 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.782 INFO:tasks.workunit.client.0.smithi070.stdout: "submit_sync_latency": { 2024-09-14T03:47:51.782 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2094, 2024-09-14T03:47:51.782 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 276326241 2024-09-14T03:47:51.782 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.782 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.782 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.782 INFO:tasks.workunit.client.0.smithi070.stdout: "mon.b1d2edfb86a8f225d9283baf06ff7497eda48d2d": { 2024-09-14T03:47:51.783 INFO:tasks.workunit.client.0.smithi070.stdout: "mon": { 2024-09-14T03:47:51.783 INFO:tasks.workunit.client.0.smithi070.stdout: "election_call": { 2024-09-14T03:47:51.783 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:51.783 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.783 INFO:tasks.workunit.client.0.smithi070.stdout: "election_lose": { 2024-09-14T03:47:51.783 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:51.783 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.783 INFO:tasks.workunit.client.0.smithi070.stdout: "election_win": { 2024-09-14T03:47:51.783 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.783 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.783 INFO:tasks.workunit.client.0.smithi070.stdout: "num_elections": { 2024-09-14T03:47:51.783 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7 2024-09-14T03:47:51.784 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.784 INFO:tasks.workunit.client.0.smithi070.stdout: "num_sessions": { 2024-09-14T03:47:51.784 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 9 2024-09-14T03:47:51.784 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.784 INFO:tasks.workunit.client.0.smithi070.stdout: "session_add": { 2024-09-14T03:47:51.784 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4 2024-09-14T03:47:51.784 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.784 INFO:tasks.workunit.client.0.smithi070.stdout: "session_rm": { 2024-09-14T03:47:51.784 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 54 2024-09-14T03:47:51.784 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.784 INFO:tasks.workunit.client.0.smithi070.stdout: "session_trim": { 2024-09-14T03:47:51.784 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.785 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.785 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.785 INFO:tasks.workunit.client.0.smithi070.stdout: "paxos": { 2024-09-14T03:47:51.785 INFO:tasks.workunit.client.0.smithi070.stdout: "accept_timeout": { 2024-09-14T03:47:51.785 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.785 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.785 INFO:tasks.workunit.client.0.smithi070.stdout: "begin": { 2024-09-14T03:47:51.785 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 834 2024-09-14T03:47:51.785 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.785 INFO:tasks.workunit.client.0.smithi070.stdout: "begin_bytes": { 2024-09-14T03:47:51.785 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 834, 2024-09-14T03:47:51.786 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4057273 2024-09-14T03:47:51.786 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.786 INFO:tasks.workunit.client.0.smithi070.stdout: "begin_keys": { 2024-09-14T03:47:51.786 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.786 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.786 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.786 INFO:tasks.workunit.client.0.smithi070.stdout: "begin_latency": { 2024-09-14T03:47:51.786 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 834, 2024-09-14T03:47:51.786 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 103998304 2024-09-14T03:47:51.786 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.786 INFO:tasks.workunit.client.0.smithi070.stdout: "collect": { 2024-09-14T03:47:51.787 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:51.787 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.787 INFO:tasks.workunit.client.0.smithi070.stdout: "collect_bytes": { 2024-09-14T03:47:51.787 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:51.787 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 146 2024-09-14T03:47:51.787 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.787 INFO:tasks.workunit.client.0.smithi070.stdout: "collect_keys": { 2024-09-14T03:47:51.787 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:51.787 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:51.787 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.787 INFO:tasks.workunit.client.0.smithi070.stdout: "collect_latency": { 2024-09-14T03:47:51.787 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:51.788 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.788 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.788 INFO:tasks.workunit.client.0.smithi070.stdout: "collect_timeout": { 2024-09-14T03:47:51.788 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.788 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.788 INFO:tasks.workunit.client.0.smithi070.stdout: "collect_uncommitted": { 2024-09-14T03:47:51.788 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.788 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.788 INFO:tasks.workunit.client.0.smithi070.stdout: "commit": { 2024-09-14T03:47:51.788 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 835 2024-09-14T03:47:51.788 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.788 INFO:tasks.workunit.client.0.smithi070.stdout: "commit_bytes": { 2024-09-14T03:47:51.789 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.789 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.789 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.789 INFO:tasks.workunit.client.0.smithi070.stdout: "commit_keys": { 2024-09-14T03:47:51.789 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.789 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.789 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.789 INFO:tasks.workunit.client.0.smithi070.stdout: "commit_latency": { 2024-09-14T03:47:51.789 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.789 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.789 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.790 INFO:tasks.workunit.client.0.smithi070.stdout: "lease_ack_timeout": { 2024-09-14T03:47:51.790 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.790 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.790 INFO:tasks.workunit.client.0.smithi070.stdout: "lease_timeout": { 2024-09-14T03:47:51.790 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.790 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.790 INFO:tasks.workunit.client.0.smithi070.stdout: "new_pn": { 2024-09-14T03:47:51.790 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.790 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.790 INFO:tasks.workunit.client.0.smithi070.stdout: "new_pn_latency": { 2024-09-14T03:47:51.790 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.790 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.790 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.791 INFO:tasks.workunit.client.0.smithi070.stdout: "refresh": { 2024-09-14T03:47:51.791 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 835 2024-09-14T03:47:51.791 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.791 INFO:tasks.workunit.client.0.smithi070.stdout: "refresh_latency": { 2024-09-14T03:47:51.791 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 835, 2024-09-14T03:47:51.791 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1665972924 2024-09-14T03:47:51.791 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.791 INFO:tasks.workunit.client.0.smithi070.stdout: "restart": { 2024-09-14T03:47:51.791 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 13 2024-09-14T03:47:51.791 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.791 INFO:tasks.workunit.client.0.smithi070.stdout: "share_state": { 2024-09-14T03:47:51.791 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.792 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.792 INFO:tasks.workunit.client.0.smithi070.stdout: "share_state_bytes": { 2024-09-14T03:47:51.792 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.792 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.792 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.792 INFO:tasks.workunit.client.0.smithi070.stdout: "share_state_keys": { 2024-09-14T03:47:51.792 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.792 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.792 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.792 INFO:tasks.workunit.client.0.smithi070.stdout: "start_leader": { 2024-09-14T03:47:51.792 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.793 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.793 INFO:tasks.workunit.client.0.smithi070.stdout: "start_peon": { 2024-09-14T03:47:51.793 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:51.793 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.793 INFO:tasks.workunit.client.0.smithi070.stdout: "store_state": { 2024-09-14T03:47:51.793 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 835 2024-09-14T03:47:51.793 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.793 INFO:tasks.workunit.client.0.smithi070.stdout: "store_state_bytes": { 2024-09-14T03:47:51.793 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 835, 2024-09-14T03:47:51.793 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7957909 2024-09-14T03:47:51.793 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.793 INFO:tasks.workunit.client.0.smithi070.stdout: "store_state_keys": { 2024-09-14T03:47:51.794 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 835, 2024-09-14T03:47:51.794 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5627 2024-09-14T03:47:51.794 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.794 INFO:tasks.workunit.client.0.smithi070.stdout: "store_state_latency": { 2024-09-14T03:47:51.794 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 835, 2024-09-14T03:47:51.794 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 114998126 2024-09-14T03:47:51.794 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.794 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.794 INFO:tasks.workunit.client.0.smithi070.stdout: "prioritycache": { 2024-09-14T03:47:51.794 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_bytes": { 2024-09-14T03:47:51.794 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1020054731 2024-09-14T03:47:51.794 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.794 INFO:tasks.workunit.client.0.smithi070.stdout: "heap_bytes": { 2024-09-14T03:47:51.795 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 49078272 2024-09-14T03:47:51.795 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.795 INFO:tasks.workunit.client.0.smithi070.stdout: "mapped_bytes": { 2024-09-14T03:47:51.795 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 33202176 2024-09-14T03:47:51.795 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.795 INFO:tasks.workunit.client.0.smithi070.stdout: "target_bytes": { 2024-09-14T03:47:51.795 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2147483648 2024-09-14T03:47:51.795 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.795 INFO:tasks.workunit.client.0.smithi070.stdout: "unmapped_bytes": { 2024-09-14T03:47:51.795 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 15876096 2024-09-14T03:47:51.795 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.796 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.796 INFO:tasks.workunit.client.0.smithi070.stdout: "prioritycache:full": { 2024-09-14T03:47:51.796 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:51.796 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 348127232 2024-09-14T03:47:51.796 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.796 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:51.796 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.796 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.796 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:51.796 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.796 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.796 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:51.797 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 277020366 2024-09-14T03:47:51.797 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.797 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:51.797 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 273925 2024-09-14T03:47:51.797 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.797 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:51.797 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.797 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.797 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:51.797 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.797 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.797 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:51.798 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.798 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.798 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:51.798 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.798 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.798 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:51.798 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.798 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.798 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:51.798 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.798 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.799 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:51.799 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.799 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.799 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:51.799 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.799 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.799 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:51.799 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 70832941 2024-09-14T03:47:51.799 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.799 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.799 INFO:tasks.workunit.client.0.smithi070.stdout: "prioritycache:inc": { 2024-09-14T03:47:51.799 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:51.800 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 348127232 2024-09-14T03:47:51.800 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.800 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:51.800 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.800 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.800 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:51.800 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.800 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.800 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:51.800 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 277020366 2024-09-14T03:47:51.800 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.800 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:51.800 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 41460 2024-09-14T03:47:51.801 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.801 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:51.801 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.801 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.801 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:51.801 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.801 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.801 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:51.801 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.801 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.801 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:51.801 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.801 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.802 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:51.802 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.802 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.802 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:51.802 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.802 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.802 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:51.802 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.802 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.802 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:51.802 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.802 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.802 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:51.803 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71065406 2024-09-14T03:47:51.803 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.803 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.803 INFO:tasks.workunit.client.0.smithi070.stdout: "prioritycache:kv": { 2024-09-14T03:47:51.803 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:51.803 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 322961408 2024-09-14T03:47:51.803 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.803 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:51.803 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:51.803 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.803 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:51.803 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.804 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.804 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:51.804 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 184680244 2024-09-14T03:47:51.804 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.804 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:51.804 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.804 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.804 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:51.804 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.804 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.804 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:51.804 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.805 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.805 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:51.805 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.805 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.805 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:51.805 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.805 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.805 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:51.805 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.805 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.805 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:51.805 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.806 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.806 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:51.806 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.806 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.806 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:51.806 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.806 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.806 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:51.806 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71172300 2024-09-14T03:47:51.806 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.806 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.807 INFO:tasks.workunit.client.0.smithi070.stdout: "rocksdb": { 2024-09-14T03:47:51.807 INFO:tasks.workunit.client.0.smithi070.stdout: "compact": { 2024-09-14T03:47:51.807 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.807 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.807 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_completed": { 2024-09-14T03:47:51.807 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1 2024-09-14T03:47:51.807 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.807 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lasted": { 2024-09-14T03:47:51.807 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 25235841 2024-09-14T03:47:51.807 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.807 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_queue_len": { 2024-09-14T03:47:51.807 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.807 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.808 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_queue_merge": { 2024-09-14T03:47:51.808 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.808 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.808 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_running": { 2024-09-14T03:47:51.808 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.808 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.808 INFO:tasks.workunit.client.0.smithi070.stdout: "get_latency": { 2024-09-14T03:47:51.808 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 35547, 2024-09-14T03:47:51.808 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 79207408 2024-09-14T03:47:51.808 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.808 INFO:tasks.workunit.client.0.smithi070.stdout: "rocksdb_write_delay_time": { 2024-09-14T03:47:51.808 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.809 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.809 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.809 INFO:tasks.workunit.client.0.smithi070.stdout: "rocksdb_write_memtable_time": { 2024-09-14T03:47:51.809 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.809 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.809 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.809 INFO:tasks.workunit.client.0.smithi070.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-14T03:47:51.809 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.809 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.809 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.809 INFO:tasks.workunit.client.0.smithi070.stdout: "rocksdb_write_wal_time": { 2024-09-14T03:47:51.810 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.810 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.810 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.810 INFO:tasks.workunit.client.0.smithi070.stdout: "submit_latency": { 2024-09-14T03:47:51.810 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.810 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.810 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.810 INFO:tasks.workunit.client.0.smithi070.stdout: "submit_sync_latency": { 2024-09-14T03:47:51.810 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1819, 2024-09-14T03:47:51.810 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 269015584 2024-09-14T03:47:51.810 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.810 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.811 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.811 INFO:tasks.workunit.client.0.smithi070.stdout: "mon.bb10d63b3f8e26c4a0c5da991c12ff03df4a20b0": { 2024-09-14T03:47:51.811 INFO:tasks.workunit.client.0.smithi070.stdout: "mon": { 2024-09-14T03:47:51.811 INFO:tasks.workunit.client.0.smithi070.stdout: "election_call": { 2024-09-14T03:47:51.811 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1 2024-09-14T03:47:51.811 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.811 INFO:tasks.workunit.client.0.smithi070.stdout: "election_lose": { 2024-09-14T03:47:51.811 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1 2024-09-14T03:47:51.811 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.811 INFO:tasks.workunit.client.0.smithi070.stdout: "election_win": { 2024-09-14T03:47:51.811 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.812 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.812 INFO:tasks.workunit.client.0.smithi070.stdout: "num_elections": { 2024-09-14T03:47:51.812 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3 2024-09-14T03:47:51.812 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.812 INFO:tasks.workunit.client.0.smithi070.stdout: "num_sessions": { 2024-09-14T03:47:51.812 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12 2024-09-14T03:47:51.812 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.812 INFO:tasks.workunit.client.0.smithi070.stdout: "session_add": { 2024-09-14T03:47:51.812 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5 2024-09-14T03:47:51.812 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.812 INFO:tasks.workunit.client.0.smithi070.stdout: "session_rm": { 2024-09-14T03:47:51.812 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 86 2024-09-14T03:47:51.813 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.813 INFO:tasks.workunit.client.0.smithi070.stdout: "session_trim": { 2024-09-14T03:47:51.813 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.813 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.813 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.813 INFO:tasks.workunit.client.0.smithi070.stdout: "paxos": { 2024-09-14T03:47:51.813 INFO:tasks.workunit.client.0.smithi070.stdout: "accept_timeout": { 2024-09-14T03:47:51.813 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.813 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.813 INFO:tasks.workunit.client.0.smithi070.stdout: "begin": { 2024-09-14T03:47:51.813 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 824 2024-09-14T03:47:51.813 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.814 INFO:tasks.workunit.client.0.smithi070.stdout: "begin_bytes": { 2024-09-14T03:47:51.814 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 824, 2024-09-14T03:47:51.814 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4030459 2024-09-14T03:47:51.814 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.814 INFO:tasks.workunit.client.0.smithi070.stdout: "begin_keys": { 2024-09-14T03:47:51.814 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.814 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.814 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.814 INFO:tasks.workunit.client.0.smithi070.stdout: "begin_latency": { 2024-09-14T03:47:51.814 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 824, 2024-09-14T03:47:51.814 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 96997151 2024-09-14T03:47:51.814 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.815 INFO:tasks.workunit.client.0.smithi070.stdout: "collect": { 2024-09-14T03:47:51.815 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1 2024-09-14T03:47:51.815 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.815 INFO:tasks.workunit.client.0.smithi070.stdout: "collect_bytes": { 2024-09-14T03:47:51.815 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:51.815 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 73 2024-09-14T03:47:51.815 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.815 INFO:tasks.workunit.client.0.smithi070.stdout: "collect_keys": { 2024-09-14T03:47:51.815 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:51.815 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1 2024-09-14T03:47:51.815 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.815 INFO:tasks.workunit.client.0.smithi070.stdout: "collect_latency": { 2024-09-14T03:47:51.816 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:51.816 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.816 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.816 INFO:tasks.workunit.client.0.smithi070.stdout: "collect_timeout": { 2024-09-14T03:47:51.816 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.816 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.816 INFO:tasks.workunit.client.0.smithi070.stdout: "collect_uncommitted": { 2024-09-14T03:47:51.816 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.816 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.816 INFO:tasks.workunit.client.0.smithi070.stdout: "commit": { 2024-09-14T03:47:51.816 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 826 2024-09-14T03:47:51.817 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.817 INFO:tasks.workunit.client.0.smithi070.stdout: "commit_bytes": { 2024-09-14T03:47:51.817 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.817 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.817 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.817 INFO:tasks.workunit.client.0.smithi070.stdout: "commit_keys": { 2024-09-14T03:47:51.817 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.817 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.817 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.817 INFO:tasks.workunit.client.0.smithi070.stdout: "commit_latency": { 2024-09-14T03:47:51.817 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.817 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.818 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.818 INFO:tasks.workunit.client.0.smithi070.stdout: "lease_ack_timeout": { 2024-09-14T03:47:51.818 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.818 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.818 INFO:tasks.workunit.client.0.smithi070.stdout: "lease_timeout": { 2024-09-14T03:47:51.818 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.818 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.818 INFO:tasks.workunit.client.0.smithi070.stdout: "new_pn": { 2024-09-14T03:47:51.818 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.818 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.818 INFO:tasks.workunit.client.0.smithi070.stdout: "new_pn_latency": { 2024-09-14T03:47:51.818 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.819 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.819 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.819 INFO:tasks.workunit.client.0.smithi070.stdout: "refresh": { 2024-09-14T03:47:51.819 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 826 2024-09-14T03:47:51.819 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.819 INFO:tasks.workunit.client.0.smithi070.stdout: "refresh_latency": { 2024-09-14T03:47:51.819 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 826, 2024-09-14T03:47:51.819 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1856945399 2024-09-14T03:47:51.819 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.819 INFO:tasks.workunit.client.0.smithi070.stdout: "restart": { 2024-09-14T03:47:51.819 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 8 2024-09-14T03:47:51.819 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.820 INFO:tasks.workunit.client.0.smithi070.stdout: "share_state": { 2024-09-14T03:47:51.820 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.820 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.820 INFO:tasks.workunit.client.0.smithi070.stdout: "share_state_bytes": { 2024-09-14T03:47:51.820 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.820 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.820 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.820 INFO:tasks.workunit.client.0.smithi070.stdout: "share_state_keys": { 2024-09-14T03:47:51.820 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.820 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.820 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.821 INFO:tasks.workunit.client.0.smithi070.stdout: "start_leader": { 2024-09-14T03:47:51.821 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.821 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.821 INFO:tasks.workunit.client.0.smithi070.stdout: "start_peon": { 2024-09-14T03:47:51.821 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1 2024-09-14T03:47:51.821 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.821 INFO:tasks.workunit.client.0.smithi070.stdout: "store_state": { 2024-09-14T03:47:51.821 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 825 2024-09-14T03:47:51.821 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.821 INFO:tasks.workunit.client.0.smithi070.stdout: "store_state_bytes": { 2024-09-14T03:47:51.821 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 825, 2024-09-14T03:47:51.821 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7906379 2024-09-14T03:47:51.822 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.822 INFO:tasks.workunit.client.0.smithi070.stdout: "store_state_keys": { 2024-09-14T03:47:51.822 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 825, 2024-09-14T03:47:51.822 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5552 2024-09-14T03:47:51.822 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.822 INFO:tasks.workunit.client.0.smithi070.stdout: "store_state_latency": { 2024-09-14T03:47:51.822 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 825, 2024-09-14T03:47:51.822 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 94997208 2024-09-14T03:47:51.822 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.822 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.822 INFO:tasks.workunit.client.0.smithi070.stdout: "prioritycache": { 2024-09-14T03:47:51.822 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_bytes": { 2024-09-14T03:47:51.823 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1020054731 2024-09-14T03:47:51.823 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.823 INFO:tasks.workunit.client.0.smithi070.stdout: "heap_bytes": { 2024-09-14T03:47:51.823 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 48029696 2024-09-14T03:47:51.823 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.823 INFO:tasks.workunit.client.0.smithi070.stdout: "mapped_bytes": { 2024-09-14T03:47:51.823 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 33873920 2024-09-14T03:47:51.823 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.823 INFO:tasks.workunit.client.0.smithi070.stdout: "target_bytes": { 2024-09-14T03:47:51.823 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2147483648 2024-09-14T03:47:51.823 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.823 INFO:tasks.workunit.client.0.smithi070.stdout: "unmapped_bytes": { 2024-09-14T03:47:51.823 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 14155776 2024-09-14T03:47:51.824 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.824 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.824 INFO:tasks.workunit.client.0.smithi070.stdout: "prioritycache:full": { 2024-09-14T03:47:51.824 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:51.824 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 348127232 2024-09-14T03:47:51.824 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.824 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:51.824 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.824 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.824 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:51.825 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.825 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.825 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:51.825 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 277020366 2024-09-14T03:47:51.825 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.825 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:51.825 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 273925 2024-09-14T03:47:51.825 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.825 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:51.825 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.825 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.825 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:51.826 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.826 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.826 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:51.826 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.826 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.827 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:51.828 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.828 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.828 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:51.828 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.828 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.828 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:51.828 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.828 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.828 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:51.828 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.829 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.829 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:51.829 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.829 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.829 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:51.829 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 70832941 2024-09-14T03:47:51.829 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.829 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.829 INFO:tasks.workunit.client.0.smithi070.stdout: "prioritycache:inc": { 2024-09-14T03:47:51.829 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:51.829 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 348127232 2024-09-14T03:47:51.829 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.829 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:51.829 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.830 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.830 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:51.830 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.830 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.830 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:51.830 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 277020366 2024-09-14T03:47:51.830 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.830 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:51.830 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 41460 2024-09-14T03:47:51.830 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.830 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:51.830 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.831 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.831 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:51.831 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.831 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.831 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:51.831 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.831 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.831 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:51.831 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.831 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.831 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:51.831 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.831 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.832 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:51.832 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.832 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.832 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:51.832 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.832 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.832 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:51.832 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.832 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.832 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:51.832 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71065406 2024-09-14T03:47:51.832 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.832 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.833 INFO:tasks.workunit.client.0.smithi070.stdout: "prioritycache:kv": { 2024-09-14T03:47:51.833 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:51.833 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 322961408 2024-09-14T03:47:51.833 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.833 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:51.833 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:51.833 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.833 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:51.833 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.833 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.833 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:51.833 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 184680244 2024-09-14T03:47:51.833 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.834 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:51.834 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.834 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.834 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:51.834 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.834 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.834 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:51.834 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.834 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.834 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:51.834 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.834 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.834 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:51.834 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.835 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.835 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:51.835 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.835 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.835 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:51.835 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.835 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.835 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:51.835 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.835 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.835 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:51.835 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.835 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.836 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:51.836 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71172300 2024-09-14T03:47:51.836 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.836 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.836 INFO:tasks.workunit.client.0.smithi070.stdout: "rocksdb": { 2024-09-14T03:47:51.836 INFO:tasks.workunit.client.0.smithi070.stdout: "compact": { 2024-09-14T03:47:51.836 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.836 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.836 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_completed": { 2024-09-14T03:47:51.836 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1 2024-09-14T03:47:51.836 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.836 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lasted": { 2024-09-14T03:47:51.836 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 24169414 2024-09-14T03:47:51.837 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.837 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_queue_len": { 2024-09-14T03:47:51.837 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.837 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.837 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_queue_merge": { 2024-09-14T03:47:51.837 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.837 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.837 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_running": { 2024-09-14T03:47:51.837 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.837 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.837 INFO:tasks.workunit.client.0.smithi070.stdout: "get_latency": { 2024-09-14T03:47:51.837 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 35177, 2024-09-14T03:47:51.837 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 85337748 2024-09-14T03:47:51.837 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.838 INFO:tasks.workunit.client.0.smithi070.stdout: "rocksdb_write_delay_time": { 2024-09-14T03:47:51.838 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.838 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.838 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.838 INFO:tasks.workunit.client.0.smithi070.stdout: "rocksdb_write_memtable_time": { 2024-09-14T03:47:51.838 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.838 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.838 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.838 INFO:tasks.workunit.client.0.smithi070.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-14T03:47:51.838 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.838 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.838 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.838 INFO:tasks.workunit.client.0.smithi070.stdout: "rocksdb_write_wal_time": { 2024-09-14T03:47:51.838 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.839 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.839 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.839 INFO:tasks.workunit.client.0.smithi070.stdout: "submit_latency": { 2024-09-14T03:47:51.839 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.839 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.839 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.839 INFO:tasks.workunit.client.0.smithi070.stdout: "submit_sync_latency": { 2024-09-14T03:47:51.839 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1771, 2024-09-14T03:47:51.839 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 203625534 2024-09-14T03:47:51.839 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.839 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.839 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.839 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.0": { 2024-09-14T03:47:51.840 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:51.840 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_slow_fallback": { 2024-09-14T03:47:51.840 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.840 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.840 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_slow_size_fallback": { 2024-09-14T03:47:51.840 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.840 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.840 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_db": { 2024-09-14T03:47:51.840 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 65536 2024-09-14T03:47:51.840 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.840 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_main": { 2024-09-14T03:47:51.840 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.841 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.841 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_wal": { 2024-09-14T03:47:51.841 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.841 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.841 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_slow": { 2024-09-14T03:47:51.841 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.841 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.841 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_sst": { 2024-09-14T03:47:51.841 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12288 2024-09-14T03:47:51.841 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.841 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_wal": { 2024-09-14T03:47:51.841 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2306048 2024-09-14T03:47:51.841 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.842 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lat": { 2024-09-14T03:47:51.842 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.842 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.842 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.842 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lock_lat": { 2024-09-14T03:47:51.842 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.842 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.842 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.842 INFO:tasks.workunit.client.0.smithi070.stdout: "db_total_bytes": { 2024-09-14T03:47:51.842 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 95995035648 2024-09-14T03:47:51.842 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.842 INFO:tasks.workunit.client.0.smithi070.stdout: "db_used_bytes": { 2024-09-14T03:47:51.842 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 304480256 2024-09-14T03:47:51.842 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.843 INFO:tasks.workunit.client.0.smithi070.stdout: "log_bytes": { 2024-09-14T03:47:51.843 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1552384 2024-09-14T03:47:51.843 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.843 INFO:tasks.workunit.client.0.smithi070.stdout: "logged_bytes": { 2024-09-14T03:47:51.843 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1413120 2024-09-14T03:47:51.843 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.843 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_db": { 2024-09-14T03:47:51.843 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 312868864 2024-09-14T03:47:51.843 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.843 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_slow": { 2024-09-14T03:47:51.843 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.843 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.843 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_wal": { 2024-09-14T03:47:51.843 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.844 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.844 INFO:tasks.workunit.client.0.smithi070.stdout: "num_files": { 2024-09-14T03:47:51.844 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 14 2024-09-14T03:47:51.844 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.844 INFO:tasks.workunit.client.0.smithi070.stdout: "read_bytes": { 2024-09-14T03:47:51.844 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 195697 2024-09-14T03:47:51.844 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.844 INFO:tasks.workunit.client.0.smithi070.stdout: "read_count": { 2024-09-14T03:47:51.844 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 23 2024-09-14T03:47:51.844 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.844 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes": { 2024-09-14T03:47:51.844 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1269760 2024-09-14T03:47:51.845 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.845 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_db": { 2024-09-14T03:47:51.845 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1273856 2024-09-14T03:47:51.845 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.845 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_slow": { 2024-09-14T03:47:51.845 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.845 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.845 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_wal": { 2024-09-14T03:47:51.845 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.845 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.845 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_count": { 2024-09-14T03:47:51.845 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 15 2024-09-14T03:47:51.846 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.846 INFO:tasks.workunit.client.0.smithi070.stdout: "read_prefetch_bytes": { 2024-09-14T03:47:51.846 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4789 2024-09-14T03:47:51.846 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.846 INFO:tasks.workunit.client.0.smithi070.stdout: "read_prefetch_count": { 2024-09-14T03:47:51.846 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4 2024-09-14T03:47:51.846 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.846 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_buffer_bytes": { 2024-09-14T03:47:51.846 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5589 2024-09-14T03:47:51.846 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.846 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_buffer_count": { 2024-09-14T03:47:51.846 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 24 2024-09-14T03:47:51.846 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.847 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_bytes": { 2024-09-14T03:47:51.847 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5589 2024-09-14T03:47:51.847 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.847 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_count": { 2024-09-14T03:47:51.847 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 24 2024-09-14T03:47:51.847 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.847 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes": { 2024-09-14T03:47:51.847 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.847 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.847 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_db": { 2024-09-14T03:47:51.847 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.847 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.847 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_slow": { 2024-09-14T03:47:51.848 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.848 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.848 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_wal": { 2024-09-14T03:47:51.848 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.848 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.848 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_count": { 2024-09-14T03:47:51.848 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.848 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.848 INFO:tasks.workunit.client.0.smithi070.stdout: "slow_total_bytes": { 2024-09-14T03:47:51.848 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.848 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.848 INFO:tasks.workunit.client.0.smithi070.stdout: "slow_used_bytes": { 2024-09-14T03:47:51.849 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.849 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.849 INFO:tasks.workunit.client.0.smithi070.stdout: "wal_total_bytes": { 2024-09-14T03:47:51.849 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.849 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.849 INFO:tasks.workunit.client.0.smithi070.stdout: "wal_used_bytes": { 2024-09-14T03:47:51.849 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.849 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.849 INFO:tasks.workunit.client.0.smithi070.stdout: "write_bytes": { 2024-09-14T03:47:51.849 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3792896 2024-09-14T03:47:51.849 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.849 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.849 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore": { 2024-09-14T03:47:51.849 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit": { 2024-09-14T03:47:51.850 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4096 2024-09-14T03:47:51.850 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.850 INFO:tasks.workunit.client.0.smithi070.stdout: "allocated": { 2024-09-14T03:47:51.850 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1302528 2024-09-14T03:47:51.850 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.850 INFO:tasks.workunit.client.0.smithi070.stdout: "clist_lat": { 2024-09-14T03:47:51.850 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 4, 2024-09-14T03:47:51.850 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 152555 2024-09-14T03:47:51.850 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.850 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_lat": { 2024-09-14T03:47:51.850 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.850 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.850 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.851 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed": { 2024-09-14T03:47:51.851 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.851 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.851 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed_allocated": { 2024-09-14T03:47:51.851 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.851 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.851 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed_original": { 2024-09-14T03:47:51.851 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.851 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.851 INFO:tasks.workunit.client.0.smithi070.stdout: "csum_lat": { 2024-09-14T03:47:51.851 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 24, 2024-09-14T03:47:51.851 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 73787 2024-09-14T03:47:51.851 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.851 INFO:tasks.workunit.client.0.smithi070.stdout: "decompress_lat": { 2024-09-14T03:47:51.852 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.852 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.852 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.852 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_commit_lat": { 2024-09-14T03:47:51.852 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 328, 2024-09-14T03:47:51.852 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 59340322 2024-09-14T03:47:51.852 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.852 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_final_lat": { 2024-09-14T03:47:51.852 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 321, 2024-09-14T03:47:51.852 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7807995 2024-09-14T03:47:51.852 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.852 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_flush_lat": { 2024-09-14T03:47:51.852 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 328, 2024-09-14T03:47:51.853 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 452089 2024-09-14T03:47:51.853 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.853 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_sync_lat": { 2024-09-14T03:47:51.853 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 328, 2024-09-14T03:47:51.853 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 59792411 2024-09-14T03:47:51.853 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.853 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_get_keys_lat": { 2024-09-14T03:47:51.853 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.853 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.853 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.853 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_get_values_lat": { 2024-09-14T03:47:51.853 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:51.853 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 26965 2024-09-14T03:47:51.853 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.854 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_lower_bound_lat": { 2024-09-14T03:47:51.854 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 5, 2024-09-14T03:47:51.854 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 11764 2024-09-14T03:47:51.854 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.854 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_next_lat": { 2024-09-14T03:47:51.854 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.854 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.854 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.854 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_seek_to_first_lat": { 2024-09-14T03:47:51.854 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.854 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.854 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.854 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_upper_bound_lat": { 2024-09-14T03:47:51.854 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 4, 2024-09-14T03:47:51.855 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 10295 2024-09-14T03:47:51.855 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.855 INFO:tasks.workunit.client.0.smithi070.stdout: "onode_hits": { 2024-09-14T03:47:51.855 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 689 2024-09-14T03:47:51.855 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.855 INFO:tasks.workunit.client.0.smithi070.stdout: "onode_misses": { 2024-09-14T03:47:51.855 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 228 2024-09-14T03:47:51.855 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.855 INFO:tasks.workunit.client.0.smithi070.stdout: "read_lat": { 2024-09-14T03:47:51.855 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 111, 2024-09-14T03:47:51.855 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1647160 2024-09-14T03:47:51.855 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.855 INFO:tasks.workunit.client.0.smithi070.stdout: "read_onode_meta_lat": { 2024-09-14T03:47:51.856 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 262, 2024-09-14T03:47:51.856 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 266412 2024-09-14T03:47:51.856 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.856 INFO:tasks.workunit.client.0.smithi070.stdout: "read_wait_aio_lat": { 2024-09-14T03:47:51.856 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 151, 2024-09-14T03:47:51.856 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 997696 2024-09-14T03:47:51.856 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.856 INFO:tasks.workunit.client.0.smithi070.stdout: "reads_with_retries": { 2024-09-14T03:47:51.856 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.856 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.856 INFO:tasks.workunit.client.0.smithi070.stdout: "remove_lat": { 2024-09-14T03:47:51.856 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:51.856 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 31068 2024-09-14T03:47:51.857 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.857 INFO:tasks.workunit.client.0.smithi070.stdout: "state_aio_wait_lat": { 2024-09-14T03:47:51.857 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 420, 2024-09-14T03:47:51.857 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 9066587 2024-09-14T03:47:51.857 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.857 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_aio_wait_lat": { 2024-09-14T03:47:51.857 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 96, 2024-09-14T03:47:51.857 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7418949 2024-09-14T03:47:51.857 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.857 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_cleanup_lat": { 2024-09-14T03:47:51.857 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 96, 2024-09-14T03:47:51.857 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 474004274278 2024-09-14T03:47:51.857 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.857 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_queued_lat": { 2024-09-14T03:47:51.858 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 96, 2024-09-14T03:47:51.858 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 160665384289 2024-09-14T03:47:51.858 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.858 INFO:tasks.workunit.client.0.smithi070.stdout: "state_done_lat": { 2024-09-14T03:47:51.858 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 420, 2024-09-14T03:47:51.858 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 64892973381 2024-09-14T03:47:51.858 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.858 INFO:tasks.workunit.client.0.smithi070.stdout: "state_finishing_lat": { 2024-09-14T03:47:51.858 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 420, 2024-09-14T03:47:51.858 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 43338 2024-09-14T03:47:51.858 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.858 INFO:tasks.workunit.client.0.smithi070.stdout: "state_io_done_lat": { 2024-09-14T03:47:51.858 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 420, 2024-09-14T03:47:51.858 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 796428 2024-09-14T03:47:51.859 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.859 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_commiting_lat": { 2024-09-14T03:47:51.859 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 420, 2024-09-14T03:47:51.859 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 119571365 2024-09-14T03:47:51.859 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.859 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_done_lat": { 2024-09-14T03:47:51.859 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 420, 2024-09-14T03:47:51.859 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 130554 2024-09-14T03:47:51.859 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.859 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_queued_lat": { 2024-09-14T03:47:51.859 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 420, 2024-09-14T03:47:51.859 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 86629140 2024-09-14T03:47:51.859 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.859 INFO:tasks.workunit.client.0.smithi070.stdout: "state_prepare_lat": { 2024-09-14T03:47:51.860 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 420, 2024-09-14T03:47:51.860 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 17467553 2024-09-14T03:47:51.860 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.860 INFO:tasks.workunit.client.0.smithi070.stdout: "stored": { 2024-09-14T03:47:51.860 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 903336 2024-09-14T03:47:51.860 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.860 INFO:tasks.workunit.client.0.smithi070.stdout: "truncate_lat": { 2024-09-14T03:47:51.860 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 56, 2024-09-14T03:47:51.860 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 264416 2024-09-14T03:47:51.860 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.860 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_commit_lat": { 2024-09-14T03:47:51.860 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 420, 2024-09-14T03:47:51.860 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 233587502 2024-09-14T03:47:51.861 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.861 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_submit_lat": { 2024-09-14T03:47:51.861 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 420, 2024-09-14T03:47:51.861 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 21081399 2024-09-14T03:47:51.861 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.861 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_throttle_lat": { 2024-09-14T03:47:51.861 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 420, 2024-09-14T03:47:51.861 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2755231 2024-09-14T03:47:51.861 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.861 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.861 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache": { 2024-09-14T03:47:51.861 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_bytes": { 2024-09-14T03:47:51.861 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1861444387 2024-09-14T03:47:51.861 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.862 INFO:tasks.workunit.client.0.smithi070.stdout: "heap_bytes": { 2024-09-14T03:47:51.862 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 272023552 2024-09-14T03:47:51.862 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.862 INFO:tasks.workunit.client.0.smithi070.stdout: "mapped_bytes": { 2024-09-14T03:47:51.862 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 271007744 2024-09-14T03:47:51.862 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.862 INFO:tasks.workunit.client.0.smithi070.stdout: "target_bytes": { 2024-09-14T03:47:51.862 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3137353830 2024-09-14T03:47:51.862 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.862 INFO:tasks.workunit.client.0.smithi070.stdout: "unmapped_bytes": { 2024-09-14T03:47:51.862 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1015808 2024-09-14T03:47:51.862 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.862 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.863 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:data": { 2024-09-14T03:47:51.863 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:51.863 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 159383552 2024-09-14T03:47:51.863 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.863 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:51.863 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.863 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.863 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:51.863 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.863 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.863 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:51.863 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 85496610 2024-09-14T03:47:51.863 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.864 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:51.864 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.864 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.864 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:51.864 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.864 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.864 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:51.864 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.864 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.864 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:51.864 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.864 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.864 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:51.865 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.865 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.865 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:51.865 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 8192 2024-09-14T03:47:51.865 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.865 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:51.865 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.865 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.865 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:51.865 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.865 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.865 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:51.865 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.866 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.866 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:51.866 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 73878750 2024-09-14T03:47:51.866 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.866 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.866 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:kv": { 2024-09-14T03:47:51.866 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:51.866 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 780140544 2024-09-14T03:47:51.866 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.866 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:51.866 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:51.866 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.866 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:51.866 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.867 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.867 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:51.867 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 641224575 2024-09-14T03:47:51.867 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.867 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:51.867 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.867 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.867 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:51.867 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.867 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.867 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:51.867 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.867 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.868 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:51.868 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.868 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.868 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:51.868 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.868 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.868 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:51.868 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2128 2024-09-14T03:47:51.868 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.868 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:51.868 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.868 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.868 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:51.869 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.869 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.869 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:51.869 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.869 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.869 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:51.869 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71804977 2024-09-14T03:47:51.869 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.869 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.869 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:kv_onode": { 2024-09-14T03:47:51.869 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:51.869 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 192937984 2024-09-14T03:47:51.869 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.870 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:51.870 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:51.870 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.870 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:51.870 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 464 2024-09-14T03:47:51.870 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.870 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:51.870 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 56997740 2024-09-14T03:47:51.870 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.870 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:51.870 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.870 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.870 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:51.871 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.871 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.871 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:51.871 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.871 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.871 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:51.871 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.871 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.871 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:51.871 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.871 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.871 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:51.871 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.871 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.872 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:51.872 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.872 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.872 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:51.872 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.872 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.872 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:51.872 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.872 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.872 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:51.872 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 68830916 2024-09-14T03:47:51.872 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.872 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.873 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:meta": { 2024-09-14T03:47:51.873 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:51.873 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 713031680 2024-09-14T03:47:51.873 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.873 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:51.873 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.873 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.873 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:51.873 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.873 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.873 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:51.873 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 641224575 2024-09-14T03:47:51.873 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.874 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:51.874 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.874 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.874 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:51.874 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2839 2024-09-14T03:47:51.874 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.874 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:51.874 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7097 2024-09-14T03:47:51.874 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.874 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:51.874 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.874 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.874 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:51.875 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.875 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.875 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:51.875 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 272550 2024-09-14T03:47:51.875 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.875 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:51.875 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.875 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.875 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:51.875 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.875 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.875 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:51.875 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.875 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.876 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:51.876 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71524619 2024-09-14T03:47:51.876 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.876 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.876 INFO:tasks.workunit.client.0.smithi070.stdout: "objecter": { 2024-09-14T03:47:51.876 INFO:tasks.workunit.client.0.smithi070.stdout: "op_active": { 2024-09-14T03:47:51.876 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.876 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.876 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r": { 2024-09-14T03:47:51.876 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.876 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.876 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rmw": { 2024-09-14T03:47:51.876 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.876 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.877 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w": { 2024-09-14T03:47:51.877 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.877 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.877 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.877 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:51.877 INFO:tasks.workunit.client.0.smithi070.stdout: "numpg": { 2024-09-14T03:47:51.877 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 45 2024-09-14T03:47:51.877 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.877 INFO:tasks.workunit.client.0.smithi070.stdout: "numpg_removing": { 2024-09-14T03:47:51.877 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.877 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.877 INFO:tasks.workunit.client.0.smithi070.stdout: "op": { 2024-09-14T03:47:51.877 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 6 2024-09-14T03:47:51.877 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.878 INFO:tasks.workunit.client.0.smithi070.stdout: "op_delayed_degraded": { 2024-09-14T03:47:51.878 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.878 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.878 INFO:tasks.workunit.client.0.smithi070.stdout: "op_delayed_unreadable": { 2024-09-14T03:47:51.878 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.878 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.878 INFO:tasks.workunit.client.0.smithi070.stdout: "op_in_bytes": { 2024-09-14T03:47:51.878 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 604 2024-09-14T03:47:51.878 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.878 INFO:tasks.workunit.client.0.smithi070.stdout: "op_latency": { 2024-09-14T03:47:51.878 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 6, 2024-09-14T03:47:51.878 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 49065134 2024-09-14T03:47:51.878 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.879 INFO:tasks.workunit.client.0.smithi070.stdout: "op_out_bytes": { 2024-09-14T03:47:51.879 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.879 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.879 INFO:tasks.workunit.client.0.smithi070.stdout: "op_prepare_latency": { 2024-09-14T03:47:51.879 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 7, 2024-09-14T03:47:51.879 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4089275 2024-09-14T03:47:51.879 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.879 INFO:tasks.workunit.client.0.smithi070.stdout: "op_process_latency": { 2024-09-14T03:47:51.879 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 6, 2024-09-14T03:47:51.879 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 15715463 2024-09-14T03:47:51.879 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.879 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r": { 2024-09-14T03:47:51.879 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.879 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.880 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_latency": { 2024-09-14T03:47:51.880 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.880 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.880 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.880 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:47:51.880 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.880 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.880 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_out_bytes": { 2024-09-14T03:47:51.880 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.880 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.880 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_prepare_latency": { 2024-09-14T03:47:51.880 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.880 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.880 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.881 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_process_latency": { 2024-09-14T03:47:51.881 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.881 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.881 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.881 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw": { 2024-09-14T03:47:51.881 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.881 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.881 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_in_bytes": { 2024-09-14T03:47:51.881 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.881 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.881 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency": { 2024-09-14T03:47:51.881 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.882 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.882 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.882 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:47:51.882 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.882 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.882 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:47:51.882 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.882 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.882 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_out_bytes": { 2024-09-14T03:47:51.882 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.882 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.882 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_prepare_latency": { 2024-09-14T03:47:51.882 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.883 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.883 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.883 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_process_latency": { 2024-09-14T03:47:51.883 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.883 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.883 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.883 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w": { 2024-09-14T03:47:51.883 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 6 2024-09-14T03:47:51.883 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.883 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_in_bytes": { 2024-09-14T03:47:51.883 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 604 2024-09-14T03:47:51.883 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.883 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_latency": { 2024-09-14T03:47:51.884 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 6, 2024-09-14T03:47:51.884 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 49065134 2024-09-14T03:47:51.884 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.884 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:47:51.884 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.884 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.884 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_prepare_latency": { 2024-09-14T03:47:51.884 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 7, 2024-09-14T03:47:51.884 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4089275 2024-09-14T03:47:51.884 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.884 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_process_latency": { 2024-09-14T03:47:51.884 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 6, 2024-09-14T03:47:51.884 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 15715463 2024-09-14T03:47:51.885 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.885 INFO:tasks.workunit.client.0.smithi070.stdout: "op_wip": { 2024-09-14T03:47:51.885 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.885 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.885 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_bytes": { 2024-09-14T03:47:51.885 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.885 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.885 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_ops": { 2024-09-14T03:47:51.885 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.885 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.885 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_bytes": { 2024-09-14T03:47:51.885 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 95995035648 2024-09-14T03:47:51.885 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.885 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_bytes_used": { 2024-09-14T03:47:51.886 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 305790976 2024-09-14T03:47:51.886 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.886 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.886 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.886 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.1": { 2024-09-14T03:47:51.886 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:51.886 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_slow_fallback": { 2024-09-14T03:47:51.886 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.886 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.886 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_slow_size_fallback": { 2024-09-14T03:47:51.886 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.886 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.886 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_db": { 2024-09-14T03:47:51.887 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 65536 2024-09-14T03:47:51.887 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.887 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_main": { 2024-09-14T03:47:51.887 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.887 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.887 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_wal": { 2024-09-14T03:47:51.887 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.887 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.887 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_slow": { 2024-09-14T03:47:51.887 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.887 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.887 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_sst": { 2024-09-14T03:47:51.887 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12288 2024-09-14T03:47:51.887 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.888 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_wal": { 2024-09-14T03:47:51.888 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1970176 2024-09-14T03:47:51.888 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.888 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lat": { 2024-09-14T03:47:51.888 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.888 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.888 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.888 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lock_lat": { 2024-09-14T03:47:51.888 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.888 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.888 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.888 INFO:tasks.workunit.client.0.smithi070.stdout: "db_total_bytes": { 2024-09-14T03:47:51.888 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 95995035648 2024-09-14T03:47:51.889 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.889 INFO:tasks.workunit.client.0.smithi070.stdout: "db_used_bytes": { 2024-09-14T03:47:51.889 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 304480256 2024-09-14T03:47:51.889 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.889 INFO:tasks.workunit.client.0.smithi070.stdout: "log_bytes": { 2024-09-14T03:47:51.889 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1306624 2024-09-14T03:47:51.889 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.889 INFO:tasks.workunit.client.0.smithi070.stdout: "logged_bytes": { 2024-09-14T03:47:51.889 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1167360 2024-09-14T03:47:51.889 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.889 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_db": { 2024-09-14T03:47:51.889 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 312868864 2024-09-14T03:47:51.889 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.889 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_slow": { 2024-09-14T03:47:51.890 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.890 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.890 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_wal": { 2024-09-14T03:47:51.890 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.890 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.890 INFO:tasks.workunit.client.0.smithi070.stdout: "num_files": { 2024-09-14T03:47:51.890 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 14 2024-09-14T03:47:51.890 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.890 INFO:tasks.workunit.client.0.smithi070.stdout: "read_bytes": { 2024-09-14T03:47:51.890 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 195697 2024-09-14T03:47:51.890 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.890 INFO:tasks.workunit.client.0.smithi070.stdout: "read_count": { 2024-09-14T03:47:51.890 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 23 2024-09-14T03:47:51.891 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.891 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes": { 2024-09-14T03:47:51.891 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1269760 2024-09-14T03:47:51.891 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.891 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_db": { 2024-09-14T03:47:51.891 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1273856 2024-09-14T03:47:51.891 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.891 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_slow": { 2024-09-14T03:47:51.891 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.891 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.891 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_wal": { 2024-09-14T03:47:51.891 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.891 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.892 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_count": { 2024-09-14T03:47:51.892 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 15 2024-09-14T03:47:51.892 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.892 INFO:tasks.workunit.client.0.smithi070.stdout: "read_prefetch_bytes": { 2024-09-14T03:47:51.892 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4789 2024-09-14T03:47:51.892 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.892 INFO:tasks.workunit.client.0.smithi070.stdout: "read_prefetch_count": { 2024-09-14T03:47:51.892 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4 2024-09-14T03:47:51.892 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.892 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_buffer_bytes": { 2024-09-14T03:47:51.892 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5589 2024-09-14T03:47:51.892 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.892 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_buffer_count": { 2024-09-14T03:47:51.893 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 24 2024-09-14T03:47:51.893 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.893 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_bytes": { 2024-09-14T03:47:51.893 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5589 2024-09-14T03:47:51.893 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.893 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_count": { 2024-09-14T03:47:51.893 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 24 2024-09-14T03:47:51.893 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.893 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes": { 2024-09-14T03:47:51.893 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.893 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.893 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_db": { 2024-09-14T03:47:51.893 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.894 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.894 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_slow": { 2024-09-14T03:47:51.894 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.894 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.894 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_wal": { 2024-09-14T03:47:51.894 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.894 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.894 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_count": { 2024-09-14T03:47:51.894 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.894 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.894 INFO:tasks.workunit.client.0.smithi070.stdout: "slow_total_bytes": { 2024-09-14T03:47:51.894 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.894 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.894 INFO:tasks.workunit.client.0.smithi070.stdout: "slow_used_bytes": { 2024-09-14T03:47:51.895 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.895 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.895 INFO:tasks.workunit.client.0.smithi070.stdout: "wal_total_bytes": { 2024-09-14T03:47:51.895 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.895 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.895 INFO:tasks.workunit.client.0.smithi070.stdout: "wal_used_bytes": { 2024-09-14T03:47:51.895 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.895 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.895 INFO:tasks.workunit.client.0.smithi070.stdout: "write_bytes": { 2024-09-14T03:47:51.895 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3211264 2024-09-14T03:47:51.895 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.895 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.895 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore": { 2024-09-14T03:47:51.895 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit": { 2024-09-14T03:47:51.896 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4096 2024-09-14T03:47:51.896 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.896 INFO:tasks.workunit.client.0.smithi070.stdout: "allocated": { 2024-09-14T03:47:51.896 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 696320 2024-09-14T03:47:51.896 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.896 INFO:tasks.workunit.client.0.smithi070.stdout: "clist_lat": { 2024-09-14T03:47:51.896 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 5, 2024-09-14T03:47:51.896 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 266520 2024-09-14T03:47:51.896 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.896 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_lat": { 2024-09-14T03:47:51.896 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.896 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.896 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.897 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed": { 2024-09-14T03:47:51.897 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.897 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.897 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed_allocated": { 2024-09-14T03:47:51.897 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.897 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.897 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed_original": { 2024-09-14T03:47:51.897 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.897 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.897 INFO:tasks.workunit.client.0.smithi070.stdout: "csum_lat": { 2024-09-14T03:47:51.897 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 22, 2024-09-14T03:47:51.897 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 70568 2024-09-14T03:47:51.897 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.897 INFO:tasks.workunit.client.0.smithi070.stdout: "decompress_lat": { 2024-09-14T03:47:51.898 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.898 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.898 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.898 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_commit_lat": { 2024-09-14T03:47:51.898 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 268, 2024-09-14T03:47:51.898 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 53199400 2024-09-14T03:47:51.898 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.898 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_final_lat": { 2024-09-14T03:47:51.898 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 265, 2024-09-14T03:47:51.898 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7667292 2024-09-14T03:47:51.898 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.898 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_flush_lat": { 2024-09-14T03:47:51.898 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 268, 2024-09-14T03:47:51.898 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 361726 2024-09-14T03:47:51.899 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.899 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_sync_lat": { 2024-09-14T03:47:51.899 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 268, 2024-09-14T03:47:51.899 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 53561126 2024-09-14T03:47:51.899 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.899 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_get_keys_lat": { 2024-09-14T03:47:51.899 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.899 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.899 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.899 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_get_values_lat": { 2024-09-14T03:47:51.899 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 3, 2024-09-14T03:47:51.899 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 34171 2024-09-14T03:47:51.900 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.900 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_lower_bound_lat": { 2024-09-14T03:47:51.900 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 6, 2024-09-14T03:47:51.900 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 15713 2024-09-14T03:47:51.900 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.900 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_next_lat": { 2024-09-14T03:47:51.900 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.900 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.900 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.900 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_seek_to_first_lat": { 2024-09-14T03:47:51.900 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.900 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.900 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.900 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_upper_bound_lat": { 2024-09-14T03:47:51.901 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 5, 2024-09-14T03:47:51.901 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 13193 2024-09-14T03:47:51.901 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.901 INFO:tasks.workunit.client.0.smithi070.stdout: "onode_hits": { 2024-09-14T03:47:51.901 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 682 2024-09-14T03:47:51.901 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.901 INFO:tasks.workunit.client.0.smithi070.stdout: "onode_misses": { 2024-09-14T03:47:51.901 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 228 2024-09-14T03:47:51.901 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.901 INFO:tasks.workunit.client.0.smithi070.stdout: "read_lat": { 2024-09-14T03:47:51.901 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 120, 2024-09-14T03:47:51.901 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1733260 2024-09-14T03:47:51.901 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.902 INFO:tasks.workunit.client.0.smithi070.stdout: "read_onode_meta_lat": { 2024-09-14T03:47:51.902 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 279, 2024-09-14T03:47:51.902 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 251501 2024-09-14T03:47:51.902 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.902 INFO:tasks.workunit.client.0.smithi070.stdout: "read_wait_aio_lat": { 2024-09-14T03:47:51.902 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 159, 2024-09-14T03:47:51.902 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 988213 2024-09-14T03:47:51.902 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.902 INFO:tasks.workunit.client.0.smithi070.stdout: "reads_with_retries": { 2024-09-14T03:47:51.902 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.902 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.902 INFO:tasks.workunit.client.0.smithi070.stdout: "remove_lat": { 2024-09-14T03:47:51.902 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 3, 2024-09-14T03:47:51.902 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 102148 2024-09-14T03:47:51.903 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.903 INFO:tasks.workunit.client.0.smithi070.stdout: "state_aio_wait_lat": { 2024-09-14T03:47:51.903 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 361, 2024-09-14T03:47:51.903 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5214151 2024-09-14T03:47:51.903 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.903 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_aio_wait_lat": { 2024-09-14T03:47:51.903 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 95, 2024-09-14T03:47:51.903 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 6796471 2024-09-14T03:47:51.903 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.903 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_cleanup_lat": { 2024-09-14T03:47:51.903 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 94, 2024-09-14T03:47:51.903 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 271815987479 2024-09-14T03:47:51.903 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.904 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_queued_lat": { 2024-09-14T03:47:51.904 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 95, 2024-09-14T03:47:51.904 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 138625203385 2024-09-14T03:47:51.904 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.904 INFO:tasks.workunit.client.0.smithi070.stdout: "state_done_lat": { 2024-09-14T03:47:51.904 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 360, 2024-09-14T03:47:51.904 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 31863399419 2024-09-14T03:47:51.904 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.904 INFO:tasks.workunit.client.0.smithi070.stdout: "state_finishing_lat": { 2024-09-14T03:47:51.904 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 360, 2024-09-14T03:47:51.904 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 36839 2024-09-14T03:47:51.904 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.904 INFO:tasks.workunit.client.0.smithi070.stdout: "state_io_done_lat": { 2024-09-14T03:47:51.905 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 361, 2024-09-14T03:47:51.905 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 358838 2024-09-14T03:47:51.905 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.905 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_commiting_lat": { 2024-09-14T03:47:51.905 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 361, 2024-09-14T03:47:51.905 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 95831917 2024-09-14T03:47:51.905 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.905 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_done_lat": { 2024-09-14T03:47:51.905 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 361, 2024-09-14T03:47:51.905 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 109066 2024-09-14T03:47:51.905 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.905 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_queued_lat": { 2024-09-14T03:47:51.905 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 361, 2024-09-14T03:47:51.905 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 83392148 2024-09-14T03:47:51.906 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.906 INFO:tasks.workunit.client.0.smithi070.stdout: "state_prepare_lat": { 2024-09-14T03:47:51.906 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 361, 2024-09-14T03:47:51.906 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 17285706 2024-09-14T03:47:51.906 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.906 INFO:tasks.workunit.client.0.smithi070.stdout: "stored": { 2024-09-14T03:47:51.906 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 311801 2024-09-14T03:47:51.906 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.906 INFO:tasks.workunit.client.0.smithi070.stdout: "truncate_lat": { 2024-09-14T03:47:51.906 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 54, 2024-09-14T03:47:51.906 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 279365 2024-09-14T03:47:51.906 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.906 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_commit_lat": { 2024-09-14T03:47:51.906 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 361, 2024-09-14T03:47:51.907 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 202128554 2024-09-14T03:47:51.907 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.907 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_submit_lat": { 2024-09-14T03:47:51.907 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 361, 2024-09-14T03:47:51.907 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 22012732 2024-09-14T03:47:51.907 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.907 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_throttle_lat": { 2024-09-14T03:47:51.907 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 361, 2024-09-14T03:47:51.907 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2404877 2024-09-14T03:47:51.907 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.907 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.907 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache": { 2024-09-14T03:47:51.907 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_bytes": { 2024-09-14T03:47:51.907 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1861444387 2024-09-14T03:47:51.908 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.908 INFO:tasks.workunit.client.0.smithi070.stdout: "heap_bytes": { 2024-09-14T03:47:51.908 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 270909440 2024-09-14T03:47:51.908 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.908 INFO:tasks.workunit.client.0.smithi070.stdout: "mapped_bytes": { 2024-09-14T03:47:51.908 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 270409728 2024-09-14T03:47:51.908 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.908 INFO:tasks.workunit.client.0.smithi070.stdout: "target_bytes": { 2024-09-14T03:47:51.908 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3137353830 2024-09-14T03:47:51.908 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.908 INFO:tasks.workunit.client.0.smithi070.stdout: "unmapped_bytes": { 2024-09-14T03:47:51.908 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 499712 2024-09-14T03:47:51.908 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.909 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.909 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:data": { 2024-09-14T03:47:51.909 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:51.909 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 159383552 2024-09-14T03:47:51.909 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.909 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:51.909 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.909 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.909 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:51.909 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.909 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.909 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:51.909 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 85497890 2024-09-14T03:47:51.910 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.910 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:51.910 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.910 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.910 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:51.910 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.910 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.910 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:51.910 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.910 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.910 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:51.910 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.910 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.910 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:51.911 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.911 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.911 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:51.911 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12288 2024-09-14T03:47:51.911 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.911 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:51.911 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.911 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.911 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:51.911 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.911 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.911 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:51.911 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.911 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.912 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:51.912 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 73873374 2024-09-14T03:47:51.912 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.912 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.912 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:kv": { 2024-09-14T03:47:51.912 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:51.912 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 780140544 2024-09-14T03:47:51.912 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.912 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:51.912 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:51.912 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.912 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:51.912 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.913 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.913 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:51.913 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 641234175 2024-09-14T03:47:51.913 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.913 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:51.913 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.913 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.913 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:51.913 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.913 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.913 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:51.913 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.913 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.913 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:51.914 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.914 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.914 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:51.914 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.914 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.914 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:51.914 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2128 2024-09-14T03:47:51.914 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.914 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:51.914 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.914 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.914 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:51.914 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.915 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.915 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:51.915 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.915 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.915 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:51.915 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71795377 2024-09-14T03:47:51.915 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.915 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.915 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:kv_onode": { 2024-09-14T03:47:51.915 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:51.915 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 192937984 2024-09-14T03:47:51.915 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.915 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:51.915 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:51.916 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.916 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:51.916 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 464 2024-09-14T03:47:51.916 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.916 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:51.916 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 56998593 2024-09-14T03:47:51.916 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.916 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:51.916 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.916 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.916 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:51.916 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.916 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.917 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:51.917 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.917 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.917 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:51.917 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.917 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.917 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:51.917 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.917 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.917 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:51.917 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.917 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.917 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:51.918 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.918 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.918 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:51.918 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.918 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.918 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:51.918 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.918 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.918 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:51.918 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 68830063 2024-09-14T03:47:51.918 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.918 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.918 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:meta": { 2024-09-14T03:47:51.918 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:51.919 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 713031680 2024-09-14T03:47:51.919 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.919 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:51.919 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.919 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.919 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:51.919 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.919 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.919 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:51.919 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 641234175 2024-09-14T03:47:51.919 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.919 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:51.919 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.920 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.920 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:51.920 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.920 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.920 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:51.920 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.920 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.920 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:51.920 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.920 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.920 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:51.920 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.920 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.921 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:51.921 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 254262 2024-09-14T03:47:51.921 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.921 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:51.921 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.921 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.921 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:51.921 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.921 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.921 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:51.921 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.921 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.921 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:51.921 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71543243 2024-09-14T03:47:51.922 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.922 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.922 INFO:tasks.workunit.client.0.smithi070.stdout: "objecter": { 2024-09-14T03:47:51.922 INFO:tasks.workunit.client.0.smithi070.stdout: "op_active": { 2024-09-14T03:47:51.922 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.922 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.922 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r": { 2024-09-14T03:47:51.922 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.922 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.922 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rmw": { 2024-09-14T03:47:51.922 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.922 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.922 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w": { 2024-09-14T03:47:51.923 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.923 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.923 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.923 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:51.923 INFO:tasks.workunit.client.0.smithi070.stdout: "numpg": { 2024-09-14T03:47:51.923 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 38 2024-09-14T03:47:51.923 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.923 INFO:tasks.workunit.client.0.smithi070.stdout: "numpg_removing": { 2024-09-14T03:47:51.923 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.923 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.923 INFO:tasks.workunit.client.0.smithi070.stdout: "op": { 2024-09-14T03:47:51.923 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 97 2024-09-14T03:47:51.923 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.923 INFO:tasks.workunit.client.0.smithi070.stdout: "op_delayed_degraded": { 2024-09-14T03:47:51.924 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.924 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.924 INFO:tasks.workunit.client.0.smithi070.stdout: "op_delayed_unreadable": { 2024-09-14T03:47:51.924 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.924 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.924 INFO:tasks.workunit.client.0.smithi070.stdout: "op_in_bytes": { 2024-09-14T03:47:51.924 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1640827 2024-09-14T03:47:51.924 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.924 INFO:tasks.workunit.client.0.smithi070.stdout: "op_latency": { 2024-09-14T03:47:51.924 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 97, 2024-09-14T03:47:51.924 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 456375647 2024-09-14T03:47:51.924 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.924 INFO:tasks.workunit.client.0.smithi070.stdout: "op_out_bytes": { 2024-09-14T03:47:51.925 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 602 2024-09-14T03:47:51.925 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.925 INFO:tasks.workunit.client.0.smithi070.stdout: "op_prepare_latency": { 2024-09-14T03:47:51.925 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 97, 2024-09-14T03:47:51.925 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 33994934 2024-09-14T03:47:51.925 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.925 INFO:tasks.workunit.client.0.smithi070.stdout: "op_process_latency": { 2024-09-14T03:47:51.925 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 97, 2024-09-14T03:47:51.925 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 105513799 2024-09-14T03:47:51.925 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.925 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r": { 2024-09-14T03:47:51.925 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 18 2024-09-14T03:47:51.925 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.925 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_latency": { 2024-09-14T03:47:51.926 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 18, 2024-09-14T03:47:51.926 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2687087 2024-09-14T03:47:51.926 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.926 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:47:51.926 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.926 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.926 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_out_bytes": { 2024-09-14T03:47:51.926 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 602 2024-09-14T03:47:51.926 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.926 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_prepare_latency": { 2024-09-14T03:47:51.926 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 18, 2024-09-14T03:47:51.926 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2366109 2024-09-14T03:47:51.926 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.927 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_process_latency": { 2024-09-14T03:47:51.927 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 18, 2024-09-14T03:47:51.927 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1914640 2024-09-14T03:47:51.927 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.927 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw": { 2024-09-14T03:47:51.927 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12 2024-09-14T03:47:51.927 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.927 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_in_bytes": { 2024-09-14T03:47:51.927 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.927 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.927 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency": { 2024-09-14T03:47:51.927 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 12, 2024-09-14T03:47:51.927 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 10442901 2024-09-14T03:47:51.927 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.928 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:47:51.928 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.928 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.928 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:47:51.928 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.928 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.928 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_out_bytes": { 2024-09-14T03:47:51.928 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.928 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.928 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_prepare_latency": { 2024-09-14T03:47:51.928 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 12, 2024-09-14T03:47:51.928 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4934481 2024-09-14T03:47:51.928 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.928 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_process_latency": { 2024-09-14T03:47:51.929 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 12, 2024-09-14T03:47:51.929 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 9786014 2024-09-14T03:47:51.929 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.929 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w": { 2024-09-14T03:47:51.929 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67 2024-09-14T03:47:51.929 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.929 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_in_bytes": { 2024-09-14T03:47:51.929 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1640827 2024-09-14T03:47:51.929 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.929 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_latency": { 2024-09-14T03:47:51.929 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 67, 2024-09-14T03:47:51.929 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 443245659 2024-09-14T03:47:51.929 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.930 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:47:51.930 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.930 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.930 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_prepare_latency": { 2024-09-14T03:47:51.930 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 67, 2024-09-14T03:47:51.930 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 26694344 2024-09-14T03:47:51.930 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.930 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_process_latency": { 2024-09-14T03:47:51.930 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 67, 2024-09-14T03:47:51.930 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 93813145 2024-09-14T03:47:51.930 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.930 INFO:tasks.workunit.client.0.smithi070.stdout: "op_wip": { 2024-09-14T03:47:51.930 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.930 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.931 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_bytes": { 2024-09-14T03:47:51.931 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 590368 2024-09-14T03:47:51.931 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.931 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_ops": { 2024-09-14T03:47:51.931 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:51.931 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.931 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_bytes": { 2024-09-14T03:47:51.931 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 95995035648 2024-09-14T03:47:51.931 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.931 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_bytes_used": { 2024-09-14T03:47:51.931 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 305188864 2024-09-14T03:47:51.931 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.931 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.931 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.932 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.2": { 2024-09-14T03:47:51.932 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:51.932 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_slow_fallback": { 2024-09-14T03:47:51.932 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.932 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.932 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_slow_size_fallback": { 2024-09-14T03:47:51.932 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.932 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.932 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_db": { 2024-09-14T03:47:51.932 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 65536 2024-09-14T03:47:51.932 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.932 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_main": { 2024-09-14T03:47:51.932 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.933 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.933 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_wal": { 2024-09-14T03:47:51.933 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.933 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.933 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_slow": { 2024-09-14T03:47:51.933 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.933 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.933 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_sst": { 2024-09-14T03:47:51.933 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12288 2024-09-14T03:47:51.933 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.933 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_wal": { 2024-09-14T03:47:51.933 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1724416 2024-09-14T03:47:51.933 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.933 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lat": { 2024-09-14T03:47:51.934 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.934 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.934 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.934 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lock_lat": { 2024-09-14T03:47:51.934 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.934 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.934 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.934 INFO:tasks.workunit.client.0.smithi070.stdout: "db_total_bytes": { 2024-09-14T03:47:51.934 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 95995035648 2024-09-14T03:47:51.934 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.934 INFO:tasks.workunit.client.0.smithi070.stdout: "db_used_bytes": { 2024-09-14T03:47:51.934 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 304480256 2024-09-14T03:47:51.934 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.934 INFO:tasks.workunit.client.0.smithi070.stdout: "log_bytes": { 2024-09-14T03:47:51.935 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1171456 2024-09-14T03:47:51.935 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.935 INFO:tasks.workunit.client.0.smithi070.stdout: "logged_bytes": { 2024-09-14T03:47:51.935 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1032192 2024-09-14T03:47:51.935 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.935 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_db": { 2024-09-14T03:47:51.935 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 312868864 2024-09-14T03:47:51.935 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.935 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_slow": { 2024-09-14T03:47:51.935 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.935 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.935 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_wal": { 2024-09-14T03:47:51.936 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.936 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.936 INFO:tasks.workunit.client.0.smithi070.stdout: "num_files": { 2024-09-14T03:47:51.936 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 14 2024-09-14T03:47:51.936 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.936 INFO:tasks.workunit.client.0.smithi070.stdout: "read_bytes": { 2024-09-14T03:47:51.936 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 195697 2024-09-14T03:47:51.936 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.936 INFO:tasks.workunit.client.0.smithi070.stdout: "read_count": { 2024-09-14T03:47:51.936 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 23 2024-09-14T03:47:51.936 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.936 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes": { 2024-09-14T03:47:51.936 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1269760 2024-09-14T03:47:51.937 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.937 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_db": { 2024-09-14T03:47:51.937 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1273856 2024-09-14T03:47:51.937 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.937 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_slow": { 2024-09-14T03:47:51.937 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.937 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.937 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_wal": { 2024-09-14T03:47:51.937 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.937 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.937 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_count": { 2024-09-14T03:47:51.937 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 15 2024-09-14T03:47:51.937 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.938 INFO:tasks.workunit.client.0.smithi070.stdout: "read_prefetch_bytes": { 2024-09-14T03:47:51.938 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4789 2024-09-14T03:47:51.938 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.938 INFO:tasks.workunit.client.0.smithi070.stdout: "read_prefetch_count": { 2024-09-14T03:47:51.938 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4 2024-09-14T03:47:51.938 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.938 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_buffer_bytes": { 2024-09-14T03:47:51.938 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5589 2024-09-14T03:47:51.938 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.938 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_buffer_count": { 2024-09-14T03:47:51.938 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 24 2024-09-14T03:47:51.938 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.938 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_bytes": { 2024-09-14T03:47:51.939 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5589 2024-09-14T03:47:51.939 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.939 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_count": { 2024-09-14T03:47:51.939 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 24 2024-09-14T03:47:51.939 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.939 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes": { 2024-09-14T03:47:51.939 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.939 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.939 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_db": { 2024-09-14T03:47:51.939 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.939 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.939 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_slow": { 2024-09-14T03:47:51.939 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.940 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.940 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_wal": { 2024-09-14T03:47:51.940 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.940 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.940 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_count": { 2024-09-14T03:47:51.940 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.940 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.940 INFO:tasks.workunit.client.0.smithi070.stdout: "slow_total_bytes": { 2024-09-14T03:47:51.940 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.940 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.940 INFO:tasks.workunit.client.0.smithi070.stdout: "slow_used_bytes": { 2024-09-14T03:47:51.940 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.941 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.941 INFO:tasks.workunit.client.0.smithi070.stdout: "wal_total_bytes": { 2024-09-14T03:47:51.941 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.941 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.941 INFO:tasks.workunit.client.0.smithi070.stdout: "wal_used_bytes": { 2024-09-14T03:47:51.941 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.941 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.941 INFO:tasks.workunit.client.0.smithi070.stdout: "write_bytes": { 2024-09-14T03:47:51.941 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2830336 2024-09-14T03:47:51.941 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.941 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.941 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore": { 2024-09-14T03:47:51.941 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit": { 2024-09-14T03:47:51.941 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4096 2024-09-14T03:47:51.942 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.942 INFO:tasks.workunit.client.0.smithi070.stdout: "allocated": { 2024-09-14T03:47:51.942 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 692224 2024-09-14T03:47:51.942 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.942 INFO:tasks.workunit.client.0.smithi070.stdout: "clist_lat": { 2024-09-14T03:47:51.942 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 4, 2024-09-14T03:47:51.942 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 200380 2024-09-14T03:47:51.942 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.942 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_lat": { 2024-09-14T03:47:51.942 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.942 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.942 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.942 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed": { 2024-09-14T03:47:51.942 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.943 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.943 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed_allocated": { 2024-09-14T03:47:51.943 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.943 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.943 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed_original": { 2024-09-14T03:47:51.943 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.943 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.943 INFO:tasks.workunit.client.0.smithi070.stdout: "csum_lat": { 2024-09-14T03:47:51.943 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 9, 2024-09-14T03:47:51.943 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 14449 2024-09-14T03:47:51.943 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.943 INFO:tasks.workunit.client.0.smithi070.stdout: "decompress_lat": { 2024-09-14T03:47:51.943 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.943 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.944 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.944 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_commit_lat": { 2024-09-14T03:47:51.944 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 235, 2024-09-14T03:47:51.944 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 37816316 2024-09-14T03:47:51.944 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.944 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_final_lat": { 2024-09-14T03:47:51.944 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 227, 2024-09-14T03:47:51.944 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 8138689 2024-09-14T03:47:51.944 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.944 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_flush_lat": { 2024-09-14T03:47:51.944 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 235, 2024-09-14T03:47:51.944 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 323761 2024-09-14T03:47:51.944 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.944 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_sync_lat": { 2024-09-14T03:47:51.945 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 235, 2024-09-14T03:47:51.945 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 38140077 2024-09-14T03:47:51.945 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.945 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_get_keys_lat": { 2024-09-14T03:47:51.945 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.945 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.945 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.945 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_get_values_lat": { 2024-09-14T03:47:51.945 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 3, 2024-09-14T03:47:51.945 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 31316 2024-09-14T03:47:51.945 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.945 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_lower_bound_lat": { 2024-09-14T03:47:51.945 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:51.946 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1597 2024-09-14T03:47:51.946 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.946 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_next_lat": { 2024-09-14T03:47:51.946 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.946 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.946 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.946 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_seek_to_first_lat": { 2024-09-14T03:47:51.946 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.946 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.946 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.946 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_upper_bound_lat": { 2024-09-14T03:47:51.946 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:51.946 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3729 2024-09-14T03:47:51.946 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.947 INFO:tasks.workunit.client.0.smithi070.stdout: "onode_hits": { 2024-09-14T03:47:51.947 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 492 2024-09-14T03:47:51.947 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.947 INFO:tasks.workunit.client.0.smithi070.stdout: "onode_misses": { 2024-09-14T03:47:51.947 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 210 2024-09-14T03:47:51.947 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.947 INFO:tasks.workunit.client.0.smithi070.stdout: "read_lat": { 2024-09-14T03:47:51.947 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 89, 2024-09-14T03:47:51.947 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 881508 2024-09-14T03:47:51.947 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.947 INFO:tasks.workunit.client.0.smithi070.stdout: "read_onode_meta_lat": { 2024-09-14T03:47:51.947 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 210, 2024-09-14T03:47:51.947 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 249409 2024-09-14T03:47:51.948 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.948 INFO:tasks.workunit.client.0.smithi070.stdout: "read_wait_aio_lat": { 2024-09-14T03:47:51.948 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 121, 2024-09-14T03:47:51.948 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 417565 2024-09-14T03:47:51.948 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.948 INFO:tasks.workunit.client.0.smithi070.stdout: "reads_with_retries": { 2024-09-14T03:47:51.948 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.948 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.948 INFO:tasks.workunit.client.0.smithi070.stdout: "remove_lat": { 2024-09-14T03:47:51.948 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 3, 2024-09-14T03:47:51.948 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 90347 2024-09-14T03:47:51.948 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.948 INFO:tasks.workunit.client.0.smithi070.stdout: "state_aio_wait_lat": { 2024-09-14T03:47:51.948 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 285, 2024-09-14T03:47:51.949 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5063422 2024-09-14T03:47:51.949 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.949 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_aio_wait_lat": { 2024-09-14T03:47:51.949 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 87, 2024-09-14T03:47:51.949 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5592582 2024-09-14T03:47:51.949 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.949 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_cleanup_lat": { 2024-09-14T03:47:51.949 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 86, 2024-09-14T03:47:51.949 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 257667352068 2024-09-14T03:47:51.949 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.949 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_queued_lat": { 2024-09-14T03:47:51.949 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 87, 2024-09-14T03:47:51.949 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 145354073905 2024-09-14T03:47:51.950 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.950 INFO:tasks.workunit.client.0.smithi070.stdout: "state_done_lat": { 2024-09-14T03:47:51.950 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 284, 2024-09-14T03:47:51.950 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 32818108531 2024-09-14T03:47:51.950 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.950 INFO:tasks.workunit.client.0.smithi070.stdout: "state_finishing_lat": { 2024-09-14T03:47:51.950 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 284, 2024-09-14T03:47:51.950 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 38139 2024-09-14T03:47:51.950 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.950 INFO:tasks.workunit.client.0.smithi070.stdout: "state_io_done_lat": { 2024-09-14T03:47:51.950 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 285, 2024-09-14T03:47:51.950 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 344294 2024-09-14T03:47:51.950 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.951 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_commiting_lat": { 2024-09-14T03:47:51.951 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 285, 2024-09-14T03:47:51.951 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 83170160 2024-09-14T03:47:51.951 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.951 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_done_lat": { 2024-09-14T03:47:51.951 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 285, 2024-09-14T03:47:51.951 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 94531 2024-09-14T03:47:51.951 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.951 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_queued_lat": { 2024-09-14T03:47:51.951 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 285, 2024-09-14T03:47:51.951 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 57059253 2024-09-14T03:47:51.951 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.951 INFO:tasks.workunit.client.0.smithi070.stdout: "state_prepare_lat": { 2024-09-14T03:47:51.951 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 285, 2024-09-14T03:47:51.951 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 14890234 2024-09-14T03:47:51.952 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.952 INFO:tasks.workunit.client.0.smithi070.stdout: "stored": { 2024-09-14T03:47:51.952 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 311956 2024-09-14T03:47:51.952 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.952 INFO:tasks.workunit.client.0.smithi070.stdout: "truncate_lat": { 2024-09-14T03:47:51.952 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 47, 2024-09-14T03:47:51.952 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 228940 2024-09-14T03:47:51.952 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.952 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_commit_lat": { 2024-09-14T03:47:51.952 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 285, 2024-09-14T03:47:51.952 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 160570020 2024-09-14T03:47:51.952 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.952 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_submit_lat": { 2024-09-14T03:47:51.953 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 285, 2024-09-14T03:47:51.953 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 18307697 2024-09-14T03:47:51.953 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.953 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_throttle_lat": { 2024-09-14T03:47:51.953 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 285, 2024-09-14T03:47:51.953 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1921189 2024-09-14T03:47:51.953 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.953 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.953 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache": { 2024-09-14T03:47:51.953 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_bytes": { 2024-09-14T03:47:51.953 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1861444387 2024-09-14T03:47:51.953 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.954 INFO:tasks.workunit.client.0.smithi070.stdout: "heap_bytes": { 2024-09-14T03:47:51.954 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 269860864 2024-09-14T03:47:51.954 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.954 INFO:tasks.workunit.client.0.smithi070.stdout: "mapped_bytes": { 2024-09-14T03:47:51.954 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 269131776 2024-09-14T03:47:51.954 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.954 INFO:tasks.workunit.client.0.smithi070.stdout: "target_bytes": { 2024-09-14T03:47:51.954 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3137353830 2024-09-14T03:47:51.954 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.954 INFO:tasks.workunit.client.0.smithi070.stdout: "unmapped_bytes": { 2024-09-14T03:47:51.954 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 729088 2024-09-14T03:47:51.954 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.954 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.955 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:data": { 2024-09-14T03:47:51.955 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:51.955 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 159383552 2024-09-14T03:47:51.955 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.955 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:51.955 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.955 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.955 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:51.955 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.955 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.955 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:51.955 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 85498746 2024-09-14T03:47:51.956 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.956 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:51.956 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.956 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.956 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:51.956 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.956 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.956 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:51.956 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.956 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.956 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:51.956 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.956 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.956 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:51.956 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.957 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.957 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:51.957 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 8192 2024-09-14T03:47:51.957 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.957 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:51.957 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.957 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.957 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:51.957 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.957 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.957 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:51.957 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.957 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.958 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:51.958 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 73876614 2024-09-14T03:47:51.958 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.958 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.958 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:kv": { 2024-09-14T03:47:51.958 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:51.958 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 780140544 2024-09-14T03:47:51.958 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.958 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:51.958 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:51.958 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.958 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:51.958 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.959 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.959 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:51.959 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 641240600 2024-09-14T03:47:51.959 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.959 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:51.959 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.959 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.959 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:51.959 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.959 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.959 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:51.959 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.959 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.959 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:51.960 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.960 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.960 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:51.960 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.960 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.960 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:51.960 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2128 2024-09-14T03:47:51.960 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.960 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:51.960 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.960 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.960 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:51.960 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.960 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.961 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:51.961 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.961 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.961 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:51.961 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71788952 2024-09-14T03:47:51.961 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.961 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.961 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:kv_onode": { 2024-09-14T03:47:51.961 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:51.961 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 192937984 2024-09-14T03:47:51.961 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.961 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:51.961 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:51.962 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.962 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:51.962 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 464 2024-09-14T03:47:51.962 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.962 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:51.962 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 56999164 2024-09-14T03:47:51.962 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.962 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:51.962 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.962 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.962 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:51.962 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.962 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.963 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:51.963 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.963 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.963 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:51.963 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.963 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.963 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:51.963 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.963 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.963 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:51.963 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.963 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.964 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:51.964 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.964 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.964 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:51.964 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.964 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.964 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:51.964 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.964 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.964 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:51.964 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 68829492 2024-09-14T03:47:51.964 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.964 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.965 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:meta": { 2024-09-14T03:47:51.965 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:51.965 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 713031680 2024-09-14T03:47:51.965 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.965 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:51.965 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.965 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.965 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:51.965 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.965 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.965 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:51.965 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 641240600 2024-09-14T03:47:51.965 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.966 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:51.966 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.966 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.966 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:51.966 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.966 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.966 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:51.966 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.966 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.966 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:51.966 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.966 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.966 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:51.967 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.967 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.967 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:51.967 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 244085 2024-09-14T03:47:51.967 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.967 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:51.967 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.967 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.967 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:51.967 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.967 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.967 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:51.967 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.967 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.968 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:51.968 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71546995 2024-09-14T03:47:51.968 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.968 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.968 INFO:tasks.workunit.client.0.smithi070.stdout: "objecter": { 2024-09-14T03:47:51.968 INFO:tasks.workunit.client.0.smithi070.stdout: "op_active": { 2024-09-14T03:47:51.968 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.968 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.968 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r": { 2024-09-14T03:47:51.968 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.968 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.968 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rmw": { 2024-09-14T03:47:51.968 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.969 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.969 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w": { 2024-09-14T03:47:51.969 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.969 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.969 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.969 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:51.969 INFO:tasks.workunit.client.0.smithi070.stdout: "numpg": { 2024-09-14T03:47:51.969 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 31 2024-09-14T03:47:51.969 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.969 INFO:tasks.workunit.client.0.smithi070.stdout: "numpg_removing": { 2024-09-14T03:47:51.969 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.969 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.969 INFO:tasks.workunit.client.0.smithi070.stdout: "op": { 2024-09-14T03:47:51.969 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 10 2024-09-14T03:47:51.970 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.970 INFO:tasks.workunit.client.0.smithi070.stdout: "op_delayed_degraded": { 2024-09-14T03:47:51.970 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.970 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.970 INFO:tasks.workunit.client.0.smithi070.stdout: "op_delayed_unreadable": { 2024-09-14T03:47:51.970 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.970 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.970 INFO:tasks.workunit.client.0.smithi070.stdout: "op_in_bytes": { 2024-09-14T03:47:51.970 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.970 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.970 INFO:tasks.workunit.client.0.smithi070.stdout: "op_latency": { 2024-09-14T03:47:51.970 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 10, 2024-09-14T03:47:51.970 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 90683861 2024-09-14T03:47:51.971 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.971 INFO:tasks.workunit.client.0.smithi070.stdout: "op_out_bytes": { 2024-09-14T03:47:51.971 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.971 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.971 INFO:tasks.workunit.client.0.smithi070.stdout: "op_prepare_latency": { 2024-09-14T03:47:51.971 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 10, 2024-09-14T03:47:51.971 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5772820 2024-09-14T03:47:51.971 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.971 INFO:tasks.workunit.client.0.smithi070.stdout: "op_process_latency": { 2024-09-14T03:47:51.971 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 10, 2024-09-14T03:47:51.971 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 72345145 2024-09-14T03:47:51.971 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.971 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r": { 2024-09-14T03:47:51.972 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.972 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.972 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_latency": { 2024-09-14T03:47:51.972 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.972 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.972 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.972 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:47:51.972 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.972 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.972 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_out_bytes": { 2024-09-14T03:47:51.972 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.972 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.973 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_prepare_latency": { 2024-09-14T03:47:51.973 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.973 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.973 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.973 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_process_latency": { 2024-09-14T03:47:51.973 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.973 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.973 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.973 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw": { 2024-09-14T03:47:51.973 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.973 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.973 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_in_bytes": { 2024-09-14T03:47:51.973 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.974 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.974 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency": { 2024-09-14T03:47:51.974 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.974 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.974 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.974 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:47:51.974 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.974 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.974 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:47:51.974 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.974 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.974 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_out_bytes": { 2024-09-14T03:47:51.974 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.975 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.975 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_prepare_latency": { 2024-09-14T03:47:51.975 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.975 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.975 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.975 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_process_latency": { 2024-09-14T03:47:51.975 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.975 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.975 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.975 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w": { 2024-09-14T03:47:51.975 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 10 2024-09-14T03:47:51.975 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.975 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_in_bytes": { 2024-09-14T03:47:51.975 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.976 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.976 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_latency": { 2024-09-14T03:47:51.976 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 10, 2024-09-14T03:47:51.976 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 90683861 2024-09-14T03:47:51.976 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.976 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:47:51.976 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.976 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.976 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_prepare_latency": { 2024-09-14T03:47:51.976 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 10, 2024-09-14T03:47:51.976 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5772820 2024-09-14T03:47:51.976 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.976 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_process_latency": { 2024-09-14T03:47:51.977 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 10, 2024-09-14T03:47:51.977 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 72345145 2024-09-14T03:47:51.977 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.977 INFO:tasks.workunit.client.0.smithi070.stdout: "op_wip": { 2024-09-14T03:47:51.977 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.977 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.977 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_bytes": { 2024-09-14T03:47:51.977 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.977 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.977 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_ops": { 2024-09-14T03:47:51.977 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.977 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.977 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_bytes": { 2024-09-14T03:47:51.978 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 95995035648 2024-09-14T03:47:51.978 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.978 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_bytes_used": { 2024-09-14T03:47:51.978 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 305184768 2024-09-14T03:47:51.978 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.978 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.978 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.978 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.3": { 2024-09-14T03:47:51.978 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:51.978 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_slow_fallback": { 2024-09-14T03:47:51.978 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.978 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.978 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_slow_size_fallback": { 2024-09-14T03:47:51.979 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.979 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.979 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_db": { 2024-09-14T03:47:51.979 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 65536 2024-09-14T03:47:51.979 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.979 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_main": { 2024-09-14T03:47:51.979 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.979 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.979 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_wal": { 2024-09-14T03:47:51.979 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.979 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.979 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_slow": { 2024-09-14T03:47:51.979 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.979 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.980 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_sst": { 2024-09-14T03:47:51.980 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12288 2024-09-14T03:47:51.980 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.980 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_wal": { 2024-09-14T03:47:51.980 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1159168 2024-09-14T03:47:51.980 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.980 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lat": { 2024-09-14T03:47:51.980 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.980 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.980 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.980 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lock_lat": { 2024-09-14T03:47:51.980 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.980 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.981 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.981 INFO:tasks.workunit.client.0.smithi070.stdout: "db_total_bytes": { 2024-09-14T03:47:51.981 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 95995035648 2024-09-14T03:47:51.981 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.981 INFO:tasks.workunit.client.0.smithi070.stdout: "db_used_bytes": { 2024-09-14T03:47:51.981 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 304480256 2024-09-14T03:47:51.981 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.981 INFO:tasks.workunit.client.0.smithi070.stdout: "log_bytes": { 2024-09-14T03:47:51.981 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 892928 2024-09-14T03:47:51.981 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.981 INFO:tasks.workunit.client.0.smithi070.stdout: "logged_bytes": { 2024-09-14T03:47:51.981 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 753664 2024-09-14T03:47:51.981 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.982 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_db": { 2024-09-14T03:47:51.982 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 312868864 2024-09-14T03:47:51.982 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.982 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_slow": { 2024-09-14T03:47:51.982 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.982 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.982 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_wal": { 2024-09-14T03:47:51.982 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.982 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.982 INFO:tasks.workunit.client.0.smithi070.stdout: "num_files": { 2024-09-14T03:47:51.982 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 14 2024-09-14T03:47:51.982 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.983 INFO:tasks.workunit.client.0.smithi070.stdout: "read_bytes": { 2024-09-14T03:47:51.983 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 195697 2024-09-14T03:47:51.983 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.983 INFO:tasks.workunit.client.0.smithi070.stdout: "read_count": { 2024-09-14T03:47:51.983 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 23 2024-09-14T03:47:51.983 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.983 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes": { 2024-09-14T03:47:51.983 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1269760 2024-09-14T03:47:51.983 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.983 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_db": { 2024-09-14T03:47:51.983 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1273856 2024-09-14T03:47:51.983 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.983 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_slow": { 2024-09-14T03:47:51.984 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.984 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.984 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_wal": { 2024-09-14T03:47:51.984 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.984 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.984 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_count": { 2024-09-14T03:47:51.984 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 15 2024-09-14T03:47:51.984 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.984 INFO:tasks.workunit.client.0.smithi070.stdout: "read_prefetch_bytes": { 2024-09-14T03:47:51.984 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4789 2024-09-14T03:47:51.984 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.984 INFO:tasks.workunit.client.0.smithi070.stdout: "read_prefetch_count": { 2024-09-14T03:47:51.984 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4 2024-09-14T03:47:51.985 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.985 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_buffer_bytes": { 2024-09-14T03:47:51.985 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5589 2024-09-14T03:47:51.985 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.985 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_buffer_count": { 2024-09-14T03:47:51.985 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 24 2024-09-14T03:47:51.985 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.985 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_bytes": { 2024-09-14T03:47:51.985 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5589 2024-09-14T03:47:51.985 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.985 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_count": { 2024-09-14T03:47:51.985 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 24 2024-09-14T03:47:51.985 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.985 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes": { 2024-09-14T03:47:51.986 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.986 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.986 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_db": { 2024-09-14T03:47:51.986 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.986 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.986 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_slow": { 2024-09-14T03:47:51.986 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.986 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.986 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_wal": { 2024-09-14T03:47:51.986 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.986 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.986 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_count": { 2024-09-14T03:47:51.986 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.987 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.987 INFO:tasks.workunit.client.0.smithi070.stdout: "slow_total_bytes": { 2024-09-14T03:47:51.987 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.987 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.987 INFO:tasks.workunit.client.0.smithi070.stdout: "slow_used_bytes": { 2024-09-14T03:47:51.987 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.987 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.987 INFO:tasks.workunit.client.0.smithi070.stdout: "wal_total_bytes": { 2024-09-14T03:47:51.987 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.987 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.987 INFO:tasks.workunit.client.0.smithi070.stdout: "wal_used_bytes": { 2024-09-14T03:47:51.987 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.987 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.987 INFO:tasks.workunit.client.0.smithi070.stdout: "write_bytes": { 2024-09-14T03:47:51.988 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1986560 2024-09-14T03:47:51.988 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:51.988 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.988 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore": { 2024-09-14T03:47:51.988 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit": { 2024-09-14T03:47:51.988 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4096 2024-09-14T03:47:51.988 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.988 INFO:tasks.workunit.client.0.smithi070.stdout: "allocated": { 2024-09-14T03:47:51.988 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 712704 2024-09-14T03:47:51.988 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.988 INFO:tasks.workunit.client.0.smithi070.stdout: "clist_lat": { 2024-09-14T03:47:51.988 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.988 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.988 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.989 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_lat": { 2024-09-14T03:47:51.989 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.989 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.989 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.989 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed": { 2024-09-14T03:47:51.989 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.989 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.989 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed_allocated": { 2024-09-14T03:47:51.989 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.989 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.989 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed_original": { 2024-09-14T03:47:51.989 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.989 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.989 INFO:tasks.workunit.client.0.smithi070.stdout: "csum_lat": { 2024-09-14T03:47:51.990 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 3, 2024-09-14T03:47:51.990 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5222 2024-09-14T03:47:51.990 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.990 INFO:tasks.workunit.client.0.smithi070.stdout: "decompress_lat": { 2024-09-14T03:47:51.990 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.990 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.990 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.990 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_commit_lat": { 2024-09-14T03:47:51.990 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 167, 2024-09-14T03:47:51.990 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 39046400 2024-09-14T03:47:51.990 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.990 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_final_lat": { 2024-09-14T03:47:51.990 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 164, 2024-09-14T03:47:51.990 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 8628994 2024-09-14T03:47:51.991 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.991 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_flush_lat": { 2024-09-14T03:47:51.991 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 167, 2024-09-14T03:47:51.991 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 241305 2024-09-14T03:47:51.991 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.991 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_sync_lat": { 2024-09-14T03:47:51.991 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 167, 2024-09-14T03:47:51.991 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 39287705 2024-09-14T03:47:51.991 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.991 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_get_keys_lat": { 2024-09-14T03:47:51.991 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.991 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.991 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.992 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_get_values_lat": { 2024-09-14T03:47:51.992 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:51.992 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 35574 2024-09-14T03:47:51.992 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.992 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_lower_bound_lat": { 2024-09-14T03:47:51.992 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:51.992 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1902 2024-09-14T03:47:51.992 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.992 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_next_lat": { 2024-09-14T03:47:51.992 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.992 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.992 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.992 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_seek_to_first_lat": { 2024-09-14T03:47:51.993 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.993 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.993 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.993 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_upper_bound_lat": { 2024-09-14T03:47:51.993 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 3, 2024-09-14T03:47:51.993 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 8538 2024-09-14T03:47:51.993 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.993 INFO:tasks.workunit.client.0.smithi070.stdout: "onode_hits": { 2024-09-14T03:47:51.993 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 332 2024-09-14T03:47:51.993 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.993 INFO:tasks.workunit.client.0.smithi070.stdout: "onode_misses": { 2024-09-14T03:47:51.993 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 218 2024-09-14T03:47:51.993 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.994 INFO:tasks.workunit.client.0.smithi070.stdout: "read_lat": { 2024-09-14T03:47:51.994 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 86, 2024-09-14T03:47:51.994 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 723816 2024-09-14T03:47:51.994 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.994 INFO:tasks.workunit.client.0.smithi070.stdout: "read_onode_meta_lat": { 2024-09-14T03:47:51.994 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 148, 2024-09-14T03:47:51.994 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 268277 2024-09-14T03:47:51.994 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.994 INFO:tasks.workunit.client.0.smithi070.stdout: "read_wait_aio_lat": { 2024-09-14T03:47:51.994 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 62, 2024-09-14T03:47:51.994 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 137814 2024-09-14T03:47:51.994 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.994 INFO:tasks.workunit.client.0.smithi070.stdout: "reads_with_retries": { 2024-09-14T03:47:51.994 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.995 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.995 INFO:tasks.workunit.client.0.smithi070.stdout: "remove_lat": { 2024-09-14T03:47:51.995 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:51.995 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:51.995 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.995 INFO:tasks.workunit.client.0.smithi070.stdout: "state_aio_wait_lat": { 2024-09-14T03:47:51.995 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 236, 2024-09-14T03:47:51.995 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3014988 2024-09-14T03:47:51.995 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.995 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_aio_wait_lat": { 2024-09-14T03:47:51.995 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 49, 2024-09-14T03:47:51.995 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3669955 2024-09-14T03:47:51.996 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.996 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_cleanup_lat": { 2024-09-14T03:47:51.996 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 46, 2024-09-14T03:47:51.996 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 324567165234 2024-09-14T03:47:51.996 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.996 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_queued_lat": { 2024-09-14T03:47:51.996 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 49, 2024-09-14T03:47:51.996 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 83532140038 2024-09-14T03:47:51.996 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.996 INFO:tasks.workunit.client.0.smithi070.stdout: "state_done_lat": { 2024-09-14T03:47:51.996 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 233, 2024-09-14T03:47:51.996 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 467320 2024-09-14T03:47:51.996 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.996 INFO:tasks.workunit.client.0.smithi070.stdout: "state_finishing_lat": { 2024-09-14T03:47:51.997 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 233, 2024-09-14T03:47:51.997 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 26633 2024-09-14T03:47:51.997 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.997 INFO:tasks.workunit.client.0.smithi070.stdout: "state_io_done_lat": { 2024-09-14T03:47:51.997 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 236, 2024-09-14T03:47:51.997 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 208789 2024-09-14T03:47:51.997 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.997 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_commiting_lat": { 2024-09-14T03:47:51.997 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 236, 2024-09-14T03:47:51.997 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 129093950 2024-09-14T03:47:51.997 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.997 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_done_lat": { 2024-09-14T03:47:51.997 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 236, 2024-09-14T03:47:51.998 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67412 2024-09-14T03:47:51.998 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.998 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_queued_lat": { 2024-09-14T03:47:51.998 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 236, 2024-09-14T03:47:51.998 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 75539469 2024-09-14T03:47:51.998 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.998 INFO:tasks.workunit.client.0.smithi070.stdout: "state_prepare_lat": { 2024-09-14T03:47:51.998 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 236, 2024-09-14T03:47:51.998 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 10861153 2024-09-14T03:47:51.998 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.998 INFO:tasks.workunit.client.0.smithi070.stdout: "stored": { 2024-09-14T03:47:51.998 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 314980 2024-09-14T03:47:51.998 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.999 INFO:tasks.workunit.client.0.smithi070.stdout: "truncate_lat": { 2024-09-14T03:47:51.999 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 40, 2024-09-14T03:47:51.999 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 219052 2024-09-14T03:47:51.999 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.999 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_commit_lat": { 2024-09-14T03:47:51.999 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 236, 2024-09-14T03:47:51.999 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 218746204 2024-09-14T03:47:51.999 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.999 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_submit_lat": { 2024-09-14T03:47:51.999 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 236, 2024-09-14T03:47:51.999 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 14422419 2024-09-14T03:47:51.999 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:51.999 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_throttle_lat": { 2024-09-14T03:47:51.999 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 236, 2024-09-14T03:47:52.000 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1491296 2024-09-14T03:47:52.000 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.000 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.000 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache": { 2024-09-14T03:47:52.000 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_bytes": { 2024-09-14T03:47:52.000 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1861444387 2024-09-14T03:47:52.000 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.000 INFO:tasks.workunit.client.0.smithi070.stdout: "heap_bytes": { 2024-09-14T03:47:52.000 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 269926400 2024-09-14T03:47:52.000 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.000 INFO:tasks.workunit.client.0.smithi070.stdout: "mapped_bytes": { 2024-09-14T03:47:52.000 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 268845056 2024-09-14T03:47:52.000 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.000 INFO:tasks.workunit.client.0.smithi070.stdout: "target_bytes": { 2024-09-14T03:47:52.001 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3137353830 2024-09-14T03:47:52.001 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.001 INFO:tasks.workunit.client.0.smithi070.stdout: "unmapped_bytes": { 2024-09-14T03:47:52.001 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1081344 2024-09-14T03:47:52.001 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.001 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.001 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:data": { 2024-09-14T03:47:52.001 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:52.001 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 159383552 2024-09-14T03:47:52.001 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.001 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:52.001 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.001 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.002 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:52.002 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.002 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.002 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:52.002 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 85498699 2024-09-14T03:47:52.002 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.002 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:52.002 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.002 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.002 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:52.002 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.002 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.002 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:52.002 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.003 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.003 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:52.003 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.003 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.003 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:52.003 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.003 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.003 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:52.003 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.003 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.003 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:52.003 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.003 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.004 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:52.004 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.004 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.004 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:52.004 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.004 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.004 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:52.004 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 73884853 2024-09-14T03:47:52.004 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.004 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.004 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:kv": { 2024-09-14T03:47:52.004 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:52.004 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 780140544 2024-09-14T03:47:52.005 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.005 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:52.005 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:52.005 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.005 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:52.005 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.005 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.005 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:52.005 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 641240246 2024-09-14T03:47:52.005 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.005 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:52.005 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.005 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.005 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:52.006 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.006 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.006 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:52.006 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.006 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.006 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:52.006 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.006 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.006 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:52.006 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.006 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.006 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:52.006 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2128 2024-09-14T03:47:52.007 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.007 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:52.007 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.007 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.007 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:52.007 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.007 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.007 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:52.007 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.007 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.007 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:52.007 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71789306 2024-09-14T03:47:52.007 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.007 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.008 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:kv_onode": { 2024-09-14T03:47:52.008 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:52.008 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 192937984 2024-09-14T03:47:52.008 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.008 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:52.008 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:52.008 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.008 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:52.008 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 464 2024-09-14T03:47:52.008 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.008 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:52.008 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 56999133 2024-09-14T03:47:52.008 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.009 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:52.009 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.009 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.009 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:52.009 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.009 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.009 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:52.009 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.009 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.009 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:52.009 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.009 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.010 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:52.010 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.010 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.010 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:52.010 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.010 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.010 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:52.010 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.010 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.010 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:52.010 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.010 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.010 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:52.010 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.011 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.011 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:52.011 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 68829523 2024-09-14T03:47:52.011 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.011 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.011 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:meta": { 2024-09-14T03:47:52.011 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:52.011 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 713031680 2024-09-14T03:47:52.011 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.011 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:52.011 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.011 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.011 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:52.011 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.012 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.012 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:52.012 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 641240246 2024-09-14T03:47:52.012 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.012 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:52.012 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.012 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.012 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:52.012 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.012 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.012 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:52.012 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.013 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.013 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:52.013 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.013 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.013 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:52.013 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 6952 2024-09-14T03:47:52.013 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.013 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:52.013 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 248902 2024-09-14T03:47:52.013 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.013 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:52.013 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.013 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.013 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:52.014 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.014 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.014 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:52.014 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.014 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.014 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:52.014 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71535580 2024-09-14T03:47:52.014 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.014 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.014 INFO:tasks.workunit.client.0.smithi070.stdout: "objecter": { 2024-09-14T03:47:52.014 INFO:tasks.workunit.client.0.smithi070.stdout: "op_active": { 2024-09-14T03:47:52.014 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.014 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.014 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r": { 2024-09-14T03:47:52.015 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.015 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.015 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rmw": { 2024-09-14T03:47:52.015 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.015 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.015 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w": { 2024-09-14T03:47:52.015 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.015 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.015 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.015 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:52.015 INFO:tasks.workunit.client.0.smithi070.stdout: "numpg": { 2024-09-14T03:47:52.015 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 37 2024-09-14T03:47:52.015 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.016 INFO:tasks.workunit.client.0.smithi070.stdout: "numpg_removing": { 2024-09-14T03:47:52.016 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.016 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.016 INFO:tasks.workunit.client.0.smithi070.stdout: "op": { 2024-09-14T03:47:52.016 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.016 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.016 INFO:tasks.workunit.client.0.smithi070.stdout: "op_delayed_degraded": { 2024-09-14T03:47:52.016 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.016 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.016 INFO:tasks.workunit.client.0.smithi070.stdout: "op_delayed_unreadable": { 2024-09-14T03:47:52.016 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.016 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.016 INFO:tasks.workunit.client.0.smithi070.stdout: "op_in_bytes": { 2024-09-14T03:47:52.017 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.017 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.017 INFO:tasks.workunit.client.0.smithi070.stdout: "op_latency": { 2024-09-14T03:47:52.017 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.017 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.017 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.017 INFO:tasks.workunit.client.0.smithi070.stdout: "op_out_bytes": { 2024-09-14T03:47:52.017 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.017 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.017 INFO:tasks.workunit.client.0.smithi070.stdout: "op_prepare_latency": { 2024-09-14T03:47:52.017 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:52.017 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 652050 2024-09-14T03:47:52.017 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.017 INFO:tasks.workunit.client.0.smithi070.stdout: "op_process_latency": { 2024-09-14T03:47:52.018 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.018 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.018 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.018 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r": { 2024-09-14T03:47:52.018 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.018 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.018 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_latency": { 2024-09-14T03:47:52.018 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.018 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.018 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.018 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:47:52.018 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.018 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.019 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_out_bytes": { 2024-09-14T03:47:52.019 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.019 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.019 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_prepare_latency": { 2024-09-14T03:47:52.019 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.019 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.019 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.019 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_process_latency": { 2024-09-14T03:47:52.019 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.019 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.019 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.019 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw": { 2024-09-14T03:47:52.019 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.019 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.020 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_in_bytes": { 2024-09-14T03:47:52.020 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.020 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.020 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency": { 2024-09-14T03:47:52.020 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.020 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.020 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.020 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:47:52.020 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.020 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.020 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:47:52.020 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.020 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.021 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_out_bytes": { 2024-09-14T03:47:52.021 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.021 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.021 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_prepare_latency": { 2024-09-14T03:47:52.021 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.021 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.021 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.021 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_process_latency": { 2024-09-14T03:47:52.021 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.021 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.021 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.021 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w": { 2024-09-14T03:47:52.021 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.022 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.022 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_in_bytes": { 2024-09-14T03:47:52.022 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.022 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.022 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_latency": { 2024-09-14T03:47:52.022 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.022 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.022 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.022 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:47:52.022 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.022 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.022 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_prepare_latency": { 2024-09-14T03:47:52.022 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:52.022 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 652050 2024-09-14T03:47:52.023 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.023 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_process_latency": { 2024-09-14T03:47:52.023 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.023 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.023 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.023 INFO:tasks.workunit.client.0.smithi070.stdout: "op_wip": { 2024-09-14T03:47:52.023 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.023 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.023 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_bytes": { 2024-09-14T03:47:52.023 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.023 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.023 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_ops": { 2024-09-14T03:47:52.023 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.023 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.024 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_bytes": { 2024-09-14T03:47:52.024 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 95995035648 2024-09-14T03:47:52.024 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.024 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_bytes_used": { 2024-09-14T03:47:52.024 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 305201152 2024-09-14T03:47:52.024 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.024 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.024 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.024 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.4": { 2024-09-14T03:47:52.024 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:52.024 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_slow_fallback": { 2024-09-14T03:47:52.024 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.024 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.025 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_slow_size_fallback": { 2024-09-14T03:47:52.025 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.025 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.025 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_db": { 2024-09-14T03:47:52.025 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 65536 2024-09-14T03:47:52.025 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.025 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_main": { 2024-09-14T03:47:52.025 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.025 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.025 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_wal": { 2024-09-14T03:47:52.025 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.025 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.025 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_slow": { 2024-09-14T03:47:52.026 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.026 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.026 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_sst": { 2024-09-14T03:47:52.026 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12288 2024-09-14T03:47:52.026 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.026 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_wal": { 2024-09-14T03:47:52.026 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1249280 2024-09-14T03:47:52.026 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.026 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lat": { 2024-09-14T03:47:52.026 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.026 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.026 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.026 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lock_lat": { 2024-09-14T03:47:52.026 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.027 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.027 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.027 INFO:tasks.workunit.client.0.smithi070.stdout: "db_total_bytes": { 2024-09-14T03:47:52.027 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 95995035648 2024-09-14T03:47:52.027 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.027 INFO:tasks.workunit.client.0.smithi070.stdout: "db_used_bytes": { 2024-09-14T03:47:52.027 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 304480256 2024-09-14T03:47:52.027 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.027 INFO:tasks.workunit.client.0.smithi070.stdout: "log_bytes": { 2024-09-14T03:47:52.027 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 958464 2024-09-14T03:47:52.027 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.027 INFO:tasks.workunit.client.0.smithi070.stdout: "logged_bytes": { 2024-09-14T03:47:52.028 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 819200 2024-09-14T03:47:52.028 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.028 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_db": { 2024-09-14T03:47:52.028 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 312868864 2024-09-14T03:47:52.028 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.028 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_slow": { 2024-09-14T03:47:52.028 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.028 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.028 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_wal": { 2024-09-14T03:47:52.028 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.028 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.028 INFO:tasks.workunit.client.0.smithi070.stdout: "num_files": { 2024-09-14T03:47:52.028 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 14 2024-09-14T03:47:52.028 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.029 INFO:tasks.workunit.client.0.smithi070.stdout: "read_bytes": { 2024-09-14T03:47:52.029 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 195697 2024-09-14T03:47:52.029 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.029 INFO:tasks.workunit.client.0.smithi070.stdout: "read_count": { 2024-09-14T03:47:52.029 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 23 2024-09-14T03:47:52.029 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.029 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes": { 2024-09-14T03:47:52.029 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1269760 2024-09-14T03:47:52.029 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.029 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_db": { 2024-09-14T03:47:52.029 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1273856 2024-09-14T03:47:52.029 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.029 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_slow": { 2024-09-14T03:47:52.030 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.030 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.030 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_wal": { 2024-09-14T03:47:52.030 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.030 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.030 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_count": { 2024-09-14T03:47:52.030 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 15 2024-09-14T03:47:52.030 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.032 INFO:tasks.workunit.client.0.smithi070.stdout: "read_prefetch_bytes": { 2024-09-14T03:47:52.032 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4789 2024-09-14T03:47:52.032 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.032 INFO:tasks.workunit.client.0.smithi070.stdout: "read_prefetch_count": { 2024-09-14T03:47:52.032 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4 2024-09-14T03:47:52.032 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.032 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_buffer_bytes": { 2024-09-14T03:47:52.032 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5589 2024-09-14T03:47:52.032 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.032 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_buffer_count": { 2024-09-14T03:47:52.032 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 24 2024-09-14T03:47:52.032 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.032 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_bytes": { 2024-09-14T03:47:52.033 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5589 2024-09-14T03:47:52.033 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.033 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_count": { 2024-09-14T03:47:52.033 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 24 2024-09-14T03:47:52.033 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.033 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes": { 2024-09-14T03:47:52.033 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.033 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.033 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_db": { 2024-09-14T03:47:52.033 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.033 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.033 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_slow": { 2024-09-14T03:47:52.033 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.034 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.034 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_wal": { 2024-09-14T03:47:52.034 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.034 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.034 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_count": { 2024-09-14T03:47:52.034 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.034 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.034 INFO:tasks.workunit.client.0.smithi070.stdout: "slow_total_bytes": { 2024-09-14T03:47:52.034 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.034 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.034 INFO:tasks.workunit.client.0.smithi070.stdout: "slow_used_bytes": { 2024-09-14T03:47:52.034 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.034 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.034 INFO:tasks.workunit.client.0.smithi070.stdout: "wal_total_bytes": { 2024-09-14T03:47:52.035 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.035 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.035 INFO:tasks.workunit.client.0.smithi070.stdout: "wal_used_bytes": { 2024-09-14T03:47:52.035 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.035 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.035 INFO:tasks.workunit.client.0.smithi070.stdout: "write_bytes": { 2024-09-14T03:47:52.035 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2142208 2024-09-14T03:47:52.035 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.035 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.035 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore": { 2024-09-14T03:47:52.035 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit": { 2024-09-14T03:47:52.035 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4096 2024-09-14T03:47:52.035 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.036 INFO:tasks.workunit.client.0.smithi070.stdout: "allocated": { 2024-09-14T03:47:52.036 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 716800 2024-09-14T03:47:52.036 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.036 INFO:tasks.workunit.client.0.smithi070.stdout: "clist_lat": { 2024-09-14T03:47:52.036 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 8, 2024-09-14T03:47:52.036 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 345825 2024-09-14T03:47:52.036 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.036 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_lat": { 2024-09-14T03:47:52.036 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.036 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.036 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.036 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed": { 2024-09-14T03:47:52.036 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.036 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.037 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed_allocated": { 2024-09-14T03:47:52.037 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.037 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.037 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed_original": { 2024-09-14T03:47:52.037 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.037 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.037 INFO:tasks.workunit.client.0.smithi070.stdout: "csum_lat": { 2024-09-14T03:47:52.037 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:52.037 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2359 2024-09-14T03:47:52.037 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.037 INFO:tasks.workunit.client.0.smithi070.stdout: "decompress_lat": { 2024-09-14T03:47:52.037 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.037 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.038 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.038 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_commit_lat": { 2024-09-14T03:47:52.038 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 183, 2024-09-14T03:47:52.038 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 40111570 2024-09-14T03:47:52.038 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.038 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_final_lat": { 2024-09-14T03:47:52.038 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 168, 2024-09-14T03:47:52.038 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 9160436 2024-09-14T03:47:52.038 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.038 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_flush_lat": { 2024-09-14T03:47:52.038 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 183, 2024-09-14T03:47:52.038 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 222326 2024-09-14T03:47:52.038 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.038 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_sync_lat": { 2024-09-14T03:47:52.039 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 183, 2024-09-14T03:47:52.039 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 40333896 2024-09-14T03:47:52.039 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.039 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_get_keys_lat": { 2024-09-14T03:47:52.039 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.039 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.039 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.039 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_get_values_lat": { 2024-09-14T03:47:52.039 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 3, 2024-09-14T03:47:52.039 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 29663 2024-09-14T03:47:52.039 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.039 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_lower_bound_lat": { 2024-09-14T03:47:52.039 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:52.040 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1276 2024-09-14T03:47:52.040 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.040 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_next_lat": { 2024-09-14T03:47:52.040 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.040 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.040 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.040 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_seek_to_first_lat": { 2024-09-14T03:47:52.040 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.040 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.040 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.040 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_upper_bound_lat": { 2024-09-14T03:47:52.040 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 3, 2024-09-14T03:47:52.040 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7892 2024-09-14T03:47:52.040 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.041 INFO:tasks.workunit.client.0.smithi070.stdout: "onode_hits": { 2024-09-14T03:47:52.041 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 357 2024-09-14T03:47:52.041 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.041 INFO:tasks.workunit.client.0.smithi070.stdout: "onode_misses": { 2024-09-14T03:47:52.041 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 237 2024-09-14T03:47:52.041 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.041 INFO:tasks.workunit.client.0.smithi070.stdout: "read_lat": { 2024-09-14T03:47:52.041 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 94, 2024-09-14T03:47:52.041 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 721550 2024-09-14T03:47:52.041 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.041 INFO:tasks.workunit.client.0.smithi070.stdout: "read_onode_meta_lat": { 2024-09-14T03:47:52.041 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 155, 2024-09-14T03:47:52.041 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 238486 2024-09-14T03:47:52.042 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.042 INFO:tasks.workunit.client.0.smithi070.stdout: "read_wait_aio_lat": { 2024-09-14T03:47:52.042 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 61, 2024-09-14T03:47:52.042 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 69229 2024-09-14T03:47:52.042 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.042 INFO:tasks.workunit.client.0.smithi070.stdout: "reads_with_retries": { 2024-09-14T03:47:52.042 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.042 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.042 INFO:tasks.workunit.client.0.smithi070.stdout: "remove_lat": { 2024-09-14T03:47:52.042 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 5, 2024-09-14T03:47:52.042 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 135227 2024-09-14T03:47:52.042 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.042 INFO:tasks.workunit.client.0.smithi070.stdout: "state_aio_wait_lat": { 2024-09-14T03:47:52.043 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 265, 2024-09-14T03:47:52.043 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3460408 2024-09-14T03:47:52.043 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.043 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_aio_wait_lat": { 2024-09-14T03:47:52.043 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 39, 2024-09-14T03:47:52.043 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2901292 2024-09-14T03:47:52.043 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.043 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_cleanup_lat": { 2024-09-14T03:47:52.043 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 38, 2024-09-14T03:47:52.043 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 228341686886 2024-09-14T03:47:52.043 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.043 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_queued_lat": { 2024-09-14T03:47:52.043 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 39, 2024-09-14T03:47:52.043 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 64303300754 2024-09-14T03:47:52.044 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.044 INFO:tasks.workunit.client.0.smithi070.stdout: "state_done_lat": { 2024-09-14T03:47:52.044 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 264, 2024-09-14T03:47:52.044 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 501637 2024-09-14T03:47:52.044 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.044 INFO:tasks.workunit.client.0.smithi070.stdout: "state_finishing_lat": { 2024-09-14T03:47:52.044 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 264, 2024-09-14T03:47:52.044 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 29802 2024-09-14T03:47:52.044 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.044 INFO:tasks.workunit.client.0.smithi070.stdout: "state_io_done_lat": { 2024-09-14T03:47:52.044 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 265, 2024-09-14T03:47:52.044 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 252745 2024-09-14T03:47:52.044 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.044 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_commiting_lat": { 2024-09-14T03:47:52.045 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 265, 2024-09-14T03:47:52.045 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 104003707 2024-09-14T03:47:52.045 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.045 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_done_lat": { 2024-09-14T03:47:52.045 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 265, 2024-09-14T03:47:52.045 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 77912 2024-09-14T03:47:52.045 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.045 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_queued_lat": { 2024-09-14T03:47:52.045 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 265, 2024-09-14T03:47:52.045 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 36691311 2024-09-14T03:47:52.045 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.045 INFO:tasks.workunit.client.0.smithi070.stdout: "state_prepare_lat": { 2024-09-14T03:47:52.045 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 265, 2024-09-14T03:47:52.046 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12786127 2024-09-14T03:47:52.046 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.046 INFO:tasks.workunit.client.0.smithi070.stdout: "stored": { 2024-09-14T03:47:52.046 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 313282 2024-09-14T03:47:52.046 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.046 INFO:tasks.workunit.client.0.smithi070.stdout: "truncate_lat": { 2024-09-14T03:47:52.046 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 37, 2024-09-14T03:47:52.046 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 184889 2024-09-14T03:47:52.046 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.046 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_commit_lat": { 2024-09-14T03:47:52.046 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 265, 2024-09-14T03:47:52.046 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 157227847 2024-09-14T03:47:52.046 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.047 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_submit_lat": { 2024-09-14T03:47:52.047 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 265, 2024-09-14T03:47:52.047 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 15411335 2024-09-14T03:47:52.047 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.047 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_throttle_lat": { 2024-09-14T03:47:52.047 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 265, 2024-09-14T03:47:52.047 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2156304 2024-09-14T03:47:52.047 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.047 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.047 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache": { 2024-09-14T03:47:52.047 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_bytes": { 2024-09-14T03:47:52.047 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1861444387 2024-09-14T03:47:52.047 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.048 INFO:tasks.workunit.client.0.smithi070.stdout: "heap_bytes": { 2024-09-14T03:47:52.048 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 269926400 2024-09-14T03:47:52.048 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.048 INFO:tasks.workunit.client.0.smithi070.stdout: "mapped_bytes": { 2024-09-14T03:47:52.048 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 269852672 2024-09-14T03:47:52.048 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.048 INFO:tasks.workunit.client.0.smithi070.stdout: "target_bytes": { 2024-09-14T03:47:52.048 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3137353830 2024-09-14T03:47:52.048 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.048 INFO:tasks.workunit.client.0.smithi070.stdout: "unmapped_bytes": { 2024-09-14T03:47:52.048 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 73728 2024-09-14T03:47:52.048 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.048 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.049 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:data": { 2024-09-14T03:47:52.049 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:52.049 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 159383552 2024-09-14T03:47:52.049 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.049 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:52.049 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.049 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.049 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:52.049 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.049 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.049 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:52.049 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 85498353 2024-09-14T03:47:52.049 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.049 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:52.050 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.050 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.050 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:52.050 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.050 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.050 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:52.050 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.050 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.050 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:52.050 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.050 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.050 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:52.050 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.051 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.051 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:52.051 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.051 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.051 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:52.051 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.051 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.051 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:52.051 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.051 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.051 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:52.051 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.051 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.052 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:52.052 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 73885199 2024-09-14T03:47:52.052 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.052 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.052 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:kv": { 2024-09-14T03:47:52.052 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:52.052 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 780140544 2024-09-14T03:47:52.052 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.052 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:52.052 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:52.052 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.052 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:52.052 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.053 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.053 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:52.053 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 641237651 2024-09-14T03:47:52.053 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.053 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:52.053 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.053 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.053 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:52.053 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.053 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.053 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:52.053 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.053 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.053 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:52.054 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.054 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.054 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:52.054 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.054 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.054 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:52.054 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2128 2024-09-14T03:47:52.054 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.054 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:52.054 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.054 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.054 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:52.054 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.055 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.055 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:52.055 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.055 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.055 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:52.055 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71791901 2024-09-14T03:47:52.055 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.055 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.055 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:kv_onode": { 2024-09-14T03:47:52.055 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:52.055 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 192937984 2024-09-14T03:47:52.055 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.055 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:52.055 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:52.056 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.056 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:52.056 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 464 2024-09-14T03:47:52.056 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.056 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:52.056 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 56998902 2024-09-14T03:47:52.056 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.056 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:52.056 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.056 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.056 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:52.056 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.056 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.057 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:52.057 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.057 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.057 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:52.057 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.057 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.057 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:52.057 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.057 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.057 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:52.057 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.057 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.057 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:52.057 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.058 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.058 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:52.058 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.058 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.058 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:52.058 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.058 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.058 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:52.058 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 68829754 2024-09-14T03:47:52.058 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.058 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.058 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:meta": { 2024-09-14T03:47:52.058 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:52.058 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 713031680 2024-09-14T03:47:52.059 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.059 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:52.059 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.059 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.059 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:52.059 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.059 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.059 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:52.059 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 641237651 2024-09-14T03:47:52.059 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.059 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:52.059 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.059 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.060 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:52.060 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.060 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.060 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:52.060 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.060 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.060 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:52.060 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.060 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.060 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:52.060 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.060 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.060 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:52.061 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 256053 2024-09-14T03:47:52.061 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.061 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:52.061 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.061 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.061 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:52.061 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.061 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.061 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:52.061 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.061 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.061 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:52.061 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71537976 2024-09-14T03:47:52.061 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.062 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.062 INFO:tasks.workunit.client.0.smithi070.stdout: "objecter": { 2024-09-14T03:47:52.062 INFO:tasks.workunit.client.0.smithi070.stdout: "op_active": { 2024-09-14T03:47:52.062 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.062 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.062 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r": { 2024-09-14T03:47:52.062 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.062 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.062 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rmw": { 2024-09-14T03:47:52.062 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.062 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.062 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w": { 2024-09-14T03:47:52.062 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.062 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.063 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.063 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:52.063 INFO:tasks.workunit.client.0.smithi070.stdout: "numpg": { 2024-09-14T03:47:52.063 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 39 2024-09-14T03:47:52.063 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.063 INFO:tasks.workunit.client.0.smithi070.stdout: "numpg_removing": { 2024-09-14T03:47:52.063 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.063 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.063 INFO:tasks.workunit.client.0.smithi070.stdout: "op": { 2024-09-14T03:47:52.063 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1 2024-09-14T03:47:52.063 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.063 INFO:tasks.workunit.client.0.smithi070.stdout: "op_delayed_degraded": { 2024-09-14T03:47:52.063 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.064 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.064 INFO:tasks.workunit.client.0.smithi070.stdout: "op_delayed_unreadable": { 2024-09-14T03:47:52.064 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.064 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.064 INFO:tasks.workunit.client.0.smithi070.stdout: "op_in_bytes": { 2024-09-14T03:47:52.064 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 22 2024-09-14T03:47:52.064 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.064 INFO:tasks.workunit.client.0.smithi070.stdout: "op_latency": { 2024-09-14T03:47:52.064 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:52.064 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7596677 2024-09-14T03:47:52.064 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.064 INFO:tasks.workunit.client.0.smithi070.stdout: "op_out_bytes": { 2024-09-14T03:47:52.064 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.064 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.065 INFO:tasks.workunit.client.0.smithi070.stdout: "op_prepare_latency": { 2024-09-14T03:47:52.065 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:52.065 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 730201 2024-09-14T03:47:52.065 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.065 INFO:tasks.workunit.client.0.smithi070.stdout: "op_process_latency": { 2024-09-14T03:47:52.065 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:52.065 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7516157 2024-09-14T03:47:52.065 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.065 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r": { 2024-09-14T03:47:52.065 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.065 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.065 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_latency": { 2024-09-14T03:47:52.066 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.066 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.066 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.066 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:47:52.066 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.066 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.066 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_out_bytes": { 2024-09-14T03:47:52.066 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.066 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.066 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_prepare_latency": { 2024-09-14T03:47:52.066 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.066 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.066 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.067 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_process_latency": { 2024-09-14T03:47:52.067 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.067 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.067 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.067 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw": { 2024-09-14T03:47:52.067 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.067 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.067 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_in_bytes": { 2024-09-14T03:47:52.067 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.067 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.067 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency": { 2024-09-14T03:47:52.067 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.067 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.067 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.068 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:47:52.068 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.068 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.068 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:47:52.068 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.068 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.068 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_out_bytes": { 2024-09-14T03:47:52.068 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.068 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.068 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_prepare_latency": { 2024-09-14T03:47:52.068 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.068 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.069 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.069 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_process_latency": { 2024-09-14T03:47:52.069 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.069 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.069 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.069 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w": { 2024-09-14T03:47:52.069 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1 2024-09-14T03:47:52.069 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.069 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_in_bytes": { 2024-09-14T03:47:52.069 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 22 2024-09-14T03:47:52.069 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.069 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_latency": { 2024-09-14T03:47:52.069 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:52.070 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7596677 2024-09-14T03:47:52.070 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.070 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:47:52.070 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.070 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.070 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_prepare_latency": { 2024-09-14T03:47:52.070 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:52.070 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 730201 2024-09-14T03:47:52.070 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.070 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_process_latency": { 2024-09-14T03:47:52.070 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:52.070 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7516157 2024-09-14T03:47:52.070 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.070 INFO:tasks.workunit.client.0.smithi070.stdout: "op_wip": { 2024-09-14T03:47:52.071 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.071 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.071 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_bytes": { 2024-09-14T03:47:52.071 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.071 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.071 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_ops": { 2024-09-14T03:47:52.071 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.071 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.071 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_bytes": { 2024-09-14T03:47:52.071 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 95995035648 2024-09-14T03:47:52.071 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.071 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_bytes_used": { 2024-09-14T03:47:52.071 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 305209344 2024-09-14T03:47:52.072 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.072 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.072 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.072 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.5": { 2024-09-14T03:47:52.072 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:52.072 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_slow_fallback": { 2024-09-14T03:47:52.072 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.072 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.072 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_slow_size_fallback": { 2024-09-14T03:47:52.072 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.072 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.072 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_db": { 2024-09-14T03:47:52.072 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 65536 2024-09-14T03:47:52.073 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.073 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_main": { 2024-09-14T03:47:52.073 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.073 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.073 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_wal": { 2024-09-14T03:47:52.073 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.073 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.073 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_slow": { 2024-09-14T03:47:52.073 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.073 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.073 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_sst": { 2024-09-14T03:47:52.073 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12288 2024-09-14T03:47:52.073 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.073 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_wal": { 2024-09-14T03:47:52.074 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1179648 2024-09-14T03:47:52.074 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.074 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lat": { 2024-09-14T03:47:52.074 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.074 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.074 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.074 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lock_lat": { 2024-09-14T03:47:52.074 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.074 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.074 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.074 INFO:tasks.workunit.client.0.smithi070.stdout: "db_total_bytes": { 2024-09-14T03:47:52.074 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 95995035648 2024-09-14T03:47:52.074 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.075 INFO:tasks.workunit.client.0.smithi070.stdout: "db_used_bytes": { 2024-09-14T03:47:52.075 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 304480256 2024-09-14T03:47:52.075 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.075 INFO:tasks.workunit.client.0.smithi070.stdout: "log_bytes": { 2024-09-14T03:47:52.075 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 880640 2024-09-14T03:47:52.075 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.075 INFO:tasks.workunit.client.0.smithi070.stdout: "logged_bytes": { 2024-09-14T03:47:52.075 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 741376 2024-09-14T03:47:52.075 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.075 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_db": { 2024-09-14T03:47:52.075 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 312868864 2024-09-14T03:47:52.075 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.075 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_slow": { 2024-09-14T03:47:52.076 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.076 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.076 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_wal": { 2024-09-14T03:47:52.076 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.076 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.076 INFO:tasks.workunit.client.0.smithi070.stdout: "num_files": { 2024-09-14T03:47:52.076 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 14 2024-09-14T03:47:52.076 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.076 INFO:tasks.workunit.client.0.smithi070.stdout: "read_bytes": { 2024-09-14T03:47:52.076 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 195697 2024-09-14T03:47:52.076 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.076 INFO:tasks.workunit.client.0.smithi070.stdout: "read_count": { 2024-09-14T03:47:52.076 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 23 2024-09-14T03:47:52.077 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.077 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes": { 2024-09-14T03:47:52.077 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1269760 2024-09-14T03:47:52.077 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.077 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_db": { 2024-09-14T03:47:52.077 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1273856 2024-09-14T03:47:52.077 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.077 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_slow": { 2024-09-14T03:47:52.077 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.077 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.077 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_wal": { 2024-09-14T03:47:52.077 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.077 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.078 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_count": { 2024-09-14T03:47:52.078 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 15 2024-09-14T03:47:52.078 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.078 INFO:tasks.workunit.client.0.smithi070.stdout: "read_prefetch_bytes": { 2024-09-14T03:47:52.078 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4789 2024-09-14T03:47:52.078 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.078 INFO:tasks.workunit.client.0.smithi070.stdout: "read_prefetch_count": { 2024-09-14T03:47:52.078 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4 2024-09-14T03:47:52.078 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.078 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_buffer_bytes": { 2024-09-14T03:47:52.078 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5589 2024-09-14T03:47:52.078 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.078 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_buffer_count": { 2024-09-14T03:47:52.079 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 24 2024-09-14T03:47:52.079 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.079 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_bytes": { 2024-09-14T03:47:52.079 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5589 2024-09-14T03:47:52.079 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.079 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_count": { 2024-09-14T03:47:52.079 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 24 2024-09-14T03:47:52.079 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.079 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes": { 2024-09-14T03:47:52.079 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.079 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.079 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_db": { 2024-09-14T03:47:52.079 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.080 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.080 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_slow": { 2024-09-14T03:47:52.080 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.080 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.080 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_wal": { 2024-09-14T03:47:52.080 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.080 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.080 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_count": { 2024-09-14T03:47:52.080 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.080 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.080 INFO:tasks.workunit.client.0.smithi070.stdout: "slow_total_bytes": { 2024-09-14T03:47:52.080 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.080 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.080 INFO:tasks.workunit.client.0.smithi070.stdout: "slow_used_bytes": { 2024-09-14T03:47:52.080 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.081 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.081 INFO:tasks.workunit.client.0.smithi070.stdout: "wal_total_bytes": { 2024-09-14T03:47:52.081 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.081 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.081 INFO:tasks.workunit.client.0.smithi070.stdout: "wal_used_bytes": { 2024-09-14T03:47:52.081 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.081 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.081 INFO:tasks.workunit.client.0.smithi070.stdout: "write_bytes": { 2024-09-14T03:47:52.081 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1994752 2024-09-14T03:47:52.081 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.081 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.081 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore": { 2024-09-14T03:47:52.082 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit": { 2024-09-14T03:47:52.082 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4096 2024-09-14T03:47:52.082 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.082 INFO:tasks.workunit.client.0.smithi070.stdout: "allocated": { 2024-09-14T03:47:52.082 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 712704 2024-09-14T03:47:52.082 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.082 INFO:tasks.workunit.client.0.smithi070.stdout: "clist_lat": { 2024-09-14T03:47:52.082 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 6, 2024-09-14T03:47:52.082 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 229396 2024-09-14T03:47:52.082 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.082 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_lat": { 2024-09-14T03:47:52.082 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.082 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.083 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.083 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed": { 2024-09-14T03:47:52.083 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.083 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.083 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed_allocated": { 2024-09-14T03:47:52.083 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.083 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.083 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed_original": { 2024-09-14T03:47:52.083 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.083 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.083 INFO:tasks.workunit.client.0.smithi070.stdout: "csum_lat": { 2024-09-14T03:47:52.083 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 3, 2024-09-14T03:47:52.083 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4234 2024-09-14T03:47:52.083 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.084 INFO:tasks.workunit.client.0.smithi070.stdout: "decompress_lat": { 2024-09-14T03:47:52.084 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.084 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.084 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.084 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_commit_lat": { 2024-09-14T03:47:52.084 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 164, 2024-09-14T03:47:52.084 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 47839247 2024-09-14T03:47:52.084 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.084 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_final_lat": { 2024-09-14T03:47:52.084 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 164, 2024-09-14T03:47:52.084 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4151205 2024-09-14T03:47:52.085 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.085 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_flush_lat": { 2024-09-14T03:47:52.085 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 164, 2024-09-14T03:47:52.085 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 175925 2024-09-14T03:47:52.085 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.085 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_sync_lat": { 2024-09-14T03:47:52.085 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 164, 2024-09-14T03:47:52.085 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 48015172 2024-09-14T03:47:52.085 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.085 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_get_keys_lat": { 2024-09-14T03:47:52.085 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.085 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.085 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.085 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_get_values_lat": { 2024-09-14T03:47:52.086 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 3, 2024-09-14T03:47:52.086 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 36969 2024-09-14T03:47:52.086 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.086 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_lower_bound_lat": { 2024-09-14T03:47:52.086 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:52.086 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1026 2024-09-14T03:47:52.086 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.086 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_next_lat": { 2024-09-14T03:47:52.086 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.086 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.086 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.086 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_seek_to_first_lat": { 2024-09-14T03:47:52.086 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.087 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.087 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.087 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_upper_bound_lat": { 2024-09-14T03:47:52.087 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:52.087 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7136 2024-09-14T03:47:52.087 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.087 INFO:tasks.workunit.client.0.smithi070.stdout: "onode_hits": { 2024-09-14T03:47:52.087 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 344 2024-09-14T03:47:52.087 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.087 INFO:tasks.workunit.client.0.smithi070.stdout: "onode_misses": { 2024-09-14T03:47:52.087 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 240 2024-09-14T03:47:52.087 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.087 INFO:tasks.workunit.client.0.smithi070.stdout: "read_lat": { 2024-09-14T03:47:52.087 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 90, 2024-09-14T03:47:52.088 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 690701 2024-09-14T03:47:52.088 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.088 INFO:tasks.workunit.client.0.smithi070.stdout: "read_onode_meta_lat": { 2024-09-14T03:47:52.088 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 146, 2024-09-14T03:47:52.088 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 261511 2024-09-14T03:47:52.088 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.088 INFO:tasks.workunit.client.0.smithi070.stdout: "read_wait_aio_lat": { 2024-09-14T03:47:52.088 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 56, 2024-09-14T03:47:52.088 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 171611 2024-09-14T03:47:52.088 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.088 INFO:tasks.workunit.client.0.smithi070.stdout: "reads_with_retries": { 2024-09-14T03:47:52.088 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.088 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.089 INFO:tasks.workunit.client.0.smithi070.stdout: "remove_lat": { 2024-09-14T03:47:52.089 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 4, 2024-09-14T03:47:52.089 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 97676 2024-09-14T03:47:52.089 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.089 INFO:tasks.workunit.client.0.smithi070.stdout: "state_aio_wait_lat": { 2024-09-14T03:47:52.089 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 274, 2024-09-14T03:47:52.089 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2801500 2024-09-14T03:47:52.089 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.089 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_aio_wait_lat": { 2024-09-14T03:47:52.089 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 41, 2024-09-14T03:47:52.089 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4051654 2024-09-14T03:47:52.089 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.089 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_cleanup_lat": { 2024-09-14T03:47:52.090 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 38, 2024-09-14T03:47:52.090 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 297657276129 2024-09-14T03:47:52.090 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.090 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_queued_lat": { 2024-09-14T03:47:52.090 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 41, 2024-09-14T03:47:52.090 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 65658052773 2024-09-14T03:47:52.090 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.090 INFO:tasks.workunit.client.0.smithi070.stdout: "state_done_lat": { 2024-09-14T03:47:52.090 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 271, 2024-09-14T03:47:52.090 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 6002782444 2024-09-14T03:47:52.090 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.090 INFO:tasks.workunit.client.0.smithi070.stdout: "state_finishing_lat": { 2024-09-14T03:47:52.090 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 271, 2024-09-14T03:47:52.090 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 30580 2024-09-14T03:47:52.091 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.091 INFO:tasks.workunit.client.0.smithi070.stdout: "state_io_done_lat": { 2024-09-14T03:47:52.091 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 274, 2024-09-14T03:47:52.091 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 274880 2024-09-14T03:47:52.091 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.091 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_commiting_lat": { 2024-09-14T03:47:52.091 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 274, 2024-09-14T03:47:52.091 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 107158317 2024-09-14T03:47:52.091 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.091 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_done_lat": { 2024-09-14T03:47:52.091 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 274, 2024-09-14T03:47:52.091 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 74514 2024-09-14T03:47:52.091 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.091 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_queued_lat": { 2024-09-14T03:47:52.092 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 274, 2024-09-14T03:47:52.092 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 102524241 2024-09-14T03:47:52.092 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.092 INFO:tasks.workunit.client.0.smithi070.stdout: "state_prepare_lat": { 2024-09-14T03:47:52.092 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 274, 2024-09-14T03:47:52.092 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 13236763 2024-09-14T03:47:52.092 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.092 INFO:tasks.workunit.client.0.smithi070.stdout: "stored": { 2024-09-14T03:47:52.092 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 314985 2024-09-14T03:47:52.092 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.092 INFO:tasks.workunit.client.0.smithi070.stdout: "truncate_lat": { 2024-09-14T03:47:52.092 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 33, 2024-09-14T03:47:52.092 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 134125 2024-09-14T03:47:52.093 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.093 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_commit_lat": { 2024-09-14T03:47:52.093 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 274, 2024-09-14T03:47:52.093 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 226027726 2024-09-14T03:47:52.093 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.093 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_submit_lat": { 2024-09-14T03:47:52.093 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 274, 2024-09-14T03:47:52.093 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 16481402 2024-09-14T03:47:52.093 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.093 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_throttle_lat": { 2024-09-14T03:47:52.093 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 274, 2024-09-14T03:47:52.093 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2140962 2024-09-14T03:47:52.093 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.094 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.094 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache": { 2024-09-14T03:47:52.094 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_bytes": { 2024-09-14T03:47:52.094 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1861444387 2024-09-14T03:47:52.094 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.094 INFO:tasks.workunit.client.0.smithi070.stdout: "heap_bytes": { 2024-09-14T03:47:52.094 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 269926400 2024-09-14T03:47:52.094 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.094 INFO:tasks.workunit.client.0.smithi070.stdout: "mapped_bytes": { 2024-09-14T03:47:52.094 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 269615104 2024-09-14T03:47:52.094 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.094 INFO:tasks.workunit.client.0.smithi070.stdout: "target_bytes": { 2024-09-14T03:47:52.094 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3137353830 2024-09-14T03:47:52.094 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.095 INFO:tasks.workunit.client.0.smithi070.stdout: "unmapped_bytes": { 2024-09-14T03:47:52.095 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 311296 2024-09-14T03:47:52.095 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.095 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.095 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:data": { 2024-09-14T03:47:52.095 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:52.095 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 159383552 2024-09-14T03:47:52.095 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.095 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:52.095 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.095 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.095 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:52.095 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.096 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.096 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:52.096 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 85498195 2024-09-14T03:47:52.096 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.096 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:52.096 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.096 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.096 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:52.096 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.096 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.096 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:52.096 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.096 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.097 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:52.097 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.097 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.097 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:52.097 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.097 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.097 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:52.097 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.097 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.097 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:52.097 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.097 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.097 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:52.097 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.098 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.098 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:52.098 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.098 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.098 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:52.098 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 73885357 2024-09-14T03:47:52.098 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.098 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.098 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:kv": { 2024-09-14T03:47:52.098 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:52.098 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 780140544 2024-09-14T03:47:52.098 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.098 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:52.099 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:52.099 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.099 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:52.099 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.099 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.099 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:52.099 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 641236466 2024-09-14T03:47:52.099 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.099 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:52.099 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.099 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.099 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:52.099 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.099 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.100 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:52.100 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.100 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.100 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:52.100 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.100 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.100 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:52.100 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.100 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.100 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:52.100 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2128 2024-09-14T03:47:52.100 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.100 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:52.101 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.101 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.101 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:52.101 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.101 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.101 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:52.101 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.101 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.101 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:52.101 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71793086 2024-09-14T03:47:52.101 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.101 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.101 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:kv_onode": { 2024-09-14T03:47:52.101 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:52.102 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 192937984 2024-09-14T03:47:52.102 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.102 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:52.102 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:52.102 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.102 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:52.102 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 464 2024-09-14T03:47:52.102 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.102 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:52.102 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 56998797 2024-09-14T03:47:52.102 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.102 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:52.103 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.103 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.103 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:52.103 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.103 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.103 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:52.103 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.103 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.103 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:52.103 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.103 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.103 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:52.103 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.104 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.104 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:52.104 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.104 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.104 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:52.104 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.104 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.104 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:52.104 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.104 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.104 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:52.104 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.104 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.104 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:52.105 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 68829859 2024-09-14T03:47:52.105 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.105 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.105 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:meta": { 2024-09-14T03:47:52.105 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:52.105 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 713031680 2024-09-14T03:47:52.105 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.105 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:52.105 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.105 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.105 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:52.105 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.105 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.106 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:52.106 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 641236466 2024-09-14T03:47:52.106 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.106 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:52.106 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.106 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.106 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:52.106 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.106 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.106 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:52.106 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.106 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.106 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:52.106 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.107 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.107 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:52.107 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 6954 2024-09-14T03:47:52.107 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.107 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:52.107 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 257300 2024-09-14T03:47:52.107 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.107 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:52.107 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.107 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.107 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:52.107 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.107 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.108 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:52.108 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.108 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.108 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:52.108 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71530960 2024-09-14T03:47:52.108 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.108 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.108 INFO:tasks.workunit.client.0.smithi070.stdout: "objecter": { 2024-09-14T03:47:52.108 INFO:tasks.workunit.client.0.smithi070.stdout: "op_active": { 2024-09-14T03:47:52.108 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.108 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.108 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r": { 2024-09-14T03:47:52.108 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.109 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.109 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rmw": { 2024-09-14T03:47:52.109 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.109 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.109 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w": { 2024-09-14T03:47:52.109 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.109 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.109 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.109 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:52.109 INFO:tasks.workunit.client.0.smithi070.stdout: "numpg": { 2024-09-14T03:47:52.109 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 41 2024-09-14T03:47:52.109 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.109 INFO:tasks.workunit.client.0.smithi070.stdout: "numpg_removing": { 2024-09-14T03:47:52.109 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.110 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.110 INFO:tasks.workunit.client.0.smithi070.stdout: "op": { 2024-09-14T03:47:52.110 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.110 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.110 INFO:tasks.workunit.client.0.smithi070.stdout: "op_delayed_degraded": { 2024-09-14T03:47:52.110 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.110 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.110 INFO:tasks.workunit.client.0.smithi070.stdout: "op_delayed_unreadable": { 2024-09-14T03:47:52.110 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.110 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.110 INFO:tasks.workunit.client.0.smithi070.stdout: "op_in_bytes": { 2024-09-14T03:47:52.110 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.110 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.111 INFO:tasks.workunit.client.0.smithi070.stdout: "op_latency": { 2024-09-14T03:47:52.111 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.111 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.111 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.111 INFO:tasks.workunit.client.0.smithi070.stdout: "op_out_bytes": { 2024-09-14T03:47:52.111 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.111 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.111 INFO:tasks.workunit.client.0.smithi070.stdout: "op_prepare_latency": { 2024-09-14T03:47:52.111 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.111 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.111 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.111 INFO:tasks.workunit.client.0.smithi070.stdout: "op_process_latency": { 2024-09-14T03:47:52.111 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.112 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.112 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.112 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r": { 2024-09-14T03:47:52.112 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.112 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.112 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_latency": { 2024-09-14T03:47:52.112 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.112 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.112 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.112 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:47:52.112 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.112 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.112 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_out_bytes": { 2024-09-14T03:47:52.112 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.113 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.113 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_prepare_latency": { 2024-09-14T03:47:52.113 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.113 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.113 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.113 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_process_latency": { 2024-09-14T03:47:52.113 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.113 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.113 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.113 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw": { 2024-09-14T03:47:52.113 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.113 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.113 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_in_bytes": { 2024-09-14T03:47:52.114 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.114 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.114 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency": { 2024-09-14T03:47:52.114 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.114 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.114 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.114 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:47:52.114 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.114 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.114 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:47:52.114 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.114 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.114 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_out_bytes": { 2024-09-14T03:47:52.114 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.115 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.115 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_prepare_latency": { 2024-09-14T03:47:52.115 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.115 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.115 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.115 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_process_latency": { 2024-09-14T03:47:52.115 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.115 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.115 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.115 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w": { 2024-09-14T03:47:52.115 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.115 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.115 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_in_bytes": { 2024-09-14T03:47:52.116 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.116 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.116 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_latency": { 2024-09-14T03:47:52.116 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.116 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.116 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.116 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:47:52.116 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.116 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.116 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_prepare_latency": { 2024-09-14T03:47:52.116 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.116 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.116 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.117 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_process_latency": { 2024-09-14T03:47:52.117 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.117 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.117 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.117 INFO:tasks.workunit.client.0.smithi070.stdout: "op_wip": { 2024-09-14T03:47:52.117 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.117 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.117 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_bytes": { 2024-09-14T03:47:52.117 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 590368 2024-09-14T03:47:52.117 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.117 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_ops": { 2024-09-14T03:47:52.117 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:52.117 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.117 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_bytes": { 2024-09-14T03:47:52.118 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 95995035648 2024-09-14T03:47:52.118 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.118 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_bytes_used": { 2024-09-14T03:47:52.118 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 305201152 2024-09-14T03:47:52.118 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.118 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.118 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.118 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.6": { 2024-09-14T03:47:52.118 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:52.118 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_slow_fallback": { 2024-09-14T03:47:52.118 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.118 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.118 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_slow_size_fallback": { 2024-09-14T03:47:52.118 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.119 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.119 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_db": { 2024-09-14T03:47:52.119 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 65536 2024-09-14T03:47:52.119 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.119 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_main": { 2024-09-14T03:47:52.119 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.119 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.119 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_wal": { 2024-09-14T03:47:52.119 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.119 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.119 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_slow": { 2024-09-14T03:47:52.119 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.119 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.119 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_sst": { 2024-09-14T03:47:52.120 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12288 2024-09-14T03:47:52.120 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.120 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_wal": { 2024-09-14T03:47:52.120 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1245184 2024-09-14T03:47:52.120 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.120 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lat": { 2024-09-14T03:47:52.120 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.120 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.120 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.120 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lock_lat": { 2024-09-14T03:47:52.120 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.120 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.121 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.121 INFO:tasks.workunit.client.0.smithi070.stdout: "db_total_bytes": { 2024-09-14T03:47:52.121 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 95995035648 2024-09-14T03:47:52.121 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.121 INFO:tasks.workunit.client.0.smithi070.stdout: "db_used_bytes": { 2024-09-14T03:47:52.121 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 304480256 2024-09-14T03:47:52.121 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.121 INFO:tasks.workunit.client.0.smithi070.stdout: "log_bytes": { 2024-09-14T03:47:52.121 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 962560 2024-09-14T03:47:52.121 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.121 INFO:tasks.workunit.client.0.smithi070.stdout: "logged_bytes": { 2024-09-14T03:47:52.121 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 823296 2024-09-14T03:47:52.121 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.121 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_db": { 2024-09-14T03:47:52.122 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 312868864 2024-09-14T03:47:52.122 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.122 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_slow": { 2024-09-14T03:47:52.122 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.122 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.122 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_wal": { 2024-09-14T03:47:52.122 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.122 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.122 INFO:tasks.workunit.client.0.smithi070.stdout: "num_files": { 2024-09-14T03:47:52.122 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 14 2024-09-14T03:47:52.122 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.122 INFO:tasks.workunit.client.0.smithi070.stdout: "read_bytes": { 2024-09-14T03:47:52.122 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 195697 2024-09-14T03:47:52.123 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.123 INFO:tasks.workunit.client.0.smithi070.stdout: "read_count": { 2024-09-14T03:47:52.123 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 23 2024-09-14T03:47:52.123 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.123 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes": { 2024-09-14T03:47:52.123 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1269760 2024-09-14T03:47:52.123 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.123 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_db": { 2024-09-14T03:47:52.123 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1273856 2024-09-14T03:47:52.123 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.123 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_slow": { 2024-09-14T03:47:52.123 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.123 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.124 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_wal": { 2024-09-14T03:47:52.124 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.124 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.124 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_count": { 2024-09-14T03:47:52.124 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 15 2024-09-14T03:47:52.124 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.124 INFO:tasks.workunit.client.0.smithi070.stdout: "read_prefetch_bytes": { 2024-09-14T03:47:52.124 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4789 2024-09-14T03:47:52.124 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.124 INFO:tasks.workunit.client.0.smithi070.stdout: "read_prefetch_count": { 2024-09-14T03:47:52.124 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4 2024-09-14T03:47:52.124 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.124 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_buffer_bytes": { 2024-09-14T03:47:52.124 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5589 2024-09-14T03:47:52.125 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.125 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_buffer_count": { 2024-09-14T03:47:52.125 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 24 2024-09-14T03:47:52.125 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.125 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_bytes": { 2024-09-14T03:47:52.125 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5589 2024-09-14T03:47:52.125 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.125 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_count": { 2024-09-14T03:47:52.125 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 24 2024-09-14T03:47:52.125 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.125 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes": { 2024-09-14T03:47:52.125 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.125 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.126 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_db": { 2024-09-14T03:47:52.126 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.126 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.126 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_slow": { 2024-09-14T03:47:52.126 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.126 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.126 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_wal": { 2024-09-14T03:47:52.126 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.126 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.126 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_count": { 2024-09-14T03:47:52.126 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.126 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.126 INFO:tasks.workunit.client.0.smithi070.stdout: "slow_total_bytes": { 2024-09-14T03:47:52.127 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.127 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.127 INFO:tasks.workunit.client.0.smithi070.stdout: "slow_used_bytes": { 2024-09-14T03:47:52.127 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.127 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.127 INFO:tasks.workunit.client.0.smithi070.stdout: "wal_total_bytes": { 2024-09-14T03:47:52.127 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.127 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.127 INFO:tasks.workunit.client.0.smithi070.stdout: "wal_used_bytes": { 2024-09-14T03:47:52.127 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.127 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.127 INFO:tasks.workunit.client.0.smithi070.stdout: "write_bytes": { 2024-09-14T03:47:52.127 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2142208 2024-09-14T03:47:52.128 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.128 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.128 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore": { 2024-09-14T03:47:52.128 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit": { 2024-09-14T03:47:52.128 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4096 2024-09-14T03:47:52.128 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.128 INFO:tasks.workunit.client.0.smithi070.stdout: "allocated": { 2024-09-14T03:47:52.128 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1298432 2024-09-14T03:47:52.128 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.128 INFO:tasks.workunit.client.0.smithi070.stdout: "clist_lat": { 2024-09-14T03:47:52.128 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:52.128 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 110641 2024-09-14T03:47:52.129 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.129 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_lat": { 2024-09-14T03:47:52.129 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.129 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.129 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.129 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed": { 2024-09-14T03:47:52.129 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.129 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.129 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed_allocated": { 2024-09-14T03:47:52.129 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.129 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.129 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed_original": { 2024-09-14T03:47:52.129 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.129 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.130 INFO:tasks.workunit.client.0.smithi070.stdout: "csum_lat": { 2024-09-14T03:47:52.130 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 10, 2024-09-14T03:47:52.130 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 33348 2024-09-14T03:47:52.130 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.130 INFO:tasks.workunit.client.0.smithi070.stdout: "decompress_lat": { 2024-09-14T03:47:52.130 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.130 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.130 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.130 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_commit_lat": { 2024-09-14T03:47:52.130 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 184, 2024-09-14T03:47:52.130 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 38425516 2024-09-14T03:47:52.130 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.130 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_final_lat": { 2024-09-14T03:47:52.131 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 171, 2024-09-14T03:47:52.131 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 16844763 2024-09-14T03:47:52.131 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.131 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_flush_lat": { 2024-09-14T03:47:52.131 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 184, 2024-09-14T03:47:52.131 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 156025 2024-09-14T03:47:52.131 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.131 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_sync_lat": { 2024-09-14T03:47:52.131 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 184, 2024-09-14T03:47:52.131 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 38581541 2024-09-14T03:47:52.131 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.131 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_get_keys_lat": { 2024-09-14T03:47:52.131 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.131 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.132 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.132 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_get_values_lat": { 2024-09-14T03:47:52.132 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:52.132 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 30588 2024-09-14T03:47:52.132 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.132 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_lower_bound_lat": { 2024-09-14T03:47:52.132 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 3, 2024-09-14T03:47:52.132 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4687 2024-09-14T03:47:52.132 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.132 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_next_lat": { 2024-09-14T03:47:52.132 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.132 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.132 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.133 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_seek_to_first_lat": { 2024-09-14T03:47:52.133 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.133 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.133 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.133 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_upper_bound_lat": { 2024-09-14T03:47:52.133 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 3, 2024-09-14T03:47:52.133 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 8549 2024-09-14T03:47:52.133 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.133 INFO:tasks.workunit.client.0.smithi070.stdout: "onode_hits": { 2024-09-14T03:47:52.133 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 353 2024-09-14T03:47:52.133 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.133 INFO:tasks.workunit.client.0.smithi070.stdout: "onode_misses": { 2024-09-14T03:47:52.133 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 244 2024-09-14T03:47:52.134 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.134 INFO:tasks.workunit.client.0.smithi070.stdout: "read_lat": { 2024-09-14T03:47:52.134 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 88, 2024-09-14T03:47:52.134 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 924026 2024-09-14T03:47:52.134 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.134 INFO:tasks.workunit.client.0.smithi070.stdout: "read_onode_meta_lat": { 2024-09-14T03:47:52.134 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 137, 2024-09-14T03:47:52.134 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 210059 2024-09-14T03:47:52.134 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.134 INFO:tasks.workunit.client.0.smithi070.stdout: "read_wait_aio_lat": { 2024-09-14T03:47:52.134 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 49, 2024-09-14T03:47:52.134 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 509522 2024-09-14T03:47:52.134 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.135 INFO:tasks.workunit.client.0.smithi070.stdout: "reads_with_retries": { 2024-09-14T03:47:52.135 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.135 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.135 INFO:tasks.workunit.client.0.smithi070.stdout: "remove_lat": { 2024-09-14T03:47:52.135 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:52.135 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 26070 2024-09-14T03:47:52.135 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.135 INFO:tasks.workunit.client.0.smithi070.stdout: "state_aio_wait_lat": { 2024-09-14T03:47:52.135 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 272, 2024-09-14T03:47:52.135 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2833419 2024-09-14T03:47:52.135 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.135 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_aio_wait_lat": { 2024-09-14T03:47:52.135 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 34, 2024-09-14T03:47:52.135 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2521038 2024-09-14T03:47:52.136 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.136 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_cleanup_lat": { 2024-09-14T03:47:52.136 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 34, 2024-09-14T03:47:52.136 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 449358501559 2024-09-14T03:47:52.136 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.136 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_queued_lat": { 2024-09-14T03:47:52.136 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 34, 2024-09-14T03:47:52.136 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 53644807972 2024-09-14T03:47:52.136 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.136 INFO:tasks.workunit.client.0.smithi070.stdout: "state_done_lat": { 2024-09-14T03:47:52.136 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 272, 2024-09-14T03:47:52.136 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 495568 2024-09-14T03:47:52.136 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.137 INFO:tasks.workunit.client.0.smithi070.stdout: "state_finishing_lat": { 2024-09-14T03:47:52.137 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 272, 2024-09-14T03:47:52.137 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 38270 2024-09-14T03:47:52.137 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.137 INFO:tasks.workunit.client.0.smithi070.stdout: "state_io_done_lat": { 2024-09-14T03:47:52.137 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 272, 2024-09-14T03:47:52.137 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 205836 2024-09-14T03:47:52.137 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.137 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_commiting_lat": { 2024-09-14T03:47:52.137 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 272, 2024-09-14T03:47:52.137 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 169304171 2024-09-14T03:47:52.137 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.137 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_done_lat": { 2024-09-14T03:47:52.137 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 272, 2024-09-14T03:47:52.138 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 84725 2024-09-14T03:47:52.138 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.138 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_queued_lat": { 2024-09-14T03:47:52.138 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 272, 2024-09-14T03:47:52.138 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 108517973 2024-09-14T03:47:52.138 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.138 INFO:tasks.workunit.client.0.smithi070.stdout: "state_prepare_lat": { 2024-09-14T03:47:52.138 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 272, 2024-09-14T03:47:52.138 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 11043922 2024-09-14T03:47:52.138 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.138 INFO:tasks.workunit.client.0.smithi070.stdout: "stored": { 2024-09-14T03:47:52.138 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 905244 2024-09-14T03:47:52.138 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.139 INFO:tasks.workunit.client.0.smithi070.stdout: "truncate_lat": { 2024-09-14T03:47:52.139 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 27, 2024-09-14T03:47:52.139 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 114486 2024-09-14T03:47:52.139 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.139 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_commit_lat": { 2024-09-14T03:47:52.139 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 272, 2024-09-14T03:47:52.139 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 291943815 2024-09-14T03:47:52.139 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.139 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_submit_lat": { 2024-09-14T03:47:52.139 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 272, 2024-09-14T03:47:52.139 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 13196129 2024-09-14T03:47:52.139 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.139 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_throttle_lat": { 2024-09-14T03:47:52.140 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 272, 2024-09-14T03:47:52.140 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1892202 2024-09-14T03:47:52.140 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.140 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.140 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache": { 2024-09-14T03:47:52.140 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_bytes": { 2024-09-14T03:47:52.140 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1861444387 2024-09-14T03:47:52.140 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.140 INFO:tasks.workunit.client.0.smithi070.stdout: "heap_bytes": { 2024-09-14T03:47:52.140 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 270974976 2024-09-14T03:47:52.140 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.140 INFO:tasks.workunit.client.0.smithi070.stdout: "mapped_bytes": { 2024-09-14T03:47:52.140 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 269418496 2024-09-14T03:47:52.141 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.141 INFO:tasks.workunit.client.0.smithi070.stdout: "target_bytes": { 2024-09-14T03:47:52.141 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3137353830 2024-09-14T03:47:52.141 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.141 INFO:tasks.workunit.client.0.smithi070.stdout: "unmapped_bytes": { 2024-09-14T03:47:52.141 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1556480 2024-09-14T03:47:52.141 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.141 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.141 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:data": { 2024-09-14T03:47:52.141 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:52.141 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 159383552 2024-09-14T03:47:52.141 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.141 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:52.142 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.142 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.142 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:52.142 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.142 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.142 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:52.142 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 85497699 2024-09-14T03:47:52.142 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.142 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:52.142 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.142 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.142 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:52.142 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.142 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.143 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:52.143 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.143 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.143 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:52.143 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.143 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.143 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:52.143 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.143 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.143 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:52.143 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 8192 2024-09-14T03:47:52.143 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.143 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:52.143 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.143 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.144 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:52.144 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.144 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.144 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:52.144 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.144 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.144 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:52.144 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 73877661 2024-09-14T03:47:52.144 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.144 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.144 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:kv": { 2024-09-14T03:47:52.144 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:52.144 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 780140544 2024-09-14T03:47:52.144 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.145 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:52.145 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:52.145 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.145 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:52.145 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.145 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.145 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:52.145 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 641232746 2024-09-14T03:47:52.145 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.145 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:52.145 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.145 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.145 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:52.146 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.146 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.146 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:52.146 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.146 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.146 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:52.146 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.146 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.146 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:52.146 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.146 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.146 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:52.146 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2128 2024-09-14T03:47:52.147 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.147 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:52.147 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.147 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.147 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:52.147 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.147 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.147 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:52.147 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.147 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.147 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:52.147 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71796806 2024-09-14T03:47:52.147 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.147 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.148 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:kv_onode": { 2024-09-14T03:47:52.148 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:52.148 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 192937984 2024-09-14T03:47:52.148 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.148 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:52.148 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:52.148 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.148 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:52.148 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 464 2024-09-14T03:47:52.148 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.148 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:52.148 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 56998466 2024-09-14T03:47:52.148 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.149 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:52.149 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.149 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.149 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:52.149 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.149 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.149 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:52.149 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.149 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.149 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:52.149 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.149 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.149 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:52.149 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.150 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.150 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:52.150 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.150 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.150 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:52.150 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.150 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.150 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:52.150 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.150 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.150 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:52.150 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.150 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.151 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:52.151 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 68830190 2024-09-14T03:47:52.151 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.151 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.151 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:meta": { 2024-09-14T03:47:52.151 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:52.151 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 713031680 2024-09-14T03:47:52.151 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.151 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:52.151 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.151 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.151 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:52.151 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.152 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.152 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:52.152 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 641232746 2024-09-14T03:47:52.152 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.152 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:52.152 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.152 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.152 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:52.152 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.152 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.152 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:52.152 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.152 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.153 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:52.153 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.153 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.153 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:52.153 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7105 2024-09-14T03:47:52.153 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.153 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:52.153 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 257223 2024-09-14T03:47:52.153 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.153 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:52.153 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.153 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.154 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:52.154 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.154 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.154 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:52.154 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.154 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.154 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:52.154 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71534606 2024-09-14T03:47:52.154 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.154 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.154 INFO:tasks.workunit.client.0.smithi070.stdout: "objecter": { 2024-09-14T03:47:52.154 INFO:tasks.workunit.client.0.smithi070.stdout: "op_active": { 2024-09-14T03:47:52.154 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.154 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.155 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r": { 2024-09-14T03:47:52.155 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.155 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.155 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rmw": { 2024-09-14T03:47:52.155 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.155 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.155 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w": { 2024-09-14T03:47:52.155 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.155 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.155 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.155 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:52.155 INFO:tasks.workunit.client.0.smithi070.stdout: "numpg": { 2024-09-14T03:47:52.155 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 38 2024-09-14T03:47:52.156 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.156 INFO:tasks.workunit.client.0.smithi070.stdout: "numpg_removing": { 2024-09-14T03:47:52.156 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.156 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.156 INFO:tasks.workunit.client.0.smithi070.stdout: "op": { 2024-09-14T03:47:52.156 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 13 2024-09-14T03:47:52.156 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.156 INFO:tasks.workunit.client.0.smithi070.stdout: "op_delayed_degraded": { 2024-09-14T03:47:52.156 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.156 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.156 INFO:tasks.workunit.client.0.smithi070.stdout: "op_delayed_unreadable": { 2024-09-14T03:47:52.156 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.156 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.156 INFO:tasks.workunit.client.0.smithi070.stdout: "op_in_bytes": { 2024-09-14T03:47:52.157 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2804 2024-09-14T03:47:52.157 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.157 INFO:tasks.workunit.client.0.smithi070.stdout: "op_latency": { 2024-09-14T03:47:52.157 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 13, 2024-09-14T03:47:52.157 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 73359531 2024-09-14T03:47:52.157 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.157 INFO:tasks.workunit.client.0.smithi070.stdout: "op_out_bytes": { 2024-09-14T03:47:52.157 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.157 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.157 INFO:tasks.workunit.client.0.smithi070.stdout: "op_prepare_latency": { 2024-09-14T03:47:52.157 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 15, 2024-09-14T03:47:52.157 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7673717 2024-09-14T03:47:52.157 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.158 INFO:tasks.workunit.client.0.smithi070.stdout: "op_process_latency": { 2024-09-14T03:47:52.158 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 13, 2024-09-14T03:47:52.158 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 26473448 2024-09-14T03:47:52.158 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.158 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r": { 2024-09-14T03:47:52.158 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.158 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.158 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_latency": { 2024-09-14T03:47:52.158 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.158 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.158 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.158 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:47:52.158 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.159 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.159 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_out_bytes": { 2024-09-14T03:47:52.159 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.159 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.159 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_prepare_latency": { 2024-09-14T03:47:52.159 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.159 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.159 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.159 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_process_latency": { 2024-09-14T03:47:52.159 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.159 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.159 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.159 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw": { 2024-09-14T03:47:52.159 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.160 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.160 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_in_bytes": { 2024-09-14T03:47:52.160 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.160 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.160 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency": { 2024-09-14T03:47:52.160 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.160 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.160 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.160 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:47:52.160 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.160 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.160 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:47:52.160 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.160 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.161 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_out_bytes": { 2024-09-14T03:47:52.161 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.161 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.161 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_prepare_latency": { 2024-09-14T03:47:52.161 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.161 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.161 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.161 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_process_latency": { 2024-09-14T03:47:52.161 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.161 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.161 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.161 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w": { 2024-09-14T03:47:52.161 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 13 2024-09-14T03:47:52.162 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.162 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_in_bytes": { 2024-09-14T03:47:52.162 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2804 2024-09-14T03:47:52.162 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.162 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_latency": { 2024-09-14T03:47:52.162 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 13, 2024-09-14T03:47:52.162 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 73359531 2024-09-14T03:47:52.162 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.162 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:47:52.162 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.162 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.162 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_prepare_latency": { 2024-09-14T03:47:52.162 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 15, 2024-09-14T03:47:52.163 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7673717 2024-09-14T03:47:52.163 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.163 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_process_latency": { 2024-09-14T03:47:52.163 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 13, 2024-09-14T03:47:52.163 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 26473448 2024-09-14T03:47:52.163 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.163 INFO:tasks.workunit.client.0.smithi070.stdout: "op_wip": { 2024-09-14T03:47:52.163 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.163 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.163 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_bytes": { 2024-09-14T03:47:52.163 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 590368 2024-09-14T03:47:52.163 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.163 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_ops": { 2024-09-14T03:47:52.164 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:52.164 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.164 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_bytes": { 2024-09-14T03:47:52.164 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 95995035648 2024-09-14T03:47:52.164 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.164 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_bytes_used": { 2024-09-14T03:47:52.164 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 305786880 2024-09-14T03:47:52.164 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.164 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.164 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.164 INFO:tasks.workunit.client.0.smithi070.stdout: "osd.7": { 2024-09-14T03:47:52.164 INFO:tasks.workunit.client.0.smithi070.stdout: "bluefs": { 2024-09-14T03:47:52.165 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_slow_fallback": { 2024-09-14T03:47:52.165 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.165 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.165 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_slow_size_fallback": { 2024-09-14T03:47:52.165 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.165 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.165 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_db": { 2024-09-14T03:47:52.165 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 65536 2024-09-14T03:47:52.165 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.165 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_main": { 2024-09-14T03:47:52.165 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.165 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.165 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit_wal": { 2024-09-14T03:47:52.166 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.166 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.166 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_slow": { 2024-09-14T03:47:52.166 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.166 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.166 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_sst": { 2024-09-14T03:47:52.166 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12288 2024-09-14T03:47:52.166 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.166 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_written_wal": { 2024-09-14T03:47:52.166 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1245184 2024-09-14T03:47:52.166 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.166 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lat": { 2024-09-14T03:47:52.166 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.167 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.167 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.167 INFO:tasks.workunit.client.0.smithi070.stdout: "compact_lock_lat": { 2024-09-14T03:47:52.167 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.167 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.167 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.167 INFO:tasks.workunit.client.0.smithi070.stdout: "db_total_bytes": { 2024-09-14T03:47:52.167 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 95995035648 2024-09-14T03:47:52.167 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.167 INFO:tasks.workunit.client.0.smithi070.stdout: "db_used_bytes": { 2024-09-14T03:47:52.167 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 304480256 2024-09-14T03:47:52.167 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.167 INFO:tasks.workunit.client.0.smithi070.stdout: "log_bytes": { 2024-09-14T03:47:52.167 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 983040 2024-09-14T03:47:52.168 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.168 INFO:tasks.workunit.client.0.smithi070.stdout: "logged_bytes": { 2024-09-14T03:47:52.168 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 843776 2024-09-14T03:47:52.168 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.168 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_db": { 2024-09-14T03:47:52.168 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 312868864 2024-09-14T03:47:52.168 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.168 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_slow": { 2024-09-14T03:47:52.168 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.168 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.168 INFO:tasks.workunit.client.0.smithi070.stdout: "max_bytes_wal": { 2024-09-14T03:47:52.168 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.168 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.169 INFO:tasks.workunit.client.0.smithi070.stdout: "num_files": { 2024-09-14T03:47:52.169 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 14 2024-09-14T03:47:52.169 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.169 INFO:tasks.workunit.client.0.smithi070.stdout: "read_bytes": { 2024-09-14T03:47:52.169 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 195697 2024-09-14T03:47:52.169 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.169 INFO:tasks.workunit.client.0.smithi070.stdout: "read_count": { 2024-09-14T03:47:52.169 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 23 2024-09-14T03:47:52.169 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.169 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes": { 2024-09-14T03:47:52.169 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1269760 2024-09-14T03:47:52.169 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.169 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_db": { 2024-09-14T03:47:52.169 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1273856 2024-09-14T03:47:52.169 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.170 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_slow": { 2024-09-14T03:47:52.170 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.170 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.170 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_bytes_wal": { 2024-09-14T03:47:52.170 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.170 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.170 INFO:tasks.workunit.client.0.smithi070.stdout: "read_disk_count": { 2024-09-14T03:47:52.170 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 15 2024-09-14T03:47:52.170 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.170 INFO:tasks.workunit.client.0.smithi070.stdout: "read_prefetch_bytes": { 2024-09-14T03:47:52.170 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4789 2024-09-14T03:47:52.170 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.170 INFO:tasks.workunit.client.0.smithi070.stdout: "read_prefetch_count": { 2024-09-14T03:47:52.170 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4 2024-09-14T03:47:52.170 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.171 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_buffer_bytes": { 2024-09-14T03:47:52.171 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5589 2024-09-14T03:47:52.171 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.171 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_buffer_count": { 2024-09-14T03:47:52.171 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 24 2024-09-14T03:47:52.171 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.171 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_bytes": { 2024-09-14T03:47:52.171 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5589 2024-09-14T03:47:52.171 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.171 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_count": { 2024-09-14T03:47:52.171 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 24 2024-09-14T03:47:52.171 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.171 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes": { 2024-09-14T03:47:52.171 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.171 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.172 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_db": { 2024-09-14T03:47:52.172 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.172 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.172 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_slow": { 2024-09-14T03:47:52.172 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.172 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.172 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_bytes_wal": { 2024-09-14T03:47:52.172 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.172 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.172 INFO:tasks.workunit.client.0.smithi070.stdout: "read_random_disk_count": { 2024-09-14T03:47:52.172 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.172 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.172 INFO:tasks.workunit.client.0.smithi070.stdout: "slow_total_bytes": { 2024-09-14T03:47:52.173 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.173 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.173 INFO:tasks.workunit.client.0.smithi070.stdout: "slow_used_bytes": { 2024-09-14T03:47:52.173 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.173 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.173 INFO:tasks.workunit.client.0.smithi070.stdout: "wal_total_bytes": { 2024-09-14T03:47:52.173 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.173 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.173 INFO:tasks.workunit.client.0.smithi070.stdout: "wal_used_bytes": { 2024-09-14T03:47:52.173 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.173 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.173 INFO:tasks.workunit.client.0.smithi070.stdout: "write_bytes": { 2024-09-14T03:47:52.173 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2162688 2024-09-14T03:47:52.173 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.174 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.174 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore": { 2024-09-14T03:47:52.174 INFO:tasks.workunit.client.0.smithi070.stdout: "alloc_unit": { 2024-09-14T03:47:52.174 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4096 2024-09-14T03:47:52.174 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.174 INFO:tasks.workunit.client.0.smithi070.stdout: "allocated": { 2024-09-14T03:47:52.174 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1298432 2024-09-14T03:47:52.174 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.174 INFO:tasks.workunit.client.0.smithi070.stdout: "clist_lat": { 2024-09-14T03:47:52.174 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 3, 2024-09-14T03:47:52.174 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 165845 2024-09-14T03:47:52.174 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.174 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_lat": { 2024-09-14T03:47:52.174 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.175 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.175 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.175 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed": { 2024-09-14T03:47:52.175 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.175 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.175 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed_allocated": { 2024-09-14T03:47:52.175 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.175 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.175 INFO:tasks.workunit.client.0.smithi070.stdout: "compressed_original": { 2024-09-14T03:47:52.175 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.175 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.175 INFO:tasks.workunit.client.0.smithi070.stdout: "csum_lat": { 2024-09-14T03:47:52.175 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 12, 2024-09-14T03:47:52.176 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 26784 2024-09-14T03:47:52.176 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.176 INFO:tasks.workunit.client.0.smithi070.stdout: "decompress_lat": { 2024-09-14T03:47:52.176 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.176 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.176 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.176 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_commit_lat": { 2024-09-14T03:47:52.176 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 189, 2024-09-14T03:47:52.176 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 44860059 2024-09-14T03:47:52.176 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.176 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_final_lat": { 2024-09-14T03:47:52.176 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 179, 2024-09-14T03:47:52.176 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 8737744 2024-09-14T03:47:52.176 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.176 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_flush_lat": { 2024-09-14T03:47:52.177 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 189, 2024-09-14T03:47:52.177 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 277025 2024-09-14T03:47:52.177 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.177 INFO:tasks.workunit.client.0.smithi070.stdout: "kv_sync_lat": { 2024-09-14T03:47:52.177 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 189, 2024-09-14T03:47:52.177 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 45137084 2024-09-14T03:47:52.177 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.177 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_get_keys_lat": { 2024-09-14T03:47:52.177 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.177 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.177 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.177 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_get_values_lat": { 2024-09-14T03:47:52.177 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:52.178 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 33804 2024-09-14T03:47:52.178 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.178 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_lower_bound_lat": { 2024-09-14T03:47:52.178 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:52.178 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1288 2024-09-14T03:47:52.178 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.178 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_next_lat": { 2024-09-14T03:47:52.178 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.178 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.178 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.178 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_seek_to_first_lat": { 2024-09-14T03:47:52.178 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 0, 2024-09-14T03:47:52.178 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.178 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.179 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_upper_bound_lat": { 2024-09-14T03:47:52.179 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 2, 2024-09-14T03:47:52.179 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 4334 2024-09-14T03:47:52.179 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.179 INFO:tasks.workunit.client.0.smithi070.stdout: "onode_hits": { 2024-09-14T03:47:52.179 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 542 2024-09-14T03:47:52.179 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.179 INFO:tasks.workunit.client.0.smithi070.stdout: "onode_misses": { 2024-09-14T03:47:52.179 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 268 2024-09-14T03:47:52.179 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.179 INFO:tasks.workunit.client.0.smithi070.stdout: "read_lat": { 2024-09-14T03:47:52.179 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 113, 2024-09-14T03:47:52.180 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1292204 2024-09-14T03:47:52.180 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.180 INFO:tasks.workunit.client.0.smithi070.stdout: "read_onode_meta_lat": { 2024-09-14T03:47:52.180 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 179, 2024-09-14T03:47:52.180 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 212712 2024-09-14T03:47:52.180 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.180 INFO:tasks.workunit.client.0.smithi070.stdout: "read_wait_aio_lat": { 2024-09-14T03:47:52.180 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 66, 2024-09-14T03:47:52.180 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 695603 2024-09-14T03:47:52.180 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.180 INFO:tasks.workunit.client.0.smithi070.stdout: "reads_with_retries": { 2024-09-14T03:47:52.180 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.181 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.181 INFO:tasks.workunit.client.0.smithi070.stdout: "remove_lat": { 2024-09-14T03:47:52.181 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 1, 2024-09-14T03:47:52.181 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 26374 2024-09-14T03:47:52.181 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.181 INFO:tasks.workunit.client.0.smithi070.stdout: "state_aio_wait_lat": { 2024-09-14T03:47:52.181 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 288, 2024-09-14T03:47:52.181 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2190213 2024-09-14T03:47:52.181 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.181 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_aio_wait_lat": { 2024-09-14T03:47:52.181 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 24, 2024-09-14T03:47:52.181 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1694874 2024-09-14T03:47:52.181 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.182 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_cleanup_lat": { 2024-09-14T03:47:52.182 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 24, 2024-09-14T03:47:52.182 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 298495756417 2024-09-14T03:47:52.182 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.182 INFO:tasks.workunit.client.0.smithi070.stdout: "state_deferred_queued_lat": { 2024-09-14T03:47:52.182 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 24, 2024-09-14T03:47:52.182 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 36302536737 2024-09-14T03:47:52.182 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.182 INFO:tasks.workunit.client.0.smithi070.stdout: "state_done_lat": { 2024-09-14T03:47:52.182 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 288, 2024-09-14T03:47:52.182 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 448360 2024-09-14T03:47:52.182 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.182 INFO:tasks.workunit.client.0.smithi070.stdout: "state_finishing_lat": { 2024-09-14T03:47:52.183 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 288, 2024-09-14T03:47:52.183 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 33332 2024-09-14T03:47:52.183 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.183 INFO:tasks.workunit.client.0.smithi070.stdout: "state_io_done_lat": { 2024-09-14T03:47:52.183 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 288, 2024-09-14T03:47:52.183 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 219535 2024-09-14T03:47:52.183 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.183 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_commiting_lat": { 2024-09-14T03:47:52.183 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 288, 2024-09-14T03:47:52.183 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 127434141 2024-09-14T03:47:52.183 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.183 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_done_lat": { 2024-09-14T03:47:52.183 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 288, 2024-09-14T03:47:52.184 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 80980 2024-09-14T03:47:52.184 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.184 INFO:tasks.workunit.client.0.smithi070.stdout: "state_kv_queued_lat": { 2024-09-14T03:47:52.184 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 288, 2024-09-14T03:47:52.184 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 92350704 2024-09-14T03:47:52.184 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.184 INFO:tasks.workunit.client.0.smithi070.stdout: "state_prepare_lat": { 2024-09-14T03:47:52.184 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 288, 2024-09-14T03:47:52.184 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 12175385 2024-09-14T03:47:52.184 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.184 INFO:tasks.workunit.client.0.smithi070.stdout: "stored": { 2024-09-14T03:47:52.184 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 903527 2024-09-14T03:47:52.184 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.185 INFO:tasks.workunit.client.0.smithi070.stdout: "truncate_lat": { 2024-09-14T03:47:52.185 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 23, 2024-09-14T03:47:52.185 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 103984 2024-09-14T03:47:52.185 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.185 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_commit_lat": { 2024-09-14T03:47:52.185 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 288, 2024-09-14T03:47:52.185 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 234404958 2024-09-14T03:47:52.185 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.185 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_submit_lat": { 2024-09-14T03:47:52.185 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 288, 2024-09-14T03:47:52.185 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 16408932 2024-09-14T03:47:52.185 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.185 INFO:tasks.workunit.client.0.smithi070.stdout: "txc_throttle_lat": { 2024-09-14T03:47:52.186 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 288, 2024-09-14T03:47:52.186 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1971853 2024-09-14T03:47:52.186 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.186 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.186 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache": { 2024-09-14T03:47:52.186 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_bytes": { 2024-09-14T03:47:52.186 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1861444387 2024-09-14T03:47:52.186 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.186 INFO:tasks.workunit.client.0.smithi070.stdout: "heap_bytes": { 2024-09-14T03:47:52.186 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 271958016 2024-09-14T03:47:52.186 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.186 INFO:tasks.workunit.client.0.smithi070.stdout: "mapped_bytes": { 2024-09-14T03:47:52.186 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 270417920 2024-09-14T03:47:52.187 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.187 INFO:tasks.workunit.client.0.smithi070.stdout: "target_bytes": { 2024-09-14T03:47:52.187 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3137353830 2024-09-14T03:47:52.187 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.187 INFO:tasks.workunit.client.0.smithi070.stdout: "unmapped_bytes": { 2024-09-14T03:47:52.187 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 1540096 2024-09-14T03:47:52.187 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.187 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.187 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:data": { 2024-09-14T03:47:52.187 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:52.187 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 159383552 2024-09-14T03:47:52.187 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.187 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:52.187 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.187 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.188 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:52.188 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.188 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.188 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:52.188 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 85480021 2024-09-14T03:47:52.188 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.188 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:52.188 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.188 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.188 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:52.188 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 151552 2024-09-14T03:47:52.188 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.188 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:52.189 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.189 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.189 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:52.189 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.189 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.189 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:52.189 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.189 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.189 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:52.189 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 135168 2024-09-14T03:47:52.189 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.189 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:52.189 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.190 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.190 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:52.190 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.190 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.190 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:52.190 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.190 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.190 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:52.190 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 73616811 2024-09-14T03:47:52.190 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.190 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.190 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:kv": { 2024-09-14T03:47:52.190 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:52.190 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 780140544 2024-09-14T03:47:52.190 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.190 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:52.191 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:52.191 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.191 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:52.191 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.191 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.191 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:52.191 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 641100162 2024-09-14T03:47:52.191 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.191 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:52.191 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.191 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.191 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:52.191 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.191 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.191 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:52.192 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.192 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.192 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:52.192 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.192 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.192 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:52.192 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.192 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.192 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:52.192 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2128 2024-09-14T03:47:52.192 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.192 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:52.192 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.192 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.193 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:52.193 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.193 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.193 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:52.193 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.193 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.193 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:52.193 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71929390 2024-09-14T03:47:52.193 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.193 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.193 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:kv_onode": { 2024-09-14T03:47:52.193 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:52.193 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 192937984 2024-09-14T03:47:52.194 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.194 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:52.194 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 67108864 2024-09-14T03:47:52.194 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.194 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:52.194 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 464 2024-09-14T03:47:52.194 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.194 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:52.194 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 56986681 2024-09-14T03:47:52.194 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.194 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:52.194 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.194 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.194 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:52.194 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.195 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.195 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:52.195 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.195 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.195 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:52.195 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.195 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.195 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:52.195 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.195 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.195 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:52.195 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.195 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.195 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:52.195 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.196 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.196 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:52.196 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.196 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.196 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:52.196 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.196 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.196 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:52.196 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 68841975 2024-09-14T03:47:52.196 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.196 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.196 INFO:tasks.workunit.client.0.smithi070.stdout: "bluestore-pricache:meta": { 2024-09-14T03:47:52.196 INFO:tasks.workunit.client.0.smithi070.stdout: "committed_bytes": { 2024-09-14T03:47:52.197 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 713031680 2024-09-14T03:47:52.197 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.197 INFO:tasks.workunit.client.0.smithi070.stdout: "pri0_bytes": { 2024-09-14T03:47:52.197 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.197 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.197 INFO:tasks.workunit.client.0.smithi070.stdout: "pri10_bytes": { 2024-09-14T03:47:52.197 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.197 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.197 INFO:tasks.workunit.client.0.smithi070.stdout: "pri11_bytes": { 2024-09-14T03:47:52.197 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 641100162 2024-09-14T03:47:52.197 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.197 INFO:tasks.workunit.client.0.smithi070.stdout: "pri1_bytes": { 2024-09-14T03:47:52.197 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.198 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.198 INFO:tasks.workunit.client.0.smithi070.stdout: "pri2_bytes": { 2024-09-14T03:47:52.198 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 15658 2024-09-14T03:47:52.198 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.198 INFO:tasks.workunit.client.0.smithi070.stdout: "pri3_bytes": { 2024-09-14T03:47:52.198 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.198 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.198 INFO:tasks.workunit.client.0.smithi070.stdout: "pri4_bytes": { 2024-09-14T03:47:52.198 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.198 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.198 INFO:tasks.workunit.client.0.smithi070.stdout: "pri5_bytes": { 2024-09-14T03:47:52.198 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.198 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.198 INFO:tasks.workunit.client.0.smithi070.stdout: "pri6_bytes": { 2024-09-14T03:47:52.199 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 264773 2024-09-14T03:47:52.199 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.199 INFO:tasks.workunit.client.0.smithi070.stdout: "pri7_bytes": { 2024-09-14T03:47:52.199 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.199 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.199 INFO:tasks.workunit.client.0.smithi070.stdout: "pri8_bytes": { 2024-09-14T03:47:52.199 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.199 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.199 INFO:tasks.workunit.client.0.smithi070.stdout: "pri9_bytes": { 2024-09-14T03:47:52.199 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.199 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.199 INFO:tasks.workunit.client.0.smithi070.stdout: "reserved_bytes": { 2024-09-14T03:47:52.199 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 71651087 2024-09-14T03:47:52.200 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.200 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.200 INFO:tasks.workunit.client.0.smithi070.stdout: "objecter": { 2024-09-14T03:47:52.200 INFO:tasks.workunit.client.0.smithi070.stdout: "op_active": { 2024-09-14T03:47:52.200 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.200 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.200 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r": { 2024-09-14T03:47:52.200 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.200 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.200 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rmw": { 2024-09-14T03:47:52.200 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.200 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.200 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w": { 2024-09-14T03:47:52.200 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.201 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.201 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.201 INFO:tasks.workunit.client.0.smithi070.stdout: "osd": { 2024-09-14T03:47:52.201 INFO:tasks.workunit.client.0.smithi070.stdout: "numpg": { 2024-09-14T03:47:52.201 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 46 2024-09-14T03:47:52.201 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.201 INFO:tasks.workunit.client.0.smithi070.stdout: "numpg_removing": { 2024-09-14T03:47:52.201 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.201 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.201 INFO:tasks.workunit.client.0.smithi070.stdout: "op": { 2024-09-14T03:47:52.201 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 87 2024-09-14T03:47:52.201 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.201 INFO:tasks.workunit.client.0.smithi070.stdout: "op_delayed_degraded": { 2024-09-14T03:47:52.201 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.202 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.202 INFO:tasks.workunit.client.0.smithi070.stdout: "op_delayed_unreadable": { 2024-09-14T03:47:52.202 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.202 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.202 INFO:tasks.workunit.client.0.smithi070.stdout: "op_in_bytes": { 2024-09-14T03:47:52.202 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.202 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.202 INFO:tasks.workunit.client.0.smithi070.stdout: "op_latency": { 2024-09-14T03:47:52.202 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 87, 2024-09-14T03:47:52.202 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 66490697 2024-09-14T03:47:52.202 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.202 INFO:tasks.workunit.client.0.smithi070.stdout: "op_out_bytes": { 2024-09-14T03:47:52.202 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 264311 2024-09-14T03:47:52.203 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.203 INFO:tasks.workunit.client.0.smithi070.stdout: "op_prepare_latency": { 2024-09-14T03:47:52.203 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 89, 2024-09-14T03:47:52.203 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 17864970 2024-09-14T03:47:52.203 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.203 INFO:tasks.workunit.client.0.smithi070.stdout: "op_process_latency": { 2024-09-14T03:47:52.203 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 87, 2024-09-14T03:47:52.203 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 49784785 2024-09-14T03:47:52.203 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.203 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r": { 2024-09-14T03:47:52.203 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 60 2024-09-14T03:47:52.203 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.203 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_latency": { 2024-09-14T03:47:52.203 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 60, 2024-09-14T03:47:52.203 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7178504 2024-09-14T03:47:52.204 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.204 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-14T03:47:52.204 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.204 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.204 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_out_bytes": { 2024-09-14T03:47:52.204 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 264311 2024-09-14T03:47:52.204 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.204 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_prepare_latency": { 2024-09-14T03:47:52.204 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 60, 2024-09-14T03:47:52.204 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 6462750 2024-09-14T03:47:52.204 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.204 INFO:tasks.workunit.client.0.smithi070.stdout: "op_r_process_latency": { 2024-09-14T03:47:52.204 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 60, 2024-09-14T03:47:52.204 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5238184 2024-09-14T03:47:52.205 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.205 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw": { 2024-09-14T03:47:52.205 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 22 2024-09-14T03:47:52.205 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.205 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_in_bytes": { 2024-09-14T03:47:52.205 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.205 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.205 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency": { 2024-09-14T03:47:52.205 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 22, 2024-09-14T03:47:52.205 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 18338763 2024-09-14T03:47:52.205 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.205 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-14T03:47:52.205 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.206 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.206 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-14T03:47:52.206 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.206 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.206 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_out_bytes": { 2024-09-14T03:47:52.206 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.206 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.206 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_prepare_latency": { 2024-09-14T03:47:52.206 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 22, 2024-09-14T03:47:52.206 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 7585805 2024-09-14T03:47:52.206 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.206 INFO:tasks.workunit.client.0.smithi070.stdout: "op_rw_process_latency": { 2024-09-14T03:47:52.206 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 22, 2024-09-14T03:47:52.206 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 17315288 2024-09-14T03:47:52.206 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.207 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w": { 2024-09-14T03:47:52.207 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 5 2024-09-14T03:47:52.207 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.207 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_in_bytes": { 2024-09-14T03:47:52.207 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.207 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.207 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_latency": { 2024-09-14T03:47:52.207 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 5, 2024-09-14T03:47:52.207 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 40973430 2024-09-14T03:47:52.207 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.207 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-14T03:47:52.207 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.207 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.207 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_prepare_latency": { 2024-09-14T03:47:52.208 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 7, 2024-09-14T03:47:52.208 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 3816415 2024-09-14T03:47:52.208 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.208 INFO:tasks.workunit.client.0.smithi070.stdout: "op_w_process_latency": { 2024-09-14T03:47:52.208 INFO:tasks.workunit.client.0.smithi070.stdout: "count": 5, 2024-09-14T03:47:52.208 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 27231313 2024-09-14T03:47:52.208 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.208 INFO:tasks.workunit.client.0.smithi070.stdout: "op_wip": { 2024-09-14T03:47:52.208 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 0 2024-09-14T03:47:52.208 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.208 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_bytes": { 2024-09-14T03:47:52.208 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 590368 2024-09-14T03:47:52.208 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.209 INFO:tasks.workunit.client.0.smithi070.stdout: "recovery_ops": { 2024-09-14T03:47:52.209 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 2 2024-09-14T03:47:52.209 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.209 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_bytes": { 2024-09-14T03:47:52.209 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 95995035648 2024-09-14T03:47:52.209 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.209 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_bytes_used": { 2024-09-14T03:47:52.209 INFO:tasks.workunit.client.0.smithi070.stdout: "value": 305786880 2024-09-14T03:47:52.209 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.209 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.209 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.209 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.209 INFO:tasks.workunit.client.0.smithi070.stdout: "pools": [ 2024-09-14T03:47:52.210 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.210 INFO:tasks.workunit.client.0.smithi070.stdout: "application": [ 2024-09-14T03:47:52.210 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr" 2024-09-14T03:47:52.210 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.210 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_mode": "none", 2024-09-14T03:47:52.210 INFO:tasks.workunit.client.0.smithi070.stdout: "erasure_code_profile": {}, 2024-09-14T03:47:52.210 INFO:tasks.workunit.client.0.smithi070.stdout: "min_size": 2, 2024-09-14T03:47:52.210 INFO:tasks.workunit.client.0.smithi070.stdout: "options": {}, 2024-09-14T03:47:52.210 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_autoscale_mode": "off", 2024-09-14T03:47:52.210 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_num": 1, 2024-09-14T03:47:52.210 INFO:tasks.workunit.client.0.smithi070.stdout: "pgp_num": 1, 2024-09-14T03:47:52.210 INFO:tasks.workunit.client.0.smithi070.stdout: "pool": 1, 2024-09-14T03:47:52.210 INFO:tasks.workunit.client.0.smithi070.stdout: "size": 3, 2024-09-14T03:47:52.211 INFO:tasks.workunit.client.0.smithi070.stdout: "stats": { 2024-09-14T03:47:52.211 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_raw": 727115933962, 2024-09-14T03:47:52.211 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_used": 1781760, 2024-09-14T03:47:52.211 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_bytes_used": 0, 2024-09-14T03:47:52.211 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_under_bytes": 0, 2024-09-14T03:47:52.211 INFO:tasks.workunit.client.0.smithi070.stdout: "data_bytes_used": 1781760, 2024-09-14T03:47:52.211 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty": 0, 2024-09-14T03:47:52.211 INFO:tasks.workunit.client.0.smithi070.stdout: "max_avail": 242371985408, 2024-09-14T03:47:52.211 INFO:tasks.workunit.client.0.smithi070.stdout: "objects": 2, 2024-09-14T03:47:52.211 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_bytes_used": 0, 2024-09-14T03:47:52.211 INFO:tasks.workunit.client.0.smithi070.stdout: "percent_used": 2.4504422526661074e-06, 2024-09-14T03:47:52.211 INFO:tasks.workunit.client.0.smithi070.stdout: "quota_bytes": 0, 2024-09-14T03:47:52.211 INFO:tasks.workunit.client.0.smithi070.stdout: "quota_objects": 0, 2024-09-14T03:47:52.212 INFO:tasks.workunit.client.0.smithi070.stdout: "rd": 272, 2024-09-14T03:47:52.212 INFO:tasks.workunit.client.0.smithi070.stdout: "rd_bytes": 490496, 2024-09-14T03:47:52.212 INFO:tasks.workunit.client.0.smithi070.stdout: "stored": 590368, 2024-09-14T03:47:52.212 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_data": 590368, 2024-09-14T03:47:52.212 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_omap": 0, 2024-09-14T03:47:52.212 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_raw": 1771104, 2024-09-14T03:47:52.212 INFO:tasks.workunit.client.0.smithi070.stdout: "wr": 145, 2024-09-14T03:47:52.212 INFO:tasks.workunit.client.0.smithi070.stdout: "wr_bytes": 1667072 2024-09-14T03:47:52.212 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.212 INFO:tasks.workunit.client.0.smithi070.stdout: "target_max_bytes": 0, 2024-09-14T03:47:52.212 INFO:tasks.workunit.client.0.smithi070.stdout: "target_max_objects": 0, 2024-09-14T03:47:52.212 INFO:tasks.workunit.client.0.smithi070.stdout: "type": "replicated" 2024-09-14T03:47:52.212 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.212 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.213 INFO:tasks.workunit.client.0.smithi070.stdout: "application": [ 2024-09-14T03:47:52.213 INFO:tasks.workunit.client.0.smithi070.stdout: "cephfs" 2024-09-14T03:47:52.213 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.213 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_mode": "none", 2024-09-14T03:47:52.213 INFO:tasks.workunit.client.0.smithi070.stdout: "erasure_code_profile": {}, 2024-09-14T03:47:52.213 INFO:tasks.workunit.client.0.smithi070.stdout: "min_size": 2, 2024-09-14T03:47:52.213 INFO:tasks.workunit.client.0.smithi070.stdout: "options": {}, 2024-09-14T03:47:52.213 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_autoscale_mode": "off", 2024-09-14T03:47:52.213 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_num": 32, 2024-09-14T03:47:52.213 INFO:tasks.workunit.client.0.smithi070.stdout: "pgp_num": 32, 2024-09-14T03:47:52.213 INFO:tasks.workunit.client.0.smithi070.stdout: "pool": 2, 2024-09-14T03:47:52.213 INFO:tasks.workunit.client.0.smithi070.stdout: "size": 3, 2024-09-14T03:47:52.213 INFO:tasks.workunit.client.0.smithi070.stdout: "stats": { 2024-09-14T03:47:52.213 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_raw": 727115933962, 2024-09-14T03:47:52.214 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_used": 98304, 2024-09-14T03:47:52.214 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_bytes_used": 0, 2024-09-14T03:47:52.214 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_under_bytes": 0, 2024-09-14T03:47:52.214 INFO:tasks.workunit.client.0.smithi070.stdout: "data_bytes_used": 98304, 2024-09-14T03:47:52.214 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty": 0, 2024-09-14T03:47:52.214 INFO:tasks.workunit.client.0.smithi070.stdout: "max_avail": 242371985408, 2024-09-14T03:47:52.214 INFO:tasks.workunit.client.0.smithi070.stdout: "objects": 22, 2024-09-14T03:47:52.214 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_bytes_used": 0, 2024-09-14T03:47:52.214 INFO:tasks.workunit.client.0.smithi070.stdout: "percent_used": 1.3519712638299097e-07, 2024-09-14T03:47:52.214 INFO:tasks.workunit.client.0.smithi070.stdout: "quota_bytes": 0, 2024-09-14T03:47:52.214 INFO:tasks.workunit.client.0.smithi070.stdout: "quota_objects": 0, 2024-09-14T03:47:52.214 INFO:tasks.workunit.client.0.smithi070.stdout: "rd": 0, 2024-09-14T03:47:52.214 INFO:tasks.workunit.client.0.smithi070.stdout: "rd_bytes": 0, 2024-09-14T03:47:52.214 INFO:tasks.workunit.client.0.smithi070.stdout: "stored": 3410, 2024-09-14T03:47:52.215 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_data": 3410, 2024-09-14T03:47:52.215 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_omap": 0, 2024-09-14T03:47:52.215 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_raw": 10230, 2024-09-14T03:47:52.215 INFO:tasks.workunit.client.0.smithi070.stdout: "wr": 51, 2024-09-14T03:47:52.215 INFO:tasks.workunit.client.0.smithi070.stdout: "wr_bytes": 19456 2024-09-14T03:47:52.215 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.215 INFO:tasks.workunit.client.0.smithi070.stdout: "target_max_bytes": 0, 2024-09-14T03:47:52.215 INFO:tasks.workunit.client.0.smithi070.stdout: "target_max_objects": 0, 2024-09-14T03:47:52.215 INFO:tasks.workunit.client.0.smithi070.stdout: "type": "replicated" 2024-09-14T03:47:52.215 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.215 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.215 INFO:tasks.workunit.client.0.smithi070.stdout: "application": [ 2024-09-14T03:47:52.215 INFO:tasks.workunit.client.0.smithi070.stdout: "cephfs" 2024-09-14T03:47:52.216 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.216 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_mode": "none", 2024-09-14T03:47:52.216 INFO:tasks.workunit.client.0.smithi070.stdout: "erasure_code_profile": {}, 2024-09-14T03:47:52.216 INFO:tasks.workunit.client.0.smithi070.stdout: "min_size": 2, 2024-09-14T03:47:52.216 INFO:tasks.workunit.client.0.smithi070.stdout: "options": {}, 2024-09-14T03:47:52.216 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_autoscale_mode": "off", 2024-09-14T03:47:52.216 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_num": 64, 2024-09-14T03:47:52.216 INFO:tasks.workunit.client.0.smithi070.stdout: "pgp_num": 64, 2024-09-14T03:47:52.216 INFO:tasks.workunit.client.0.smithi070.stdout: "pool": 3, 2024-09-14T03:47:52.216 INFO:tasks.workunit.client.0.smithi070.stdout: "size": 3, 2024-09-14T03:47:52.216 INFO:tasks.workunit.client.0.smithi070.stdout: "stats": { 2024-09-14T03:47:52.216 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_raw": 727115933962, 2024-09-14T03:47:52.216 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_used": 0, 2024-09-14T03:47:52.216 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_bytes_used": 0, 2024-09-14T03:47:52.216 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_under_bytes": 0, 2024-09-14T03:47:52.217 INFO:tasks.workunit.client.0.smithi070.stdout: "data_bytes_used": 0, 2024-09-14T03:47:52.217 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty": 0, 2024-09-14T03:47:52.217 INFO:tasks.workunit.client.0.smithi070.stdout: "max_avail": 242371985408, 2024-09-14T03:47:52.217 INFO:tasks.workunit.client.0.smithi070.stdout: "objects": 0, 2024-09-14T03:47:52.217 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_bytes_used": 0, 2024-09-14T03:47:52.217 INFO:tasks.workunit.client.0.smithi070.stdout: "percent_used": 0.0, 2024-09-14T03:47:52.217 INFO:tasks.workunit.client.0.smithi070.stdout: "quota_bytes": 0, 2024-09-14T03:47:52.217 INFO:tasks.workunit.client.0.smithi070.stdout: "quota_objects": 0, 2024-09-14T03:47:52.217 INFO:tasks.workunit.client.0.smithi070.stdout: "rd": 0, 2024-09-14T03:47:52.217 INFO:tasks.workunit.client.0.smithi070.stdout: "rd_bytes": 0, 2024-09-14T03:47:52.217 INFO:tasks.workunit.client.0.smithi070.stdout: "stored": 0, 2024-09-14T03:47:52.217 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_data": 0, 2024-09-14T03:47:52.217 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_omap": 0, 2024-09-14T03:47:52.218 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_raw": 0, 2024-09-14T03:47:52.218 INFO:tasks.workunit.client.0.smithi070.stdout: "wr": 0, 2024-09-14T03:47:52.218 INFO:tasks.workunit.client.0.smithi070.stdout: "wr_bytes": 0 2024-09-14T03:47:52.218 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.218 INFO:tasks.workunit.client.0.smithi070.stdout: "target_max_bytes": 0, 2024-09-14T03:47:52.218 INFO:tasks.workunit.client.0.smithi070.stdout: "target_max_objects": 0, 2024-09-14T03:47:52.218 INFO:tasks.workunit.client.0.smithi070.stdout: "type": "replicated" 2024-09-14T03:47:52.218 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.218 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.218 INFO:tasks.workunit.client.0.smithi070.stdout: "application": [ 2024-09-14T03:47:52.218 INFO:tasks.workunit.client.0.smithi070.stdout: "rbd" 2024-09-14T03:47:52.218 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.218 INFO:tasks.workunit.client.0.smithi070.stdout: "cache_mode": "none", 2024-09-14T03:47:52.218 INFO:tasks.workunit.client.0.smithi070.stdout: "erasure_code_profile": {}, 2024-09-14T03:47:52.219 INFO:tasks.workunit.client.0.smithi070.stdout: "min_size": 2, 2024-09-14T03:47:52.219 INFO:tasks.workunit.client.0.smithi070.stdout: "options": {}, 2024-09-14T03:47:52.219 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_autoscale_mode": "off", 2024-09-14T03:47:52.219 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_num": 8, 2024-09-14T03:47:52.219 INFO:tasks.workunit.client.0.smithi070.stdout: "pgp_num": 8, 2024-09-14T03:47:52.219 INFO:tasks.workunit.client.0.smithi070.stdout: "pool": 4, 2024-09-14T03:47:52.219 INFO:tasks.workunit.client.0.smithi070.stdout: "size": 3, 2024-09-14T03:47:52.219 INFO:tasks.workunit.client.0.smithi070.stdout: "stats": { 2024-09-14T03:47:52.219 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_raw": 727115933962, 2024-09-14T03:47:52.219 INFO:tasks.workunit.client.0.smithi070.stdout: "bytes_used": 12288, 2024-09-14T03:47:52.219 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_bytes_used": 0, 2024-09-14T03:47:52.219 INFO:tasks.workunit.client.0.smithi070.stdout: "compress_under_bytes": 0, 2024-09-14T03:47:52.219 INFO:tasks.workunit.client.0.smithi070.stdout: "data_bytes_used": 12288, 2024-09-14T03:47:52.220 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty": 0, 2024-09-14T03:47:52.220 INFO:tasks.workunit.client.0.smithi070.stdout: "max_avail": 242371985408, 2024-09-14T03:47:52.220 INFO:tasks.workunit.client.0.smithi070.stdout: "objects": 2, 2024-09-14T03:47:52.220 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_bytes_used": 0, 2024-09-14T03:47:52.220 INFO:tasks.workunit.client.0.smithi070.stdout: "percent_used": 1.689964257423071e-08, 2024-09-14T03:47:52.220 INFO:tasks.workunit.client.0.smithi070.stdout: "quota_bytes": 0, 2024-09-14T03:47:52.220 INFO:tasks.workunit.client.0.smithi070.stdout: "quota_objects": 0, 2024-09-14T03:47:52.220 INFO:tasks.workunit.client.0.smithi070.stdout: "rd": 0, 2024-09-14T03:47:52.220 INFO:tasks.workunit.client.0.smithi070.stdout: "rd_bytes": 0, 2024-09-14T03:47:52.220 INFO:tasks.workunit.client.0.smithi070.stdout: "stored": 19, 2024-09-14T03:47:52.220 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_data": 19, 2024-09-14T03:47:52.220 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_omap": 0, 2024-09-14T03:47:52.220 INFO:tasks.workunit.client.0.smithi070.stdout: "stored_raw": 57, 2024-09-14T03:47:52.221 INFO:tasks.workunit.client.0.smithi070.stdout: "wr": 2, 2024-09-14T03:47:52.221 INFO:tasks.workunit.client.0.smithi070.stdout: "wr_bytes": 2048 2024-09-14T03:47:52.221 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.221 INFO:tasks.workunit.client.0.smithi070.stdout: "target_max_bytes": 0, 2024-09-14T03:47:52.221 INFO:tasks.workunit.client.0.smithi070.stdout: "target_max_objects": 0, 2024-09-14T03:47:52.221 INFO:tasks.workunit.client.0.smithi070.stdout: "type": "replicated" 2024-09-14T03:47:52.221 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.221 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.221 INFO:tasks.workunit.client.0.smithi070.stdout: "rbd": { 2024-09-14T03:47:52.221 INFO:tasks.workunit.client.0.smithi070.stdout: "mirroring_by_pool": [ 2024-09-14T03:47:52.221 INFO:tasks.workunit.client.0.smithi070.stdout: false 2024-09-14T03:47:52.221 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.221 INFO:tasks.workunit.client.0.smithi070.stdout: "num_images_by_pool": [ 2024-09-14T03:47:52.222 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.222 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.222 INFO:tasks.workunit.client.0.smithi070.stdout: "num_pools": 1 2024-09-14T03:47:52.222 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.222 INFO:tasks.workunit.client.0.smithi070.stdout: "report_id": "04c26ace-6e52-41dd-9ad2-e917daf8e93d", 2024-09-14T03:47:52.222 INFO:tasks.workunit.client.0.smithi070.stdout: "report_timestamp": "2024-09-14T03:47:50.461926", 2024-09-14T03:47:52.222 INFO:tasks.workunit.client.0.smithi070.stdout: "report_version": 1, 2024-09-14T03:47:52.222 INFO:tasks.workunit.client.0.smithi070.stdout: "rocksdb_stats": { 2024-09-14T03:47:52.222 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "6.15.5" 2024-09-14T03:47:52.222 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.222 INFO:tasks.workunit.client.0.smithi070.stdout: "rook": { 2024-09-14T03:47:52.222 INFO:tasks.workunit.client.0.smithi070.stdout: "cluster": { 2024-09-14T03:47:52.222 INFO:tasks.workunit.client.0.smithi070.stdout: "external-mode": null, 2024-09-14T03:47:52.222 INFO:tasks.workunit.client.0.smithi070.stdout: "mon": { 2024-09-14T03:47:52.222 INFO:tasks.workunit.client.0.smithi070.stdout: "allow-multiple-per-node": null, 2024-09-14T03:47:52.223 INFO:tasks.workunit.client.0.smithi070.stdout: "count": null, 2024-09-14T03:47:52.223 INFO:tasks.workunit.client.0.smithi070.stdout: "max-id": null, 2024-09-14T03:47:52.223 INFO:tasks.workunit.client.0.smithi070.stdout: "pvc": { 2024-09-14T03:47:52.223 INFO:tasks.workunit.client.0.smithi070.stdout: "enabled": null 2024-09-14T03:47:52.223 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.223 INFO:tasks.workunit.client.0.smithi070.stdout: "stretch": { 2024-09-14T03:47:52.223 INFO:tasks.workunit.client.0.smithi070.stdout: "enabled": null 2024-09-14T03:47:52.223 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.223 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.223 INFO:tasks.workunit.client.0.smithi070.stdout: "network": { 2024-09-14T03:47:52.223 INFO:tasks.workunit.client.0.smithi070.stdout: "provider": null 2024-09-14T03:47:52.223 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.223 INFO:tasks.workunit.client.0.smithi070.stdout: "storage": { 2024-09-14T03:47:52.223 INFO:tasks.workunit.client.0.smithi070.stdout: "device-set": { 2024-09-14T03:47:52.224 INFO:tasks.workunit.client.0.smithi070.stdout: "count": { 2024-09-14T03:47:52.224 INFO:tasks.workunit.client.0.smithi070.stdout: "non-portable": null, 2024-09-14T03:47:52.224 INFO:tasks.workunit.client.0.smithi070.stdout: "portable": null, 2024-09-14T03:47:52.224 INFO:tasks.workunit.client.0.smithi070.stdout: "total": null 2024-09-14T03:47:52.224 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.224 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.224 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.224 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.224 INFO:tasks.workunit.client.0.smithi070.stdout: "csi": { 2024-09-14T03:47:52.224 INFO:tasks.workunit.client.0.smithi070.stdout: "version": null 2024-09-14T03:47:52.224 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.224 INFO:tasks.workunit.client.0.smithi070.stdout: "kubernetes": { 2024-09-14T03:47:52.224 INFO:tasks.workunit.client.0.smithi070.stdout: "version": null 2024-09-14T03:47:52.225 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.225 INFO:tasks.workunit.client.0.smithi070.stdout: "node": { 2024-09-14T03:47:52.225 INFO:tasks.workunit.client.0.smithi070.stdout: "count": { 2024-09-14T03:47:52.225 INFO:tasks.workunit.client.0.smithi070.stdout: "kubernetes-total": null, 2024-09-14T03:47:52.225 INFO:tasks.workunit.client.0.smithi070.stdout: "with-ceph-daemons": null, 2024-09-14T03:47:52.225 INFO:tasks.workunit.client.0.smithi070.stdout: "with-csi-cephfs-plugin": null, 2024-09-14T03:47:52.225 INFO:tasks.workunit.client.0.smithi070.stdout: "with-csi-nfs-plugin": null, 2024-09-14T03:47:52.225 INFO:tasks.workunit.client.0.smithi070.stdout: "with-csi-rbd-plugin": null 2024-09-14T03:47:52.225 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.225 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.225 INFO:tasks.workunit.client.0.smithi070.stdout: "usage": { 2024-09-14T03:47:52.225 INFO:tasks.workunit.client.0.smithi070.stdout: "storage-class": { 2024-09-14T03:47:52.225 INFO:tasks.workunit.client.0.smithi070.stdout: "count": { 2024-09-14T03:47:52.226 INFO:tasks.workunit.client.0.smithi070.stdout: "bucket": null, 2024-09-14T03:47:52.226 INFO:tasks.workunit.client.0.smithi070.stdout: "cephfs": null, 2024-09-14T03:47:52.226 INFO:tasks.workunit.client.0.smithi070.stdout: "nfs": null, 2024-09-14T03:47:52.226 INFO:tasks.workunit.client.0.smithi070.stdout: "rbd": null, 2024-09-14T03:47:52.226 INFO:tasks.workunit.client.0.smithi070.stdout: "total": null 2024-09-14T03:47:52.226 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.226 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:52.226 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.226 INFO:tasks.workunit.client.0.smithi070.stdout: "version": null 2024-09-14T03:47:52.226 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.226 INFO:tasks.workunit.client.0.smithi070.stdout: "services": {}, 2024-09-14T03:47:52.226 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_per_pg": [ 2024-09-14T03:47:52.226 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.226 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.226 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.227 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.227 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.227 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.227 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:52.227 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.227 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.227 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.227 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.227 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.227 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.227 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.661024+0000", 2024-09-14T03:47:52.227 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.882481+0000", 2024-09-14T03:47:52.227 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.882481+0000", 2024-09-14T03:47:52.227 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.882774+0000", 2024-09-14T03:47:52.227 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.661024+0000", 2024-09-14T03:47:52.227 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.228 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.228 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.228 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.228 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.661024+0000", 2024-09-14T03:47:52.228 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.661024+0000", 2024-09-14T03:47:52.228 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.661024+0000", 2024-09-14T03:47:52.228 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.228 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.228 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.228 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.661024+0000", 2024-09-14T03:47:52.228 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.661024+0000", 2024-09-14T03:47:52.228 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.228 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.228 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.229 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.229 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.229 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.229 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.229 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.229 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.229 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.229 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.229 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.229 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.229 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.3f", 2024-09-14T03:47:52.229 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.229 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.229 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.230 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:52.230 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.230 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:49:25.209573+0000", 2024-09-14T03:47:52.230 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.230 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.230 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.230 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.230 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.230 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.230 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.230 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.230 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.230 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.230 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.231 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.231 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.231 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.231 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.231 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.231 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.231 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.231 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.231 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.231 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.231 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.231 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.231 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.231 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.232 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.232 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.232 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.232 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.232 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.232 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.232 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.232 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.232 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.232 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.232 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.232 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.232 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.232 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.233 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.233 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.233 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.233 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.233 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.233 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.233 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.233 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.233 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.233 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.233 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.233 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.233 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:52.233 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.234 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.234 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.234 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.234 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.234 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.234 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:52.234 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.234 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 1, 2024-09-14T03:47:52.234 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.234 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.234 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.234 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.234 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.234 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.234 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.458995+0000", 2024-09-14T03:47:52.236 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.868152+0000", 2024-09-14T03:47:52.236 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.868152+0000", 2024-09-14T03:47:52.236 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.868415+0000", 2024-09-14T03:47:52.236 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.458995+0000", 2024-09-14T03:47:52.236 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.236 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.236 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.236 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.236 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.458995+0000", 2024-09-14T03:47:52.236 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.458995+0000", 2024-09-14T03:47:52.236 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.458995+0000", 2024-09-14T03:47:52.236 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.236 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.236 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.237 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.458995+0000", 2024-09-14T03:47:52.237 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.458995+0000", 2024-09-14T03:47:52.237 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.237 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.237 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.237 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.237 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.237 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.237 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.237 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.237 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.237 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.237 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.237 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.238 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.238 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.3e", 2024-09-14T03:47:52.238 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.238 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.238 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.238 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:52.238 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.238 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T03:56:00.270070+0000", 2024-09-14T03:47:52.238 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.238 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.238 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.238 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.238 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.238 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.238 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.239 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.239 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.239 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.239 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.239 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.239 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.239 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.239 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.239 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.239 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.239 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.239 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.239 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.239 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.240 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.240 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.240 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.240 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.240 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.240 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.240 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.240 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.240 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.240 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.240 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.240 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.240 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.240 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.241 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.241 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.241 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.241 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.241 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.241 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.241 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.241 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.241 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.241 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.241 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.241 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.241 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.241 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.241 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:52.242 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.242 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 1, 2024-09-14T03:47:52.242 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.242 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.242 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.242 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.242 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.242 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:52.242 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:52.242 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.242 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 0, 2024-09-14T03:47:52.242 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.242 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.242 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.242 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.243 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.243 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.243 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.684432+0000", 2024-09-14T03:47:52.243 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.870749+0000", 2024-09-14T03:47:52.243 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.870749+0000", 2024-09-14T03:47:52.243 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.871459+0000", 2024-09-14T03:47:52.243 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.684432+0000", 2024-09-14T03:47:52.243 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.243 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.243 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.243 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.243 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.684432+0000", 2024-09-14T03:47:52.243 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.684432+0000", 2024-09-14T03:47:52.243 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.684432+0000", 2024-09-14T03:47:52.243 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.244 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.244 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.244 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.684432+0000", 2024-09-14T03:47:52.244 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.684432+0000", 2024-09-14T03:47:52.244 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.244 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.244 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.244 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.244 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.244 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.244 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.244 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.244 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.244 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.245 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.245 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.245 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.245 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.3d", 2024-09-14T03:47:52.245 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.245 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.245 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.245 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:52.245 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.245 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:40:54.391363+0000", 2024-09-14T03:47:52.245 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.245 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.245 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.245 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.245 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.246 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.246 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.246 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.246 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.246 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.246 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.246 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.246 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.246 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.246 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.246 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.246 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.246 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.246 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.246 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.247 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.247 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.247 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.247 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.247 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.247 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.247 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.247 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.247 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.247 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.247 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.247 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.247 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.247 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.247 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.248 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.248 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.248 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.248 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.248 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.248 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.248 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.248 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.248 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.248 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.248 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.248 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.248 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.249 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:52.249 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:52.249 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.249 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 0, 2024-09-14T03:47:52.249 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.249 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.249 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.249 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.249 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.249 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.249 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:52.249 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.249 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 5, 2024-09-14T03:47:52.249 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.250 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.250 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.250 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.250 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.250 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.250 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.461823+0000", 2024-09-14T03:47:52.250 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.882250+0000", 2024-09-14T03:47:52.250 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.882250+0000", 2024-09-14T03:47:52.250 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.882503+0000", 2024-09-14T03:47:52.250 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.461823+0000", 2024-09-14T03:47:52.250 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.250 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.250 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.251 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.251 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.461823+0000", 2024-09-14T03:47:52.251 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.461823+0000", 2024-09-14T03:47:52.251 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.461823+0000", 2024-09-14T03:47:52.251 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.251 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.251 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.251 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.461823+0000", 2024-09-14T03:47:52.251 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.461823+0000", 2024-09-14T03:47:52.251 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.251 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.251 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.251 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.251 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.251 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.252 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.252 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.252 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.252 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.252 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.252 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.252 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.252 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.3c", 2024-09-14T03:47:52.252 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.252 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.252 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.252 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:52.253 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.253 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:46:29.551285+0000", 2024-09-14T03:47:52.253 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.253 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.253 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.253 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.253 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.253 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.253 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.253 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.253 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.253 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.253 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.253 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.254 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.254 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.254 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.254 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.254 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.254 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.254 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.254 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.254 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.254 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.254 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.254 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.254 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.254 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.254 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.255 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.255 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.255 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.255 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.255 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.255 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.255 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.255 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.255 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.255 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.255 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.255 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.255 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.256 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.256 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.256 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.256 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.256 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.256 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.256 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.256 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.256 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.256 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:52.256 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.257 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 5, 2024-09-14T03:47:52.257 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.257 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.257 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.257 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.257 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.257 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:52.257 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:52.257 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.257 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:52.257 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.257 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.257 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.258 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.258 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.258 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.258 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.660633+0000", 2024-09-14T03:47:52.258 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.880010+0000", 2024-09-14T03:47:52.258 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.880010+0000", 2024-09-14T03:47:52.258 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.880333+0000", 2024-09-14T03:47:52.258 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.660633+0000", 2024-09-14T03:47:52.258 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.258 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.258 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.258 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.258 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.660633+0000", 2024-09-14T03:47:52.258 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.660633+0000", 2024-09-14T03:47:52.259 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.660633+0000", 2024-09-14T03:47:52.259 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.259 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.259 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.259 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.660633+0000", 2024-09-14T03:47:52.259 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.660633+0000", 2024-09-14T03:47:52.259 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.259 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.259 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.259 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.259 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.259 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.259 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.259 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.259 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.260 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.260 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.260 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.260 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.260 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.3b", 2024-09-14T03:47:52.260 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.260 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.260 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.260 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:52.260 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.260 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:45:02.741425+0000", 2024-09-14T03:47:52.260 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.260 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.261 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.261 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.261 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.261 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.261 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.261 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.261 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.261 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.261 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.261 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.261 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.261 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.261 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.261 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.261 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.262 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.262 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.262 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.262 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.262 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.262 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.262 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.262 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.262 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.262 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.262 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.262 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.262 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.262 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.262 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.263 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.263 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.263 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.263 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.263 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.263 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.263 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.263 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.263 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.263 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.263 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.263 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.263 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.263 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.263 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.264 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.264 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:52.264 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:52.264 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.264 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:52.264 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.264 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.264 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.264 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.264 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.264 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.264 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:52.264 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.265 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 5, 2024-09-14T03:47:52.265 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.265 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.265 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.265 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.265 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.265 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.265 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.461594+0000", 2024-09-14T03:47:52.265 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.880237+0000", 2024-09-14T03:47:52.265 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.880237+0000", 2024-09-14T03:47:52.265 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.880710+0000", 2024-09-14T03:47:52.265 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.461594+0000", 2024-09-14T03:47:52.265 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.266 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.266 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.266 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.266 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.461594+0000", 2024-09-14T03:47:52.266 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.461594+0000", 2024-09-14T03:47:52.266 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.461594+0000", 2024-09-14T03:47:52.266 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.266 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.266 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.266 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.461594+0000", 2024-09-14T03:47:52.266 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.461594+0000", 2024-09-14T03:47:52.266 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.266 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.266 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.266 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.267 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.267 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.267 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.267 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.267 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.267 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.267 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.267 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.267 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.267 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.3a", 2024-09-14T03:47:52.267 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.267 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.267 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.267 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:52.268 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.268 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T03:56:49.808707+0000", 2024-09-14T03:47:52.268 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.268 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.268 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.268 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.268 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.268 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.268 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.268 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.268 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.268 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.268 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.269 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.269 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.269 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.269 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.269 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.269 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.269 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.269 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.269 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.269 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.269 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.269 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.269 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.270 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.270 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.270 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.270 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.270 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.270 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.270 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.270 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.270 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.270 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.270 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.270 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.270 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.271 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.271 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.271 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.271 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.271 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.271 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.271 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.271 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.271 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.271 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.271 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.271 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.271 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:52.271 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.272 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 5, 2024-09-14T03:47:52.272 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.272 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.272 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.272 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.272 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.272 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.272 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:52.272 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.272 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:52.272 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.272 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.272 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.272 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.273 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.273 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.273 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.660415+0000", 2024-09-14T03:47:52.273 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.878281+0000", 2024-09-14T03:47:52.273 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.878281+0000", 2024-09-14T03:47:52.273 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.878590+0000", 2024-09-14T03:47:52.273 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.660415+0000", 2024-09-14T03:47:52.273 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.273 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.273 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.273 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.273 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.660415+0000", 2024-09-14T03:47:52.273 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.660415+0000", 2024-09-14T03:47:52.274 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.660415+0000", 2024-09-14T03:47:52.274 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.274 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.274 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.274 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.660415+0000", 2024-09-14T03:47:52.274 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.660415+0000", 2024-09-14T03:47:52.274 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.274 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.274 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.274 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.274 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.274 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.274 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.275 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.275 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.275 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.275 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.275 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.275 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.275 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.39", 2024-09-14T03:47:52.275 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.275 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.275 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.275 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:52.275 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.275 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:41:57.164227+0000", 2024-09-14T03:47:52.276 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.276 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.276 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.276 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.276 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.276 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.276 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.276 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.276 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.276 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.276 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.276 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.276 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.276 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.277 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.277 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.277 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.277 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.277 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.277 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.277 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.277 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.277 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.277 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.277 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.277 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.277 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.277 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.278 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.278 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.278 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.278 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.278 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.278 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.278 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.278 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.278 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.278 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.278 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.278 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.278 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.278 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.278 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.279 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.279 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.279 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.279 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.279 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.279 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.279 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:52.279 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.279 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:52.279 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.279 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.279 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.279 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.280 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.280 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.280 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:52.280 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.280 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 3, 2024-09-14T03:47:52.280 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.280 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.280 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.280 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.280 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.280 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.280 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.857411+0000", 2024-09-14T03:47:52.280 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.876727+0000", 2024-09-14T03:47:52.280 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.876727+0000", 2024-09-14T03:47:52.281 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.877057+0000", 2024-09-14T03:47:52.281 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.857411+0000", 2024-09-14T03:47:52.281 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.281 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.281 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.281 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.281 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.857411+0000", 2024-09-14T03:47:52.281 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.857411+0000", 2024-09-14T03:47:52.281 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.857411+0000", 2024-09-14T03:47:52.281 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.281 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.281 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.281 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.857411+0000", 2024-09-14T03:47:52.281 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.857411+0000", 2024-09-14T03:47:52.282 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.282 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.282 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.282 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.282 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.282 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.282 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.282 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.282 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.282 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.282 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.282 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.282 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.283 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.38", 2024-09-14T03:47:52.283 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.283 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.283 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.283 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:52.283 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.283 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:31:24.716316+0000", 2024-09-14T03:47:52.283 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.283 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.283 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.283 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.283 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.283 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.283 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.284 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.284 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.284 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.284 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.284 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.284 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.284 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.284 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.284 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.284 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.284 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.284 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.284 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.285 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.285 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.285 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.285 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.285 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.285 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.285 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.285 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.285 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.285 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.285 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.285 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.285 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.286 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.286 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.286 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.286 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.286 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.286 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.286 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.286 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.286 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.286 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.286 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.286 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.286 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.286 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.287 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.287 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.287 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:52.287 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.287 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 3, 2024-09-14T03:47:52.287 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.287 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.287 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.287 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.287 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.287 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.287 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.287 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.288 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:52.288 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.288 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.288 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.288 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.288 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.288 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.288 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.661211+0000", 2024-09-14T03:47:52.288 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.884136+0000", 2024-09-14T03:47:52.288 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.884136+0000", 2024-09-14T03:47:52.288 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.884865+0000", 2024-09-14T03:47:52.288 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.661211+0000", 2024-09-14T03:47:52.288 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.289 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.289 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.289 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.289 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.661211+0000", 2024-09-14T03:47:52.289 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.661211+0000", 2024-09-14T03:47:52.289 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.661211+0000", 2024-09-14T03:47:52.289 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.289 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.289 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.289 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.661211+0000", 2024-09-14T03:47:52.289 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.661211+0000", 2024-09-14T03:47:52.289 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.289 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.289 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.290 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.290 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.290 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.290 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.290 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.290 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.290 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.290 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.290 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.290 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.290 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.37", 2024-09-14T03:47:52.290 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.290 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.291 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.291 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:52.291 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.291 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:14:59.903382+0000", 2024-09-14T03:47:52.291 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.291 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.291 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.291 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.291 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.291 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.291 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.291 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.291 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.291 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.292 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.292 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.292 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.292 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.292 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.292 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.292 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.292 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.292 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.292 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.292 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.292 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.292 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.293 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.293 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.293 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.293 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.293 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.293 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.293 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.293 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.293 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.293 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.293 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.293 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.293 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.293 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.294 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.294 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.294 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.294 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.294 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.294 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.294 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.294 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.294 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.294 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.294 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.294 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.294 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.294 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.294 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:52.295 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.295 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.295 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.295 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.295 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.295 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:52.295 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:52.295 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.295 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 6, 2024-09-14T03:47:52.295 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.295 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.295 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.295 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.295 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.295 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.296 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.544938+0000", 2024-09-14T03:47:52.296 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:26.429658+0000", 2024-09-14T03:47:52.296 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:26.429658+0000", 2024-09-14T03:47:52.296 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:26.430052+0000", 2024-09-14T03:47:52.296 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.544938+0000", 2024-09-14T03:47:52.296 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.296 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.296 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.296 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 60, 2024-09-14T03:47:52.296 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.544938+0000", 2024-09-14T03:47:52.296 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.544938+0000", 2024-09-14T03:47:52.296 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.544938+0000", 2024-09-14T03:47:52.296 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.296 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.297 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.297 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.544938+0000", 2024-09-14T03:47:52.297 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.544938+0000", 2024-09-14T03:47:52.297 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.297 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.297 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.297 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.297 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 59, 2024-09-14T03:47:52.297 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.297 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.297 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.297 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.297 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.297 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.298 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.298 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.298 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.36", 2024-09-14T03:47:52.298 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.298 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.298 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.298 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 30, 2024-09-14T03:47:52.298 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.298 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T03:59:34.312039+0000", 2024-09-14T03:47:52.298 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.298 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.298 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.298 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.299 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.299 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.299 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.299 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.299 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.299 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.299 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.299 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.299 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.299 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.299 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.299 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.299 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.299 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.300 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.300 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.300 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.300 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.300 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.300 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.300 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.300 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.300 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.300 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.300 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.300 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.301 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.301 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.301 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.301 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.301 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.301 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.301 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.301 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.301 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.301 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.301 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.301 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.301 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.301 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.302 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.302 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.302 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.302 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.302 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:52.302 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:52.302 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.302 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 6, 2024-09-14T03:47:52.302 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.302 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.302 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.302 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.302 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.303 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.303 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.303 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.303 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 6, 2024-09-14T03:47:52.303 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.303 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.303 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.303 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.303 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.303 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.303 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.544941+0000", 2024-09-14T03:47:52.303 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.874687+0000", 2024-09-14T03:47:52.303 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.874687+0000", 2024-09-14T03:47:52.303 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.875017+0000", 2024-09-14T03:47:52.304 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.544941+0000", 2024-09-14T03:47:52.304 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.304 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.304 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.304 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.304 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.544941+0000", 2024-09-14T03:47:52.304 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.544941+0000", 2024-09-14T03:47:52.304 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.544941+0000", 2024-09-14T03:47:52.304 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.304 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.304 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.304 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.544941+0000", 2024-09-14T03:47:52.304 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.544941+0000", 2024-09-14T03:47:52.305 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.305 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.305 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.305 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.305 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.305 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.305 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.305 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.305 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.305 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.305 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.305 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.305 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.305 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.35", 2024-09-14T03:47:52.306 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.306 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.306 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.306 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:52.306 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.306 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:29:09.035342+0000", 2024-09-14T03:47:52.306 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.306 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.306 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.306 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.306 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.306 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.306 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.306 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.307 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.307 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.307 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.307 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.307 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.307 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.307 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.307 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.307 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.307 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.307 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.307 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.307 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.307 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.308 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.308 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.308 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.308 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.308 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.308 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.308 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.308 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.308 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.308 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.308 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.308 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.308 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.309 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.309 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.309 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.309 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.309 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.309 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.309 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.309 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.309 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.309 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.309 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.309 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.309 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.309 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.310 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.310 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.310 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 6, 2024-09-14T03:47:52.310 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.310 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.310 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.310 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.310 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.310 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.310 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.310 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.310 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 4, 2024-09-14T03:47:52.310 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.311 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.311 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.311 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.311 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.311 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.311 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.458577+0000", 2024-09-14T03:47:52.311 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.882770+0000", 2024-09-14T03:47:52.311 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.882770+0000", 2024-09-14T03:47:52.311 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.883313+0000", 2024-09-14T03:47:52.311 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.458577+0000", 2024-09-14T03:47:52.311 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.311 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.311 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.311 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.312 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.458577+0000", 2024-09-14T03:47:52.312 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.458577+0000", 2024-09-14T03:47:52.312 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.458577+0000", 2024-09-14T03:47:52.312 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.312 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.312 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.312 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.458577+0000", 2024-09-14T03:47:52.312 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.458577+0000", 2024-09-14T03:47:52.312 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.312 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.312 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.312 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.313 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.313 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.313 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.313 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.313 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.313 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.313 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.313 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.313 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.313 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.34", 2024-09-14T03:47:52.313 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.313 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.313 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.314 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:52.314 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.314 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T04:57:19.402319+0000", 2024-09-14T03:47:52.314 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.314 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.314 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.314 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.314 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.314 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.314 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.314 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.314 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.314 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.314 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.315 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.315 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.315 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.315 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.315 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.315 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.315 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.315 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.315 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.315 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.315 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.315 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.315 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.316 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.316 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.316 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.316 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.316 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.316 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.316 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.316 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.316 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.316 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.316 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.316 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.316 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.317 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.317 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.317 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.317 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.317 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.317 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.317 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.317 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.317 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.317 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.317 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.317 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.317 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.318 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.318 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 4, 2024-09-14T03:47:52.318 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.318 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.318 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.318 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.318 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.318 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.318 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:52.318 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.318 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 3, 2024-09-14T03:47:52.318 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.318 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.319 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.319 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.319 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.319 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.319 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.857631+0000", 2024-09-14T03:47:52.319 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.877861+0000", 2024-09-14T03:47:52.319 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.877861+0000", 2024-09-14T03:47:52.319 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.878245+0000", 2024-09-14T03:47:52.319 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.857631+0000", 2024-09-14T03:47:52.319 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.319 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.319 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.319 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.319 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.857631+0000", 2024-09-14T03:47:52.320 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.857631+0000", 2024-09-14T03:47:52.320 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.857631+0000", 2024-09-14T03:47:52.320 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.320 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.320 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.320 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.857631+0000", 2024-09-14T03:47:52.320 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.857631+0000", 2024-09-14T03:47:52.320 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.320 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.320 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.320 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.320 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.320 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.321 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.321 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.321 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.321 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.321 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.321 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.321 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.321 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.33", 2024-09-14T03:47:52.321 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.321 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.321 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.321 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:52.321 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.322 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:08:29.902090+0000", 2024-09-14T03:47:52.322 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.322 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.322 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.322 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.322 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.322 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.322 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.322 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.322 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.322 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.322 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.322 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.322 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.323 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.323 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.323 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.323 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.323 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.323 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.323 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.323 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.323 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.323 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.323 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.323 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.323 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.323 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.324 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.324 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.324 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.324 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.324 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.324 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.324 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.324 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.324 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.324 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.324 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.324 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.324 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.325 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.325 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.325 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.325 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.325 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.325 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.325 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.325 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.325 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.325 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:52.325 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.325 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 3, 2024-09-14T03:47:52.325 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.326 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.326 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.326 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.326 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.326 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.326 INFO:tasks.workunit.client.0.smithi070.stdout: 5 2024-09-14T03:47:52.326 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.326 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 0, 2024-09-14T03:47:52.326 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.326 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.326 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.326 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.326 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.326 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.327 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.684034+0000", 2024-09-14T03:47:52.327 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.876797+0000", 2024-09-14T03:47:52.327 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.876797+0000", 2024-09-14T03:47:52.327 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.877391+0000", 2024-09-14T03:47:52.327 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.684034+0000", 2024-09-14T03:47:52.327 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.327 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.327 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.327 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.327 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.684034+0000", 2024-09-14T03:47:52.327 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.684034+0000", 2024-09-14T03:47:52.327 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.684034+0000", 2024-09-14T03:47:52.327 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.327 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.328 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.328 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.684034+0000", 2024-09-14T03:47:52.328 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.684034+0000", 2024-09-14T03:47:52.328 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.328 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.328 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.328 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.328 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.328 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.328 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.328 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.328 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.328 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.329 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.329 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.329 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.329 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.32", 2024-09-14T03:47:52.329 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.329 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.329 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.329 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:52.329 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.329 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:06:02.891762+0000", 2024-09-14T03:47:52.329 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.329 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.329 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.329 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.330 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.330 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.330 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.330 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.330 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.330 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.330 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.330 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.330 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.330 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.330 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.330 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.330 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.330 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.330 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.331 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.331 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.331 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.331 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.331 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.331 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.331 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.331 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.331 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.331 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.331 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.331 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.331 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.331 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.332 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.332 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.332 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.332 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.332 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.332 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.332 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.332 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.332 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.332 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.332 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.332 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.332 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.332 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.333 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.333 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.333 INFO:tasks.workunit.client.0.smithi070.stdout: 5 2024-09-14T03:47:52.333 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.333 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 0, 2024-09-14T03:47:52.333 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.333 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.333 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.333 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.333 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.333 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.333 INFO:tasks.workunit.client.0.smithi070.stdout: 5 2024-09-14T03:47:52.333 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.333 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 1, 2024-09-14T03:47:52.334 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.334 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.334 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.334 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.334 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.334 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.334 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.458588+0000", 2024-09-14T03:47:52.334 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.876091+0000", 2024-09-14T03:47:52.334 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.876091+0000", 2024-09-14T03:47:52.334 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.876700+0000", 2024-09-14T03:47:52.334 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.458588+0000", 2024-09-14T03:47:52.334 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.334 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.334 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.335 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.335 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.458588+0000", 2024-09-14T03:47:52.335 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.458588+0000", 2024-09-14T03:47:52.335 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.458588+0000", 2024-09-14T03:47:52.335 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.335 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.335 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.335 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.458588+0000", 2024-09-14T03:47:52.335 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.458588+0000", 2024-09-14T03:47:52.335 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.335 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.335 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.335 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.335 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.335 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.336 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.336 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.336 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.336 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.336 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.336 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.336 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.336 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.31", 2024-09-14T03:47:52.336 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.336 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.336 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.336 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:52.336 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.336 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:25:41.750127+0000", 2024-09-14T03:47:52.337 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.337 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.337 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.337 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.337 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.337 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.337 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.337 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.337 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.337 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.337 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.337 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.337 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.338 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.338 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.338 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.338 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.338 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.338 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.338 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.338 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.338 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.338 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.338 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.338 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.338 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.338 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.338 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.338 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.339 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.339 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.339 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.339 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.339 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.339 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.339 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.339 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.339 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.339 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.339 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.339 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.339 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.339 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.340 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.340 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.340 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.340 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.340 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.340 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.340 INFO:tasks.workunit.client.0.smithi070.stdout: 5 2024-09-14T03:47:52.340 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.340 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 1, 2024-09-14T03:47:52.340 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.340 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.340 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.340 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.341 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.341 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.341 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:52.341 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.341 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 5, 2024-09-14T03:47:52.341 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.341 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.341 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.341 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.341 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.341 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.341 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.461481+0000", 2024-09-14T03:47:52.341 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.876793+0000", 2024-09-14T03:47:52.342 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.876793+0000", 2024-09-14T03:47:52.342 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.877120+0000", 2024-09-14T03:47:52.342 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.461481+0000", 2024-09-14T03:47:52.342 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.342 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.342 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.342 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.342 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.461481+0000", 2024-09-14T03:47:52.342 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.461481+0000", 2024-09-14T03:47:52.342 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.461481+0000", 2024-09-14T03:47:52.342 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.342 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.342 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.342 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.461481+0000", 2024-09-14T03:47:52.343 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.461481+0000", 2024-09-14T03:47:52.343 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.343 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.343 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.343 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.343 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.343 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.343 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.343 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.343 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.343 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.343 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.343 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.344 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.344 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.30", 2024-09-14T03:47:52.344 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.344 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.344 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.344 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:52.344 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.344 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T05:23:56.450728+0000", 2024-09-14T03:47:52.344 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.344 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.344 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.344 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.344 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.345 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.345 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.345 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.345 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.345 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.345 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.345 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.345 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.345 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.345 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.345 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.345 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.345 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.345 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.346 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.346 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.346 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.346 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.346 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.346 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.346 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.346 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.346 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.346 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.346 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.346 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.346 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.346 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.347 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.347 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.347 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.347 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.347 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.347 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.347 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.347 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.347 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.347 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.347 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.347 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.347 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.347 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.348 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.348 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.348 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:52.348 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.348 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 5, 2024-09-14T03:47:52.348 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.348 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.348 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.348 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.348 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.348 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.348 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:52.348 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.348 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 5, 2024-09-14T03:47:52.349 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.349 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.349 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.349 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.349 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.349 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.349 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.462251+0000", 2024-09-14T03:47:52.349 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.884857+0000", 2024-09-14T03:47:52.349 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.884857+0000", 2024-09-14T03:47:52.349 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.885108+0000", 2024-09-14T03:47:52.349 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.462251+0000", 2024-09-14T03:47:52.349 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.349 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.350 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.350 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.350 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.462251+0000", 2024-09-14T03:47:52.350 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.462251+0000", 2024-09-14T03:47:52.350 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.462251+0000", 2024-09-14T03:47:52.350 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.350 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.350 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.350 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.462251+0000", 2024-09-14T03:47:52.350 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.462251+0000", 2024-09-14T03:47:52.350 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.350 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.350 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.350 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.351 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.351 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.351 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.351 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.351 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.351 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.351 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.351 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.351 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.351 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.2f", 2024-09-14T03:47:52.351 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.351 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.351 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.351 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:52.351 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.352 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T07:09:31.448617+0000", 2024-09-14T03:47:52.352 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.352 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.352 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.352 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.352 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.352 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.352 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.352 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.352 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.352 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.352 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.352 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.352 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.353 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.353 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.353 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.353 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.353 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.353 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.353 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.353 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.353 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.353 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.353 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.353 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.353 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.354 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.354 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.354 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.354 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.354 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.354 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.354 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.354 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.354 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.354 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.354 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.354 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.354 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.354 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.355 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.355 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.355 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.355 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.355 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.355 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.355 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.355 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.355 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.355 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:52.355 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.355 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 5, 2024-09-14T03:47:52.355 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.355 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.356 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.356 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.356 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:52.356 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.356 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:52.356 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.356 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 2, 2024-09-14T03:47:52.356 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.356 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.356 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.356 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.356 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.356 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.357 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:15.135790+0000", 2024-09-14T03:47:52.357 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.876227+0000", 2024-09-14T03:47:52.357 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.876227+0000", 2024-09-14T03:47:52.357 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.876503+0000", 2024-09-14T03:47:52.357 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:15.135790+0000", 2024-09-14T03:47:52.357 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.357 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.357 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.357 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.357 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:15.135790+0000", 2024-09-14T03:47:52.357 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:15.135790+0000", 2024-09-14T03:47:52.357 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:15.135790+0000", 2024-09-14T03:47:52.357 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.357 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.357 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.358 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:15.135790+0000", 2024-09-14T03:47:52.358 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:15.135790+0000", 2024-09-14T03:47:52.358 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.358 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.358 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.358 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.358 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.358 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.358 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.358 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.358 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.358 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.358 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.358 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.359 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.359 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.2e", 2024-09-14T03:47:52.359 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.359 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.359 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.359 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:52.359 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.359 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T04:44:50.132152+0000", 2024-09-14T03:47:52.359 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.359 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.359 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.359 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.359 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.359 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.359 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.360 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.360 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.360 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.360 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.360 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.360 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.360 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.360 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.360 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.360 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.360 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.360 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.360 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.360 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.361 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.361 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.361 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.361 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.361 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.361 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.361 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.361 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.361 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.361 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.361 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.361 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.361 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.361 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.362 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.362 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.362 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.362 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.362 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.362 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.362 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.362 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.362 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.362 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.362 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.362 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.362 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:52.362 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.362 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:52.363 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.363 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 2, 2024-09-14T03:47:52.363 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.363 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.363 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.363 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.363 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.363 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:52.363 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:52.363 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.363 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:52.363 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.363 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.363 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.363 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.364 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.364 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.364 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.661079+0000", 2024-09-14T03:47:52.364 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.883440+0000", 2024-09-14T03:47:52.364 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.883440+0000", 2024-09-14T03:47:52.364 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.883708+0000", 2024-09-14T03:47:52.364 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.661079+0000", 2024-09-14T03:47:52.364 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.364 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.364 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.364 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.364 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.661079+0000", 2024-09-14T03:47:52.364 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.661079+0000", 2024-09-14T03:47:52.364 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.661079+0000", 2024-09-14T03:47:52.365 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.365 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.365 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.365 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.661079+0000", 2024-09-14T03:47:52.365 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.661079+0000", 2024-09-14T03:47:52.365 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.365 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.365 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.365 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.365 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.365 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.365 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.365 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.366 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.366 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.366 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.366 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.366 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.366 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.2d", 2024-09-14T03:47:52.366 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.366 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.366 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.366 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:52.366 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.366 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:10:34.658062+0000", 2024-09-14T03:47:52.366 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.366 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.367 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.367 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.367 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.367 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.367 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.367 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.367 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.367 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.367 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.367 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.367 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.367 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.367 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.368 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.368 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.368 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.368 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.368 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.368 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.368 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.368 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.368 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.368 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.368 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.368 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.368 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.369 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.369 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.369 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.369 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.369 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.369 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.369 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.369 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.369 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.369 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.369 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.369 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.369 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.369 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.370 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.370 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.370 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.370 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.370 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.370 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.370 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:52.370 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:52.370 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.370 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:52.370 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.370 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.370 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.370 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.371 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.371 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.371 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.371 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.371 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 6, 2024-09-14T03:47:52.371 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.371 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.371 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.371 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.371 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.371 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.371 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.544854+0000", 2024-09-14T03:47:52.371 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.888191+0000", 2024-09-14T03:47:52.371 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.888191+0000", 2024-09-14T03:47:52.372 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.888565+0000", 2024-09-14T03:47:52.372 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.544854+0000", 2024-09-14T03:47:52.372 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.372 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.372 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.372 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.372 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.544854+0000", 2024-09-14T03:47:52.372 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.544854+0000", 2024-09-14T03:47:52.372 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.544854+0000", 2024-09-14T03:47:52.372 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.372 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.372 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.372 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.544854+0000", 2024-09-14T03:47:52.373 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.544854+0000", 2024-09-14T03:47:52.373 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.373 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.373 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.373 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.373 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.373 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.373 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.373 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.373 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.373 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.373 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.373 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.373 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.374 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.2c", 2024-09-14T03:47:52.374 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.374 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.374 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.374 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:52.374 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.374 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T07:52:23.782664+0000", 2024-09-14T03:47:52.374 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.374 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.374 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.374 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.374 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.374 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.374 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.375 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.375 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.375 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.375 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.375 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.375 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.375 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.375 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.375 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.375 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.375 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.375 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.375 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.375 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.376 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.376 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.376 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.376 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.376 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.376 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.376 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.376 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.376 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.376 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.376 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.376 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.376 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.376 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.377 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.377 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.377 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.377 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.377 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.377 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.377 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.377 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.377 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.377 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.377 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.377 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.377 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.378 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.378 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.378 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.378 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 6, 2024-09-14T03:47:52.378 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.378 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.378 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.378 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.378 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.378 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.378 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:52.378 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.378 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 0, 2024-09-14T03:47:52.378 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.378 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.379 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.379 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.379 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.379 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.379 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.684144+0000", 2024-09-14T03:47:52.379 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.874384+0000", 2024-09-14T03:47:52.379 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.874384+0000", 2024-09-14T03:47:52.379 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.874642+0000", 2024-09-14T03:47:52.379 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.684144+0000", 2024-09-14T03:47:52.379 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.379 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.379 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.379 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.379 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.684144+0000", 2024-09-14T03:47:52.379 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.684144+0000", 2024-09-14T03:47:52.380 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.684144+0000", 2024-09-14T03:47:52.380 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.380 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.380 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.380 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.684144+0000", 2024-09-14T03:47:52.380 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.684144+0000", 2024-09-14T03:47:52.380 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.380 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.380 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.380 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.380 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.380 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.380 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.380 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.381 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.381 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.381 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.381 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.381 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.381 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.2b", 2024-09-14T03:47:52.381 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.381 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.381 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.381 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:52.381 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.381 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T15:35:05.535173+0000", 2024-09-14T03:47:52.381 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.382 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.382 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.382 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.382 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.382 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.382 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.382 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.382 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.382 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.382 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.382 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.382 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.382 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.383 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.383 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.383 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.383 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.383 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.383 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.383 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.383 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.383 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.383 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.383 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.383 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.383 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.383 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.383 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.384 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.384 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.384 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.384 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.384 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.384 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.384 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.384 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.384 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.384 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.384 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.384 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.384 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.385 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.385 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.385 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.385 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.385 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.385 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.385 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.385 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:52.385 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.385 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 0, 2024-09-14T03:47:52.385 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.385 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.385 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.385 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.386 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.386 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.386 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:52.386 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.386 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 5, 2024-09-14T03:47:52.386 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.386 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.386 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.386 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.386 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.386 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.386 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.461689+0000", 2024-09-14T03:47:52.386 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.886594+0000", 2024-09-14T03:47:52.386 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.886594+0000", 2024-09-14T03:47:52.387 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.886893+0000", 2024-09-14T03:47:52.387 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.461689+0000", 2024-09-14T03:47:52.387 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.387 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.387 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.387 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.387 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.461689+0000", 2024-09-14T03:47:52.387 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.461689+0000", 2024-09-14T03:47:52.387 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.461689+0000", 2024-09-14T03:47:52.387 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.387 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.387 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.387 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.461689+0000", 2024-09-14T03:47:52.388 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.461689+0000", 2024-09-14T03:47:52.388 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.388 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.388 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.388 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.388 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.388 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.388 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.388 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.388 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.388 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.388 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.388 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.388 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.389 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.2a", 2024-09-14T03:47:52.389 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.389 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.389 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.389 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:52.389 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.389 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:07:37.154358+0000", 2024-09-14T03:47:52.389 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.389 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.389 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.389 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.389 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.389 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.390 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.390 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.390 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.390 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.390 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.390 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.390 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.390 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.390 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.390 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.390 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.390 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.390 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.390 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.390 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.391 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.391 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.391 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.391 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.391 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.391 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.391 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.391 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.391 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.391 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.391 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.391 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.391 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.391 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.391 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.392 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.392 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.392 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.392 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.392 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.392 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.392 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.392 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.392 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.392 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.392 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.392 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.392 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.392 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:52.392 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.393 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 5, 2024-09-14T03:47:52.393 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.393 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.393 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.393 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.393 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.393 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.393 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:52.393 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.393 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 3, 2024-09-14T03:47:52.393 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.393 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.393 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.393 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.394 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.394 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.394 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.857420+0000", 2024-09-14T03:47:52.394 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.868442+0000", 2024-09-14T03:47:52.394 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.868442+0000", 2024-09-14T03:47:52.394 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.868698+0000", 2024-09-14T03:47:52.394 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.857420+0000", 2024-09-14T03:47:52.394 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.394 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.394 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.394 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.394 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.857420+0000", 2024-09-14T03:47:52.394 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.857420+0000", 2024-09-14T03:47:52.394 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.857420+0000", 2024-09-14T03:47:52.395 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.395 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.395 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.395 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.857420+0000", 2024-09-14T03:47:52.395 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.857420+0000", 2024-09-14T03:47:52.395 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.395 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.395 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.395 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.395 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.395 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.395 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.395 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.395 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.396 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.396 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.396 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.396 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.396 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.29", 2024-09-14T03:47:52.396 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.396 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.396 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.396 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:52.396 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.396 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:41:18.296424+0000", 2024-09-14T03:47:52.396 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.396 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.396 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.396 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.397 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.397 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.397 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.397 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.397 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.397 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.397 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.397 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.397 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.397 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.397 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.397 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.397 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.397 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.398 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.398 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.398 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.398 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.398 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.398 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.398 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.398 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.398 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.398 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.398 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.398 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.398 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.398 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.399 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.399 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.399 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.399 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.399 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.399 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.399 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.399 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.399 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.399 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.399 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.399 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.399 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.399 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.399 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.400 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.400 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.400 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:52.400 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.400 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 3, 2024-09-14T03:47:52.400 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.400 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.400 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.400 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.400 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.400 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.400 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:52.400 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.401 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 5, 2024-09-14T03:47:52.401 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.401 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.401 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.401 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.401 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.401 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.401 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.461524+0000", 2024-09-14T03:47:52.401 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.882443+0000", 2024-09-14T03:47:52.401 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.882443+0000", 2024-09-14T03:47:52.401 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.882708+0000", 2024-09-14T03:47:52.401 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.461524+0000", 2024-09-14T03:47:52.401 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.401 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.402 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.402 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.402 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.461524+0000", 2024-09-14T03:47:52.402 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.461524+0000", 2024-09-14T03:47:52.402 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.461524+0000", 2024-09-14T03:47:52.402 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.402 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.402 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.402 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.461524+0000", 2024-09-14T03:47:52.402 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.461524+0000", 2024-09-14T03:47:52.402 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.402 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.402 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.402 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.403 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.403 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.403 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.403 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.403 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.403 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.403 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.403 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.403 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.403 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.28", 2024-09-14T03:47:52.403 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.403 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.403 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.403 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:52.404 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.404 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:46:48.306263+0000", 2024-09-14T03:47:52.404 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.404 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.404 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.404 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.404 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.404 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.404 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.404 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.404 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.404 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.404 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.404 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.405 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.405 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.405 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.405 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.405 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.405 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.405 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.405 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.405 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.405 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.405 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.405 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.405 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.405 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.406 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.406 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.406 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.406 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.406 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.406 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.406 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.406 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.406 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.406 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.406 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.406 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.406 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.406 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.406 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.407 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.407 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.407 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.407 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.407 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.407 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.407 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.407 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.407 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:52.407 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.407 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 5, 2024-09-14T03:47:52.407 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.407 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.407 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.408 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.408 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.408 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.408 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:52.408 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.408 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 3, 2024-09-14T03:47:52.408 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.408 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.408 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.408 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.408 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.408 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.408 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.857790+0000", 2024-09-14T03:47:52.408 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.884304+0000", 2024-09-14T03:47:52.409 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.884304+0000", 2024-09-14T03:47:52.409 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.884940+0000", 2024-09-14T03:47:52.409 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.857790+0000", 2024-09-14T03:47:52.409 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.409 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.409 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.409 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.409 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.857790+0000", 2024-09-14T03:47:52.409 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.857790+0000", 2024-09-14T03:47:52.409 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.857790+0000", 2024-09-14T03:47:52.409 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.409 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.409 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.409 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.857790+0000", 2024-09-14T03:47:52.410 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.857790+0000", 2024-09-14T03:47:52.410 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.410 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.410 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.410 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.410 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.410 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.410 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.410 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.410 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.410 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.410 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.410 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.410 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.411 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.27", 2024-09-14T03:47:52.411 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.411 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.411 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.411 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:52.411 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.411 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:01:49.061637+0000", 2024-09-14T03:47:52.411 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.411 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.411 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.411 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.411 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.411 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.411 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.411 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.411 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.412 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.412 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.412 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.412 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.412 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.412 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.412 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.412 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.412 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.412 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.412 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.412 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.412 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.412 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.413 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.413 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.413 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.413 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.413 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.413 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.413 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.413 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.413 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.413 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.413 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.413 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.413 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.413 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.414 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.414 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.414 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.414 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.414 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.414 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.414 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.414 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.414 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.414 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.414 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.414 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.414 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:52.414 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.414 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 3, 2024-09-14T03:47:52.415 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.415 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.415 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.415 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.415 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.415 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.415 INFO:tasks.workunit.client.0.smithi070.stdout: 3 2024-09-14T03:47:52.415 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.415 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 6, 2024-09-14T03:47:52.415 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.415 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.415 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.415 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.415 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.415 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.416 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.544999+0000", 2024-09-14T03:47:52.416 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:26.430426+0000", 2024-09-14T03:47:52.416 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:26.430426+0000", 2024-09-14T03:47:52.416 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:26.430891+0000", 2024-09-14T03:47:52.416 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.544999+0000", 2024-09-14T03:47:52.416 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.416 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.416 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.416 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 60, 2024-09-14T03:47:52.416 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.544999+0000", 2024-09-14T03:47:52.416 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.544999+0000", 2024-09-14T03:47:52.416 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.544999+0000", 2024-09-14T03:47:52.416 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.416 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.417 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.417 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.544999+0000", 2024-09-14T03:47:52.417 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.544999+0000", 2024-09-14T03:47:52.417 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.417 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.417 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.417 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.417 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 59, 2024-09-14T03:47:52.417 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.417 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.417 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.417 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.417 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.417 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.418 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.418 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.418 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.26", 2024-09-14T03:47:52.418 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.418 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.418 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.418 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 30, 2024-09-14T03:47:52.418 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.418 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T05:00:53.758328+0000", 2024-09-14T03:47:52.418 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.418 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.418 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.418 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.418 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.419 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.419 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.419 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.419 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.419 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.419 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.419 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.419 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.419 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.419 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.419 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.419 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.419 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.419 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.420 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.420 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.420 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.420 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.420 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.420 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.420 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.420 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.420 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.420 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.420 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.420 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.420 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.420 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.421 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.421 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.421 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.421 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.421 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.421 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.421 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.421 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.421 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.421 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.421 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.421 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.421 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.422 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.422 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.422 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.422 INFO:tasks.workunit.client.0.smithi070.stdout: 3 2024-09-14T03:47:52.422 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.422 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 6, 2024-09-14T03:47:52.422 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.422 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.422 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.422 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.422 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.422 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.423 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:52.423 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.423 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 1, 2024-09-14T03:47:52.423 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.423 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.423 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.423 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.423 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.423 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.423 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.458426+0000", 2024-09-14T03:47:52.423 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.874297+0000", 2024-09-14T03:47:52.423 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.874297+0000", 2024-09-14T03:47:52.423 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.874564+0000", 2024-09-14T03:47:52.424 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.458426+0000", 2024-09-14T03:47:52.424 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.424 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.424 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.424 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.424 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.458426+0000", 2024-09-14T03:47:52.424 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.458426+0000", 2024-09-14T03:47:52.424 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.458426+0000", 2024-09-14T03:47:52.424 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.424 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.424 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.424 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.458426+0000", 2024-09-14T03:47:52.424 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.458426+0000", 2024-09-14T03:47:52.425 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.425 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.425 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.425 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.425 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.425 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.425 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.425 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.425 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.425 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.425 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.425 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.425 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.425 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.25", 2024-09-14T03:47:52.426 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.426 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.426 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.426 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:52.426 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.426 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:28:07.027610+0000", 2024-09-14T03:47:52.426 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.426 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.426 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.426 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.426 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.426 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.426 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.427 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.427 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.427 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.427 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.427 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.427 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.427 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.427 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.427 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.427 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.427 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.427 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.427 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.427 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.428 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.428 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.428 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.428 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.428 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.428 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.428 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.428 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.428 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.428 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.428 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.428 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.428 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.428 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.428 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.429 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.429 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.429 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.429 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.429 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.429 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.429 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.429 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.429 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.429 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.429 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.429 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.429 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.429 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:52.430 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.430 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 1, 2024-09-14T03:47:52.430 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.430 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.430 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.430 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.430 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.430 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.430 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.430 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.430 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 3, 2024-09-14T03:47:52.430 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.430 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.430 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.431 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.431 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.431 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.431 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.857969+0000", 2024-09-14T03:47:52.431 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.884218+0000", 2024-09-14T03:47:52.431 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.884218+0000", 2024-09-14T03:47:52.431 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.884486+0000", 2024-09-14T03:47:52.431 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.857969+0000", 2024-09-14T03:47:52.431 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.431 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.431 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.431 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.431 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.857969+0000", 2024-09-14T03:47:52.431 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.857969+0000", 2024-09-14T03:47:52.432 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.857969+0000", 2024-09-14T03:47:52.432 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.432 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.432 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.432 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.857969+0000", 2024-09-14T03:47:52.432 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.857969+0000", 2024-09-14T03:47:52.432 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.432 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.432 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.432 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.432 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.432 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.432 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.433 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.433 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.433 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.433 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.433 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.433 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.433 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.24", 2024-09-14T03:47:52.433 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.433 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.433 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.433 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:52.433 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.433 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T07:59:18.153239+0000", 2024-09-14T03:47:52.434 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.434 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.434 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.434 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.434 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.434 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.434 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.434 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.434 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.434 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.434 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.434 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.434 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.434 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.435 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.435 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.435 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.435 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.435 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.435 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.435 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.435 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.435 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.435 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.435 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.435 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.435 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.435 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.435 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.436 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.436 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.436 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.436 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.436 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.436 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.436 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.436 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.436 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.436 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.436 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.436 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.436 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.436 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.437 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.437 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.437 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.437 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.437 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.437 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.437 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.437 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.437 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 3, 2024-09-14T03:47:52.437 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.437 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.437 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.437 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.438 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.438 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.438 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:52.438 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.438 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 0, 2024-09-14T03:47:52.438 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.438 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.438 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.438 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.438 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.438 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.438 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.684242+0000", 2024-09-14T03:47:52.439 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.878504+0000", 2024-09-14T03:47:52.439 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.878504+0000", 2024-09-14T03:47:52.440 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.878737+0000", 2024-09-14T03:47:52.440 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.684242+0000", 2024-09-14T03:47:52.440 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.440 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.440 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.440 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.440 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.684242+0000", 2024-09-14T03:47:52.440 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.684242+0000", 2024-09-14T03:47:52.440 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.684242+0000", 2024-09-14T03:47:52.440 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.440 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.440 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.441 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.684242+0000", 2024-09-14T03:47:52.441 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.684242+0000", 2024-09-14T03:47:52.441 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.441 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.441 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.441 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.441 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.441 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.441 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.441 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.441 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.441 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.441 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.442 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.442 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.442 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.23", 2024-09-14T03:47:52.442 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.442 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.442 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.442 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:52.442 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.442 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T04:41:35.420957+0000", 2024-09-14T03:47:52.442 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.442 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.442 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.442 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.442 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.442 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.443 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.443 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.443 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.443 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.443 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.443 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.443 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.443 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.443 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.443 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.443 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.443 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.443 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.443 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.444 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.444 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.444 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.444 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.444 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.444 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.444 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.444 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.444 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.444 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.444 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.444 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.444 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.444 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.445 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.445 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.445 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.445 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.445 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.445 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.445 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.445 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.445 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.445 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.445 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.445 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.445 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.446 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.446 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.446 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:52.446 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.446 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 0, 2024-09-14T03:47:52.446 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.446 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.446 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.446 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.446 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.446 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.446 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.446 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.446 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 5, 2024-09-14T03:47:52.447 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.447 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.447 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.447 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.447 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.447 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.447 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.461691+0000", 2024-09-14T03:47:52.447 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.879855+0000", 2024-09-14T03:47:52.447 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.879855+0000", 2024-09-14T03:47:52.447 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.880509+0000", 2024-09-14T03:47:52.447 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.461691+0000", 2024-09-14T03:47:52.447 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.447 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.447 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.448 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.448 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.461691+0000", 2024-09-14T03:47:52.448 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.461691+0000", 2024-09-14T03:47:52.448 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.461691+0000", 2024-09-14T03:47:52.448 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.448 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.448 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.448 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.461691+0000", 2024-09-14T03:47:52.448 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.461691+0000", 2024-09-14T03:47:52.448 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.448 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.448 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.448 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.448 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.449 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.449 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.449 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.449 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.449 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.449 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.449 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.449 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.449 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.22", 2024-09-14T03:47:52.449 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.449 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.449 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.449 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:52.449 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.450 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T05:05:31.166902+0000", 2024-09-14T03:47:52.450 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.450 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.450 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.450 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.450 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.450 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.450 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.450 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.450 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.450 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.450 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.451 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.451 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.451 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.451 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.451 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.451 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.451 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.451 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.451 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.451 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.451 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.451 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.451 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.451 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.451 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.452 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.452 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.452 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.452 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.452 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.452 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.452 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.452 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.452 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.452 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.452 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.452 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.453 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.453 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.453 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.453 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.453 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.453 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.453 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.453 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.453 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.453 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.453 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.453 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.453 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.454 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 5, 2024-09-14T03:47:52.454 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.454 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.454 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.454 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.454 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:52.454 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.454 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:52.454 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.454 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 2, 2024-09-14T03:47:52.454 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.454 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.454 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.454 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.455 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.455 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.455 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:15.135594+0000", 2024-09-14T03:47:52.455 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.869225+0000", 2024-09-14T03:47:52.455 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.869225+0000", 2024-09-14T03:47:52.455 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.869523+0000", 2024-09-14T03:47:52.455 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:15.135594+0000", 2024-09-14T03:47:52.455 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.455 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.455 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.455 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.455 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:15.135594+0000", 2024-09-14T03:47:52.455 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:15.135594+0000", 2024-09-14T03:47:52.456 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:15.135594+0000", 2024-09-14T03:47:52.456 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.456 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.456 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.456 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:15.135594+0000", 2024-09-14T03:47:52.456 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:15.135594+0000", 2024-09-14T03:47:52.456 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.456 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.456 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.456 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.456 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.456 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.456 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.456 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.457 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.457 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.457 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.457 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.457 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.457 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.21", 2024-09-14T03:47:52.457 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.457 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.457 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.457 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:52.457 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.457 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T10:43:15.638187+0000", 2024-09-14T03:47:52.457 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.457 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.458 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.458 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.458 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.458 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.458 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.458 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.458 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.458 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.458 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.458 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.458 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.458 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.458 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.459 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.459 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.459 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.459 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.459 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.459 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.459 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.459 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.459 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.459 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.459 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.459 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.459 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.459 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.460 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.460 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.460 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.460 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.460 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.460 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.460 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.460 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.460 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.460 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.460 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.460 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.460 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.460 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.461 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.461 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.461 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.461 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.461 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:52.461 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.461 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:52.461 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.461 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 2, 2024-09-14T03:47:52.461 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.461 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.461 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.461 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.461 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.462 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.462 INFO:tasks.workunit.client.0.smithi070.stdout: 5 2024-09-14T03:47:52.462 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.462 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 6, 2024-09-14T03:47:52.462 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.462 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.462 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.462 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.462 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.462 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.462 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.544409+0000", 2024-09-14T03:47:52.462 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.875496+0000", 2024-09-14T03:47:52.462 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.875496+0000", 2024-09-14T03:47:52.462 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.876014+0000", 2024-09-14T03:47:52.462 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.544409+0000", 2024-09-14T03:47:52.463 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.463 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.463 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.463 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.463 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.544409+0000", 2024-09-14T03:47:52.463 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.544409+0000", 2024-09-14T03:47:52.463 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.544409+0000", 2024-09-14T03:47:52.463 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.463 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.463 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.463 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.544409+0000", 2024-09-14T03:47:52.463 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.544409+0000", 2024-09-14T03:47:52.463 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.464 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.464 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.464 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.464 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.464 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.464 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.464 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.464 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.464 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.464 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.464 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.464 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.464 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.20", 2024-09-14T03:47:52.465 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.465 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.465 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.465 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:52.465 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.465 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:03:45.344199+0000", 2024-09-14T03:47:52.465 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.465 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.465 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.465 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.465 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.465 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.465 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.466 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.466 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.466 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.466 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.466 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.466 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.466 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.466 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.466 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.466 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.466 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.466 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.466 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.466 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.467 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.467 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.467 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.467 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.467 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.467 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.467 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.467 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.467 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.467 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.467 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.467 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.467 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.468 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.468 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.468 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.468 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.468 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.468 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.468 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.468 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.468 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.468 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.468 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.468 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.468 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.469 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.469 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.469 INFO:tasks.workunit.client.0.smithi070.stdout: 5 2024-09-14T03:47:52.469 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.469 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 6, 2024-09-14T03:47:52.469 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.469 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.469 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.469 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.469 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.469 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.469 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:52.469 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.470 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 4, 2024-09-14T03:47:52.470 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.470 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.470 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.470 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.470 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.470 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.470 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.458305+0000", 2024-09-14T03:47:52.470 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.865190+0000", 2024-09-14T03:47:52.470 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.865190+0000", 2024-09-14T03:47:52.470 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.865496+0000", 2024-09-14T03:47:52.470 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.458305+0000", 2024-09-14T03:47:52.470 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.471 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.471 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.471 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.471 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.458305+0000", 2024-09-14T03:47:52.471 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.458305+0000", 2024-09-14T03:47:52.471 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.458305+0000", 2024-09-14T03:47:52.471 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.471 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.471 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.471 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.458305+0000", 2024-09-14T03:47:52.471 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.458305+0000", 2024-09-14T03:47:52.471 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.471 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.472 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.472 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.472 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.472 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.472 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.472 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.472 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.472 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.472 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.472 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.472 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.472 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.1a", 2024-09-14T03:47:52.472 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.473 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.473 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.473 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:52.473 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.473 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:18:50.037675+0000", 2024-09-14T03:47:52.473 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.473 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.473 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.473 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.473 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.473 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.473 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.473 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.474 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.474 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.474 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.474 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.474 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.474 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.474 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.474 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.474 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.474 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.474 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.474 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.474 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.474 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.475 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.475 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.475 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.475 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.475 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.475 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.475 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.475 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.475 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.475 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.475 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.475 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.475 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.476 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.476 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.476 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.476 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.476 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.476 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.476 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.476 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.476 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.476 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.476 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.476 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.476 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.477 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.477 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:52.477 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.477 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 4, 2024-09-14T03:47:52.477 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.477 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.477 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.477 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.477 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.477 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.477 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:52.477 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.477 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 3, 2024-09-14T03:47:52.478 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.478 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.478 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:52.478 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.478 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.478 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.478 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.857788+0000", 2024-09-14T03:47:52.478 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.728927+0000", 2024-09-14T03:47:52.478 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.728927+0000", 2024-09-14T03:47:52.478 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.729188+0000", 2024-09-14T03:47:52.478 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.857788+0000", 2024-09-14T03:47:52.478 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.478 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.478 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.479 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:52.479 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.857788+0000", 2024-09-14T03:47:52.479 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.857788+0000", 2024-09-14T03:47:52.479 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.857788+0000", 2024-09-14T03:47:52.479 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.479 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.479 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.479 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.857788+0000", 2024-09-14T03:47:52.479 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.857788+0000", 2024-09-14T03:47:52.479 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.479 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 1, 2024-09-14T03:47:52.479 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.480 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.480 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:52.480 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.480 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.480 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.480 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.480 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 1, 2024-09-14T03:47:52.480 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.480 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.480 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.480 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.1b", 2024-09-14T03:47:52.480 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.480 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.480 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.481 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 24, 2024-09-14T03:47:52.481 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.481 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:13:16.419701+0000", 2024-09-14T03:47:52.481 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.481 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.481 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.481 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.481 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.481 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.481 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.481 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.481 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.481 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.482 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.482 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.482 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.482 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.482 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.482 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.482 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.482 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.482 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.482 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.482 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.482 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.482 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.482 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.483 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.483 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.483 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.483 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.483 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.483 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.483 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.483 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.483 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.483 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.483 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.483 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.483 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.484 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.484 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.484 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.484 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.484 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.484 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.484 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.484 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.484 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.484 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.484 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.484 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.484 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:52.485 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.485 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 3, 2024-09-14T03:47:52.485 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "58'1" 2024-09-14T03:47:52.485 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.485 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.485 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.485 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.485 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.485 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:52.485 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.485 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 0, 2024-09-14T03:47:52.485 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.485 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.485 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.486 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.486 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.486 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.486 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.684318+0000", 2024-09-14T03:47:52.486 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.878801+0000", 2024-09-14T03:47:52.486 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.878801+0000", 2024-09-14T03:47:52.486 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.879101+0000", 2024-09-14T03:47:52.486 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.684318+0000", 2024-09-14T03:47:52.486 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.486 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.486 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.486 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.486 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.684318+0000", 2024-09-14T03:47:52.486 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.684318+0000", 2024-09-14T03:47:52.487 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.684318+0000", 2024-09-14T03:47:52.487 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.487 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.487 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.487 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.684318+0000", 2024-09-14T03:47:52.487 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.684318+0000", 2024-09-14T03:47:52.487 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.487 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.487 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.487 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.487 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.487 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.487 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.488 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.488 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.488 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.488 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.488 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.488 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.488 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.1b", 2024-09-14T03:47:52.488 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.488 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.488 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.488 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:52.488 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.488 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:59:13.447251+0000", 2024-09-14T03:47:52.489 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.489 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.489 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.489 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.489 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.489 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.489 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.489 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.489 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.489 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.489 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.489 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.489 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.489 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.490 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.490 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.490 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.490 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.490 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.490 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.490 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.490 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.490 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.490 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.490 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.490 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.490 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.490 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.491 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.491 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.491 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.491 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.491 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.491 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.491 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.491 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.491 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.491 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.491 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.491 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.491 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.491 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.492 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.492 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.492 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.492 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.492 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.492 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.492 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.492 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:52.492 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.492 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 0, 2024-09-14T03:47:52.492 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.492 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.492 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.493 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.493 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.493 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.493 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:52.493 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.493 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 6, 2024-09-14T03:47:52.493 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.493 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.493 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:52.493 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.493 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.493 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.493 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.544625+0000", 2024-09-14T03:47:52.494 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.730172+0000", 2024-09-14T03:47:52.494 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.730172+0000", 2024-09-14T03:47:52.494 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.730444+0000", 2024-09-14T03:47:52.494 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.544625+0000", 2024-09-14T03:47:52.494 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.494 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.494 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.494 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:52.494 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.544625+0000", 2024-09-14T03:47:52.494 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.544625+0000", 2024-09-14T03:47:52.494 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.544625+0000", 2024-09-14T03:47:52.494 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.494 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.495 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.495 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.544625+0000", 2024-09-14T03:47:52.495 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.544625+0000", 2024-09-14T03:47:52.495 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.495 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 1, 2024-09-14T03:47:52.495 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.495 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.495 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:52.495 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.495 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.495 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.495 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.495 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 1, 2024-09-14T03:47:52.495 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.496 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.496 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.496 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.1a", 2024-09-14T03:47:52.496 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.496 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.496 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.496 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 24, 2024-09-14T03:47:52.496 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.496 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T10:03:44.810562+0000", 2024-09-14T03:47:52.496 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.496 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.496 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.496 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.497 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.497 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.497 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.497 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.497 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.497 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.497 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.497 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.497 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.497 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.497 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.497 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.497 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.498 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.498 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.498 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.498 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.498 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.498 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.498 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.498 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.498 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.498 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.498 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.498 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.498 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.498 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.499 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.499 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.499 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.499 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.499 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.499 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.499 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.499 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.499 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.499 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.499 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.499 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.499 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.500 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.500 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.500 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.500 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.500 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.500 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:52.500 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.500 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 6, 2024-09-14T03:47:52.500 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "58'1" 2024-09-14T03:47:52.500 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.500 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.500 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.500 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.500 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.501 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.501 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.501 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 3, 2024-09-14T03:47:52.501 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.501 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.501 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:52.501 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.501 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.501 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.501 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.857500+0000", 2024-09-14T03:47:52.501 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.731206+0000", 2024-09-14T03:47:52.501 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.731206+0000", 2024-09-14T03:47:52.501 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.731458+0000", 2024-09-14T03:47:52.502 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.857500+0000", 2024-09-14T03:47:52.502 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.502 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.502 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.502 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:52.502 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.857500+0000", 2024-09-14T03:47:52.502 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.857500+0000", 2024-09-14T03:47:52.502 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.857500+0000", 2024-09-14T03:47:52.502 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.502 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.502 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.502 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.857500+0000", 2024-09-14T03:47:52.502 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.857500+0000", 2024-09-14T03:47:52.502 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.503 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.503 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.503 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.503 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:52.503 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.503 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.503 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.503 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.503 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.503 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.503 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.503 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.503 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.19", 2024-09-14T03:47:52.503 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.504 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.504 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.504 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 23, 2024-09-14T03:47:52.504 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.504 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:37:06.670539+0000", 2024-09-14T03:47:52.504 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.504 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.504 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.504 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.504 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.504 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.504 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.504 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.505 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.505 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.505 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.505 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.505 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.505 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.505 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.505 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.505 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.505 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.505 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.505 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.505 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.505 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.506 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.506 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.506 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.506 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.506 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.506 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.506 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.506 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.506 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.506 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.506 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.506 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.506 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.506 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.507 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.507 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.507 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.507 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.507 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.507 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.507 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.507 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.507 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.507 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.507 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.507 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.507 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.508 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.508 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.508 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 3, 2024-09-14T03:47:52.508 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.508 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.508 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.508 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.508 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.508 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.508 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:52.508 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.508 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 3, 2024-09-14T03:47:52.508 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.509 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.509 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.509 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.509 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.509 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.509 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.857453+0000", 2024-09-14T03:47:52.509 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.869521+0000", 2024-09-14T03:47:52.509 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.869521+0000", 2024-09-14T03:47:52.509 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.869781+0000", 2024-09-14T03:47:52.509 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.857453+0000", 2024-09-14T03:47:52.509 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.509 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.509 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.510 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.510 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.857453+0000", 2024-09-14T03:47:52.510 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.857453+0000", 2024-09-14T03:47:52.510 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.857453+0000", 2024-09-14T03:47:52.510 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.510 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.510 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.510 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.857453+0000", 2024-09-14T03:47:52.510 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.857453+0000", 2024-09-14T03:47:52.510 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.510 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.510 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.510 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.510 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.511 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.511 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.511 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.511 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.511 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.511 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.511 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.511 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.511 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.18", 2024-09-14T03:47:52.511 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.511 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.511 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.511 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:52.511 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.512 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T04:35:18.565206+0000", 2024-09-14T03:47:52.512 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.512 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.512 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.512 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.512 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.512 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.512 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.512 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.512 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.512 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.512 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.512 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.513 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.513 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.513 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.513 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.513 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.513 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.513 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.513 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.513 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.513 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.513 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.513 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.513 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.514 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.514 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.514 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.514 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.514 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.514 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.514 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.514 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.514 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.514 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.514 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.514 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.514 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.514 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.515 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.515 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.515 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.515 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.515 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.515 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.515 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.515 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.515 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.515 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.515 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:52.515 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.515 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 3, 2024-09-14T03:47:52.515 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.516 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.516 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.516 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.516 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.516 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.516 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:52.516 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.516 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 5, 2024-09-14T03:47:52.516 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.516 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.516 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:52.516 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.516 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.517 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.517 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.461579+0000", 2024-09-14T03:47:52.517 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:25.932288+0000", 2024-09-14T03:47:52.517 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:25.932288+0000", 2024-09-14T03:47:52.517 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:25.932578+0000", 2024-09-14T03:47:52.517 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.461579+0000", 2024-09-14T03:47:52.517 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.517 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.517 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.517 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 60, 2024-09-14T03:47:52.517 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.461579+0000", 2024-09-14T03:47:52.517 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.461579+0000", 2024-09-14T03:47:52.517 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.461579+0000", 2024-09-14T03:47:52.517 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.518 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.518 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.518 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.461579+0000", 2024-09-14T03:47:52.518 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.461579+0000", 2024-09-14T03:47:52.518 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.518 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.518 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.518 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.518 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 59, 2024-09-14T03:47:52.518 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.518 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.518 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.518 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.518 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.519 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.519 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.519 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.519 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.18", 2024-09-14T03:47:52.519 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.519 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.519 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.519 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 33, 2024-09-14T03:47:52.519 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.519 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:24:41.091353+0000", 2024-09-14T03:47:52.519 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.519 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.519 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.519 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.520 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.520 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.520 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.520 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.520 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.520 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.520 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.520 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.520 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.520 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.520 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.520 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.520 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.521 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.521 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.521 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.521 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.521 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.521 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.521 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.521 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.521 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.521 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.521 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.521 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.521 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.522 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.522 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.522 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.522 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.522 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.522 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.522 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.522 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.522 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.522 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.522 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.522 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.522 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.522 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.523 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.523 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.523 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.523 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.523 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.523 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:52.523 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.523 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 5, 2024-09-14T03:47:52.523 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.523 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.523 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.523 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.523 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.524 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.524 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:52.524 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.524 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 1, 2024-09-14T03:47:52.524 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.524 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.524 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.524 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.524 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.524 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.524 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.457866+0000", 2024-09-14T03:47:52.524 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.873473+0000", 2024-09-14T03:47:52.524 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.873473+0000", 2024-09-14T03:47:52.525 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.873748+0000", 2024-09-14T03:47:52.525 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.457866+0000", 2024-09-14T03:47:52.525 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.525 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.525 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.525 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.525 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.457866+0000", 2024-09-14T03:47:52.525 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.457866+0000", 2024-09-14T03:47:52.525 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.457866+0000", 2024-09-14T03:47:52.525 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.525 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.525 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.525 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.457866+0000", 2024-09-14T03:47:52.526 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.457866+0000", 2024-09-14T03:47:52.526 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.526 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.526 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.526 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.526 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.526 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.526 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.526 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.526 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.526 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.526 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.526 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.527 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.527 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.19", 2024-09-14T03:47:52.527 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.527 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.527 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.527 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:52.527 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.527 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:04:39.289903+0000", 2024-09-14T03:47:52.527 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.527 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.527 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.527 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.527 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.527 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.528 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.528 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.528 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.528 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.528 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.528 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.528 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.528 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.528 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.528 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.528 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.528 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.529 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.529 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.529 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.529 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.529 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.529 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.529 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.529 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.529 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.529 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.529 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.529 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.529 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.529 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.529 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.530 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.530 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.530 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.530 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.530 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.530 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.530 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.530 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.530 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.530 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.530 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.530 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.530 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.530 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.531 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.531 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.531 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:52.531 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.531 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 1, 2024-09-14T03:47:52.531 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.531 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.531 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.531 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.531 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.531 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.531 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:52.531 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.531 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 6, 2024-09-14T03:47:52.532 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.532 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.532 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:52.532 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.532 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.532 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.532 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.544953+0000", 2024-09-14T03:47:52.532 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.735301+0000", 2024-09-14T03:47:52.532 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.735301+0000", 2024-09-14T03:47:52.532 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.735608+0000", 2024-09-14T03:47:52.532 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.544953+0000", 2024-09-14T03:47:52.532 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.532 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.532 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.533 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:52.533 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.544953+0000", 2024-09-14T03:47:52.533 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.544953+0000", 2024-09-14T03:47:52.533 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.544953+0000", 2024-09-14T03:47:52.533 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.533 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.533 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.533 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.544953+0000", 2024-09-14T03:47:52.533 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.544953+0000", 2024-09-14T03:47:52.533 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.533 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.533 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.533 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.534 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:52.534 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.534 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.534 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.534 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.534 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.534 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.534 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.534 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.534 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.17", 2024-09-14T03:47:52.534 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.534 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.534 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.534 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 23, 2024-09-14T03:47:52.535 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.535 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:44:56.523073+0000", 2024-09-14T03:47:52.535 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.535 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.535 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.535 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.535 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.535 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.535 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.535 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.535 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.535 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.535 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.535 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.535 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.536 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.536 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.536 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.536 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.536 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.536 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.536 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.536 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.536 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.536 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.536 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.536 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.536 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.536 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.537 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.537 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.537 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.537 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.537 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.537 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.537 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.537 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.537 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.537 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.537 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.537 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.537 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.537 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.538 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.538 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.538 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.538 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.538 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.538 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.538 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.538 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.538 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:52.538 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.538 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 6, 2024-09-14T03:47:52.538 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.538 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.539 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.539 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.539 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.539 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.539 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:52.539 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.539 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 5, 2024-09-14T03:47:52.539 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.539 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.539 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.539 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.539 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.539 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.539 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.462078+0000", 2024-09-14T03:47:52.540 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.874473+0000", 2024-09-14T03:47:52.540 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.874473+0000", 2024-09-14T03:47:52.540 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.874777+0000", 2024-09-14T03:47:52.540 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.462078+0000", 2024-09-14T03:47:52.540 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.540 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.540 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.540 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.540 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.462078+0000", 2024-09-14T03:47:52.540 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.462078+0000", 2024-09-14T03:47:52.540 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.462078+0000", 2024-09-14T03:47:52.540 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.540 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.540 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.541 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.462078+0000", 2024-09-14T03:47:52.541 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.462078+0000", 2024-09-14T03:47:52.541 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.541 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.541 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.541 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.541 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.541 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.541 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.541 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.541 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.541 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.541 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.542 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.542 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.542 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.16", 2024-09-14T03:47:52.542 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.542 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.542 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.542 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:52.542 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.542 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T04:49:02.042635+0000", 2024-09-14T03:47:52.542 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.542 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.542 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.542 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.542 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.543 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.543 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.543 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.543 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.543 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.543 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.543 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.543 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.543 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.543 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.543 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.543 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.543 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.543 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.543 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.544 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.544 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.544 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.544 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.544 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.544 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.544 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.544 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.544 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.544 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.544 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.544 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.544 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.544 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.545 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.545 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.545 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.545 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.545 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.545 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.545 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.545 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.545 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.545 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.545 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.545 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.546 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.546 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.546 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.546 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:52.546 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.546 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 5, 2024-09-14T03:47:52.546 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.546 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.546 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.546 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.546 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.546 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.546 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:52.546 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.546 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 5, 2024-09-14T03:47:52.547 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.547 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.547 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:52.547 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.547 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.547 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.547 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.462172+0000", 2024-09-14T03:47:52.547 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.732492+0000", 2024-09-14T03:47:52.547 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.732492+0000", 2024-09-14T03:47:52.547 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.732969+0000", 2024-09-14T03:47:52.547 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.462172+0000", 2024-09-14T03:47:52.547 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.547 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.547 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.548 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:52.548 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.462172+0000", 2024-09-14T03:47:52.548 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.462172+0000", 2024-09-14T03:47:52.548 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.462172+0000", 2024-09-14T03:47:52.548 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.548 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.548 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.548 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.462172+0000", 2024-09-14T03:47:52.548 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.462172+0000", 2024-09-14T03:47:52.548 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.548 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.548 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.548 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.548 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:52.548 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.549 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.549 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.549 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.549 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.549 INFO:tasks.workunit.client.0.smithi070.stderr:+ echo OK 2024-09-14T03:47:52.552 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.552 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.552 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.552 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.16", 2024-09-14T03:47:52.552 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.552 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.553 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.553 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 23, 2024-09-14T03:47:52.553 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.553 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:50:28.719447+0000", 2024-09-14T03:47:52.553 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.553 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.553 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.553 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.553 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.553 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.553 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.553 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.553 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.554 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.554 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.554 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.554 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.554 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.554 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.554 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.554 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.554 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.554 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.554 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.554 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.554 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.554 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.554 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.555 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.555 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.555 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.555 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.555 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.555 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.555 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.555 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.555 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.555 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.555 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.555 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.555 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.556 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.556 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.556 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.556 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.556 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.556 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.556 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.556 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.556 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.556 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.556 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.556 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.556 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:52.557 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.557 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 5, 2024-09-14T03:47:52.557 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.557 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.557 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.557 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.557 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.557 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.557 INFO:tasks.workunit.client.0.smithi070.stdout: 3 2024-09-14T03:47:52.557 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.557 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 0, 2024-09-14T03:47:52.557 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.557 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.558 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.558 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.558 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.558 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.558 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.684384+0000", 2024-09-14T03:47:52.558 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.875422+0000", 2024-09-14T03:47:52.558 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.875422+0000", 2024-09-14T03:47:52.558 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.875679+0000", 2024-09-14T03:47:52.558 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.684384+0000", 2024-09-14T03:47:52.558 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.558 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.558 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.558 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.558 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.684384+0000", 2024-09-14T03:47:52.559 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.684384+0000", 2024-09-14T03:47:52.559 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.684384+0000", 2024-09-14T03:47:52.559 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.559 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.559 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.559 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.684384+0000", 2024-09-14T03:47:52.559 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.684384+0000", 2024-09-14T03:47:52.559 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.559 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.559 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.559 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.559 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.559 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.559 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.560 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.560 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.560 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.560 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.560 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.560 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.560 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.17", 2024-09-14T03:47:52.560 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.560 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.560 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.560 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:52.560 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.560 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:13:00.349373+0000", 2024-09-14T03:47:52.560 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.561 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.561 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.561 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.561 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.561 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.561 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.561 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.561 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.561 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.561 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.561 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.561 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.561 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.561 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.562 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.562 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.562 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.562 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.562 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.562 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.562 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.562 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.562 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.562 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.562 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.562 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.562 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.562 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.562 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.563 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.563 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.563 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.563 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.563 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.563 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.563 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.563 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.563 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.563 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.563 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.563 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.563 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.563 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.564 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.564 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.564 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.564 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.564 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.564 INFO:tasks.workunit.client.0.smithi070.stdout: 3 2024-09-14T03:47:52.564 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.564 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 0, 2024-09-14T03:47:52.564 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.564 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.564 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.564 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.564 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.564 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.565 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.565 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.565 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 1, 2024-09-14T03:47:52.565 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.565 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.565 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:52.565 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.565 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.565 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.565 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.458567+0000", 2024-09-14T03:47:52.565 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.734514+0000", 2024-09-14T03:47:52.565 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.734514+0000", 2024-09-14T03:47:52.565 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.735066+0000", 2024-09-14T03:47:52.565 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.458567+0000", 2024-09-14T03:47:52.566 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.566 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.566 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.566 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:52.566 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.458567+0000", 2024-09-14T03:47:52.566 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.458567+0000", 2024-09-14T03:47:52.566 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.458567+0000", 2024-09-14T03:47:52.566 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.566 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.566 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.566 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.458567+0000", 2024-09-14T03:47:52.566 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.458567+0000", 2024-09-14T03:47:52.567 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.567 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 3, 2024-09-14T03:47:52.567 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.567 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.567 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:52.567 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.567 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.567 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.567 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.567 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 3, 2024-09-14T03:47:52.567 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.567 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.567 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.567 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.15", 2024-09-14T03:47:52.567 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.568 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.568 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.568 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 26, 2024-09-14T03:47:52.568 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.568 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T09:41:55.478295+0000", 2024-09-14T03:47:52.568 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.568 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.568 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.568 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 90, 2024-09-14T03:47:52.568 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.568 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.568 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.568 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.568 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.569 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.569 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.569 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.569 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.569 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.569 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.569 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.569 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.569 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.569 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.569 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 6, 2024-09-14T03:47:52.569 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 2, 2024-09-14T03:47:52.569 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.570 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 2, 2024-09-14T03:47:52.570 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.570 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.570 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.570 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.570 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.570 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 1, 2024-09-14T03:47:52.570 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.570 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.570 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.570 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.570 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.570 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.570 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.571 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.571 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.571 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.571 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.571 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.571 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 4, 2024-09-14T03:47:52.571 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 1 2024-09-14T03:47:52.571 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.571 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.571 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.571 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.571 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.571 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.571 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.572 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.572 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 1, 2024-09-14T03:47:52.572 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "58'3" 2024-09-14T03:47:52.572 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.572 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.572 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.572 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.572 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.572 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:52.572 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.572 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 4, 2024-09-14T03:47:52.572 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.572 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.572 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.573 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.573 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.573 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.573 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.458619+0000", 2024-09-14T03:47:52.573 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.881032+0000", 2024-09-14T03:47:52.573 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.881032+0000", 2024-09-14T03:47:52.573 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.881319+0000", 2024-09-14T03:47:52.573 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.458619+0000", 2024-09-14T03:47:52.573 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.573 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.573 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.573 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.573 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.458619+0000", 2024-09-14T03:47:52.574 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.458619+0000", 2024-09-14T03:47:52.574 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.458619+0000", 2024-09-14T03:47:52.574 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.574 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.574 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.574 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.458619+0000", 2024-09-14T03:47:52.574 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.458619+0000", 2024-09-14T03:47:52.574 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.574 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.574 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.574 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.574 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.574 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.574 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.574 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.575 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.575 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.575 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.575 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.575 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.575 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.14", 2024-09-14T03:47:52.575 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.575 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.575 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.575 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:52.575 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.575 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T15:35:41.762014+0000", 2024-09-14T03:47:52.575 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.576 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.576 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.576 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.576 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.576 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.576 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.576 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.576 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.576 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.576 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.576 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.576 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.576 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.577 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.577 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.577 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.577 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.577 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.577 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.577 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.577 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.577 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.577 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.577 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.577 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.577 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.577 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.577 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.578 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.578 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.578 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.578 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.578 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.578 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.578 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.578 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.578 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.578 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.578 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.578 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.578 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.578 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.579 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.579 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.579 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.579 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.579 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.579 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.579 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:52.579 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.579 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 4, 2024-09-14T03:47:52.579 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.579 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.579 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.579 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.579 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.579 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.579 INFO:tasks.workunit.client.0.smithi070.stdout: 5 2024-09-14T03:47:52.580 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.580 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 6, 2024-09-14T03:47:52.580 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.580 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.580 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:52.580 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.580 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.580 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.580 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:46:36.633586+0000", 2024-09-14T03:47:52.580 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.734380+0000", 2024-09-14T03:47:52.580 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.734380+0000", 2024-09-14T03:47:52.580 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.734778+0000", 2024-09-14T03:47:52.581 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:46:36.633586+0000", 2024-09-14T03:47:52.581 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.581 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.581 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.581 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:52.581 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:46:36.633586+0000", 2024-09-14T03:47:52.581 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:46:36.633586+0000", 2024-09-14T03:47:52.581 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:46:36.633586+0000", 2024-09-14T03:47:52.581 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.581 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.581 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.581 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:46:36.633586+0000", 2024-09-14T03:47:52.581 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:46:36.633586+0000", 2024-09-14T03:47:52.581 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.581 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 12, 2024-09-14T03:47:52.582 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.582 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.582 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:52.582 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.582 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.582 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.582 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.582 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 12, 2024-09-14T03:47:52.582 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.582 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.582 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.582 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.14", 2024-09-14T03:47:52.582 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.582 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.582 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 68, 2024-09-14T03:47:52.583 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 35, 2024-09-14T03:47:52.583 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.583 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:55:13.561065+0000", 2024-09-14T03:47:52.583 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.583 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.583 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.583 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 2624, 2024-09-14T03:47:52.583 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.583 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.583 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.583 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.583 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.583 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.584 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.584 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.584 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.584 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.584 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.584 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.584 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.584 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.584 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.584 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 12, 2024-09-14T03:47:52.584 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 4, 2024-09-14T03:47:52.584 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.584 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 4, 2024-09-14T03:47:52.584 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.584 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.585 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.585 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.585 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.585 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 1, 2024-09-14T03:47:52.585 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.585 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.585 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.585 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.585 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.585 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.585 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.585 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.585 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.585 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.586 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.586 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.586 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 12, 2024-09-14T03:47:52.586 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 9 2024-09-14T03:47:52.586 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.586 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.586 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.586 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.586 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.586 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.586 INFO:tasks.workunit.client.0.smithi070.stdout: 5 2024-09-14T03:47:52.586 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.586 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 6, 2024-09-14T03:47:52.586 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "68'12" 2024-09-14T03:47:52.587 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.587 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.587 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.587 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.587 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.587 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:52.587 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.587 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:52.587 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.587 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.587 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.587 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.587 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.588 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.588 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.660824+0000", 2024-09-14T03:47:52.588 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.879840+0000", 2024-09-14T03:47:52.588 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.879840+0000", 2024-09-14T03:47:52.588 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.880139+0000", 2024-09-14T03:47:52.588 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.660824+0000", 2024-09-14T03:47:52.588 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.588 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.588 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.588 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.588 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.660824+0000", 2024-09-14T03:47:52.588 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.660824+0000", 2024-09-14T03:47:52.588 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.660824+0000", 2024-09-14T03:47:52.589 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.589 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.589 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.589 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.660824+0000", 2024-09-14T03:47:52.589 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.660824+0000", 2024-09-14T03:47:52.589 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.589 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.589 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.589 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.589 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.589 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.589 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.589 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.590 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.590 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.590 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.590 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.590 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.590 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.15", 2024-09-14T03:47:52.590 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.590 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.590 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.590 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:52.590 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.590 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:31:34.003676+0000", 2024-09-14T03:47:52.590 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.590 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.591 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.591 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.591 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.591 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.591 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.591 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.591 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.591 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.591 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.591 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.591 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.591 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.591 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.591 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.592 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.592 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.592 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.592 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.592 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.592 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.592 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.592 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.592 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.592 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.592 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.592 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.592 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.592 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.592 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.593 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.593 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.593 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.593 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.593 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.593 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.593 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.593 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.593 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.593 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.593 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.593 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.593 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.593 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.594 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.594 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.594 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.594 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.594 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:52.594 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.594 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:52.594 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.594 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.594 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.594 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.594 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.595 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.595 INFO:tasks.workunit.client.0.smithi070.stdout: 3 2024-09-14T03:47:52.595 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.595 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 0, 2024-09-14T03:47:52.595 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.595 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.595 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:52.595 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.595 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.595 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.595 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.684403+0000", 2024-09-14T03:47:52.595 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.727893+0000", 2024-09-14T03:47:52.595 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.727893+0000", 2024-09-14T03:47:52.595 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.728768+0000", 2024-09-14T03:47:52.596 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.684403+0000", 2024-09-14T03:47:52.596 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.596 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.596 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.596 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:52.596 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.684403+0000", 2024-09-14T03:47:52.596 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.684403+0000", 2024-09-14T03:47:52.596 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.684403+0000", 2024-09-14T03:47:52.596 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.596 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.596 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.596 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.684403+0000", 2024-09-14T03:47:52.596 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.684403+0000", 2024-09-14T03:47:52.596 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.597 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 2, 2024-09-14T03:47:52.597 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.597 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.597 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:52.597 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.597 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.597 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.597 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.597 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 2, 2024-09-14T03:47:52.597 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.597 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.597 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.597 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.13", 2024-09-14T03:47:52.597 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.598 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.598 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.598 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 25, 2024-09-14T03:47:52.598 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.598 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T04:54:48.554486+0000", 2024-09-14T03:47:52.598 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.598 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.598 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.598 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 34, 2024-09-14T03:47:52.598 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.598 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.598 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.598 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.598 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.599 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.599 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.599 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.599 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.599 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.599 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.599 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.599 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.599 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.599 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.599 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 6, 2024-09-14T03:47:52.599 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 2, 2024-09-14T03:47:52.599 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.600 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 2, 2024-09-14T03:47:52.600 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.600 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.600 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.600 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.600 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.600 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 1, 2024-09-14T03:47:52.600 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.600 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.600 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.600 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.600 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.600 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.600 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.601 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.601 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.601 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.601 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.601 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.601 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 3, 2024-09-14T03:47:52.601 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 6 2024-09-14T03:47:52.601 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.601 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.601 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.601 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.601 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.601 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.602 INFO:tasks.workunit.client.0.smithi070.stdout: 3 2024-09-14T03:47:52.602 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.602 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 0, 2024-09-14T03:47:52.602 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "58'2" 2024-09-14T03:47:52.602 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.602 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.602 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.602 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.602 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.602 INFO:tasks.workunit.client.0.smithi070.stdout: 3 2024-09-14T03:47:52.602 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.602 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 0, 2024-09-14T03:47:52.602 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.602 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.602 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.603 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.603 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.603 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.603 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.684119+0000", 2024-09-14T03:47:52.603 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.877147+0000", 2024-09-14T03:47:52.603 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.877147+0000", 2024-09-14T03:47:52.603 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.877540+0000", 2024-09-14T03:47:52.603 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.684119+0000", 2024-09-14T03:47:52.603 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.603 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.603 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.603 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.603 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.684119+0000", 2024-09-14T03:47:52.604 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.684119+0000", 2024-09-14T03:47:52.604 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.684119+0000", 2024-09-14T03:47:52.604 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.604 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.604 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.604 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.684119+0000", 2024-09-14T03:47:52.604 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.684119+0000", 2024-09-14T03:47:52.604 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.604 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.604 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.604 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.604 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.604 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.604 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.605 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.605 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.605 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.605 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.605 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.605 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.605 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.12", 2024-09-14T03:47:52.605 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.605 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.605 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.605 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:52.605 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.605 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T07:17:57.724648+0000", 2024-09-14T03:47:52.605 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.606 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.606 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.606 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.606 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.606 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.606 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.606 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.606 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.606 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.606 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.606 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.606 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.606 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.606 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.607 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.607 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.607 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.607 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.607 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.607 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.607 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.607 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.607 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.607 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.607 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.607 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.607 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.607 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.608 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.608 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.608 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.608 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.608 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.608 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.608 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.608 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.608 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.608 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.608 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.608 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.608 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.608 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.609 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.609 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.609 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.609 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.609 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.609 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.609 INFO:tasks.workunit.client.0.smithi070.stdout: 3 2024-09-14T03:47:52.609 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.609 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 0, 2024-09-14T03:47:52.609 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.609 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.609 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.609 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.609 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.609 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.610 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:52.610 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.610 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 5, 2024-09-14T03:47:52.610 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.610 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.610 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:52.610 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.610 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.610 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.610 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.461815+0000", 2024-09-14T03:47:52.610 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.732537+0000", 2024-09-14T03:47:52.610 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.732537+0000", 2024-09-14T03:47:52.610 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.732812+0000", 2024-09-14T03:47:52.610 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.461815+0000", 2024-09-14T03:47:52.611 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.611 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.611 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.611 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:52.611 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.461815+0000", 2024-09-14T03:47:52.611 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.461815+0000", 2024-09-14T03:47:52.611 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.461815+0000", 2024-09-14T03:47:52.611 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.611 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.611 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.611 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.461815+0000", 2024-09-14T03:47:52.611 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.461815+0000", 2024-09-14T03:47:52.611 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.611 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.612 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.612 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.612 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:52.612 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.612 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.612 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.612 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.612 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.612 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.612 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.612 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.612 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.12", 2024-09-14T03:47:52.612 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.613 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.613 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.613 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 23, 2024-09-14T03:47:52.613 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.613 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:26:11.551182+0000", 2024-09-14T03:47:52.613 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.613 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.613 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.613 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.613 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.613 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.613 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.613 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.613 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.614 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.614 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.614 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.614 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.614 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.614 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.614 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.614 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.614 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.614 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.614 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.614 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.614 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.614 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.615 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.615 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.615 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.615 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.615 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.615 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.615 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.615 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.615 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.615 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.615 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.615 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.616 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.616 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.616 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.616 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.616 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.616 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.616 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.616 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.616 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.616 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.616 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.616 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.616 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.616 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.617 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:52.617 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.617 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 5, 2024-09-14T03:47:52.617 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.617 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.617 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.617 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.617 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.617 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.617 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:52.617 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.617 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:52.617 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.618 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.618 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.618 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.618 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.618 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.618 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.660730+0000", 2024-09-14T03:47:52.618 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.880868+0000", 2024-09-14T03:47:52.618 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.880868+0000", 2024-09-14T03:47:52.618 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.881140+0000", 2024-09-14T03:47:52.618 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.660730+0000", 2024-09-14T03:47:52.618 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.618 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.618 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.618 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.619 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.660730+0000", 2024-09-14T03:47:52.619 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.660730+0000", 2024-09-14T03:47:52.619 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.660730+0000", 2024-09-14T03:47:52.619 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.619 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.619 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.619 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.660730+0000", 2024-09-14T03:47:52.619 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.660730+0000", 2024-09-14T03:47:52.619 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.619 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.619 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.619 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.619 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.619 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.619 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.620 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.620 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.620 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.620 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.620 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.620 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.620 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.13", 2024-09-14T03:47:52.620 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.620 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.620 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.620 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:52.620 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.620 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:29:05.729804+0000", 2024-09-14T03:47:52.620 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.621 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.621 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.621 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.621 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.621 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.621 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.621 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.621 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.621 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.621 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.621 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.621 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.621 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.622 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.622 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.622 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.622 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.622 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.622 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.622 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.622 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.622 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.622 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.622 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.622 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.622 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.622 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.622 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.623 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.623 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.623 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.623 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.623 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.623 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.623 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.623 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.623 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.623 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.623 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.623 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.623 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.623 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.624 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.624 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.624 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.624 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.624 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.624 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.624 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:52.624 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.624 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:52.624 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.624 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.624 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.624 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.624 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.625 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.625 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:52.625 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.625 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 6, 2024-09-14T03:47:52.625 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.625 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.625 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:52.625 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.625 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.625 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.625 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.544699+0000", 2024-09-14T03:47:52.625 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.729821+0000", 2024-09-14T03:47:52.625 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.729821+0000", 2024-09-14T03:47:52.625 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.730505+0000", 2024-09-14T03:47:52.625 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.544699+0000", 2024-09-14T03:47:52.626 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.626 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.626 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.626 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:52.626 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.544699+0000", 2024-09-14T03:47:52.626 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.544699+0000", 2024-09-14T03:47:52.626 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.544699+0000", 2024-09-14T03:47:52.626 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.626 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.626 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.626 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.544699+0000", 2024-09-14T03:47:52.626 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.544699+0000", 2024-09-14T03:47:52.626 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.626 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.627 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.627 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.627 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:52.627 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.627 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.627 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.627 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.627 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.627 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.627 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.627 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.627 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.11", 2024-09-14T03:47:52.627 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.627 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.628 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.628 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 23, 2024-09-14T03:47:52.628 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.628 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T10:17:44.091667+0000", 2024-09-14T03:47:52.628 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.628 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.628 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.628 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.628 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.628 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.628 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.628 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.628 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.629 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.629 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.629 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.629 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.629 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.629 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.629 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.629 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.629 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.629 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.629 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.629 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.629 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.629 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.630 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.630 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.630 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.630 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.630 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.630 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.630 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.630 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.630 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.630 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.630 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.630 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.630 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.630 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.630 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.631 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.631 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.631 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.631 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.631 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.631 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.631 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.631 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.631 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.631 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.631 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.631 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:52.631 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.631 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 6, 2024-09-14T03:47:52.632 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.632 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.632 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.632 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.632 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.632 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.632 INFO:tasks.workunit.client.0.smithi070.stdout: 5 2024-09-14T03:47:52.632 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.632 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 6, 2024-09-14T03:47:52.632 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.632 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.632 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.632 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.633 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.633 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.633 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.544408+0000", 2024-09-14T03:47:52.633 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.873714+0000", 2024-09-14T03:47:52.633 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.873714+0000", 2024-09-14T03:47:52.633 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.874477+0000", 2024-09-14T03:47:52.633 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.544408+0000", 2024-09-14T03:47:52.633 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.633 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.633 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.633 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.633 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.544408+0000", 2024-09-14T03:47:52.633 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.544408+0000", 2024-09-14T03:47:52.633 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.544408+0000", 2024-09-14T03:47:52.634 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.634 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.634 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.634 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.544408+0000", 2024-09-14T03:47:52.634 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.544408+0000", 2024-09-14T03:47:52.634 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.634 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.634 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.634 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.634 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.634 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.634 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.634 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.635 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.635 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.635 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.635 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.635 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.635 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.10", 2024-09-14T03:47:52.635 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.635 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.635 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.635 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:52.635 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.635 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:54:21.601768+0000", 2024-09-14T03:47:52.635 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.635 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.636 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.636 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.636 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.636 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.636 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.636 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.636 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.636 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.636 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.636 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.636 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.636 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.636 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.637 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.637 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.637 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.637 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.637 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.637 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.637 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.637 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.637 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.637 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.637 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.637 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.637 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.638 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.638 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.638 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.638 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.638 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.638 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.638 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.638 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.638 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.638 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.638 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.638 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.638 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.638 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.639 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.639 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.639 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.639 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.639 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.639 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.639 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.639 INFO:tasks.workunit.client.0.smithi070.stdout: 5 2024-09-14T03:47:52.639 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.639 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 6, 2024-09-14T03:47:52.639 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.639 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.639 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.639 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.640 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:52.640 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.640 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.640 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.640 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 2, 2024-09-14T03:47:52.640 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.640 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.640 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:52.640 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.640 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.640 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.640 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:15.135490+0000", 2024-09-14T03:47:52.640 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.726115+0000", 2024-09-14T03:47:52.640 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.726115+0000", 2024-09-14T03:47:52.641 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.726387+0000", 2024-09-14T03:47:52.641 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:15.135490+0000", 2024-09-14T03:47:52.641 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.641 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.641 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.641 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:52.641 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:15.135490+0000", 2024-09-14T03:47:52.641 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:15.135490+0000", 2024-09-14T03:47:52.641 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:15.135490+0000", 2024-09-14T03:47:52.641 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.641 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.641 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.641 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:15.135490+0000", 2024-09-14T03:47:52.641 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:15.135490+0000", 2024-09-14T03:47:52.642 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.642 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 8, 2024-09-14T03:47:52.642 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.642 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.642 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:52.642 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.642 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.643 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.643 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.644 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 8, 2024-09-14T03:47:52.644 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.644 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.644 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.644 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.10", 2024-09-14T03:47:52.644 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.644 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.644 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.644 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 31, 2024-09-14T03:47:52.644 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.644 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:18:04.176970+0000", 2024-09-14T03:47:52.644 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.644 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.645 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.645 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.645 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.645 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.645 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.645 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.645 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.645 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.645 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.645 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.645 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.645 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.645 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.645 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.646 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.646 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.646 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.646 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 12, 2024-09-14T03:47:52.646 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 4, 2024-09-14T03:47:52.646 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.646 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 4, 2024-09-14T03:47:52.646 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.646 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.646 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.646 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.646 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.646 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 4, 2024-09-14T03:47:52.646 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.646 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.647 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.647 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.647 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.647 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.647 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.647 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.647 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.647 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.647 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.647 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.647 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 12, 2024-09-14T03:47:52.647 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.647 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.647 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.648 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.648 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.648 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:52.648 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.648 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.648 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.648 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 2, 2024-09-14T03:47:52.648 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "58'8" 2024-09-14T03:47:52.648 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.648 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.648 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.648 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.648 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.648 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:52.649 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.649 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:52.649 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.649 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.649 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.649 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.649 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.649 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.649 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.660478+0000", 2024-09-14T03:47:52.649 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.878343+0000", 2024-09-14T03:47:52.649 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.878343+0000", 2024-09-14T03:47:52.649 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.878624+0000", 2024-09-14T03:47:52.649 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.660478+0000", 2024-09-14T03:47:52.649 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.650 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.650 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.650 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.650 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.660478+0000", 2024-09-14T03:47:52.650 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.660478+0000", 2024-09-14T03:47:52.650 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.660478+0000", 2024-09-14T03:47:52.650 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.650 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.650 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.650 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.660478+0000", 2024-09-14T03:47:52.650 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.660478+0000", 2024-09-14T03:47:52.650 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.650 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.651 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.651 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.651 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.651 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.651 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.651 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.651 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.651 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.651 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.651 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.651 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.651 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.11", 2024-09-14T03:47:52.651 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.651 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.652 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.652 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:52.652 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.652 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T10:44:05.205671+0000", 2024-09-14T03:47:52.652 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.652 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.652 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.652 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.652 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.652 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.652 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.652 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.652 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.652 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.653 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.653 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.653 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.653 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.653 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.653 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.653 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.653 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.653 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.653 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.653 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.653 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.653 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.653 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.654 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.654 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.654 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.654 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.654 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.654 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.654 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.654 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.654 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.654 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.654 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.654 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.654 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.654 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.655 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.655 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.655 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.655 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.655 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.655 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.655 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.655 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.655 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.655 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.655 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.655 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:52.655 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.655 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:52.656 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.656 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.656 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.656 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.656 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.656 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.656 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:52.656 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.656 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 4, 2024-09-14T03:47:52.656 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.656 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.656 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:52.656 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.656 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.657 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.657 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.458928+0000", 2024-09-14T03:47:52.657 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.728182+0000", 2024-09-14T03:47:52.657 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.728182+0000", 2024-09-14T03:47:52.657 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.728489+0000", 2024-09-14T03:47:52.657 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.458928+0000", 2024-09-14T03:47:52.657 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.657 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.657 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.657 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:52.657 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.458928+0000", 2024-09-14T03:47:52.657 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.458928+0000", 2024-09-14T03:47:52.658 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.458928+0000", 2024-09-14T03:47:52.658 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.658 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.658 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.658 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.458928+0000", 2024-09-14T03:47:52.658 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.458928+0000", 2024-09-14T03:47:52.658 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.658 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 1, 2024-09-14T03:47:52.658 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.658 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.658 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:52.658 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.658 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.658 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.659 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.659 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 1, 2024-09-14T03:47:52.659 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.659 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.659 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.659 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.f", 2024-09-14T03:47:52.659 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.659 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.659 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.659 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 23, 2024-09-14T03:47:52.659 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.659 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T09:15:56.310710+0000", 2024-09-14T03:47:52.659 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.659 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.660 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.660 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 22, 2024-09-14T03:47:52.660 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.660 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.660 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.660 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.660 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.660 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.660 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.660 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.660 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.660 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.660 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.660 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.661 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.661 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.661 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.661 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 3, 2024-09-14T03:47:52.661 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 1, 2024-09-14T03:47:52.661 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.661 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 1, 2024-09-14T03:47:52.661 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.661 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.661 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.661 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.661 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.661 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.661 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.662 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.662 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.662 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.662 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.662 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.662 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.662 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.662 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.662 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.662 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.662 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.662 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 1, 2024-09-14T03:47:52.662 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 1 2024-09-14T03:47:52.662 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.663 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.663 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.663 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.663 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.663 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.663 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:52.663 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.663 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 4, 2024-09-14T03:47:52.663 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "58'1" 2024-09-14T03:47:52.663 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.663 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.663 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.663 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.663 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.664 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:52.664 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.664 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:52.664 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.664 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.664 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.664 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.664 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.664 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.664 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.661108+0000", 2024-09-14T03:47:52.664 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.880483+0000", 2024-09-14T03:47:52.664 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.880483+0000", 2024-09-14T03:47:52.664 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.880754+0000", 2024-09-14T03:47:52.665 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.661108+0000", 2024-09-14T03:47:52.665 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.665 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.665 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.665 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.665 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.661108+0000", 2024-09-14T03:47:52.665 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.661108+0000", 2024-09-14T03:47:52.665 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.661108+0000", 2024-09-14T03:47:52.665 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.665 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.665 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.665 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.661108+0000", 2024-09-14T03:47:52.665 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.661108+0000", 2024-09-14T03:47:52.665 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.666 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.666 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.666 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.666 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.666 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.666 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.666 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.666 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.666 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.666 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.666 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.666 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.666 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.e", 2024-09-14T03:47:52.666 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.667 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.667 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.667 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:52.667 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.667 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:32:53.245007+0000", 2024-09-14T03:47:52.667 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.667 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.667 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.667 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.667 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.667 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.667 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.667 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.668 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.668 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.668 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.668 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.668 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.668 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.668 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.668 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.668 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.668 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.668 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.668 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.668 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.668 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.669 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.669 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.669 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.669 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.669 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.669 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.669 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.669 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.669 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.669 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.669 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.669 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.669 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.669 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.670 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.670 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.670 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.670 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.670 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.670 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.670 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.670 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.670 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.670 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.670 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.670 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.670 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.670 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:52.671 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.671 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:52.671 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.671 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.671 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.671 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.671 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:52.671 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.671 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:52.671 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.671 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 2, 2024-09-14T03:47:52.671 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.671 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.671 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:52.672 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.672 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.672 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.672 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:15.135841+0000", 2024-09-14T03:47:52.672 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.726090+0000", 2024-09-14T03:47:52.672 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.726090+0000", 2024-09-14T03:47:52.672 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.726366+0000", 2024-09-14T03:47:52.672 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:15.135841+0000", 2024-09-14T03:47:52.672 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.672 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.672 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.672 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:52.672 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:15.135841+0000", 2024-09-14T03:47:52.672 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:15.135841+0000", 2024-09-14T03:47:52.673 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:15.135841+0000", 2024-09-14T03:47:52.673 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.673 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.673 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.673 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:15.135841+0000", 2024-09-14T03:47:52.673 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:15.135841+0000", 2024-09-14T03:47:52.673 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.673 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 2, 2024-09-14T03:47:52.673 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.673 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.673 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:52.673 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.673 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.673 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.674 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.674 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 2, 2024-09-14T03:47:52.674 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.674 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.674 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.674 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.e", 2024-09-14T03:47:52.674 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.674 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.674 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.674 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 25, 2024-09-14T03:47:52.674 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.674 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:45:33.572820+0000", 2024-09-14T03:47:52.674 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.674 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.675 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.675 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.675 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.675 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.675 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.675 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.675 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.675 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.675 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.675 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.675 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.675 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.675 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.675 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.676 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.676 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.676 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.676 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 3, 2024-09-14T03:47:52.676 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 1, 2024-09-14T03:47:52.676 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.676 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 1, 2024-09-14T03:47:52.676 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.676 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.676 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.676 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.676 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.676 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 1, 2024-09-14T03:47:52.676 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.677 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.677 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.677 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.677 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.677 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.677 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.677 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.677 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.677 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.677 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.677 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.677 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 3, 2024-09-14T03:47:52.677 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.678 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.678 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.678 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.678 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.678 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:52.678 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.678 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:52.678 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.678 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 2, 2024-09-14T03:47:52.678 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "58'2" 2024-09-14T03:47:52.678 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.678 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.679 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.679 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.679 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.679 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.679 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.679 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:52.679 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.679 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.679 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.679 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.679 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.679 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.679 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.661089+0000", 2024-09-14T03:47:52.679 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.883448+0000", 2024-09-14T03:47:52.680 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.883448+0000", 2024-09-14T03:47:52.680 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.883704+0000", 2024-09-14T03:47:52.680 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.661089+0000", 2024-09-14T03:47:52.680 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.680 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.680 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.680 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.680 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.661089+0000", 2024-09-14T03:47:52.680 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.661089+0000", 2024-09-14T03:47:52.680 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.661089+0000", 2024-09-14T03:47:52.680 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.680 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.681 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.681 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.661089+0000", 2024-09-14T03:47:52.681 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.661089+0000", 2024-09-14T03:47:52.681 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.681 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.681 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.681 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.681 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.681 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.681 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.681 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.681 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.681 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.682 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.682 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.682 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.682 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.f", 2024-09-14T03:47:52.682 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.682 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.682 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.682 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:52.682 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.682 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:16:27.946770+0000", 2024-09-14T03:47:52.682 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.682 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.682 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.683 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.683 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.683 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.683 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.683 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.683 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.683 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.683 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.683 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.683 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.683 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.683 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.683 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.683 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.684 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.684 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.684 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.684 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.684 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.684 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.684 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.684 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.684 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.684 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.684 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.684 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.685 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.685 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.685 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.685 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.685 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.685 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.685 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.685 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.685 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.685 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.685 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.685 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.685 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.685 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.686 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.686 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.686 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.686 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.686 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.686 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.686 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.686 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.686 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:52.686 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.686 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.686 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.686 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.686 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.687 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.687 INFO:tasks.workunit.client.0.smithi070.stdout: 3 2024-09-14T03:47:52.687 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.687 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 1, 2024-09-14T03:47:52.687 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.687 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.687 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:52.687 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.687 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.687 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.687 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.458573+0000", 2024-09-14T03:47:52.687 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.732929+0000", 2024-09-14T03:47:52.687 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.732929+0000", 2024-09-14T03:47:52.687 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.733205+0000", 2024-09-14T03:47:52.687 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.458573+0000", 2024-09-14T03:47:52.688 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.688 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.688 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.688 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:52.688 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.458573+0000", 2024-09-14T03:47:52.688 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.458573+0000", 2024-09-14T03:47:52.688 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.458573+0000", 2024-09-14T03:47:52.688 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.688 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.688 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.688 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.458573+0000", 2024-09-14T03:47:52.688 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.458573+0000", 2024-09-14T03:47:52.688 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.689 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 1, 2024-09-14T03:47:52.689 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.689 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.689 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:52.689 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.689 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.689 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.689 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.689 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 1, 2024-09-14T03:47:52.689 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.689 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.689 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.689 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.d", 2024-09-14T03:47:52.690 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.690 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.690 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.690 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 24, 2024-09-14T03:47:52.690 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.690 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T15:31:13.660965+0000", 2024-09-14T03:47:52.690 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.690 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.690 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.690 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 70, 2024-09-14T03:47:52.690 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.690 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.690 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.690 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.691 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.691 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.691 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.691 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.691 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.691 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.691 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.691 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.691 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.691 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.691 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.691 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 3, 2024-09-14T03:47:52.691 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 1, 2024-09-14T03:47:52.692 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.692 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 1, 2024-09-14T03:47:52.692 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.692 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.692 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.692 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.692 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.692 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.692 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.692 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.692 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.692 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.692 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.693 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.693 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.693 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.693 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.693 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.693 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.693 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.693 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 1, 2024-09-14T03:47:52.693 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 1 2024-09-14T03:47:52.693 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.693 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.693 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.693 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.694 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.694 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.694 INFO:tasks.workunit.client.0.smithi070.stdout: 3 2024-09-14T03:47:52.694 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.694 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 1, 2024-09-14T03:47:52.694 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "58'1" 2024-09-14T03:47:52.694 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.694 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.694 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.694 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.694 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.694 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:52.694 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.694 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 5, 2024-09-14T03:47:52.695 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.695 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.695 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.695 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.695 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.695 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.695 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.461995+0000", 2024-09-14T03:47:52.695 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.884747+0000", 2024-09-14T03:47:52.695 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.884747+0000", 2024-09-14T03:47:52.695 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.886158+0000", 2024-09-14T03:47:52.695 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.461995+0000", 2024-09-14T03:47:52.695 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.695 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.695 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.696 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.696 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.461995+0000", 2024-09-14T03:47:52.696 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.461995+0000", 2024-09-14T03:47:52.696 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.461995+0000", 2024-09-14T03:47:52.696 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.696 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.696 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.696 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.461995+0000", 2024-09-14T03:47:52.696 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.461995+0000", 2024-09-14T03:47:52.696 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.696 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.696 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.696 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.697 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.697 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.697 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.697 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.697 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.697 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.697 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.697 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.697 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.697 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.c", 2024-09-14T03:47:52.697 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.697 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.697 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.698 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:52.698 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.698 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:49:56.265783+0000", 2024-09-14T03:47:52.698 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.698 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.698 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.698 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.698 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.698 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.698 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.698 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.698 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.698 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.699 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.699 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.699 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.699 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.699 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.699 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.699 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.699 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.699 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.699 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.699 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.699 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.699 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.699 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.700 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.700 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.700 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.700 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.700 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.700 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.700 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.700 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.700 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.700 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.700 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.700 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.700 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.700 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.701 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.701 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.701 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.701 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.701 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.701 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.701 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.701 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.701 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.701 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.701 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.701 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:52.701 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.701 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 5, 2024-09-14T03:47:52.702 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.702 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.702 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.702 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.702 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:52.702 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.702 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.702 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.702 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 2, 2024-09-14T03:47:52.702 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.702 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.702 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:52.702 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.702 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.702 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.703 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:15.135633+0000", 2024-09-14T03:47:52.703 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.732067+0000", 2024-09-14T03:47:52.703 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.732067+0000", 2024-09-14T03:47:52.703 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.732401+0000", 2024-09-14T03:47:52.703 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:15.135633+0000", 2024-09-14T03:47:52.703 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.703 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.703 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.703 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:52.703 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:15.135633+0000", 2024-09-14T03:47:52.703 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:15.135633+0000", 2024-09-14T03:47:52.703 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:15.135633+0000", 2024-09-14T03:47:52.703 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.703 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.704 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.704 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:15.135633+0000", 2024-09-14T03:47:52.704 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:15.135633+0000", 2024-09-14T03:47:52.704 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.704 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.704 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.704 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.704 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:52.704 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.704 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.704 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.704 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.704 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.704 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.705 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.705 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.705 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.c", 2024-09-14T03:47:52.705 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.705 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.705 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.705 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 23, 2024-09-14T03:47:52.705 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.705 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T09:34:00.566245+0000", 2024-09-14T03:47:52.705 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.705 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.705 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.705 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.705 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.706 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.706 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.706 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.706 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.706 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.706 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.706 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.706 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.706 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.706 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.706 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.706 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.706 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.706 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.707 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.707 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.707 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.707 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.707 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.707 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.707 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.707 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.707 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.707 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.707 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.707 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.707 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.707 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.708 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.708 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.708 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.708 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.708 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.708 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.708 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.708 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.708 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.708 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.708 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.708 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.708 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.708 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.709 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:52.709 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.709 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.709 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.709 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 2, 2024-09-14T03:47:52.709 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.709 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.709 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.709 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.709 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.709 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.709 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:52.709 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.709 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:52.710 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.710 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.710 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.710 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.710 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.710 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.710 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.660858+0000", 2024-09-14T03:47:52.710 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.881219+0000", 2024-09-14T03:47:52.710 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.881219+0000", 2024-09-14T03:47:52.710 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.881736+0000", 2024-09-14T03:47:52.710 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.660858+0000", 2024-09-14T03:47:52.710 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.710 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.710 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.711 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.711 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.660858+0000", 2024-09-14T03:47:52.711 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.660858+0000", 2024-09-14T03:47:52.711 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.660858+0000", 2024-09-14T03:47:52.711 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.711 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.711 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.711 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.660858+0000", 2024-09-14T03:47:52.711 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.660858+0000", 2024-09-14T03:47:52.711 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.711 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.711 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.711 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.711 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.712 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.712 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.712 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.712 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.712 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.712 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.712 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.712 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.712 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.d", 2024-09-14T03:47:52.712 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.712 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.712 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.712 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:52.713 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.713 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T09:24:23.342066+0000", 2024-09-14T03:47:52.713 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.713 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.713 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.713 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.713 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.713 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.713 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.713 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.713 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.713 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.713 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.713 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.714 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.714 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.714 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.714 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.714 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.714 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.714 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.714 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.714 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.714 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.714 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.714 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.714 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.715 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.715 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.715 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.715 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.715 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.715 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.715 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.715 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.715 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.715 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.715 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.715 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.715 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.715 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.716 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.716 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.716 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.716 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.716 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.716 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.716 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.716 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.716 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.716 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.716 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:52.716 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.716 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:52.717 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.717 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.717 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.717 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.717 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.717 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.717 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:52.717 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.717 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 6, 2024-09-14T03:47:52.717 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.717 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.717 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 62, 2024-09-14T03:47:52.717 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.717 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.718 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.718 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:20.503002+0000", 2024-09-14T03:47:52.718 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:45:09.755484+0000", 2024-09-14T03:47:52.718 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:45:09.755484+0000", 2024-09-14T03:47:52.718 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:45:14.545163+0000", 2024-09-14T03:47:52.718 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:20.503002+0000", 2024-09-14T03:47:52.718 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:52.718 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.718 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:52.718 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 63, 2024-09-14T03:47:52.718 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:20.503002+0000", 2024-09-14T03:47:52.718 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:20.503002+0000", 2024-09-14T03:47:52.718 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:20.503002+0000", 2024-09-14T03:47:52.719 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.719 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.719 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:52.719 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:20.503002+0000", 2024-09-14T03:47:52.719 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:20.503002+0000", 2024-09-14T03:47:52.719 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.719 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.719 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.719 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.719 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 62, 2024-09-14T03:47:52.719 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.719 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.719 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.719 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.719 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.720 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.720 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.720 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.720 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "4.5", 2024-09-14T03:47:52.720 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.720 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.720 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 68, 2024-09-14T03:47:52.720 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 19, 2024-09-14T03:47:52.720 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.720 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:18:49.100636+0000", 2024-09-14T03:47:52.720 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.000393397, 2024-09-14T03:47:52.720 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.720 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.721 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.721 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.721 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.721 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.721 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.721 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.721 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.721 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.721 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.721 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.721 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.721 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.721 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.721 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.722 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.722 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.722 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.722 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.722 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.722 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.722 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.722 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.722 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.722 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.722 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.722 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.722 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.722 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.723 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.723 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.723 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.723 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.723 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.723 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.723 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.723 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.723 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.723 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.723 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.723 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.723 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.723 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.724 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.724 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.724 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.724 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.724 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:52.724 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.724 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 6, 2024-09-14T03:47:52.724 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.724 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.724 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.724 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.724 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.724 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.725 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:52.725 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.725 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:52.725 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.725 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.725 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 20, 2024-09-14T03:47:52.725 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.725 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.725 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.725 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:47:47.131642+0000", 2024-09-14T03:47:52.725 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:36.141953+0000", 2024-09-14T03:47:52.725 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:36.141953+0000", 2024-09-14T03:47:52.725 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:36.147227+0000", 2024-09-14T03:47:52.726 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:47:47.131642+0000", 2024-09-14T03:47:52.726 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:41:39.458815+0000", 2024-09-14T03:47:52.726 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.726 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:41:39.458815+0000", 2024-09-14T03:47:52.726 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 51, 2024-09-14T03:47:52.726 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:47:47.131642+0000", 2024-09-14T03:47:52.726 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:47:47.131642+0000", 2024-09-14T03:47:52.726 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:47:47.131642+0000", 2024-09-14T03:47:52.726 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.726 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.726 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:41:39.458815+0000", 2024-09-14T03:47:52.726 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:47:47.131642+0000", 2024-09-14T03:47:52.726 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:47:47.131642+0000", 2024-09-14T03:47:52.726 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.727 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 95, 2024-09-14T03:47:52.727 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.727 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.727 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 50, 2024-09-14T03:47:52.727 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.727 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.727 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.727 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.727 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 95, 2024-09-14T03:47:52.727 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.727 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.727 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.727 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "1.0", 2024-09-14T03:47:52.727 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.728 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.728 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 68, 2024-09-14T03:47:52.728 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 116, 2024-09-14T03:47:52.728 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.728 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T05:38:43.056328+0000", 2024-09-14T03:47:52.728 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.728 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.728 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.728 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 590368, 2024-09-14T03:47:52.728 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.728 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 590368, 2024-09-14T03:47:52.728 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.728 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.728 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.729 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.729 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.729 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.729 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.729 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.729 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.729 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.729 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.729 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.729 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.729 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 6, 2024-09-14T03:47:52.729 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 2, 2024-09-14T03:47:52.729 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.730 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 2, 2024-09-14T03:47:52.730 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.730 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.730 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.730 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.730 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.730 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.730 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.730 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 2, 2024-09-14T03:47:52.730 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.730 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.730 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.730 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.730 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.731 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 272, 2024-09-14T03:47:52.731 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 479, 2024-09-14T03:47:52.731 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.731 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.731 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.731 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 145, 2024-09-14T03:47:52.731 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 1628 2024-09-14T03:47:52.731 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.731 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.731 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.731 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.731 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.731 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.731 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:52.732 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.732 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:52.732 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "68'95" 2024-09-14T03:47:52.732 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.732 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.732 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.732 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.732 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:52.732 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:52.732 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.732 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 5, 2024-09-14T03:47:52.732 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.732 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.732 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:52.733 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.733 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.733 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.733 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.461867+0000", 2024-09-14T03:47:52.733 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.733256+0000", 2024-09-14T03:47:52.733 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.733256+0000", 2024-09-14T03:47:52.733 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.733557+0000", 2024-09-14T03:47:52.733 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.461867+0000", 2024-09-14T03:47:52.733 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.733 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.733 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.733 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:52.733 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.461867+0000", 2024-09-14T03:47:52.733 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.461867+0000", 2024-09-14T03:47:52.734 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.461867+0000", 2024-09-14T03:47:52.734 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.734 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.734 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.734 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.461867+0000", 2024-09-14T03:47:52.734 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.461867+0000", 2024-09-14T03:47:52.734 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.734 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.734 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.734 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.734 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:52.734 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.735 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.735 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.735 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.735 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.735 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.735 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.735 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.735 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.3", 2024-09-14T03:47:52.735 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.735 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.735 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.735 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 23, 2024-09-14T03:47:52.735 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.736 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T10:09:12.987156+0000", 2024-09-14T03:47:52.736 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.736 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.736 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.736 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.736 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.736 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.736 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.736 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.736 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.736 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.736 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.736 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.736 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.737 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.737 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.737 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.737 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.737 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.737 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.737 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.737 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.737 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.737 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.737 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.737 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.737 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.737 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.738 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.738 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.738 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.738 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.738 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.738 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.738 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.738 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.738 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.738 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.738 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.738 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.738 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.738 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.739 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.739 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.739 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.739 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.739 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.739 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.739 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.739 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:52.739 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:52.739 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.739 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 5, 2024-09-14T03:47:52.739 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.739 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.739 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.740 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.740 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.740 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.740 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:52.740 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.740 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 3, 2024-09-14T03:47:52.740 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.740 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.740 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.740 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.740 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.740 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.740 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.857593+0000", 2024-09-14T03:47:52.741 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.875241+0000", 2024-09-14T03:47:52.741 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.875241+0000", 2024-09-14T03:47:52.741 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.875525+0000", 2024-09-14T03:47:52.741 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.857593+0000", 2024-09-14T03:47:52.741 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.741 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.741 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.741 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.741 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.857593+0000", 2024-09-14T03:47:52.741 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.857593+0000", 2024-09-14T03:47:52.741 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.857593+0000", 2024-09-14T03:47:52.741 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.741 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.742 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.742 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.857593+0000", 2024-09-14T03:47:52.742 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.857593+0000", 2024-09-14T03:47:52.742 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.742 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.742 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.742 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.742 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.742 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.742 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.742 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.742 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.742 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.742 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.743 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.743 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.743 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.2", 2024-09-14T03:47:52.743 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.743 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.743 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.743 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:52.743 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.743 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:34:47.346963+0000", 2024-09-14T03:47:52.743 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.743 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.743 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.743 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.744 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.744 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.744 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.744 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.744 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.744 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.744 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.744 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.744 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.744 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.744 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.744 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.744 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.744 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.745 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.745 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.745 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.745 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.745 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.745 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.745 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.745 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.745 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.745 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.745 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.745 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.745 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.746 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.746 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.746 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.746 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.746 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.746 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.746 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.746 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.746 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.746 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.746 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.746 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.746 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.747 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.747 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.747 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.747 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.747 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.747 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:52.747 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.747 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 3, 2024-09-14T03:47:52.747 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.747 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.747 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.747 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.747 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.747 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.747 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:52.748 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.748 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 0, 2024-09-14T03:47:52.748 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.748 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.748 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 62, 2024-09-14T03:47:52.748 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.748 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.748 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.748 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:20.503141+0000", 2024-09-14T03:47:52.748 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:45:09.711005+0000", 2024-09-14T03:47:52.748 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:45:09.711005+0000", 2024-09-14T03:47:52.748 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:45:14.684739+0000", 2024-09-14T03:47:52.749 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:20.503141+0000", 2024-09-14T03:47:52.749 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:52.749 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.749 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:52.749 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 63, 2024-09-14T03:47:52.749 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:20.503141+0000", 2024-09-14T03:47:52.749 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:20.503141+0000", 2024-09-14T03:47:52.749 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:20.503141+0000", 2024-09-14T03:47:52.749 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.749 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.749 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:52.749 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:20.503141+0000", 2024-09-14T03:47:52.749 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:20.503141+0000", 2024-09-14T03:47:52.750 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.750 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.750 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.750 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.750 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 62, 2024-09-14T03:47:52.750 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.750 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.750 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.750 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.750 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.750 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.750 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.750 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.750 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "4.6", 2024-09-14T03:47:52.751 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.751 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.751 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 68, 2024-09-14T03:47:52.751 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 19, 2024-09-14T03:47:52.751 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.751 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T09:20:35.312720+0000", 2024-09-14T03:47:52.751 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.00019552, 2024-09-14T03:47:52.751 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.751 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.751 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.751 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.751 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.751 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.751 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.752 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.752 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.752 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.752 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.752 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.752 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.752 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.752 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.752 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.752 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.752 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.752 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.752 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.753 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.753 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.753 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.753 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.753 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.753 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.753 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.753 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.753 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.753 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.753 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.753 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.753 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.754 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.754 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.754 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.754 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.754 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.754 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.754 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.754 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.754 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.754 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.754 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.754 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.754 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.755 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.755 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.755 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:52.755 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.755 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 0, 2024-09-14T03:47:52.755 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.755 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.755 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.755 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.755 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.755 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.755 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.755 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.755 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:52.756 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.756 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.756 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:52.756 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.756 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.756 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.756 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.660472+0000", 2024-09-14T03:47:52.756 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.731812+0000", 2024-09-14T03:47:52.756 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.731812+0000", 2024-09-14T03:47:52.756 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.732376+0000", 2024-09-14T03:47:52.756 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.660472+0000", 2024-09-14T03:47:52.756 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.756 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.757 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.757 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:52.757 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.660472+0000", 2024-09-14T03:47:52.757 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.660472+0000", 2024-09-14T03:47:52.757 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.660472+0000", 2024-09-14T03:47:52.757 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.757 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.757 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.757 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.660472+0000", 2024-09-14T03:47:52.757 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.660472+0000", 2024-09-14T03:47:52.757 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.757 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.757 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.758 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.758 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:52.758 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.758 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.758 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.758 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.758 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.758 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.758 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.758 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.758 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.0", 2024-09-14T03:47:52.758 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.758 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.758 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.759 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 24, 2024-09-14T03:47:52.759 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.759 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:35:26.262383+0000", 2024-09-14T03:47:52.759 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.759 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.759 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.759 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.759 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.759 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.759 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.759 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.759 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.759 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.759 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.760 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.760 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.760 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.760 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.760 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.760 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.760 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.760 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.760 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.760 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.760 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.760 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.760 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.761 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.761 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.761 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.761 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.761 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.761 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.761 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.761 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.761 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.761 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.761 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.761 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.761 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.761 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.762 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.762 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.762 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.762 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.762 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.762 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.762 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.762 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.762 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.762 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.762 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.762 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.762 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.762 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:52.763 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.763 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.763 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.763 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.763 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.763 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.763 INFO:tasks.workunit.client.0.smithi070.stdout: 3 2024-09-14T03:47:52.763 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.763 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 0, 2024-09-14T03:47:52.763 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.763 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.763 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.763 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.763 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.764 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.764 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.683969+0000", 2024-09-14T03:47:52.764 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.876225+0000", 2024-09-14T03:47:52.764 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.876225+0000", 2024-09-14T03:47:52.764 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.876511+0000", 2024-09-14T03:47:52.764 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.683969+0000", 2024-09-14T03:47:52.764 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.764 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.764 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.764 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.764 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.683969+0000", 2024-09-14T03:47:52.764 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.683969+0000", 2024-09-14T03:47:52.764 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.683969+0000", 2024-09-14T03:47:52.764 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.765 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.765 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.765 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.683969+0000", 2024-09-14T03:47:52.765 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.683969+0000", 2024-09-14T03:47:52.765 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.765 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.765 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.765 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.765 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.765 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.765 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.765 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.765 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.766 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.766 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.766 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.766 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.766 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.1", 2024-09-14T03:47:52.766 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.766 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.766 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.766 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:52.766 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.766 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T10:36:40.283622+0000", 2024-09-14T03:47:52.766 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.766 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.767 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.767 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.767 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.767 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.767 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.767 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.767 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.767 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.767 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.767 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.767 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.767 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.767 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.768 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.768 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.768 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.768 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.768 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.768 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.768 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.768 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.768 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.768 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.768 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.768 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.769 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.769 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.769 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.769 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.769 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.769 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.769 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.769 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.769 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.769 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.769 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.769 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.769 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.770 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.770 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.770 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.770 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.770 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.770 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.770 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.770 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.770 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.770 INFO:tasks.workunit.client.0.smithi070.stdout: 3 2024-09-14T03:47:52.770 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.770 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 0, 2024-09-14T03:47:52.770 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.771 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.771 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.771 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.771 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.771 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.771 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.771 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.771 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 1, 2024-09-14T03:47:52.771 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.771 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.771 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 62, 2024-09-14T03:47:52.771 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.771 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.771 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.772 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:20.464110+0000", 2024-09-14T03:47:52.772 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:45:09.879190+0000", 2024-09-14T03:47:52.772 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:45:09.879190+0000", 2024-09-14T03:47:52.772 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:45:14.458990+0000", 2024-09-14T03:47:52.772 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:20.464110+0000", 2024-09-14T03:47:52.772 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:52.772 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.772 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:52.772 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 63, 2024-09-14T03:47:52.772 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:20.464110+0000", 2024-09-14T03:47:52.772 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:20.464110+0000", 2024-09-14T03:47:52.772 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:20.464110+0000", 2024-09-14T03:47:52.772 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.772 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.773 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:52.773 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:20.464110+0000", 2024-09-14T03:47:52.773 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:20.464110+0000", 2024-09-14T03:47:52.773 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.773 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.773 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.773 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.773 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 62, 2024-09-14T03:47:52.773 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.773 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.773 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.773 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.773 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.774 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.774 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.774 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.774 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "4.7", 2024-09-14T03:47:52.774 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.774 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.774 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 68, 2024-09-14T03:47:52.774 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 19, 2024-09-14T03:47:52.774 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.774 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T07:23:17.328924+0000", 2024-09-14T03:47:52.774 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.000571641, 2024-09-14T03:47:52.774 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.775 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.775 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.775 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.775 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.775 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.775 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.775 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.775 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.775 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.775 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.775 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.775 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.775 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.776 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.776 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.776 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.776 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.776 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.776 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.776 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.776 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.776 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.776 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.776 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.776 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.777 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.777 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.777 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.777 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.777 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.777 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.777 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.777 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.777 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.777 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.777 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.777 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.777 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.777 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.778 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.778 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.778 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.778 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.778 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.778 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.778 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.778 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.778 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.778 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.778 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 1, 2024-09-14T03:47:52.778 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.778 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.779 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.779 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.779 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:52.779 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.779 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.779 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.779 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 2, 2024-09-14T03:47:52.779 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.779 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.779 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:52.779 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.779 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.779 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.780 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:15.135684+0000", 2024-09-14T03:47:52.780 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.726879+0000", 2024-09-14T03:47:52.780 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.726879+0000", 2024-09-14T03:47:52.780 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.727129+0000", 2024-09-14T03:47:52.780 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:15.135684+0000", 2024-09-14T03:47:52.780 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.780 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.780 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.780 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:52.780 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:15.135684+0000", 2024-09-14T03:47:52.780 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:15.135684+0000", 2024-09-14T03:47:52.780 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:15.135684+0000", 2024-09-14T03:47:52.780 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.781 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.781 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.781 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:15.135684+0000", 2024-09-14T03:47:52.781 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:15.135684+0000", 2024-09-14T03:47:52.781 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.781 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.781 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.781 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.781 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:52.781 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.781 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.781 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.782 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.782 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.782 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.782 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.782 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.782 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.1", 2024-09-14T03:47:52.782 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.782 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.782 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.782 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 23, 2024-09-14T03:47:52.782 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.782 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T09:54:41.748651+0000", 2024-09-14T03:47:52.782 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.782 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.783 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.783 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.783 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.783 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.783 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.783 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.783 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.783 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.783 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.783 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.783 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.783 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.783 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.784 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.784 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.784 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.784 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.784 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.784 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.784 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.784 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.784 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.784 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.784 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.784 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.785 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.785 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.785 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.785 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.785 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.785 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.785 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.785 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.785 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.785 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.785 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.785 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.785 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.786 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.786 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.786 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.786 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.786 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.786 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.786 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.786 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:52.786 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.786 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.786 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.786 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 2, 2024-09-14T03:47:52.786 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.786 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.787 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.787 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.787 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.787 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:52.787 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:52.787 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.787 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 1, 2024-09-14T03:47:52.787 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.787 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.787 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.787 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.787 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.787 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.787 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.457983+0000", 2024-09-14T03:47:52.788 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.870707+0000", 2024-09-14T03:47:52.788 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.870707+0000", 2024-09-14T03:47:52.788 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.871005+0000", 2024-09-14T03:47:52.788 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.457983+0000", 2024-09-14T03:47:52.788 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.788 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.788 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.788 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.788 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.457983+0000", 2024-09-14T03:47:52.788 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.457983+0000", 2024-09-14T03:47:52.788 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.457983+0000", 2024-09-14T03:47:52.788 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.788 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.788 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.788 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.457983+0000", 2024-09-14T03:47:52.789 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.457983+0000", 2024-09-14T03:47:52.789 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.789 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.789 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.789 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.789 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.789 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.789 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.789 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.789 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.789 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.789 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.789 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.790 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.790 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.0", 2024-09-14T03:47:52.790 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.790 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.790 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.790 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:52.790 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.790 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:30:22.758581+0000", 2024-09-14T03:47:52.790 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.790 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.790 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.790 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.790 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.790 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.790 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.791 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.791 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.791 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.791 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.791 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.791 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.791 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.791 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.791 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.791 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.791 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.791 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.791 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.791 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.792 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.792 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.792 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.792 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.792 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.792 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.792 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.792 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.792 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.792 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.792 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.792 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.792 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.792 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.793 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.793 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.793 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.793 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.793 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.793 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.793 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.793 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.793 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.793 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.793 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.793 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.793 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.794 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:52.794 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:52.794 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.794 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 1, 2024-09-14T03:47:52.794 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.794 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.794 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.794 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.794 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.794 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:52.794 INFO:tasks.workunit.client.0.smithi070.stdout: 3 2024-09-14T03:47:52.794 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.794 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 1, 2024-09-14T03:47:52.795 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.795 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.795 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 62, 2024-09-14T03:47:52.795 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.795 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.795 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.795 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:20.464062+0000", 2024-09-14T03:47:52.795 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:45:09.711634+0000", 2024-09-14T03:47:52.795 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:45:09.711634+0000", 2024-09-14T03:47:52.795 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:45:14.458757+0000", 2024-09-14T03:47:52.795 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:20.464062+0000", 2024-09-14T03:47:52.795 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:52.795 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.795 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:52.796 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 63, 2024-09-14T03:47:52.796 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:20.464062+0000", 2024-09-14T03:47:52.796 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:20.464062+0000", 2024-09-14T03:47:52.796 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:20.464062+0000", 2024-09-14T03:47:52.796 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.796 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.796 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:52.796 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:20.464062+0000", 2024-09-14T03:47:52.796 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:20.464062+0000", 2024-09-14T03:47:52.796 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.796 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.796 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.796 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.797 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 62, 2024-09-14T03:47:52.797 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.797 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.797 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.797 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.797 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.797 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.797 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.797 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.797 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "4.4", 2024-09-14T03:47:52.797 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.797 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.797 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 68, 2024-09-14T03:47:52.798 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 19, 2024-09-14T03:47:52.798 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.798 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:42:58.495982+0000", 2024-09-14T03:47:52.798 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.000547481, 2024-09-14T03:47:52.798 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.798 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.798 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.798 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.798 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.798 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.798 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.798 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.798 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.798 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.798 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.799 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.799 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.799 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.799 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.799 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.799 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.799 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.799 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.799 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.799 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.799 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.799 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.799 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.799 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.800 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.800 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.800 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.800 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.800 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.800 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.800 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.800 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.800 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.800 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.800 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.800 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.800 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.801 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.801 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.801 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.801 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.801 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.801 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.801 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.801 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.801 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.801 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:52.801 INFO:tasks.workunit.client.0.smithi070.stdout: 3 2024-09-14T03:47:52.801 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.801 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 1, 2024-09-14T03:47:52.802 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.802 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.802 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.802 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.802 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.802 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.802 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:52.802 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.802 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 5, 2024-09-14T03:47:52.802 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.802 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.802 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:52.802 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.803 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.803 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.803 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.461874+0000", 2024-09-14T03:47:52.803 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.732753+0000", 2024-09-14T03:47:52.803 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.732753+0000", 2024-09-14T03:47:52.803 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.733038+0000", 2024-09-14T03:47:52.803 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.461874+0000", 2024-09-14T03:47:52.803 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.803 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.803 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.803 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:52.803 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.461874+0000", 2024-09-14T03:47:52.803 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.461874+0000", 2024-09-14T03:47:52.803 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.461874+0000", 2024-09-14T03:47:52.804 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.804 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.804 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.804 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.461874+0000", 2024-09-14T03:47:52.804 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.461874+0000", 2024-09-14T03:47:52.804 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.804 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.804 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.804 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.804 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:52.804 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.804 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.804 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.805 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.805 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.805 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.805 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.805 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.805 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.2", 2024-09-14T03:47:52.805 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.805 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.805 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.805 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 23, 2024-09-14T03:47:52.805 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.805 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:40:49.754211+0000", 2024-09-14T03:47:52.805 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.806 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.806 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.806 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.806 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.806 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.806 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.806 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.806 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.806 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.806 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.806 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.806 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.806 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.806 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.807 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.807 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.807 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.807 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.807 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.807 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.807 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.807 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.807 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.807 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.807 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.807 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.807 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.807 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.808 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.808 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.808 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.808 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.808 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.808 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.808 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.808 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.808 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.808 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.808 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.808 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.808 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.809 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.809 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.809 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.809 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.809 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.809 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.809 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.809 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:52.809 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.809 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 5, 2024-09-14T03:47:52.809 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.809 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.809 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.809 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.810 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.810 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.810 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:52.810 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.810 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 4, 2024-09-14T03:47:52.810 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.810 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.810 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.810 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.810 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.810 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.810 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.458431+0000", 2024-09-14T03:47:52.810 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:26.429935+0000", 2024-09-14T03:47:52.811 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:26.429935+0000", 2024-09-14T03:47:52.811 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:26.430320+0000", 2024-09-14T03:47:52.811 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.458431+0000", 2024-09-14T03:47:52.811 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.811 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.811 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.811 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 60, 2024-09-14T03:47:52.811 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.458431+0000", 2024-09-14T03:47:52.811 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.458431+0000", 2024-09-14T03:47:52.811 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.458431+0000", 2024-09-14T03:47:52.811 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.811 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.811 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.812 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.458431+0000", 2024-09-14T03:47:52.812 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.458431+0000", 2024-09-14T03:47:52.812 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.812 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.812 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.812 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.812 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 59, 2024-09-14T03:47:52.812 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.812 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.812 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.812 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.812 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.812 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.813 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.813 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.813 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.3", 2024-09-14T03:47:52.813 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.813 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.813 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.813 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 30, 2024-09-14T03:47:52.813 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.813 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T15:12:35.577538+0000", 2024-09-14T03:47:52.813 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.813 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.813 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.813 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.813 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.814 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.814 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.814 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.814 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.814 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.814 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.814 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.814 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.814 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.814 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.814 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.814 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.814 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.815 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.815 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.815 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.815 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.815 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.815 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.815 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.815 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.815 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.815 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.815 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.815 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.815 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.815 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.816 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.816 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.816 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.816 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.816 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.816 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.816 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.816 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.816 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.816 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.816 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.816 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.816 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.817 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.817 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.817 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.817 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.817 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:52.817 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.817 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 4, 2024-09-14T03:47:52.817 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.817 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.817 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.817 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.817 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.817 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.817 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:52.818 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.818 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 1, 2024-09-14T03:47:52.818 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.818 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.818 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 62, 2024-09-14T03:47:52.818 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.818 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.818 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.818 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:20.464076+0000", 2024-09-14T03:47:52.818 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:45:09.879248+0000", 2024-09-14T03:47:52.818 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:45:09.879248+0000", 2024-09-14T03:47:52.818 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:45:14.458846+0000", 2024-09-14T03:47:52.818 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:20.464076+0000", 2024-09-14T03:47:52.818 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:52.819 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.819 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:52.819 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 63, 2024-09-14T03:47:52.819 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:20.464076+0000", 2024-09-14T03:47:52.819 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:20.464076+0000", 2024-09-14T03:47:52.819 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:20.464076+0000", 2024-09-14T03:47:52.819 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.819 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.819 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:52.819 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:20.464076+0000", 2024-09-14T03:47:52.819 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:20.464076+0000", 2024-09-14T03:47:52.819 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.820 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 2, 2024-09-14T03:47:52.820 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.820 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.820 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 62, 2024-09-14T03:47:52.820 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.820 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.820 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.820 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.820 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 2, 2024-09-14T03:47:52.820 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.820 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.820 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.820 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "4.2", 2024-09-14T03:47:52.820 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.821 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.821 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 68, 2024-09-14T03:47:52.821 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:52.821 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.821 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:02:23.634325+0000", 2024-09-14T03:47:52.821 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.000682786, 2024-09-14T03:47:52.821 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.821 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.821 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 19, 2024-09-14T03:47:52.821 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.821 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.821 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.821 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.822 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.822 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.822 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.822 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.822 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.822 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.822 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.822 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.822 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.822 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.822 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.822 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 3, 2024-09-14T03:47:52.822 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 1, 2024-09-14T03:47:52.822 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.823 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 1, 2024-09-14T03:47:52.823 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.823 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.823 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.823 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.823 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.823 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.823 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.823 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.823 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.823 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.823 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.823 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.824 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.824 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.824 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.824 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.824 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.824 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.824 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 2, 2024-09-14T03:47:52.824 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 2 2024-09-14T03:47:52.824 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.824 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.824 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.824 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.824 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.824 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.825 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:52.825 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.825 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 1, 2024-09-14T03:47:52.825 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "67'2" 2024-09-14T03:47:52.825 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.825 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.825 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.825 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.825 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.825 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:52.825 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.825 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 1, 2024-09-14T03:47:52.825 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.826 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.826 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:52.826 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.826 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.826 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.826 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.458338+0000", 2024-09-14T03:47:52.826 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.733357+0000", 2024-09-14T03:47:52.826 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.733357+0000", 2024-09-14T03:47:52.826 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.733621+0000", 2024-09-14T03:47:52.826 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.458338+0000", 2024-09-14T03:47:52.826 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.826 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.826 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.826 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:52.827 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.458338+0000", 2024-09-14T03:47:52.827 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.458338+0000", 2024-09-14T03:47:52.827 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.458338+0000", 2024-09-14T03:47:52.827 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.827 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.827 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.827 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.458338+0000", 2024-09-14T03:47:52.827 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.458338+0000", 2024-09-14T03:47:52.827 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.827 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.827 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.827 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.827 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:52.828 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.828 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.828 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.828 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.828 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.828 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.828 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.828 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.828 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.4", 2024-09-14T03:47:52.828 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.828 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.828 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.828 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 23, 2024-09-14T03:47:52.828 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.829 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T05:25:53.791926+0000", 2024-09-14T03:47:52.829 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.829 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.829 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.829 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.829 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.829 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.829 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.829 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.829 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.829 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.829 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.829 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.829 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.830 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.830 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.830 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.830 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.830 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.830 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.830 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.830 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.830 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.830 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.830 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.830 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.830 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.830 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.831 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.831 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.831 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.831 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.831 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.831 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.831 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.831 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.831 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.831 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.831 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.831 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.831 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.831 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.832 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.832 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.832 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.832 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.832 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.832 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.832 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.832 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.832 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:52.832 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.832 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 1, 2024-09-14T03:47:52.832 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.832 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.832 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.833 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.833 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.833 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.833 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:52.833 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.833 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 5, 2024-09-14T03:47:52.833 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.833 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.833 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.833 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.833 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.833 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.833 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.462075+0000", 2024-09-14T03:47:52.834 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.879925+0000", 2024-09-14T03:47:52.834 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.879925+0000", 2024-09-14T03:47:52.834 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.880200+0000", 2024-09-14T03:47:52.834 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.462075+0000", 2024-09-14T03:47:52.834 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.834 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.834 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.834 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.834 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.462075+0000", 2024-09-14T03:47:52.834 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.462075+0000", 2024-09-14T03:47:52.834 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.462075+0000", 2024-09-14T03:47:52.834 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.834 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.834 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.834 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.462075+0000", 2024-09-14T03:47:52.835 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.462075+0000", 2024-09-14T03:47:52.835 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.835 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.835 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.835 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.835 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.835 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.835 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.835 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.835 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.835 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.835 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.835 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.836 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.836 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.5", 2024-09-14T03:47:52.836 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.836 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.836 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.836 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:52.836 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.836 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T10:55:24.738215+0000", 2024-09-14T03:47:52.836 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.836 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.836 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.836 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.836 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.837 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.837 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.837 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.837 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.837 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.837 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.837 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.837 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.837 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.837 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.837 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.837 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.837 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.838 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.838 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.838 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.838 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.838 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.838 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.838 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.838 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.838 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.838 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.838 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.838 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.838 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.839 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.839 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.839 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.839 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.839 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.839 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.839 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.839 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.839 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.839 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.839 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.839 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.839 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.839 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.840 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.840 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.840 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.840 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.840 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:52.840 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.840 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 5, 2024-09-14T03:47:52.840 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.840 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.840 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.840 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.840 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.840 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.841 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:52.841 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.841 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 0, 2024-09-14T03:47:52.841 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.841 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.841 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 62, 2024-09-14T03:47:52.841 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.841 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.841 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.841 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:20.502304+0000", 2024-09-14T03:47:52.841 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:45:09.879229+0000", 2024-09-14T03:47:52.841 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:45:09.879229+0000", 2024-09-14T03:47:52.841 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:45:14.684595+0000", 2024-09-14T03:47:52.842 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:20.502304+0000", 2024-09-14T03:47:52.842 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:52.842 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.842 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:52.842 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 63, 2024-09-14T03:47:52.842 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:20.502304+0000", 2024-09-14T03:47:52.842 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:20.502304+0000", 2024-09-14T03:47:52.842 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:20.502304+0000", 2024-09-14T03:47:52.842 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.842 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.842 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:52.842 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:20.502304+0000", 2024-09-14T03:47:52.842 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:20.502304+0000", 2024-09-14T03:47:52.842 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.843 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 1, 2024-09-14T03:47:52.843 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.843 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.843 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 62, 2024-09-14T03:47:52.843 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.843 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.843 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.843 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.843 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 1, 2024-09-14T03:47:52.843 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.843 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.843 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.843 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "4.3", 2024-09-14T03:47:52.844 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.844 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.844 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 68, 2024-09-14T03:47:52.844 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:52.844 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.844 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T10:49:58.555610+0000", 2024-09-14T03:47:52.844 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.000406004, 2024-09-14T03:47:52.844 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.844 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.844 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.844 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.844 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.844 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.845 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.845 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.845 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.845 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.845 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.845 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.845 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.845 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.845 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.845 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.845 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.845 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.845 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 3, 2024-09-14T03:47:52.845 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 1, 2024-09-14T03:47:52.846 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.846 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 1, 2024-09-14T03:47:52.846 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.846 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.846 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.846 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.846 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.847 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.848 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.848 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.848 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.848 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.848 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.848 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.848 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.848 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.848 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.848 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.848 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.848 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.848 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.849 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.849 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.849 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.849 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.849 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.849 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.849 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.849 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:52.849 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.849 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 0, 2024-09-14T03:47:52.849 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "65'1" 2024-09-14T03:47:52.849 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.849 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.849 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.850 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.850 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.850 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:52.850 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.850 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:52.850 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.850 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.850 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:52.850 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.850 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.850 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.850 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.661082+0000", 2024-09-14T03:47:52.850 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.732868+0000", 2024-09-14T03:47:52.851 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.732868+0000", 2024-09-14T03:47:52.851 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.733230+0000", 2024-09-14T03:47:52.851 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.661082+0000", 2024-09-14T03:47:52.851 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.851 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.851 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.851 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:52.851 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.661082+0000", 2024-09-14T03:47:52.851 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.661082+0000", 2024-09-14T03:47:52.851 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.661082+0000", 2024-09-14T03:47:52.851 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.851 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.851 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.852 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.661082+0000", 2024-09-14T03:47:52.852 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.661082+0000", 2024-09-14T03:47:52.852 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.852 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.852 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.852 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.852 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:52.852 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.852 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.852 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.852 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.852 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.852 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.853 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.853 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.853 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.5", 2024-09-14T03:47:52.853 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.853 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.853 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.853 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 24, 2024-09-14T03:47:52.853 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.853 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:15:12.184851+0000", 2024-09-14T03:47:52.853 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.853 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.853 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.853 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.854 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.854 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.854 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.854 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.854 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.854 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.854 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.854 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.854 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.854 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.854 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.854 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.854 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.854 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.855 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.855 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.855 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.855 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.855 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.855 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.855 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.855 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.855 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.855 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.855 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.855 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.855 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.856 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.856 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.856 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.856 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.856 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.856 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.856 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.856 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.856 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.856 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.856 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.856 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.856 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.857 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.857 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.857 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.857 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.857 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.857 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:52.857 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.857 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:52.857 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.857 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.857 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.857 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.858 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.858 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:52.858 INFO:tasks.workunit.client.0.smithi070.stdout: 5 2024-09-14T03:47:52.858 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.858 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 1, 2024-09-14T03:47:52.858 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.858 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.858 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.858 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.858 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.858 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.858 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.458306+0000", 2024-09-14T03:47:52.858 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.877390+0000", 2024-09-14T03:47:52.858 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.877390+0000", 2024-09-14T03:47:52.858 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.878736+0000", 2024-09-14T03:47:52.859 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.458306+0000", 2024-09-14T03:47:52.859 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.859 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.859 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.859 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.859 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.458306+0000", 2024-09-14T03:47:52.859 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.458306+0000", 2024-09-14T03:47:52.859 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.458306+0000", 2024-09-14T03:47:52.859 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.859 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.859 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.859 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.458306+0000", 2024-09-14T03:47:52.859 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.458306+0000", 2024-09-14T03:47:52.859 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.860 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.860 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.860 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.860 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.860 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.860 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.860 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.860 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.860 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.860 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.860 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.860 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.860 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.4", 2024-09-14T03:47:52.861 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.861 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.861 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.861 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:52.861 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.861 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T09:17:42.812575+0000", 2024-09-14T03:47:52.861 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.861 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.861 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.861 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.861 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.861 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.862 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.862 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.862 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.862 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.862 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.862 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.862 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.862 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.862 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.862 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.862 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.862 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.862 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.862 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.863 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.863 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.863 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.863 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.863 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.863 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.863 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.863 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.863 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.863 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.863 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.863 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.863 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.864 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.864 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.864 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.864 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.864 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.864 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.864 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.864 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.864 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.864 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.864 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.864 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.864 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.865 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.865 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.865 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:52.865 INFO:tasks.workunit.client.0.smithi070.stdout: 5 2024-09-14T03:47:52.865 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.865 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 1, 2024-09-14T03:47:52.865 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.865 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.865 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.865 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.865 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.865 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.865 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.866 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.866 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 3, 2024-09-14T03:47:52.866 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.866 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.866 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 62, 2024-09-14T03:47:52.866 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.866 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.866 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.866 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:20.501909+0000", 2024-09-14T03:47:52.866 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:45:09.710318+0000", 2024-09-14T03:47:52.866 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:45:09.710318+0000", 2024-09-14T03:47:52.866 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:45:14.857657+0000", 2024-09-14T03:47:52.866 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:20.501909+0000", 2024-09-14T03:47:52.867 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:52.867 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.867 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:52.867 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 63, 2024-09-14T03:47:52.867 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:20.501909+0000", 2024-09-14T03:47:52.867 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:20.501909+0000", 2024-09-14T03:47:52.867 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:20.501909+0000", 2024-09-14T03:47:52.867 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.867 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.867 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:52.867 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:20.501909+0000", 2024-09-14T03:47:52.867 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:20.501909+0000", 2024-09-14T03:47:52.867 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.868 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.868 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.868 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.868 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 62, 2024-09-14T03:47:52.868 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.868 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.868 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.868 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.868 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.868 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.868 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.868 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.868 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "4.0", 2024-09-14T03:47:52.869 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.869 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.869 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 68, 2024-09-14T03:47:52.869 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 19, 2024-09-14T03:47:52.869 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.869 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:37:23.130384+0000", 2024-09-14T03:47:52.869 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.000315721, 2024-09-14T03:47:52.869 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.869 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.869 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.869 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.869 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.869 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.870 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.870 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.870 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.870 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.870 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.870 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.870 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.870 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.870 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.870 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.870 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.870 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.870 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.870 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.871 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.871 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.871 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.871 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.871 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.871 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.871 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.871 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.871 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.871 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.871 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.871 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.871 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.872 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.872 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.872 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.872 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.872 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.872 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.872 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.872 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.872 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.872 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.872 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.872 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.872 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.873 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.873 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.873 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.873 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.873 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 3, 2024-09-14T03:47:52.873 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.873 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.873 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.873 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.873 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.873 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.873 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:52.873 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.873 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 1, 2024-09-14T03:47:52.874 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.874 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.874 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:52.874 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.874 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.874 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.874 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.459070+0000", 2024-09-14T03:47:52.874 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.733457+0000", 2024-09-14T03:47:52.874 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.733457+0000", 2024-09-14T03:47:52.874 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.733688+0000", 2024-09-14T03:47:52.874 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.459070+0000", 2024-09-14T03:47:52.874 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.874 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.875 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.875 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:52.875 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.459070+0000", 2024-09-14T03:47:52.875 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.459070+0000", 2024-09-14T03:47:52.875 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.459070+0000", 2024-09-14T03:47:52.875 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.875 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.875 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.875 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.459070+0000", 2024-09-14T03:47:52.875 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.459070+0000", 2024-09-14T03:47:52.875 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.875 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 1, 2024-09-14T03:47:52.875 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.875 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.876 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:52.876 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.876 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.876 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.876 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.876 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 1, 2024-09-14T03:47:52.876 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.876 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.876 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.876 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.6", 2024-09-14T03:47:52.876 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.876 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.876 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.876 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 24, 2024-09-14T03:47:52.877 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.877 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:09:01.069988+0000", 2024-09-14T03:47:52.877 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.877 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.877 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.877 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.877 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.877 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.877 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.877 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.877 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.877 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.877 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.878 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.878 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.878 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.878 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.878 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.878 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.878 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.878 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.878 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 3, 2024-09-14T03:47:52.878 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 1, 2024-09-14T03:47:52.878 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.878 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 1, 2024-09-14T03:47:52.878 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.879 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.879 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.879 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.879 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.879 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 1, 2024-09-14T03:47:52.879 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.879 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.879 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.879 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.879 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.879 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.879 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.879 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.879 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.880 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.880 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.880 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.880 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 1, 2024-09-14T03:47:52.880 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.880 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.880 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.880 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.880 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.880 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.880 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.880 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:52.881 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.881 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 1, 2024-09-14T03:47:52.881 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "58'1" 2024-09-14T03:47:52.881 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.881 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.881 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.881 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.881 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.881 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.881 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.881 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 3, 2024-09-14T03:47:52.881 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.881 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.881 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.882 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.882 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.882 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.882 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.857916+0000", 2024-09-14T03:47:52.882 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.869780+0000", 2024-09-14T03:47:52.882 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.869780+0000", 2024-09-14T03:47:52.882 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.870045+0000", 2024-09-14T03:47:52.882 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.857916+0000", 2024-09-14T03:47:52.882 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.882 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.882 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.882 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.882 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.857916+0000", 2024-09-14T03:47:52.883 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.857916+0000", 2024-09-14T03:47:52.883 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.857916+0000", 2024-09-14T03:47:52.883 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.883 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.883 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.883 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.857916+0000", 2024-09-14T03:47:52.883 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.857916+0000", 2024-09-14T03:47:52.883 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.883 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.883 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.883 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.883 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.883 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.883 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.884 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.884 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.884 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.884 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.884 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.884 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.884 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.7", 2024-09-14T03:47:52.884 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.884 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.884 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.884 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:52.884 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.884 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T09:04:58.634704+0000", 2024-09-14T03:47:52.885 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.885 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.885 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.885 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.885 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.885 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.885 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.885 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.885 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.885 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.885 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.885 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.885 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.886 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.886 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.886 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.886 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.886 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.886 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.886 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.886 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.886 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.886 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.886 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.886 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.886 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.886 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.887 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.887 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.887 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.887 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.887 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.887 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.887 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.887 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.887 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.887 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.887 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.887 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.887 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.887 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.888 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.888 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.888 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.888 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.888 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.888 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.888 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.888 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.888 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.888 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.888 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 3, 2024-09-14T03:47:52.888 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.888 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.889 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.889 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.889 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.889 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.889 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:52.889 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.889 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 4, 2024-09-14T03:47:52.889 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.889 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.889 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 62, 2024-09-14T03:47:52.889 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.889 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.889 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.889 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:20.502067+0000", 2024-09-14T03:47:52.890 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:45:09.879281+0000", 2024-09-14T03:47:52.890 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:45:09.879281+0000", 2024-09-14T03:47:52.890 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:45:14.459017+0000", 2024-09-14T03:47:52.890 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:20.502067+0000", 2024-09-14T03:47:52.890 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:52.890 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.890 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:52.890 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 63, 2024-09-14T03:47:52.890 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:20.502067+0000", 2024-09-14T03:47:52.890 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:20.502067+0000", 2024-09-14T03:47:52.890 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:20.502067+0000", 2024-09-14T03:47:52.890 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.890 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.890 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:45:08.693776+0000", 2024-09-14T03:47:52.891 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:20.502067+0000", 2024-09-14T03:47:52.891 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:20.502067+0000", 2024-09-14T03:47:52.891 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.891 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.891 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.891 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.891 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 62, 2024-09-14T03:47:52.891 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.891 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.891 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.891 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.891 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.891 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.892 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.892 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.892 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "4.1", 2024-09-14T03:47:52.892 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.892 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.892 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 68, 2024-09-14T03:47:52.892 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 19, 2024-09-14T03:47:52.892 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.892 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:10:03.506895+0000", 2024-09-14T03:47:52.892 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.000827153, 2024-09-14T03:47:52.892 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.892 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.893 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.893 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.893 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.893 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.893 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.893 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.893 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.893 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.893 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.893 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.893 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.893 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.893 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.893 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.894 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.894 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.894 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.894 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.894 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.894 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.894 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.894 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.894 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.894 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.894 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.894 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.894 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.895 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.895 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.895 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.895 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.895 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.895 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.895 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.895 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.895 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.895 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.895 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.895 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.895 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.895 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.896 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.896 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.896 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.896 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.896 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.896 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:52.896 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.896 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 4, 2024-09-14T03:47:52.896 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.896 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.896 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.896 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.896 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.897 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.897 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:52.897 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.897 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 6, 2024-09-14T03:47:52.897 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.897 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.897 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:52.897 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.897 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.897 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.897 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.545051+0000", 2024-09-14T03:47:52.897 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.734833+0000", 2024-09-14T03:47:52.898 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.734833+0000", 2024-09-14T03:47:52.898 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.735364+0000", 2024-09-14T03:47:52.898 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.545051+0000", 2024-09-14T03:47:52.898 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.898 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.898 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.898 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:52.898 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.545051+0000", 2024-09-14T03:47:52.898 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.545051+0000", 2024-09-14T03:47:52.898 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.545051+0000", 2024-09-14T03:47:52.898 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.898 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.898 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.899 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.545051+0000", 2024-09-14T03:47:52.899 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.545051+0000", 2024-09-14T03:47:52.899 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.899 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 2, 2024-09-14T03:47:52.899 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.899 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.899 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:52.899 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.899 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.899 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.899 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.899 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 2, 2024-09-14T03:47:52.899 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.899 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.900 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.900 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.7", 2024-09-14T03:47:52.900 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.900 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.900 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.900 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 25, 2024-09-14T03:47:52.900 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.900 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T10:58:37.428713+0000", 2024-09-14T03:47:52.900 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.900 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.900 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.900 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.900 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.900 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.901 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.901 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.901 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.901 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.901 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.901 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.901 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.901 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.901 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.901 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.901 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.901 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.901 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.901 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 3, 2024-09-14T03:47:52.902 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 1, 2024-09-14T03:47:52.902 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.902 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 1, 2024-09-14T03:47:52.902 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.902 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.902 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.902 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.902 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.902 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 1, 2024-09-14T03:47:52.902 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.902 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.902 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.902 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.903 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.903 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.903 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.903 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.903 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.903 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.903 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.903 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.903 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 3, 2024-09-14T03:47:52.903 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.903 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.903 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.903 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.903 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.904 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.904 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.904 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:52.904 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.904 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 6, 2024-09-14T03:47:52.904 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "58'2" 2024-09-14T03:47:52.904 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.904 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.904 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.904 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.904 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.904 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:52.904 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.905 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 0, 2024-09-14T03:47:52.905 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.905 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.905 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.905 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.905 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.905 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.905 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.684514+0000", 2024-09-14T03:47:52.905 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.878316+0000", 2024-09-14T03:47:52.905 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.878316+0000", 2024-09-14T03:47:52.905 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.878573+0000", 2024-09-14T03:47:52.905 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.684514+0000", 2024-09-14T03:47:52.905 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.906 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.906 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.906 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.906 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.684514+0000", 2024-09-14T03:47:52.906 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.684514+0000", 2024-09-14T03:47:52.906 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.684514+0000", 2024-09-14T03:47:52.906 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.906 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.906 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.906 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.684514+0000", 2024-09-14T03:47:52.906 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.684514+0000", 2024-09-14T03:47:52.906 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.906 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.907 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.907 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.907 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.907 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.907 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.907 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.907 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.907 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.907 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.907 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.907 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.907 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.6", 2024-09-14T03:47:52.907 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.907 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.908 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.908 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:52.908 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.908 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:45:21.129163+0000", 2024-09-14T03:47:52.908 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.908 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.908 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.908 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.908 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.908 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.908 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.908 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.908 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.909 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.909 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.909 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.909 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.909 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.909 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.909 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.909 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.909 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.909 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.909 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.909 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.909 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.910 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.910 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.910 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.910 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.910 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.910 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.910 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.910 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.910 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.910 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.910 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.910 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.910 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.910 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.911 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.911 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.911 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.911 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.911 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.911 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.911 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.911 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.911 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.911 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.911 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.911 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.911 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.912 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:52.912 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.912 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 0, 2024-09-14T03:47:52.912 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.912 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.912 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.912 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.912 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.912 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.912 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:52.912 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.912 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:52.912 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.913 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.913 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:52.913 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.913 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.913 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.913 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.660416+0000", 2024-09-14T03:47:52.913 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.730538+0000", 2024-09-14T03:47:52.913 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.730538+0000", 2024-09-14T03:47:52.913 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.730843+0000", 2024-09-14T03:47:52.913 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.660416+0000", 2024-09-14T03:47:52.913 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.913 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.913 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.914 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:52.914 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.660416+0000", 2024-09-14T03:47:52.914 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.660416+0000", 2024-09-14T03:47:52.914 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.660416+0000", 2024-09-14T03:47:52.914 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.914 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.914 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.914 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.660416+0000", 2024-09-14T03:47:52.914 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.660416+0000", 2024-09-14T03:47:52.914 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.914 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.914 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.914 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.914 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:52.915 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.915 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.915 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.915 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.915 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.915 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.915 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.915 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.915 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.8", 2024-09-14T03:47:52.915 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.915 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.915 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.915 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 24, 2024-09-14T03:47:52.915 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.916 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T04:04:57.059345+0000", 2024-09-14T03:47:52.916 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.916 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.916 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.916 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.916 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.916 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.916 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.916 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.916 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.916 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.916 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.916 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.917 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.917 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.917 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.917 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.917 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.917 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.917 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.917 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.917 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.917 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.917 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.917 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.917 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.918 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.918 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.918 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.918 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.918 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.918 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.918 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.918 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.918 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.918 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.918 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.918 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.918 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.918 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.919 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.919 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.919 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.919 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.919 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.919 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.919 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.919 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.919 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.919 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:52.919 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:52.919 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.919 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:52.919 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.920 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.920 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.920 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.920 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.920 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:52.920 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:52.920 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.920 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 4, 2024-09-14T03:47:52.920 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.920 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.920 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.920 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.920 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.921 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.921 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.458548+0000", 2024-09-14T03:47:52.921 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.874049+0000", 2024-09-14T03:47:52.921 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.874049+0000", 2024-09-14T03:47:52.921 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.874355+0000", 2024-09-14T03:47:52.921 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.458548+0000", 2024-09-14T03:47:52.921 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.921 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.921 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.921 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.921 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.458548+0000", 2024-09-14T03:47:52.921 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.458548+0000", 2024-09-14T03:47:52.921 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.458548+0000", 2024-09-14T03:47:52.922 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.922 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.922 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.922 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.458548+0000", 2024-09-14T03:47:52.922 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.458548+0000", 2024-09-14T03:47:52.922 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.922 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.922 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.922 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.922 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.922 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.922 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.922 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.922 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.922 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.923 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.923 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.923 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.923 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.9", 2024-09-14T03:47:52.923 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.923 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.923 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.923 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:52.923 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.923 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:53:30.306175+0000", 2024-09-14T03:47:52.923 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.923 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.923 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.923 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.924 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.924 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.924 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.924 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.924 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.924 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.924 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.924 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.924 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.924 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.924 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.924 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.924 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.925 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.925 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.925 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.925 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.925 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.925 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.925 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.925 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.925 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.925 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.925 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.925 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.925 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.925 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.926 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.926 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.926 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.926 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.926 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.926 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.926 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.926 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.926 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.926 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.926 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.926 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.926 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.926 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.927 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.927 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.927 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.927 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:52.927 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:52.927 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.927 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 4, 2024-09-14T03:47:52.927 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.927 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.927 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.927 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.927 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.927 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.927 INFO:tasks.workunit.client.0.smithi070.stdout: 3 2024-09-14T03:47:52.928 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.928 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 1, 2024-09-14T03:47:52.928 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.928 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.928 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:52.928 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.928 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.928 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.928 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.458659+0000", 2024-09-14T03:47:52.928 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.727820+0000", 2024-09-14T03:47:52.928 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.727820+0000", 2024-09-14T03:47:52.928 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.728080+0000", 2024-09-14T03:47:52.928 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.458659+0000", 2024-09-14T03:47:52.928 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.929 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.929 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.929 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:52.929 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.458659+0000", 2024-09-14T03:47:52.929 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.458659+0000", 2024-09-14T03:47:52.929 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.458659+0000", 2024-09-14T03:47:52.929 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.929 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.929 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.929 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.458659+0000", 2024-09-14T03:47:52.929 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.458659+0000", 2024-09-14T03:47:52.929 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.929 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.930 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.930 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.930 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:52.930 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.930 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.930 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.930 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.930 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.930 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.930 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.930 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.930 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.9", 2024-09-14T03:47:52.930 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.930 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.930 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.931 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 23, 2024-09-14T03:47:52.931 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.931 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T12:37:58.166430+0000", 2024-09-14T03:47:52.931 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.931 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.931 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.931 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.931 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.931 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.931 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.931 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.931 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.931 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.932 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.932 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.932 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.932 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.932 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.932 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.932 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.932 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.932 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.932 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.932 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.932 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.932 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.933 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.933 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.933 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.933 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.933 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.933 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.933 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.933 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.933 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.933 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.933 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.933 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.933 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.933 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.933 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.934 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.934 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.934 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.934 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.934 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.934 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.934 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.934 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.934 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.934 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.934 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.934 INFO:tasks.workunit.client.0.smithi070.stdout: 3 2024-09-14T03:47:52.934 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.934 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 1, 2024-09-14T03:47:52.935 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.935 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.935 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.935 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.935 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.935 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.935 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:52.935 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.935 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 3, 2024-09-14T03:47:52.935 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.935 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.935 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.935 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.935 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.936 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.936 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.857492+0000", 2024-09-14T03:47:52.936 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.869065+0000", 2024-09-14T03:47:52.936 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.869065+0000", 2024-09-14T03:47:52.936 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.869311+0000", 2024-09-14T03:47:52.936 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.857492+0000", 2024-09-14T03:47:52.936 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.936 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.936 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.936 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.936 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.857492+0000", 2024-09-14T03:47:52.936 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.857492+0000", 2024-09-14T03:47:52.936 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.857492+0000", 2024-09-14T03:47:52.936 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.937 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.937 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.937 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.857492+0000", 2024-09-14T03:47:52.937 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.857492+0000", 2024-09-14T03:47:52.937 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.937 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.937 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.937 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.937 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.937 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.937 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.937 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.937 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.938 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.938 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.938 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.938 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.938 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.8", 2024-09-14T03:47:52.938 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.938 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.938 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.938 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:52.938 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.938 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T03:45:09.508019+0000", 2024-09-14T03:47:52.938 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.938 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.938 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.938 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.939 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.939 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.939 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.939 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.939 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.939 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.939 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.939 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.939 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.939 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.939 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.939 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.939 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.939 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.940 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.940 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.940 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.940 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.940 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.940 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.940 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.940 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.940 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.940 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.940 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.940 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.940 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.941 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.941 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.941 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.941 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.941 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.941 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.941 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.941 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.941 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.941 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.941 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.941 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.941 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.941 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.942 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.942 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.942 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.942 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.942 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:52.942 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.942 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 3, 2024-09-14T03:47:52.942 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.942 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.942 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.942 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.942 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.942 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.942 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:52.943 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.943 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 1, 2024-09-14T03:47:52.943 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.943 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.943 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:52.943 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.943 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.943 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.943 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.459148+0000", 2024-09-14T03:47:52.943 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.728647+0000", 2024-09-14T03:47:52.943 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.728647+0000", 2024-09-14T03:47:52.943 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.728931+0000", 2024-09-14T03:47:52.943 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.459148+0000", 2024-09-14T03:47:52.943 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.943 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.944 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.944 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:52.944 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.459148+0000", 2024-09-14T03:47:52.944 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.459148+0000", 2024-09-14T03:47:52.944 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.459148+0000", 2024-09-14T03:47:52.944 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.944 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.944 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.944 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.459148+0000", 2024-09-14T03:47:52.944 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.459148+0000", 2024-09-14T03:47:52.944 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.944 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.944 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.944 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.945 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:52.945 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.945 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.945 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.945 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.945 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.945 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.945 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.945 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.945 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.a", 2024-09-14T03:47:52.945 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.945 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.945 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.946 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 23, 2024-09-14T03:47:52.946 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.946 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:09:38.769523+0000", 2024-09-14T03:47:52.946 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.946 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.946 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.946 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.946 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.946 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.946 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.946 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.946 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.946 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.946 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.947 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.947 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.947 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.947 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.947 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.947 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.947 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.947 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.947 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.947 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.947 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.947 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.947 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.948 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.948 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.948 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.948 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.948 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.948 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.948 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.948 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.948 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.948 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.948 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.948 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.948 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.949 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.949 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.949 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.949 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.949 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.949 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.949 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.949 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.949 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.949 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.949 INFO:tasks.workunit.client.0.smithi070.stdout: 1, 2024-09-14T03:47:52.949 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.949 INFO:tasks.workunit.client.0.smithi070.stdout: 7 2024-09-14T03:47:52.950 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.950 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 1, 2024-09-14T03:47:52.950 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.950 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.950 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.950 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.950 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.950 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:52.950 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:52.950 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.950 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 3, 2024-09-14T03:47:52.950 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.950 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.951 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.951 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.951 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.951 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.951 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.857689+0000", 2024-09-14T03:47:52.951 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:26.429880+0000", 2024-09-14T03:47:52.951 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:26.429880+0000", 2024-09-14T03:47:52.951 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:26.430230+0000", 2024-09-14T03:47:52.951 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.857689+0000", 2024-09-14T03:47:52.951 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.951 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.951 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.951 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 60, 2024-09-14T03:47:52.952 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.857689+0000", 2024-09-14T03:47:52.952 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.857689+0000", 2024-09-14T03:47:52.952 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.857689+0000", 2024-09-14T03:47:52.952 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.952 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.952 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.952 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.857689+0000", 2024-09-14T03:47:52.952 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.857689+0000", 2024-09-14T03:47:52.952 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.952 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.952 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.952 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.952 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 59, 2024-09-14T03:47:52.953 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.953 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.953 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.953 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.953 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.953 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.953 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.953 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.953 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.b", 2024-09-14T03:47:52.953 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.953 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.953 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.953 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 31, 2024-09-14T03:47:52.954 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.954 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:17:39.132572+0000", 2024-09-14T03:47:52.954 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.954 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.954 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.954 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.954 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.954 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.954 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.954 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.954 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.954 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.954 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.954 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.955 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.955 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.955 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.955 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.955 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.955 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.955 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.955 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.955 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.955 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.955 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.955 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.956 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.956 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.956 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.956 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.956 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.956 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.956 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.956 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.956 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.956 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.956 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.956 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.956 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.956 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.957 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.957 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.957 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.957 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.957 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.957 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.957 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.957 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.957 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.957 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:52.957 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:52.957 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:52.957 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.958 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 3, 2024-09-14T03:47:52.958 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.958 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.958 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.958 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.958 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.958 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.958 INFO:tasks.workunit.client.0.smithi070.stdout: 5 2024-09-14T03:47:52.958 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.958 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:52.958 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.958 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.958 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:52.958 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.959 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.959 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.959 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.660988+0000", 2024-09-14T03:47:52.959 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.736546+0000", 2024-09-14T03:47:52.959 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.736546+0000", 2024-09-14T03:47:52.959 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.737705+0000", 2024-09-14T03:47:52.959 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.660988+0000", 2024-09-14T03:47:52.959 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.959 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.959 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.959 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:52.959 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.660988+0000", 2024-09-14T03:47:52.959 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.660988+0000", 2024-09-14T03:47:52.959 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.660988+0000", 2024-09-14T03:47:52.960 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.960 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.960 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.960 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.660988+0000", 2024-09-14T03:47:52.960 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.660988+0000", 2024-09-14T03:47:52.960 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.960 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 2, 2024-09-14T03:47:52.960 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.960 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.960 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:52.960 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.960 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.960 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.961 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.961 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 2, 2024-09-14T03:47:52.961 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.961 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.961 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.961 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.b", 2024-09-14T03:47:52.961 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.961 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.961 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.961 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 26, 2024-09-14T03:47:52.961 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.961 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:27:12.542706+0000", 2024-09-14T03:47:52.961 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.962 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.962 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.962 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.962 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.962 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.962 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.962 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.962 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.962 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.962 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.962 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.962 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.962 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.963 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.963 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.963 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.963 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.963 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.963 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 3, 2024-09-14T03:47:52.963 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 1, 2024-09-14T03:47:52.963 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.963 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 1, 2024-09-14T03:47:52.963 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.963 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.963 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.963 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.963 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.964 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 1, 2024-09-14T03:47:52.964 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.964 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.964 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.964 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.964 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.964 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.964 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.964 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.964 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.964 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.964 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.964 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.965 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 1, 2024-09-14T03:47:52.965 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.965 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.965 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.965 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.965 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.965 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.965 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.965 INFO:tasks.workunit.client.0.smithi070.stdout: 5 2024-09-14T03:47:52.965 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.965 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:52.965 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "58'2" 2024-09-14T03:47:52.965 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.965 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.966 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.966 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:52.966 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.966 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:52.966 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.966 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 2, 2024-09-14T03:47:52.966 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.966 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.966 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.966 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.966 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.966 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.966 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:15.135501+0000", 2024-09-14T03:47:52.966 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:26.429364+0000", 2024-09-14T03:47:52.967 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:26.429364+0000", 2024-09-14T03:47:52.967 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:26.429678+0000", 2024-09-14T03:47:52.967 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:15.135501+0000", 2024-09-14T03:47:52.967 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.967 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.967 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.967 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 60, 2024-09-14T03:47:52.967 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:15.135501+0000", 2024-09-14T03:47:52.967 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:15.135501+0000", 2024-09-14T03:47:52.967 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:15.135501+0000", 2024-09-14T03:47:52.967 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.967 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.967 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.968 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:15.135501+0000", 2024-09-14T03:47:52.968 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:15.135501+0000", 2024-09-14T03:47:52.968 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.968 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.968 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.968 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.968 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 59, 2024-09-14T03:47:52.968 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.968 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.968 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.968 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.968 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.968 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.969 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.969 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.969 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.a", 2024-09-14T03:47:52.969 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.969 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.969 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.969 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 18, 2024-09-14T03:47:52.969 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.969 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:29:31.230108+0000", 2024-09-14T03:47:52.969 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.969 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.969 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.969 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.970 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.970 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.970 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.970 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.970 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.970 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.970 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.970 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.970 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.970 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.970 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.970 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.970 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.970 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.971 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.971 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.971 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.971 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.971 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.971 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.971 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.971 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.971 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.971 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.971 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.971 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.971 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.971 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.972 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.972 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.972 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.972 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.972 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.972 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.972 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.972 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.972 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.972 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.972 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.972 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.972 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.973 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.973 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.973 INFO:tasks.workunit.client.0.smithi070.stdout: 2, 2024-09-14T03:47:52.973 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.973 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:52.973 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.973 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 2, 2024-09-14T03:47:52.973 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.973 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.973 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.973 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.973 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.973 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.974 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:52.974 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.974 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 5, 2024-09-14T03:47:52.974 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.974 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.974 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.974 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.974 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.974 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.974 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.462051+0000", 2024-09-14T03:47:52.974 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.888257+0000", 2024-09-14T03:47:52.974 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.888257+0000", 2024-09-14T03:47:52.974 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.888591+0000", 2024-09-14T03:47:52.974 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.462051+0000", 2024-09-14T03:47:52.975 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.975 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.975 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.975 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.975 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.462051+0000", 2024-09-14T03:47:52.975 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.462051+0000", 2024-09-14T03:47:52.975 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.462051+0000", 2024-09-14T03:47:52.975 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.975 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.975 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.975 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.462051+0000", 2024-09-14T03:47:52.975 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.462051+0000", 2024-09-14T03:47:52.975 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.975 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.976 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.976 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.976 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.976 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.976 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.976 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.976 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.976 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.976 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.976 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.976 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.976 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.1d", 2024-09-14T03:47:52.976 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.977 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.977 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.977 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:52.977 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.977 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:24:40.992358+0000", 2024-09-14T03:47:52.977 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.977 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.977 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.977 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.977 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.977 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.977 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.977 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.977 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.978 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.978 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.978 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.978 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.978 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.978 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.978 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.978 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.978 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.978 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.978 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.978 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.978 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.979 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.979 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.979 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.979 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.979 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.979 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.979 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.979 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.979 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.979 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.979 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.979 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.979 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.979 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.980 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.980 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.980 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.980 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.980 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.980 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.980 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.980 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.980 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.980 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.980 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.980 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.980 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.980 INFO:tasks.workunit.client.0.smithi070.stdout: 6 2024-09-14T03:47:52.981 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.981 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 5, 2024-09-14T03:47:52.981 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.981 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.981 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.981 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.981 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.981 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.981 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:52.981 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.981 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:52.981 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.982 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.982 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:52.982 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.982 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.982 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.982 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.660896+0000", 2024-09-14T03:47:52.982 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.734408+0000", 2024-09-14T03:47:52.982 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.734408+0000", 2024-09-14T03:47:52.982 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.734680+0000", 2024-09-14T03:47:52.982 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.660896+0000", 2024-09-14T03:47:52.982 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.982 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.982 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.982 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:52.983 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.660896+0000", 2024-09-14T03:47:52.983 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.660896+0000", 2024-09-14T03:47:52.983 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.660896+0000", 2024-09-14T03:47:52.983 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.983 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.983 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.983 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.660896+0000", 2024-09-14T03:47:52.983 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.660896+0000", 2024-09-14T03:47:52.983 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.983 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 2, 2024-09-14T03:47:52.983 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.983 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.983 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:52.983 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.984 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.984 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.984 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.984 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 2, 2024-09-14T03:47:52.984 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.984 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.984 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.984 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.1c", 2024-09-14T03:47:52.984 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.984 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.984 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.984 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 26, 2024-09-14T03:47:52.984 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.985 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T06:49:02.536404+0000", 2024-09-14T03:47:52.985 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.985 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.985 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.985 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.985 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.985 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.985 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.985 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.985 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.985 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.985 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.986 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.986 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.986 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.986 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.986 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.986 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.986 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.986 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.986 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 3, 2024-09-14T03:47:52.986 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 1, 2024-09-14T03:47:52.986 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.986 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 1, 2024-09-14T03:47:52.986 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.986 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.987 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.987 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.987 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.987 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 1, 2024-09-14T03:47:52.987 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.987 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.987 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.987 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.987 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.987 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.987 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.987 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.987 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.987 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.988 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.988 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.988 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 3, 2024-09-14T03:47:52.988 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.988 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.988 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.988 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.988 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.988 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.988 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.988 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:52.988 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.988 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:52.988 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "58'2" 2024-09-14T03:47:52.989 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.989 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.989 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.989 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.989 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.989 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:52.989 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.989 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 5, 2024-09-14T03:47:52.989 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.989 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.989 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:52.989 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.989 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.989 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.990 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.461880+0000", 2024-09-14T03:47:52.990 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.885110+0000", 2024-09-14T03:47:52.990 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.885110+0000", 2024-09-14T03:47:52.990 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.885572+0000", 2024-09-14T03:47:52.990 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.461880+0000", 2024-09-14T03:47:52.990 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.990 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.990 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.990 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:52.990 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.461880+0000", 2024-09-14T03:47:52.990 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.461880+0000", 2024-09-14T03:47:52.990 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.461880+0000", 2024-09-14T03:47:52.990 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.991 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.991 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:52.991 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.461880+0000", 2024-09-14T03:47:52.991 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.461880+0000", 2024-09-14T03:47:52.991 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.991 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:52.991 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.991 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.991 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:52.991 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.991 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.991 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.991 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.991 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:52.992 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.992 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.992 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.992 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.1c", 2024-09-14T03:47:52.992 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.992 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.992 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.992 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:52.992 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:52.992 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T08:16:10.439239+0000", 2024-09-14T03:47:52.992 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:52.992 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:52.992 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:52.992 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:52.993 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:52.993 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:52.993 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:52.993 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:52.993 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:52.993 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:52.993 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:52.993 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:52.993 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:52.993 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:52.993 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:52.993 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:52.993 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:52.993 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:52.994 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:52.994 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:52.994 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:52.994 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:52.994 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:52.994 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:52.994 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:52.994 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:52.994 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:52.994 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:52.994 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:52.994 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:52.994 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:52.994 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:52.995 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:52.995 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:52.995 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:52.995 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:52.995 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:52.995 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:52.995 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:52.995 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:52.995 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:52.995 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:52.995 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:52.995 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.995 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:52.996 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:52.996 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:52.996 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:52.996 INFO:tasks.workunit.client.0.smithi070.stdout: 4, 2024-09-14T03:47:52.996 INFO:tasks.workunit.client.0.smithi070.stdout: 1 2024-09-14T03:47:52.996 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.996 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 5, 2024-09-14T03:47:52.996 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:52.996 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:52.996 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:52.996 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:52.996 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:52.996 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:52.997 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:52.997 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:52.997 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 7, 2024-09-14T03:47:52.997 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:52.997 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:52.997 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:52.997 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:52.997 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:52.997 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:52.997 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.660935+0000", 2024-09-14T03:47:52.997 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.730951+0000", 2024-09-14T03:47:52.997 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.730951+0000", 2024-09-14T03:47:52.997 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.731296+0000", 2024-09-14T03:47:52.997 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.660935+0000", 2024-09-14T03:47:52.997 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.998 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:52.998 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.998 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:52.998 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.660935+0000", 2024-09-14T03:47:52.998 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.660935+0000", 2024-09-14T03:47:52.998 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.660935+0000", 2024-09-14T03:47:52.998 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:52.998 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:52.998 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:52.998 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.660935+0000", 2024-09-14T03:47:52.998 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.660935+0000", 2024-09-14T03:47:52.998 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:52.998 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 2, 2024-09-14T03:47:52.998 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:52.999 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:52.999 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:52.999 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:52.999 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:52.999 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:52.999 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:52.999 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 2, 2024-09-14T03:47:52.999 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:52.999 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:52.999 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:52.999 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.1d", 2024-09-14T03:47:52.999 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:52.999 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:52.999 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:52.999 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 26, 2024-09-14T03:47:53.000 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:53.000 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T13:26:14.801414+0000", 2024-09-14T03:47:53.000 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:53.000 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:53.000 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:53.000 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:53.000 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:53.000 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:53.000 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:53.000 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:53.000 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:53.000 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:53.000 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:53.001 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:53.001 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:53.001 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:53.001 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:53.001 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:53.001 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:53.001 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:53.001 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:53.001 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 3, 2024-09-14T03:47:53.001 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 1, 2024-09-14T03:47:53.001 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:53.001 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 1, 2024-09-14T03:47:53.001 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:53.001 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:53.002 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:53.002 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:53.002 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:53.002 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 1, 2024-09-14T03:47:53.002 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:53.002 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:53.002 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:53.002 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:53.002 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:53.002 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:53.002 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:53.002 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:53.002 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:53.002 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:53.003 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:53.003 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:53.003 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 3, 2024-09-14T03:47:53.003 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:53.003 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:53.003 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:53.003 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:53.003 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:53.003 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:53.003 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:53.003 INFO:tasks.workunit.client.0.smithi070.stdout: 0 2024-09-14T03:47:53.003 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:53.003 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 7, 2024-09-14T03:47:53.004 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "58'2" 2024-09-14T03:47:53.004 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:53.004 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:53.004 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:53.004 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:53.004 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:53.004 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:53.004 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:53.004 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 0, 2024-09-14T03:47:53.004 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:53.004 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:53.004 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:53.004 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:53.004 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:53.005 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:53.005 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.684378+0000", 2024-09-14T03:47:53.005 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.874668+0000", 2024-09-14T03:47:53.005 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.874668+0000", 2024-09-14T03:47:53.005 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.874935+0000", 2024-09-14T03:47:53.005 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.684378+0000", 2024-09-14T03:47:53.005 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:53.005 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:53.005 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:53.005 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:53.005 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.684378+0000", 2024-09-14T03:47:53.005 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.684378+0000", 2024-09-14T03:47:53.005 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.684378+0000", 2024-09-14T03:47:53.006 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:53.006 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:53.006 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:53.006 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.684378+0000", 2024-09-14T03:47:53.006 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.684378+0000", 2024-09-14T03:47:53.006 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:53.006 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:53.006 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:53.006 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:53.006 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:53.006 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:53.006 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:53.006 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:53.007 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:53.007 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:53.007 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:53.007 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:53.007 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:53.007 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.1f", 2024-09-14T03:47:53.007 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:53.007 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:53.007 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:53.007 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 20, 2024-09-14T03:47:53.007 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:53.007 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T15:24:31.404958+0000", 2024-09-14T03:47:53.007 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:53.007 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:53.007 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:53.008 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:53.008 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:53.008 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:53.008 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:53.008 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:53.008 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:53.008 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:53.008 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:53.008 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:53.008 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:53.008 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:53.008 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:53.008 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:53.009 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:53.009 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:53.009 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:53.009 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:53.009 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:53.009 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:53.009 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:53.009 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:53.009 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:53.009 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:53.009 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:53.009 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:53.009 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:53.010 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:53.010 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:53.010 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:53.010 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:53.010 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:53.010 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:53.010 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:53.010 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:53.010 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:53.010 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:53.010 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:53.010 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:53.010 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:53.010 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:53.010 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:53.011 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:53.011 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:53.011 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:53.011 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:53.011 INFO:tasks.workunit.client.0.smithi070.stdout: 5, 2024-09-14T03:47:53.011 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:53.011 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:53.011 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 0, 2024-09-14T03:47:53.011 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:53.011 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:53.011 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:53.011 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:53.011 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:53.011 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:53.011 INFO:tasks.workunit.client.0.smithi070.stdout: 5 2024-09-14T03:47:53.012 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:53.012 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 3, 2024-09-14T03:47:53.012 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:53.012 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:53.012 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:53.012 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:53.012 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:53.012 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:53.012 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.857877+0000", 2024-09-14T03:47:53.012 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.733960+0000", 2024-09-14T03:47:53.012 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.733960+0000", 2024-09-14T03:47:53.012 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.734206+0000", 2024-09-14T03:47:53.012 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.857877+0000", 2024-09-14T03:47:53.013 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:53.013 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:53.013 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:53.013 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:53.013 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.857877+0000", 2024-09-14T03:47:53.013 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.857877+0000", 2024-09-14T03:47:53.013 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.857877+0000", 2024-09-14T03:47:53.013 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:53.013 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:53.013 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:53.013 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.857877+0000", 2024-09-14T03:47:53.013 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.857877+0000", 2024-09-14T03:47:53.013 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:53.013 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 1, 2024-09-14T03:47:53.014 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:53.014 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:53.014 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:53.014 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:53.014 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:53.014 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:53.014 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:53.014 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 1, 2024-09-14T03:47:53.014 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:53.014 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:53.014 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:53.014 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.1e", 2024-09-14T03:47:53.014 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:53.014 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:53.015 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:53.015 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 24, 2024-09-14T03:47:53.015 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:53.015 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T11:36:44.073503+0000", 2024-09-14T03:47:53.015 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:53.015 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:53.015 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:53.015 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:53.015 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:53.015 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:53.015 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:53.015 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:53.015 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:53.015 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:53.016 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:53.016 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:53.016 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:53.016 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:53.016 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:53.016 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:53.016 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:53.016 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:53.016 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:53.016 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:53.016 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:53.016 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:53.016 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:53.016 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:53.017 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:53.017 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:53.017 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:53.017 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:53.017 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:53.017 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:53.017 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:53.017 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:53.017 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:53.017 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:53.017 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:53.017 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:53.017 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:53.017 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:53.018 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:53.018 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:53.018 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:53.018 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:53.018 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:53.018 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:53.018 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:53.018 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:53.018 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:53.018 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:53.018 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:53.018 INFO:tasks.workunit.client.0.smithi070.stdout: 5 2024-09-14T03:47:53.018 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:53.018 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 3, 2024-09-14T03:47:53.018 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "58'1" 2024-09-14T03:47:53.019 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:53.019 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:53.019 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:53.019 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:53.019 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:53.019 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:53.019 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:53.019 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 3, 2024-09-14T03:47:53.019 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:53.019 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:53.019 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 55, 2024-09-14T03:47:53.019 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:53.019 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:53.019 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:53.019 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.857768+0000", 2024-09-14T03:47:53.020 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:44:01.883265+0000", 2024-09-14T03:47:53.020 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:44:01.883265+0000", 2024-09-14T03:47:53.020 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:44:01.883550+0000", 2024-09-14T03:47:53.020 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.857768+0000", 2024-09-14T03:47:53.020 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:53.020 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:53.020 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:53.020 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 56, 2024-09-14T03:47:53.020 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.857768+0000", 2024-09-14T03:47:53.020 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.857768+0000", 2024-09-14T03:47:53.020 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.857768+0000", 2024-09-14T03:47:53.020 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:53.020 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:53.020 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:44:00.853212+0000", 2024-09-14T03:47:53.021 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.857768+0000", 2024-09-14T03:47:53.021 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.857768+0000", 2024-09-14T03:47:53.021 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:53.021 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:53.021 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:53.021 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:53.021 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 55, 2024-09-14T03:47:53.021 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:53.021 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:53.021 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:53.021 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:53.021 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:53.021 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:53.021 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:53.022 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:53.022 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "3.1e", 2024-09-14T03:47:53.022 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:53.022 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:53.022 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:53.022 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 21, 2024-09-14T03:47:53.022 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:53.022 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T03:52:04.512990+0000", 2024-09-14T03:47:53.022 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:53.022 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:53.022 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:53.022 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:53.022 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:53.022 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:53.023 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:53.023 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:53.023 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:53.023 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:53.023 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:53.023 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:53.023 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:53.023 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:53.023 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:53.023 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:53.023 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:53.023 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:53.023 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:53.023 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:53.024 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:53.024 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:53.024 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:53.024 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:53.024 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:53.024 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:53.024 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:53.024 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:53.024 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:53.024 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:53.024 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:53.024 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:53.024 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:53.024 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:53.025 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:53.025 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:53.025 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:53.025 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:53.025 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:53.025 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:53.025 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:53.025 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:53.025 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:53.025 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:53.025 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:53.025 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:53.025 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:53.025 INFO:tasks.workunit.client.0.smithi070.stdout: 3, 2024-09-14T03:47:53.025 INFO:tasks.workunit.client.0.smithi070.stdout: 6, 2024-09-14T03:47:53.026 INFO:tasks.workunit.client.0.smithi070.stdout: 2 2024-09-14T03:47:53.026 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:53.026 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 3, 2024-09-14T03:47:53.026 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "0'0" 2024-09-14T03:47:53.026 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:53.026 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:53.026 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": [ 2024-09-14T03:47:53.026 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:53.026 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:53.026 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:53.026 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:53.026 INFO:tasks.workunit.client.0.smithi070.stdout: "acting_primary": 0, 2024-09-14T03:47:53.026 INFO:tasks.workunit.client.0.smithi070.stdout: "avail_no_missing": [], 2024-09-14T03:47:53.026 INFO:tasks.workunit.client.0.smithi070.stdout: "blocked_by": [], 2024-09-14T03:47:53.027 INFO:tasks.workunit.client.0.smithi070.stdout: "created": 52, 2024-09-14T03:47:53.027 INFO:tasks.workunit.client.0.smithi070.stdout: "dirty_stats_invalid": false, 2024-09-14T03:47:53.027 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_bytes_stats_invalid": false, 2024-09-14T03:47:53.027 INFO:tasks.workunit.client.0.smithi070.stdout: "hitset_stats_invalid": false, 2024-09-14T03:47:53.027 INFO:tasks.workunit.client.0.smithi070.stdout: "last_active": "2024-09-14T03:45:14.684569+0000", 2024-09-14T03:47:53.027 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_active": "2024-09-14T03:43:58.731688+0000", 2024-09-14T03:47:53.027 INFO:tasks.workunit.client.0.smithi070.stdout: "last_became_peered": "2024-09-14T03:43:58.731688+0000", 2024-09-14T03:47:53.027 INFO:tasks.workunit.client.0.smithi070.stdout: "last_change": "2024-09-14T03:43:58.731954+0000", 2024-09-14T03:47:53.027 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean": "2024-09-14T03:45:14.684569+0000", 2024-09-14T03:47:53.027 INFO:tasks.workunit.client.0.smithi070.stdout: "last_clean_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:53.027 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub": "0'0", 2024-09-14T03:47:53.027 INFO:tasks.workunit.client.0.smithi070.stdout: "last_deep_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:53.027 INFO:tasks.workunit.client.0.smithi070.stdout: "last_epoch_clean": 53, 2024-09-14T03:47:53.027 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fresh": "2024-09-14T03:45:14.684569+0000", 2024-09-14T03:47:53.028 INFO:tasks.workunit.client.0.smithi070.stdout: "last_fullsized": "2024-09-14T03:45:14.684569+0000", 2024-09-14T03:47:53.028 INFO:tasks.workunit.client.0.smithi070.stdout: "last_peered": "2024-09-14T03:45:14.684569+0000", 2024-09-14T03:47:53.028 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub": "0'0", 2024-09-14T03:47:53.028 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_duration": 0, 2024-09-14T03:47:53.028 INFO:tasks.workunit.client.0.smithi070.stdout: "last_scrub_stamp": "2024-09-14T03:43:57.713731+0000", 2024-09-14T03:47:53.028 INFO:tasks.workunit.client.0.smithi070.stdout: "last_undegraded": "2024-09-14T03:45:14.684569+0000", 2024-09-14T03:47:53.028 INFO:tasks.workunit.client.0.smithi070.stdout: "last_unstale": "2024-09-14T03:45:14.684569+0000", 2024-09-14T03:47:53.028 INFO:tasks.workunit.client.0.smithi070.stdout: "log_dups_size": 0, 2024-09-14T03:47:53.028 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 4, 2024-09-14T03:47:53.028 INFO:tasks.workunit.client.0.smithi070.stdout: "log_start": "0'0", 2024-09-14T03:47:53.028 INFO:tasks.workunit.client.0.smithi070.stdout: "manifest_stats_invalid": false, 2024-09-14T03:47:53.028 INFO:tasks.workunit.client.0.smithi070.stdout: "mapping_epoch": 52, 2024-09-14T03:47:53.028 INFO:tasks.workunit.client.0.smithi070.stdout: "object_location_counts": [], 2024-09-14T03:47:53.028 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_scrubbed": 0, 2024-09-14T03:47:53.029 INFO:tasks.workunit.client.0.smithi070.stdout: "objects_trimmed": 0, 2024-09-14T03:47:53.029 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_stats_invalid": false, 2024-09-14T03:47:53.029 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 4, 2024-09-14T03:47:53.029 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_start": "0'0", 2024-09-14T03:47:53.029 INFO:tasks.workunit.client.0.smithi070.stdout: "parent": "0.0", 2024-09-14T03:47:53.029 INFO:tasks.workunit.client.0.smithi070.stdout: "parent_split_bits": 0, 2024-09-14T03:47:53.029 INFO:tasks.workunit.client.0.smithi070.stdout: "pgid": "2.1f", 2024-09-14T03:47:53.029 INFO:tasks.workunit.client.0.smithi070.stdout: "pin_stats_invalid": false, 2024-09-14T03:47:53.029 INFO:tasks.workunit.client.0.smithi070.stdout: "purged_snaps": [], 2024-09-14T03:47:53.029 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_epoch": 67, 2024-09-14T03:47:53.029 INFO:tasks.workunit.client.0.smithi070.stdout: "reported_seq": 27, 2024-09-14T03:47:53.029 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_duration": 0.0, 2024-09-14T03:47:53.029 INFO:tasks.workunit.client.0.smithi070.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-15T14:30:10.811609+0000", 2024-09-14T03:47:53.029 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrim_duration": 0.0, 2024-09-14T03:47:53.030 INFO:tasks.workunit.client.0.smithi070.stdout: "snaptrimq_len": 0, 2024-09-14T03:47:53.030 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:53.030 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 570, 2024-09-14T03:47:53.030 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:53.030 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:53.030 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:53.030 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:53.030 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:53.030 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:53.030 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:53.030 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:53.030 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:53.030 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:53.030 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:53.030 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:53.031 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:53.031 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:53.031 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:53.031 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 6, 2024-09-14T03:47:53.031 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 2, 2024-09-14T03:47:53.031 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:53.031 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 2, 2024-09-14T03:47:53.031 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:53.031 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:53.031 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:53.031 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:53.031 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:53.031 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 1, 2024-09-14T03:47:53.031 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:53.032 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:53.032 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:53.032 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:53.032 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:53.032 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:53.032 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:53.032 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:53.032 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:53.032 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:53.032 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:53.032 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:53.032 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 4, 2024-09-14T03:47:53.032 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 1 2024-09-14T03:47:53.033 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:53.033 INFO:tasks.workunit.client.0.smithi070.stdout: "state": "active+clean", 2024-09-14T03:47:53.033 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_invalid": false, 2024-09-14T03:47:53.033 INFO:tasks.workunit.client.0.smithi070.stdout: "up": [ 2024-09-14T03:47:53.033 INFO:tasks.workunit.client.0.smithi070.stdout: 0, 2024-09-14T03:47:53.033 INFO:tasks.workunit.client.0.smithi070.stdout: 7, 2024-09-14T03:47:53.033 INFO:tasks.workunit.client.0.smithi070.stdout: 4 2024-09-14T03:47:53.033 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:53.033 INFO:tasks.workunit.client.0.smithi070.stdout: "up_primary": 0, 2024-09-14T03:47:53.033 INFO:tasks.workunit.client.0.smithi070.stdout: "version": "58'4" 2024-09-14T03:47:53.033 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:53.033 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:53.033 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_per_pool": [ 2024-09-14T03:47:53.034 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:53.034 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": 24, 2024-09-14T03:47:53.034 INFO:tasks.workunit.client.0.smithi070.stdout: "application": [ 2024-09-14T03:47:53.034 INFO:tasks.workunit.client.0.smithi070.stdout: "rbd" 2024-09-14T03:47:53.034 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:53.034 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 3, 2024-09-14T03:47:53.034 INFO:tasks.workunit.client.0.smithi070.stdout: "num_pg": 8, 2024-09-14T03:47:53.034 INFO:tasks.workunit.client.0.smithi070.stdout: "num_store_stats": 8, 2024-09-14T03:47:53.034 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 3, 2024-09-14T03:47:53.034 INFO:tasks.workunit.client.0.smithi070.stdout: "poolid": 4, 2024-09-14T03:47:53.034 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:53.034 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 19, 2024-09-14T03:47:53.034 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:53.034 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:53.035 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:53.035 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:53.035 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:53.035 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:53.035 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:53.035 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:53.035 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:53.035 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:53.035 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:53.035 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:53.035 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:53.035 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:53.035 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:53.036 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 6, 2024-09-14T03:47:53.036 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 2, 2024-09-14T03:47:53.036 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:53.036 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 2, 2024-09-14T03:47:53.036 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:53.036 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:53.036 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:53.036 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:53.036 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:53.036 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:53.036 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:53.036 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:53.036 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:53.037 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:53.037 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:53.037 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:53.037 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:53.037 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:53.037 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:53.037 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:53.037 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:53.037 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:53.037 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 2, 2024-09-14T03:47:53.037 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 2 2024-09-14T03:47:53.037 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:53.037 INFO:tasks.workunit.client.0.smithi070.stdout: "store_stats": { 2024-09-14T03:47:53.038 INFO:tasks.workunit.client.0.smithi070.stdout: "allocated": 12288, 2024-09-14T03:47:53.038 INFO:tasks.workunit.client.0.smithi070.stdout: "available": 0, 2024-09-14T03:47:53.038 INFO:tasks.workunit.client.0.smithi070.stdout: "data_compressed": 0, 2024-09-14T03:47:53.038 INFO:tasks.workunit.client.0.smithi070.stdout: "data_compressed_allocated": 0, 2024-09-14T03:47:53.038 INFO:tasks.workunit.client.0.smithi070.stdout: "data_compressed_original": 0, 2024-09-14T03:47:53.038 INFO:tasks.workunit.client.0.smithi070.stdout: "data_stored": 57, 2024-09-14T03:47:53.038 INFO:tasks.workunit.client.0.smithi070.stdout: "internal_metadata": 0, 2024-09-14T03:47:53.038 INFO:tasks.workunit.client.0.smithi070.stdout: "internally_reserved": 0, 2024-09-14T03:47:53.038 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_allocated": 0, 2024-09-14T03:47:53.038 INFO:tasks.workunit.client.0.smithi070.stdout: "total": 0 2024-09-14T03:47:53.038 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:53.038 INFO:tasks.workunit.client.0.smithi070.stdout: "up": 24 2024-09-14T03:47:53.038 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:53.038 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:53.039 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": 192, 2024-09-14T03:47:53.039 INFO:tasks.workunit.client.0.smithi070.stdout: "application": [ 2024-09-14T03:47:53.039 INFO:tasks.workunit.client.0.smithi070.stdout: "cephfs" 2024-09-14T03:47:53.039 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:53.039 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 0, 2024-09-14T03:47:53.039 INFO:tasks.workunit.client.0.smithi070.stdout: "num_pg": 64, 2024-09-14T03:47:53.039 INFO:tasks.workunit.client.0.smithi070.stdout: "num_store_stats": 8, 2024-09-14T03:47:53.039 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 0, 2024-09-14T03:47:53.039 INFO:tasks.workunit.client.0.smithi070.stdout: "poolid": 3, 2024-09-14T03:47:53.039 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:53.039 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 0, 2024-09-14T03:47:53.039 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:53.039 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:53.040 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:53.040 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:53.040 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:53.040 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:53.040 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:53.040 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:53.040 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:53.040 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:53.040 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:53.040 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:53.040 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:53.040 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:53.040 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:53.041 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 0, 2024-09-14T03:47:53.041 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 0, 2024-09-14T03:47:53.041 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:53.041 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 0, 2024-09-14T03:47:53.041 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:53.041 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:53.041 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:53.041 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:53.041 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:53.041 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:53.041 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:53.041 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:53.041 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:53.042 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:53.042 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:53.042 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:53.042 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:53.042 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:53.042 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:53.042 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:53.042 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:53.042 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:53.042 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 0, 2024-09-14T03:47:53.042 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 0 2024-09-14T03:47:53.042 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:53.042 INFO:tasks.workunit.client.0.smithi070.stdout: "store_stats": { 2024-09-14T03:47:53.042 INFO:tasks.workunit.client.0.smithi070.stdout: "allocated": 0, 2024-09-14T03:47:53.043 INFO:tasks.workunit.client.0.smithi070.stdout: "available": 0, 2024-09-14T03:47:53.043 INFO:tasks.workunit.client.0.smithi070.stdout: "data_compressed": 0, 2024-09-14T03:47:53.043 INFO:tasks.workunit.client.0.smithi070.stdout: "data_compressed_allocated": 0, 2024-09-14T03:47:53.043 INFO:tasks.workunit.client.0.smithi070.stdout: "data_compressed_original": 0, 2024-09-14T03:47:53.043 INFO:tasks.workunit.client.0.smithi070.stdout: "data_stored": 0, 2024-09-14T03:47:53.043 INFO:tasks.workunit.client.0.smithi070.stdout: "internal_metadata": 0, 2024-09-14T03:47:53.043 INFO:tasks.workunit.client.0.smithi070.stdout: "internally_reserved": 0, 2024-09-14T03:47:53.043 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_allocated": 0, 2024-09-14T03:47:53.043 INFO:tasks.workunit.client.0.smithi070.stdout: "total": 0 2024-09-14T03:47:53.043 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:53.043 INFO:tasks.workunit.client.0.smithi070.stdout: "up": 192 2024-09-14T03:47:53.043 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:53.044 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:53.044 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": 96, 2024-09-14T03:47:53.044 INFO:tasks.workunit.client.0.smithi070.stdout: "application": [ 2024-09-14T03:47:53.044 INFO:tasks.workunit.client.0.smithi070.stdout: "cephfs" 2024-09-14T03:47:53.044 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:53.044 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 45, 2024-09-14T03:47:53.044 INFO:tasks.workunit.client.0.smithi070.stdout: "num_pg": 32, 2024-09-14T03:47:53.044 INFO:tasks.workunit.client.0.smithi070.stdout: "num_store_stats": 8, 2024-09-14T03:47:53.044 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 45, 2024-09-14T03:47:53.044 INFO:tasks.workunit.client.0.smithi070.stdout: "poolid": 2, 2024-09-14T03:47:53.044 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:53.044 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 3410, 2024-09-14T03:47:53.044 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:53.045 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 0, 2024-09-14T03:47:53.045 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:53.045 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:53.045 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:53.045 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:53.045 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:53.045 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:53.045 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:53.045 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:53.045 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:53.045 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:53.045 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:53.045 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:53.045 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:53.046 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 66, 2024-09-14T03:47:53.046 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 22, 2024-09-14T03:47:53.046 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:53.046 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 22, 2024-09-14T03:47:53.046 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:53.046 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:53.046 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:53.046 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:53.046 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:53.046 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 14, 2024-09-14T03:47:53.046 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:53.046 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 0, 2024-09-14T03:47:53.046 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:53.046 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:53.046 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:53.047 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:53.047 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:53.047 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 0, 2024-09-14T03:47:53.047 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 0, 2024-09-14T03:47:53.047 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:53.047 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:53.047 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:53.047 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 51, 2024-09-14T03:47:53.047 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 19 2024-09-14T03:47:53.047 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:53.047 INFO:tasks.workunit.client.0.smithi070.stdout: "store_stats": { 2024-09-14T03:47:53.047 INFO:tasks.workunit.client.0.smithi070.stdout: "allocated": 98304, 2024-09-14T03:47:53.047 INFO:tasks.workunit.client.0.smithi070.stdout: "available": 0, 2024-09-14T03:47:53.047 INFO:tasks.workunit.client.0.smithi070.stdout: "data_compressed": 0, 2024-09-14T03:47:53.048 INFO:tasks.workunit.client.0.smithi070.stdout: "data_compressed_allocated": 0, 2024-09-14T03:47:53.048 INFO:tasks.workunit.client.0.smithi070.stdout: "data_compressed_original": 0, 2024-09-14T03:47:53.048 INFO:tasks.workunit.client.0.smithi070.stdout: "data_stored": 10230, 2024-09-14T03:47:53.048 INFO:tasks.workunit.client.0.smithi070.stdout: "internal_metadata": 0, 2024-09-14T03:47:53.048 INFO:tasks.workunit.client.0.smithi070.stdout: "internally_reserved": 0, 2024-09-14T03:47:53.048 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_allocated": 0, 2024-09-14T03:47:53.048 INFO:tasks.workunit.client.0.smithi070.stdout: "total": 0 2024-09-14T03:47:53.048 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:53.048 INFO:tasks.workunit.client.0.smithi070.stdout: "up": 96 2024-09-14T03:47:53.048 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:53.048 INFO:tasks.workunit.client.0.smithi070.stdout: { 2024-09-14T03:47:53.048 INFO:tasks.workunit.client.0.smithi070.stdout: "acting": 3, 2024-09-14T03:47:53.048 INFO:tasks.workunit.client.0.smithi070.stdout: "application": [ 2024-09-14T03:47:53.048 INFO:tasks.workunit.client.0.smithi070.stdout: "mgr" 2024-09-14T03:47:53.049 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:53.049 INFO:tasks.workunit.client.0.smithi070.stdout: "log_size": 95, 2024-09-14T03:47:53.049 INFO:tasks.workunit.client.0.smithi070.stdout: "num_pg": 1, 2024-09-14T03:47:53.049 INFO:tasks.workunit.client.0.smithi070.stdout: "num_store_stats": 3, 2024-09-14T03:47:53.049 INFO:tasks.workunit.client.0.smithi070.stdout: "ondisk_log_size": 95, 2024-09-14T03:47:53.049 INFO:tasks.workunit.client.0.smithi070.stdout: "poolid": 1, 2024-09-14T03:47:53.049 INFO:tasks.workunit.client.0.smithi070.stdout: "stat_sum": { 2024-09-14T03:47:53.049 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes": 590368, 2024-09-14T03:47:53.049 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_hit_set_archive": 0, 2024-09-14T03:47:53.049 INFO:tasks.workunit.client.0.smithi070.stdout: "num_bytes_recovered": 590368, 2024-09-14T03:47:53.049 INFO:tasks.workunit.client.0.smithi070.stdout: "num_deep_scrub_errors": 0, 2024-09-14T03:47:53.049 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict": 0, 2024-09-14T03:47:53.049 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_kb": 0, 2024-09-14T03:47:53.050 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_full": 0, 2024-09-14T03:47:53.050 INFO:tasks.workunit.client.0.smithi070.stdout: "num_evict_mode_some": 0, 2024-09-14T03:47:53.051 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush": 0, 2024-09-14T03:47:53.052 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_kb": 0, 2024-09-14T03:47:53.052 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_high": 0, 2024-09-14T03:47:53.052 INFO:tasks.workunit.client.0.smithi070.stdout: "num_flush_mode_low": 0, 2024-09-14T03:47:53.052 INFO:tasks.workunit.client.0.smithi070.stdout: "num_keys_recovered": 0, 2024-09-14T03:47:53.052 INFO:tasks.workunit.client.0.smithi070.stdout: "num_large_omap_objects": 0, 2024-09-14T03:47:53.052 INFO:tasks.workunit.client.0.smithi070.stdout: "num_legacy_snapsets": 0, 2024-09-14T03:47:53.052 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_clones": 0, 2024-09-14T03:47:53.052 INFO:tasks.workunit.client.0.smithi070.stdout: "num_object_copies": 6, 2024-09-14T03:47:53.052 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects": 2, 2024-09-14T03:47:53.052 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_degraded": 0, 2024-09-14T03:47:53.052 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_dirty": 2, 2024-09-14T03:47:53.052 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_hit_set_archive": 0, 2024-09-14T03:47:53.052 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_manifest": 0, 2024-09-14T03:47:53.053 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_misplaced": 0, 2024-09-14T03:47:53.053 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing": 0, 2024-09-14T03:47:53.053 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_missing_on_primary": 0, 2024-09-14T03:47:53.053 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_omap": 0, 2024-09-14T03:47:53.053 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_pinned": 0, 2024-09-14T03:47:53.053 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_recovered": 2, 2024-09-14T03:47:53.053 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_repaired": 0, 2024-09-14T03:47:53.053 INFO:tasks.workunit.client.0.smithi070.stdout: "num_objects_unfound": 0, 2024-09-14T03:47:53.053 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_bytes": 0, 2024-09-14T03:47:53.053 INFO:tasks.workunit.client.0.smithi070.stdout: "num_omap_keys": 0, 2024-09-14T03:47:53.053 INFO:tasks.workunit.client.0.smithi070.stdout: "num_promote": 0, 2024-09-14T03:47:53.053 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read": 272, 2024-09-14T03:47:53.053 INFO:tasks.workunit.client.0.smithi070.stdout: "num_read_kb": 479, 2024-09-14T03:47:53.053 INFO:tasks.workunit.client.0.smithi070.stdout: "num_scrub_errors": 0, 2024-09-14T03:47:53.054 INFO:tasks.workunit.client.0.smithi070.stdout: "num_shallow_scrub_errors": 0, 2024-09-14T03:47:53.054 INFO:tasks.workunit.client.0.smithi070.stdout: "num_whiteouts": 0, 2024-09-14T03:47:53.054 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write": 145, 2024-09-14T03:47:53.054 INFO:tasks.workunit.client.0.smithi070.stdout: "num_write_kb": 1628 2024-09-14T03:47:53.054 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:53.054 INFO:tasks.workunit.client.0.smithi070.stdout: "store_stats": { 2024-09-14T03:47:53.054 INFO:tasks.workunit.client.0.smithi070.stdout: "allocated": 1781760, 2024-09-14T03:47:53.054 INFO:tasks.workunit.client.0.smithi070.stdout: "available": 0, 2024-09-14T03:47:53.054 INFO:tasks.workunit.client.0.smithi070.stdout: "data_compressed": 0, 2024-09-14T03:47:53.054 INFO:tasks.workunit.client.0.smithi070.stdout: "data_compressed_allocated": 0, 2024-09-14T03:47:53.054 INFO:tasks.workunit.client.0.smithi070.stdout: "data_compressed_original": 0, 2024-09-14T03:47:53.054 INFO:tasks.workunit.client.0.smithi070.stdout: "data_stored": 1771104, 2024-09-14T03:47:53.055 INFO:tasks.workunit.client.0.smithi070.stdout: "internal_metadata": 0, 2024-09-14T03:47:53.055 INFO:tasks.workunit.client.0.smithi070.stdout: "internally_reserved": 0, 2024-09-14T03:47:53.055 INFO:tasks.workunit.client.0.smithi070.stdout: "omap_allocated": 0, 2024-09-14T03:47:53.055 INFO:tasks.workunit.client.0.smithi070.stdout: "total": 0 2024-09-14T03:47:53.055 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:53.055 INFO:tasks.workunit.client.0.smithi070.stdout: "up": 3 2024-09-14T03:47:53.055 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:53.055 INFO:tasks.workunit.client.0.smithi070.stdout: ], 2024-09-14T03:47:53.055 INFO:tasks.workunit.client.0.smithi070.stdout: "usage": { 2024-09-14T03:47:53.055 INFO:tasks.workunit.client.0.smithi070.stdout: "pg_num": 105, 2024-09-14T03:47:53.055 INFO:tasks.workunit.client.0.smithi070.stdout: "pools": 4, 2024-09-14T03:47:53.055 INFO:tasks.workunit.client.0.smithi070.stdout: "stats_by_class": { 2024-09-14T03:47:53.055 INFO:tasks.workunit.client.0.smithi070.stdout: "ssd": { 2024-09-14T03:47:53.055 INFO:tasks.workunit.client.0.smithi070.stdout: "total_avail_bytes": 765516935168, 2024-09-14T03:47:53.055 INFO:tasks.workunit.client.0.smithi070.stdout: "total_bytes": 767960285184, 2024-09-14T03:47:53.056 INFO:tasks.workunit.client.0.smithi070.stdout: "total_used_bytes": 2443350016, 2024-09-14T03:47:53.056 INFO:tasks.workunit.client.0.smithi070.stdout: "total_used_raw_bytes": 2443350016, 2024-09-14T03:47:53.056 INFO:tasks.workunit.client.0.smithi070.stdout: "total_used_raw_ratio": 0.0031816097907721996 2024-09-14T03:47:53.056 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:53.056 INFO:tasks.workunit.client.0.smithi070.stdout: }, 2024-09-14T03:47:53.056 INFO:tasks.workunit.client.0.smithi070.stdout: "total_avail_bytes": 765516935168, 2024-09-14T03:47:53.056 INFO:tasks.workunit.client.0.smithi070.stdout: "total_bytes": 767960285184, 2024-09-14T03:47:53.056 INFO:tasks.workunit.client.0.smithi070.stdout: "total_used_bytes": 2443350016 2024-09-14T03:47:53.056 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:53.056 INFO:tasks.workunit.client.0.smithi070.stdout: } 2024-09-14T03:47:53.056 INFO:tasks.workunit.client.0.smithi070.stdout:} 2024-09-14T03:47:53.056 INFO:tasks.workunit.client.0.smithi070.stdout:OK 2024-09-14T03:47:53.057 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-14T03:47:53.057 DEBUG:teuthology.orchestra.run.smithi070:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-14T03:47:53.058 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='client.24770 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:53.058 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='client.24788 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:53.058 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='client.24788 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:53.058 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='client.14886 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:53.058 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:53.058 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:53.058 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:53.058 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:53.059 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:53.059 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:53.059 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:53.059 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:53.059 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:53.059 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:53.059 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:53.059 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:53.059 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:53.059 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:53.059 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:53.059 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:53.059 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:53.059 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:53.060 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:53.060 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:53.060 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:53.060 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:53.060 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:53.060 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:47:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:47:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:47:53.060 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='client.24770 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:53.060 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='client.24788 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:53.060 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='client.24788 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:53.061 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='client.14886 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:53.061 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:53.061 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:53.061 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:53.061 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:53.061 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:53.061 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:53.061 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:53.061 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:53.061 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:53.061 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:53.061 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:53.061 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:53.062 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:53.062 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:53.062 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:53.062 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:53.062 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:53.062 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:53.062 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:53.062 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:53.062 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:53.062 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:53.062 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:53.062 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:51 smithi094 ceph-mon[31053]: from='client.14886 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:53.063 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:51 smithi094 ceph-mon[31053]: pgmap v92: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-14T03:47:53.063 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:51 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:47:53.063 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:53.063 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:53.063 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:53.063 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:53.063 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:53.063 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:53.063 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:53.063 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:53.064 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:53.064 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:53.064 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:53.064 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:53.064 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:53.064 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:53.064 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:53.064 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:53.064 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:53.064 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:53.064 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:53.064 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:53.064 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:53.065 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:53.065 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:53.065 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:53.065 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:53.065 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:53.065 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:53.065 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:53.065 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:53.065 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:53.065 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:53.066 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:53.066 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:53.066 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:47:53.066 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:53.066 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:53.066 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:53.066 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:53.066 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:53.066 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:53.066 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:53.066 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:53.066 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:53.066 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:53.067 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:53.067 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:53.067 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:53.067 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:53.067 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:53.067 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:53.067 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:53.067 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:53.067 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:53.067 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:47:53.067 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:47:53.067 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:53.067 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:53.068 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:50 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:53.068 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:51 smithi070 ceph-mon[39288]: from='client.14886 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:53.068 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:51 smithi070 ceph-mon[39288]: pgmap v92: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-14T03:47:53.068 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:51 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:47:53.068 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:51 smithi070 ceph-mon[27495]: from='client.14886 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:53.068 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:51 smithi070 ceph-mon[27495]: pgmap v92: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-14T03:47:53.069 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:51 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:47:53.101 INFO:tasks.workunit:Stopping ['test_telemetry_quincy.sh'] on client.0... 2024-09-14T03:47:53.101 DEBUG:teuthology.orchestra.run.smithi070:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-09-14T03:47:53.413 DEBUG:teuthology.parallel:result is None 2024-09-14T03:47:53.414 DEBUG:teuthology.orchestra.run.smithi070:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2024-09-14T03:47:53.439 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2024-09-14T03:47:53.439 DEBUG:teuthology.orchestra.run.smithi070:> rmdir -- /home/ubuntu/cephtest/mnt.0 2024-09-14T03:47:53.500 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2024-09-14T03:47:53.500 INFO:teuthology.run_tasks:Running task print... 2024-09-14T03:47:53.509 INFO:teuthology.task.print:**** done end telemetry quincy... 2024-09-14T03:47:53.510 INFO:teuthology.run_tasks:Running task print... 2024-09-14T03:47:53.524 INFO:teuthology.task.print:**** done start parallel 2024-09-14T03:47:53.524 INFO:teuthology.run_tasks:Running task parallel... 2024-09-14T03:47:53.540 INFO:teuthology.task.parallel:starting parallel... 2024-09-14T03:47:53.540 INFO:teuthology.task.parallel:In parallel, running task full_sequential... 2024-09-14T03:47:53.541 INFO:teuthology.task.full_sequential:In full_sequential, running task print... 2024-09-14T03:47:53.541 INFO:teuthology.task.print:**** done start ec-rados-default.yaml 2024-09-14T03:47:53.541 INFO:teuthology.task.full_sequential:In full_sequential, running task rados... 2024-09-14T03:47:53.544 INFO:tasks.rados:Beginning rados... 2024-09-14T03:47:53.544 INFO:tasks.rados:config is {'clients': ['client.0'], 'ec_pool': True, 'objects': 50, 'op_weights': {'append': 100, 'copy_from': 50, 'delete': 50, 'read': 100, 'rmattr': 25, 'rollback': 50, 'setattr': 25, 'snap_create': 50, 'snap_remove': 50, 'write': 0}, 'ops': 4000, 'write_append_excl': False} 2024-09-14T03:47:53.544 INFO:tasks.rados:overrides is {'admin_socket': {'branch': 'main'}, 'ceph': {'conf': {'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(POOL_APP_NOT_ENABLED\\)', 'OSD_DOWN', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'OSDMAP_FLAGS', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_AVAILABILITY'], 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'extra_system_packages': ['python3-pytest'], 'flavor': 'default', 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'main', 'sha1': '1c941a66c1b420259b8fc5ac19e792dbbf60aded'}} 2024-09-14T03:47:53.544 INFO:tasks.rados:config is {'clients': ['client.0'], 'ec_pool': True, 'objects': 50, 'op_weights': {'append': 100, 'copy_from': 50, 'delete': 50, 'read': 100, 'rmattr': 25, 'rollback': 50, 'setattr': 25, 'snap_create': 50, 'snap_remove': 50, 'write': 0}, 'ops': 4000, 'write_append_excl': False} 2024-09-14T03:47:53.544 INFO:tasks.rados:joining rados 2024-09-14T03:47:53.544 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-14T03:47:53.545 INFO:teuthology.task.sequential:In sequential, running task print... 2024-09-14T03:47:53.545 INFO:teuthology.task.print:**** done start upgrade, wait 2024-09-14T03:47:53.545 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-14T03:47:53.545 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi070.front.sepia.ceph.com 2024-09-14T03:47:53.546 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-14T03:47:53.546 INFO:tasks.rados:clients are ['client.0', 'client.1'] 2024-09-14T03:47:53.546 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph osd erasure-code-profile set teuthologyprofile k=2 m=1 crush-failure-domain=osd 2024-09-14T03:47:53.803 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:47:53.803 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:47:54.188 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:53 smithi070 ceph-mon[27495]: pgmap v93: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-14T03:47:54.189 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:53 smithi070 ceph-mon[39288]: pgmap v93: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-14T03:47:54.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:53 smithi094 ceph-mon[31053]: pgmap v93: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-14T03:47:56.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:55 smithi070 ceph-mon[27495]: pgmap v94: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-14T03:47:56.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:55 smithi070 ceph-mon[39288]: pgmap v94: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-14T03:47:56.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:55 smithi094 ceph-mon[31053]: pgmap v94: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-14T03:47:56.440 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:47:56 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:47:56] "GET /metrics HTTP/1.1" 200 236462 "" "Prometheus/2.43.0" 2024-09-14T03:47:57.981 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-14T03:47:58.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:57 smithi094 ceph-mon[31053]: pgmap v95: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-14T03:47:58.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:57 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/1906511163' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-14T03:47:58.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:57 smithi094 ceph-mon[31053]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-14T03:47:58.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:57 smithi070 ceph-mon[27495]: pgmap v95: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-14T03:47:58.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:57 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/1906511163' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-14T03:47:58.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:57 smithi070 ceph-mon[27495]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-14T03:47:58.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:57 smithi070 ceph-mon[39288]: pgmap v95: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-14T03:47:58.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:57 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/1906511163' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-14T03:47:58.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:57 smithi070 ceph-mon[39288]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-14T03:47:58.629 INFO:tasks.rados:starting run 0 out of 1 2024-09-14T03:47:58.629 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2024-09-14T03:47:58.629 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_0 16 16 erasure teuthologyprofile 2024-09-14T03:47:58.662 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:47:59.191 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:58 smithi070 ceph-mon[27495]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2024-09-14T03:47:59.191 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:58 smithi070 ceph-mon[27495]: osdmap e69: 8 total, 8 up, 8 in 2024-09-14T03:47:59.191 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:58 smithi070 ceph-mon[39288]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2024-09-14T03:47:59.191 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:58 smithi070 ceph-mon[39288]: osdmap e69: 8 total, 8 up, 8 in 2024-09-14T03:47:59.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:58 smithi094 ceph-mon[31053]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2024-09-14T03:47:59.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:58 smithi094 ceph-mon[31053]: osdmap e69: 8 total, 8 up, 8 in 2024-09-14T03:47:59.383 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:48:00.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:47:59 smithi070 ceph-mon[27495]: pgmap v97: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 35 op/s 2024-09-14T03:48:00.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:47:59 smithi070 ceph-mon[39288]: pgmap v97: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 35 op/s 2024-09-14T03:48:00.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:47:59 smithi094 ceph-mon[31053]: pgmap v97: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 35 op/s 2024-09-14T03:48:01.809 INFO:teuthology.orchestra.run.smithi070.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:48:01.938 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:48:01 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:48:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:48:02.186 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:01 smithi070 ceph-mon[27495]: pgmap v98: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 35 op/s 2024-09-14T03:48:02.186 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:01 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:48:02.186 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:01 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:48:02.186 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:01 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:02.186 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:01 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:48:02.186 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:01 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:48:02.186 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:01 smithi070 ceph-mon[39288]: pgmap v98: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 35 op/s 2024-09-14T03:48:02.187 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:01 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:48:02.187 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:01 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:48:02.187 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:01 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:02.187 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:01 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:48:02.187 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:01 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:48:02.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:01 smithi094 ceph-mon[31053]: pgmap v98: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 35 op/s 2024-09-14T03:48:02.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:01 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:48:02.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:01 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:48:02.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:01 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:02.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:01 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:48:02.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:01 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:48:02.552 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-14T03:48:02.852 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:48:03.273 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:02 smithi070 ceph-mon[27495]: from='client.24824 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:03.273 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:02 smithi070 ceph-mon[27495]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:48:03.273 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:02 smithi070 ceph-mon[27495]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:48:03.274 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:02 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/2292319355' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]: dispatch 2024-09-14T03:48:03.274 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:02 smithi070 ceph-mon[27495]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]: dispatch 2024-09-14T03:48:03.274 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:02 smithi070 ceph-mon[39288]: from='client.24824 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:03.274 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:02 smithi070 ceph-mon[39288]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:48:03.274 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:02 smithi070 ceph-mon[39288]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:48:03.274 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:02 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/2292319355' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]: dispatch 2024-09-14T03:48:03.274 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:02 smithi070 ceph-mon[39288]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]: dispatch 2024-09-14T03:48:03.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:02 smithi094 ceph-mon[31053]: from='client.24824 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:03.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:02 smithi094 ceph-mon[31053]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:48:03.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:02 smithi094 ceph-mon[31053]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:48:03.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:02 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/2292319355' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]: dispatch 2024-09-14T03:48:03.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:02 smithi094 ceph-mon[31053]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]: dispatch 2024-09-14T03:48:03.956 INFO:teuthology.orchestra.run.smithi070.stderr:pool 'unique_pool_0' created 2024-09-14T03:48:04.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:03 smithi070 ceph-mon[27495]: pgmap v99: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:48:04.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:03 smithi070 ceph-mon[27495]: osdmap e70: 8 total, 8 up, 8 in 2024-09-14T03:48:04.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:03 smithi070 ceph-mon[39288]: pgmap v99: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:48:04.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:03 smithi070 ceph-mon[39288]: osdmap e70: 8 total, 8 up, 8 in 2024-09-14T03:48:04.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:03 smithi094 ceph-mon[31053]: pgmap v99: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:48:04.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:03 smithi094 ceph-mon[31053]: osdmap e70: 8 total, 8 up, 8 in 2024-09-14T03:48:04.730 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2024-09-14T03:48:04.990 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:04 smithi070 ceph-mon[27495]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]': finished 2024-09-14T03:48:04.991 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:04 smithi070 ceph-mon[27495]: osdmap e71: 8 total, 8 up, 8 in 2024-09-14T03:48:04.991 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:04 smithi070 ceph-mon[39288]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]': finished 2024-09-14T03:48:04.991 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:04 smithi070 ceph-mon[39288]: osdmap e71: 8 total, 8 up, 8 in 2024-09-14T03:48:05.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:04 smithi094 ceph-mon[31053]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]': finished 2024-09-14T03:48:05.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:04 smithi094 ceph-mon[31053]: osdmap e71: 8 total, 8 up, 8 in 2024-09-14T03:48:05.287 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:48:05.446 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-09-14T03:48:06.039 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:48:06.039 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (103s) 85s ago 2m 15.1M - 0.25.0 c8568f914cd2 fef584d3d5c6 2024-09-14T03:48:06.039 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi094 *:3000 running (2m) 87s ago 2m 85.0M - 9.4.7 954c08fa6188 846770f48257 2024-09-14T03:48:06.039 INFO:teuthology.orchestra.run.smithi070.stdout:mds.a smithi070 running (4m) 85s ago 4m 16.7M - 17.2.7-1629-geb16b97b ab51d1bebd12 52e0dbb7be50 2024-09-14T03:48:06.039 INFO:teuthology.orchestra.run.smithi070.stdout:mds.b smithi094 running (4m) 87s ago 4m 13.6M - 17.2.7-1629-geb16b97b ab51d1bebd12 e3b5e3e00189 2024-09-14T03:48:06.039 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi094 *:8443 running (7m) 87s ago 7m 399M - 17.2.7-1629-geb16b97b ab51d1bebd12 8a5c729a2096 2024-09-14T03:48:06.040 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:9283,8765 running (9m) 85s ago 9m 447M - 17.2.7-1629-geb16b97b ab51d1bebd12 dc71810017b9 2024-09-14T03:48:06.040 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (9m) 85s ago 9m 44.3M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 12d19db91f54 2024-09-14T03:48:06.040 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi094 running (8m) 87s ago 8m 37.9M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 90ea2e164909 2024-09-14T03:48:06.040 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (7m) 85s ago 7m 38.1M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 44ee80c29ee0 2024-09-14T03:48:06.040 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (2m) 85s ago 2m 13.3M - 1.5.0 0da6a335fe13 315c3f32399f 2024-09-14T03:48:06.040 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi094 *:9100 running (2m) 87s ago 2m 3577k - 1.5.0 0da6a335fe13 3dcf1723ea35 2024-09-14T03:48:06.040 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (7m) 85s ago 7m 44.0M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 741c06b6adbf 2024-09-14T03:48:06.040 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (6m) 85s ago 6m 44.6M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 af459ea7ef68 2024-09-14T03:48:06.040 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (6m) 85s ago 6m 42.4M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 109b66e4e60f 2024-09-14T03:48:06.040 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (6m) 85s ago 6m 43.7M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 2e4199b58392 2024-09-14T03:48:06.040 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi094 running (5m) 87s ago 5m 42.0M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 52b64527e24c 2024-09-14T03:48:06.040 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi094 running (5m) 87s ago 5m 42.4M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 b543f152ba87 2024-09-14T03:48:06.040 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi094 running (5m) 87s ago 5m 42.0M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 e66b592ac940 2024-09-14T03:48:06.041 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi094 running (4m) 87s ago 4m 44.9M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 037c6208e9ce 2024-09-14T03:48:06.041 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi094 *:9095 running (98s) 87s ago 3m 24.5M - 2.43.0 a07b618ecd1d 9c88e2923e9d 2024-09-14T03:48:06.041 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:05 smithi070 ceph-mon[27495]: pgmap v102: 121 pgs: 16 unknown, 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:48:06.041 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:05 smithi070 ceph-mon[27495]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:48:06.041 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:05 smithi070 ceph-mon[27495]: osdmap e72: 8 total, 8 up, 8 in 2024-09-14T03:48:06.041 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:05 smithi070 ceph-mon[39288]: pgmap v102: 121 pgs: 16 unknown, 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:48:06.041 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:05 smithi070 ceph-mon[39288]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:48:06.041 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:05 smithi070 ceph-mon[39288]: osdmap e72: 8 total, 8 up, 8 in 2024-09-14T03:48:06.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:05 smithi094 ceph-mon[31053]: pgmap v102: 121 pgs: 16 unknown, 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:48:06.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:05 smithi094 ceph-mon[31053]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:48:06.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:05 smithi094 ceph-mon[31053]: osdmap e72: 8 total, 8 up, 8 in 2024-09-14T03:48:06.385 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:48:06.385 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-09-14T03:48:06.385 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 3 2024-09-14T03:48:06.385 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:48:06.386 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-09-14T03:48:06.386 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:48:06.386 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:48:06.386 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-09-14T03:48:06.386 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 8 2024-09-14T03:48:06.386 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:48:06.386 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": { 2024-09-14T03:48:06.386 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:48:06.386 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:48:06.386 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-09-14T03:48:06.386 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 15 2024-09-14T03:48:06.386 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-09-14T03:48:06.386 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:48:06.602 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:48:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:48:06] "GET /metrics HTTP/1.1" 200 236462 "" "Prometheus/2.43.0" 2024-09-14T03:48:06.714 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:48:06.714 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", 2024-09-14T03:48:06.714 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-09-14T03:48:06.714 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:48:06.715 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [], 2024-09-14T03:48:06.715 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "", 2024-09-14T03:48:06.715 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 image", 2024-09-14T03:48:06.715 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-09-14T03:48:06.715 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:48:07.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:06 smithi094 ceph-mon[31053]: from='client.24836 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:07.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:06 smithi094 ceph-mon[31053]: from='client.24842 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:07.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:06 smithi094 ceph-mon[31053]: osdmap e73: 8 total, 8 up, 8 in 2024-09-14T03:48:07.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:06 smithi094 ceph-mon[31053]: from='client.24848 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:07.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:06 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/3274072966' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:07.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:06 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:48:07.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:06 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:48:07.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:06 smithi070 ceph-mon[27495]: from='client.24836 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:07.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:06 smithi070 ceph-mon[27495]: from='client.24842 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:07.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:06 smithi070 ceph-mon[27495]: osdmap e73: 8 total, 8 up, 8 in 2024-09-14T03:48:07.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:06 smithi070 ceph-mon[27495]: from='client.24848 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:07.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:06 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/3274072966' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:07.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:06 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:48:07.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:06 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:48:07.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:06 smithi070 ceph-mon[39288]: from='client.24836 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:07.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:06 smithi070 ceph-mon[39288]: from='client.24842 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:07.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:06 smithi070 ceph-mon[39288]: osdmap e73: 8 total, 8 up, 8 in 2024-09-14T03:48:07.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:06 smithi070 ceph-mon[39288]: from='client.24848 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:07.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:06 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/3274072966' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:07.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:06 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:48:07.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:06 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:48:07.966 INFO:teuthology.orchestra.run.smithi070.stderr:enabled application 'rados' on pool 'unique_pool_0' 2024-09-14T03:48:08.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:07 smithi094 ceph-mon[31053]: pgmap v105: 121 pgs: 16 unknown, 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:48:08.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:07 smithi094 ceph-mon[31053]: from='client.14952 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:08.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:07 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/184799645' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:48:08.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:07 smithi070 ceph-mon[39288]: pgmap v105: 121 pgs: 16 unknown, 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:48:08.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:07 smithi070 ceph-mon[39288]: from='client.14952 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:08.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:07 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/184799645' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:48:08.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:07 smithi070 ceph-mon[27495]: pgmap v105: 121 pgs: 16 unknown, 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:48:08.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:07 smithi070 ceph-mon[27495]: from='client.14952 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:08.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:07 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/184799645' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:48:08.973 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:08 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/184799645' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T03:48:08.973 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:08 smithi070 ceph-mon[39288]: osdmap e74: 8 total, 8 up, 8 in 2024-09-14T03:48:09.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:08 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/184799645' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T03:48:09.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:08 smithi094 ceph-mon[31053]: osdmap e74: 8 total, 8 up, 8 in 2024-09-14T03:48:09.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:08 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/184799645' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T03:48:09.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:08 smithi070 ceph-mon[27495]: osdmap e74: 8 total, 8 up, 8 in 2024-09-14T03:48:09.705 DEBUG:teuthology.orchestra.run.smithi070:> CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --no-omap --ec-pool --max-ops 4000 --objects 50 --max-in-flight 16 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 0 --op read 100 --op write 0 --op delete 50 --op snap_create 50 --op snap_remove 50 --op rollback 50 --op setattr 25 --op rmattr 25 --op copy_from 50 --op append 100 --pool unique_pool_0 2024-09-14T03:48:09.732 INFO:tasks.rados.rados.0.smithi070.stdout:adding op weight read -> 100 2024-09-14T03:48:09.732 INFO:tasks.rados.rados.0.smithi070.stdout:adding op weight delete -> 50 2024-09-14T03:48:09.732 INFO:tasks.rados.rados.0.smithi070.stdout:adding op weight snap_create -> 50 2024-09-14T03:48:09.732 INFO:tasks.rados.rados.0.smithi070.stdout:adding op weight snap_remove -> 50 2024-09-14T03:48:09.732 INFO:tasks.rados.rados.0.smithi070.stdout:adding op weight rollback -> 50 2024-09-14T03:48:09.732 INFO:tasks.rados.rados.0.smithi070.stdout:adding op weight setattr -> 25 2024-09-14T03:48:09.732 INFO:tasks.rados.rados.0.smithi070.stdout:adding op weight rmattr -> 25 2024-09-14T03:48:09.732 INFO:tasks.rados.rados.0.smithi070.stdout:adding op weight copy_from -> 50 2024-09-14T03:48:09.733 INFO:tasks.rados.rados.0.smithi070.stdout:adding op weight append -> 100 2024-09-14T03:48:09.733 INFO:tasks.rados.rados.0.smithi070.stdout:ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable) 2024-09-14T03:48:09.733 INFO:tasks.rados.rados.0.smithi070.stdout:Configuration: 2024-09-14T03:48:09.733 INFO:tasks.rados.rados.0.smithi070.stdout: Number of operations: 4000 2024-09-14T03:48:09.733 INFO:tasks.rados.rados.0.smithi070.stdout: Number of objects: 50 2024-09-14T03:48:09.733 INFO:tasks.rados.rados.0.smithi070.stdout: Max in flight operations: 16 2024-09-14T03:48:09.733 INFO:tasks.rados.rados.0.smithi070.stdout: Object size (in bytes): 4000000 2024-09-14T03:48:09.733 INFO:tasks.rados.rados.0.smithi070.stdout: Write stride min: 400000 2024-09-14T03:48:09.733 INFO:tasks.rados.rados.0.smithi070.stdout: Write stride max: 800000 2024-09-14T03:48:09.981 INFO:tasks.rados.rados.0.smithi070.stdout:1: write initial oid 1 2024-09-14T03:48:09.981 INFO:tasks.rados.rados.0.smithi070.stdout:1: seq_num 0 ranges {0=24576} 2024-09-14T03:48:09.981 INFO:tasks.rados.rados.0.smithi070.stdout:1: writing smithi07085154-1 from 0 to 24576 tid 1 2024-09-14T03:48:09.981 INFO:tasks.rados.rados.0.smithi070.stdout:2: write initial oid 2 2024-09-14T03:48:09.981 INFO:tasks.rados.rados.0.smithi070.stdout:2: seq_num 1 ranges {0=24576} 2024-09-14T03:48:09.981 INFO:tasks.rados.rados.0.smithi070.stdout:2: writing smithi07085154-2 from 0 to 24576 tid 1 2024-09-14T03:48:09.982 INFO:tasks.rados.rados.0.smithi070.stdout:3: write initial oid 3 2024-09-14T03:48:09.982 INFO:tasks.rados.rados.0.smithi070.stdout:3: seq_num 2 ranges {0=40960} 2024-09-14T03:48:09.982 INFO:tasks.rados.rados.0.smithi070.stdout:3: writing smithi07085154-3 from 0 to 40960 tid 1 2024-09-14T03:48:09.982 INFO:tasks.rados.rados.0.smithi070.stdout:4: write initial oid 4 2024-09-14T03:48:09.982 INFO:tasks.rados.rados.0.smithi070.stdout:4: seq_num 3 ranges {0=57344} 2024-09-14T03:48:09.982 INFO:tasks.rados.rados.0.smithi070.stdout:4: writing smithi07085154-4 from 0 to 57344 tid 1 2024-09-14T03:48:09.982 INFO:tasks.rados.rados.0.smithi070.stdout:5: write initial oid 5 2024-09-14T03:48:09.982 INFO:tasks.rados.rados.0.smithi070.stdout:5: seq_num 4 ranges {0=73728} 2024-09-14T03:48:09.983 INFO:tasks.rados.rados.0.smithi070.stdout:5: writing smithi07085154-5 from 0 to 73728 tid 1 2024-09-14T03:48:09.983 INFO:tasks.rados.rados.0.smithi070.stdout:6: write initial oid 6 2024-09-14T03:48:09.983 INFO:tasks.rados.rados.0.smithi070.stdout:6: seq_num 5 ranges {0=90112} 2024-09-14T03:48:09.984 INFO:tasks.rados.rados.0.smithi070.stdout:6: writing smithi07085154-6 from 0 to 90112 tid 1 2024-09-14T03:48:09.984 INFO:tasks.rados.rados.0.smithi070.stdout:7: write initial oid 7 2024-09-14T03:48:09.984 INFO:tasks.rados.rados.0.smithi070.stdout:7: seq_num 6 ranges {0=106496} 2024-09-14T03:48:09.984 INFO:tasks.rados.rados.0.smithi070.stdout:7: writing smithi07085154-7 from 0 to 106496 tid 1 2024-09-14T03:48:09.984 INFO:tasks.rados.rados.0.smithi070.stdout:8: write initial oid 8 2024-09-14T03:48:09.984 INFO:tasks.rados.rados.0.smithi070.stdout:8: seq_num 7 ranges {0=122880} 2024-09-14T03:48:09.984 INFO:tasks.rados.rados.0.smithi070.stdout:8: writing smithi07085154-8 from 0 to 122880 tid 1 2024-09-14T03:48:09.985 INFO:tasks.rados.rados.0.smithi070.stdout:9: write initial oid 9 2024-09-14T03:48:09.985 INFO:tasks.rados.rados.0.smithi070.stdout:9: seq_num 8 ranges {0=139264} 2024-09-14T03:48:09.986 INFO:tasks.rados.rados.0.smithi070.stdout:9: writing smithi07085154-9 from 0 to 139264 tid 1 2024-09-14T03:48:09.986 INFO:tasks.rados.rados.0.smithi070.stdout:10: write initial oid 10 2024-09-14T03:48:09.986 INFO:tasks.rados.rados.0.smithi070.stdout:10: seq_num 9 ranges {0=155648} 2024-09-14T03:48:09.987 INFO:tasks.rados.rados.0.smithi070.stdout:10: writing smithi07085154-10 from 0 to 155648 tid 1 2024-09-14T03:48:09.988 INFO:tasks.rados.rados.0.smithi070.stdout:11: write initial oid 11 2024-09-14T03:48:09.988 INFO:tasks.rados.rados.0.smithi070.stdout:11: seq_num 10 ranges {0=172032} 2024-09-14T03:48:09.989 INFO:tasks.rados.rados.0.smithi070.stdout:11: writing smithi07085154-11 from 0 to 172032 tid 1 2024-09-14T03:48:09.990 INFO:tasks.rados.rados.0.smithi070.stdout:12: write initial oid 12 2024-09-14T03:48:09.990 INFO:tasks.rados.rados.0.smithi070.stdout:12: seq_num 11 ranges {0=188416} 2024-09-14T03:48:09.992 INFO:tasks.rados.rados.0.smithi070.stdout:12: writing smithi07085154-12 from 0 to 188416 tid 1 2024-09-14T03:48:09.992 INFO:tasks.rados.rados.0.smithi070.stdout:1: finishing write tid 1 to smithi07085154-1 2024-09-14T03:48:09.992 INFO:tasks.rados.rados.0.smithi070.stdout:2: finishing write tid 1 to smithi07085154-2 2024-09-14T03:48:09.992 INFO:tasks.rados.rados.0.smithi070.stdout:1: finishing write tid 2 to smithi07085154-1 2024-09-14T03:48:09.992 INFO:tasks.rados.rados.0.smithi070.stdout:1: finishing write tid 3 to smithi07085154-1 2024-09-14T03:48:09.992 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 1 v 2 (ObjNum 0 snap 0 seq_num 0) dirty exists 2024-09-14T03:48:09.992 INFO:tasks.rados.rados.0.smithi070.stdout:1: left oid 1 (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:48:09.992 INFO:tasks.rados.rados.0.smithi070.stdout:3: finishing write tid 1 to smithi07085154-3 2024-09-14T03:48:09.992 INFO:tasks.rados.rados.0.smithi070.stdout:3: finishing write tid 2 to smithi07085154-3 2024-09-14T03:48:09.993 INFO:tasks.rados.rados.0.smithi070.stdout:2: finishing write tid 2 to smithi07085154-2 2024-09-14T03:48:09.993 INFO:tasks.rados.rados.0.smithi070.stdout:6: finishing write tid 1 to smithi07085154-6 2024-09-14T03:48:09.993 INFO:tasks.rados.rados.0.smithi070.stdout:6: finishing write tid 2 to smithi07085154-6 2024-09-14T03:48:09.993 INFO:tasks.rados.rados.0.smithi070.stdout:4: finishing write tid 1 to smithi07085154-4 2024-09-14T03:48:09.993 INFO:tasks.rados.rados.0.smithi070.stdout:5: finishing write tid 1 to smithi07085154-5 2024-09-14T03:48:09.993 INFO:tasks.rados.rados.0.smithi070.stdout:4: finishing write tid 2 to smithi07085154-4 2024-09-14T03:48:09.993 INFO:tasks.rados.rados.0.smithi070.stdout:3: finishing write tid 3 to smithi07085154-3 2024-09-14T03:48:09.993 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 3 v 2 (ObjNum 2 snap 0 seq_num 2) dirty exists 2024-09-14T03:48:09.993 INFO:tasks.rados.rados.0.smithi070.stdout:3: left oid 3 (ObjNum 2 snap 0 seq_num 2) 2024-09-14T03:48:09.993 INFO:tasks.rados.rados.0.smithi070.stdout:6: finishing write tid 3 to smithi07085154-6 2024-09-14T03:48:09.993 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 2 (ObjNum 5 snap 0 seq_num 5) dirty exists 2024-09-14T03:48:09.993 INFO:tasks.rados.rados.0.smithi070.stdout:6: left oid 6 (ObjNum 5 snap 0 seq_num 5) 2024-09-14T03:48:09.993 INFO:tasks.rados.rados.0.smithi070.stdout:5: finishing write tid 2 to smithi07085154-5 2024-09-14T03:48:09.993 INFO:tasks.rados.rados.0.smithi070.stdout:7: finishing write tid 1 to smithi07085154-7 2024-09-14T03:48:09.994 INFO:tasks.rados.rados.0.smithi070.stdout:7: finishing write tid 2 to smithi07085154-7 2024-09-14T03:48:09.994 INFO:tasks.rados.rados.0.smithi070.stdout:2: finishing write tid 3 to smithi07085154-2 2024-09-14T03:48:09.994 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 2 v 2 (ObjNum 1 snap 0 seq_num 1) dirty exists 2024-09-14T03:48:09.994 INFO:tasks.rados.rados.0.smithi070.stdout:2: left oid 2 (ObjNum 1 snap 0 seq_num 1) 2024-09-14T03:48:09.994 INFO:tasks.rados.rados.0.smithi070.stdout:8: finishing write tid 1 to smithi07085154-8 2024-09-14T03:48:09.994 INFO:tasks.rados.rados.0.smithi070.stdout:8: finishing write tid 2 to smithi07085154-8 2024-09-14T03:48:09.994 INFO:tasks.rados.rados.0.smithi070.stdout:9: finishing write tid 1 to smithi07085154-9 2024-09-14T03:48:09.994 INFO:tasks.rados.rados.0.smithi070.stdout:9: finishing write tid 2 to smithi07085154-9 2024-09-14T03:48:09.994 INFO:tasks.rados.rados.0.smithi070.stdout:4: finishing write tid 3 to smithi07085154-4 2024-09-14T03:48:09.994 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 2 (ObjNum 3 snap 0 seq_num 3) dirty exists 2024-09-14T03:48:09.994 INFO:tasks.rados.rados.0.smithi070.stdout:4: left oid 4 (ObjNum 3 snap 0 seq_num 3) 2024-09-14T03:48:09.994 INFO:tasks.rados.rados.0.smithi070.stdout:10: finishing write tid 1 to smithi07085154-10 2024-09-14T03:48:09.994 INFO:tasks.rados.rados.0.smithi070.stdout:1: done (11 left) 2024-09-14T03:48:09.995 INFO:tasks.rados.rados.0.smithi070.stdout:2: done (10 left) 2024-09-14T03:48:09.995 INFO:tasks.rados.rados.0.smithi070.stdout:3: done (9 left) 2024-09-14T03:48:09.995 INFO:tasks.rados.rados.0.smithi070.stdout:4: done (8 left) 2024-09-14T03:48:09.995 INFO:tasks.rados.rados.0.smithi070.stdout:6: done (7 left) 2024-09-14T03:48:09.995 INFO:tasks.rados.rados.0.smithi070.stdout:13: write initial oid 13 2024-09-14T03:48:09.995 INFO:tasks.rados.rados.0.smithi070.stdout:13: seq_num 12 ranges {0=204800} 2024-09-14T03:48:09.996 INFO:tasks.rados.rados.0.smithi070.stdout:13: writing smithi07085154-13 from 0 to 204800 tid 1 2024-09-14T03:48:09.996 INFO:tasks.rados.rados.0.smithi070.stdout:14: write initial oid 14 2024-09-14T03:48:09.996 INFO:tasks.rados.rados.0.smithi070.stdout:14: seq_num 13 ranges {0=221184} 2024-09-14T03:48:09.996 INFO:tasks.rados.rados.0.smithi070.stdout:14: writing smithi07085154-14 from 0 to 221184 tid 1 2024-09-14T03:48:09.996 INFO:tasks.rados.rados.0.smithi070.stdout:10: finishing write tid 2 to smithi07085154-10 2024-09-14T03:48:09.996 INFO:tasks.rados.rados.0.smithi070.stdout:5: finishing write tid 3 to smithi07085154-5 2024-09-14T03:48:09.997 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 2 (ObjNum 4 snap 0 seq_num 4) dirty exists 2024-09-14T03:48:09.997 INFO:tasks.rados.rados.0.smithi070.stdout:5: left oid 5 (ObjNum 4 snap 0 seq_num 4) 2024-09-14T03:48:09.997 INFO:tasks.rados.rados.0.smithi070.stdout:9: finishing write tid 3 to smithi07085154-9 2024-09-14T03:48:09.997 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 4 (ObjNum 8 snap 0 seq_num 8) dirty exists 2024-09-14T03:48:09.997 INFO:tasks.rados.rados.0.smithi070.stdout:9: left oid 9 (ObjNum 8 snap 0 seq_num 8) 2024-09-14T03:48:09.997 INFO:tasks.rados.rados.0.smithi070.stdout:7: finishing write tid 3 to smithi07085154-7 2024-09-14T03:48:09.997 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 4 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-09-14T03:48:09.997 INFO:tasks.rados.rados.0.smithi070.stdout:7: left oid 7 (ObjNum 6 snap 0 seq_num 6) 2024-09-14T03:48:09.997 INFO:tasks.rados.rados.0.smithi070.stdout:8: finishing write tid 3 to smithi07085154-8 2024-09-14T03:48:09.997 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 8 v 2 (ObjNum 7 snap 0 seq_num 7) dirty exists 2024-09-14T03:48:09.997 INFO:tasks.rados.rados.0.smithi070.stdout:8: left oid 8 (ObjNum 7 snap 0 seq_num 7) 2024-09-14T03:48:09.997 INFO:tasks.rados.rados.0.smithi070.stdout:10: finishing write tid 3 to smithi07085154-10 2024-09-14T03:48:09.997 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 6 (ObjNum 9 snap 0 seq_num 9) dirty exists 2024-09-14T03:48:09.998 INFO:tasks.rados.rados.0.smithi070.stdout:10: left oid 10 (ObjNum 9 snap 0 seq_num 9) 2024-09-14T03:48:09.998 INFO:tasks.rados.rados.0.smithi070.stdout:5: done (8 left) 2024-09-14T03:48:09.998 INFO:tasks.rados.rados.0.smithi070.stdout:7: done (7 left) 2024-09-14T03:48:09.998 INFO:tasks.rados.rados.0.smithi070.stdout:8: done (6 left) 2024-09-14T03:48:09.998 INFO:tasks.rados.rados.0.smithi070.stdout:9: done (5 left) 2024-09-14T03:48:09.998 INFO:tasks.rados.rados.0.smithi070.stdout:10: done (4 left) 2024-09-14T03:48:09.998 INFO:tasks.rados.rados.0.smithi070.stdout:15: write initial oid 15 2024-09-14T03:48:09.998 INFO:tasks.rados.rados.0.smithi070.stdout:15: seq_num 14 ranges {0=237568} 2024-09-14T03:48:09.999 INFO:tasks.rados.rados.0.smithi070.stdout:15: writing smithi07085154-15 from 0 to 237568 tid 1 2024-09-14T03:48:09.999 INFO:tasks.rados.rados.0.smithi070.stdout:16: write initial oid 16 2024-09-14T03:48:09.999 INFO:tasks.rados.rados.0.smithi070.stdout:12: finishing write tid 1 to smithi07085154-12 2024-09-14T03:48:09.999 INFO:tasks.rados.rados.0.smithi070.stdout:12: finishing write tid 2 to smithi07085154-12 2024-09-14T03:48:09.999 INFO:tasks.rados.rados.0.smithi070.stdout:11: finishing write tid 1 to smithi07085154-11 2024-09-14T03:48:10.000 INFO:tasks.rados.rados.0.smithi070.stdout:11: finishing write tid 2 to smithi07085154-11 2024-09-14T03:48:10.000 INFO:tasks.rados.rados.0.smithi070.stdout:13: finishing write tid 1 to smithi07085154-13 2024-09-14T03:48:10.000 INFO:tasks.rados.rados.0.smithi070.stdout:13: finishing write tid 2 to smithi07085154-13 2024-09-14T03:48:10.000 INFO:tasks.rados.rados.0.smithi070.stdout:12: finishing write tid 3 to smithi07085154-12 2024-09-14T03:48:10.000 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 2 (ObjNum 11 snap 0 seq_num 11) dirty exists 2024-09-14T03:48:10.000 INFO:tasks.rados.rados.0.smithi070.stdout:12: left oid 12 (ObjNum 11 snap 0 seq_num 11) 2024-09-14T03:48:10.000 INFO:tasks.rados.rados.0.smithi070.stdout:16: seq_num 15 ranges {0=253952} 2024-09-14T03:48:10.000 INFO:tasks.rados.rados.0.smithi070.stdout:16: writing smithi07085154-16 from 0 to 253952 tid 1 2024-09-14T03:48:10.000 INFO:tasks.rados.rados.0.smithi070.stdout:13: finishing write tid 3 to smithi07085154-13 2024-09-14T03:48:10.000 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 4 (ObjNum 12 snap 0 seq_num 12) dirty exists 2024-09-14T03:48:10.000 INFO:tasks.rados.rados.0.smithi070.stdout:13: left oid 13 (ObjNum 12 snap 0 seq_num 12) 2024-09-14T03:48:10.000 INFO:tasks.rados.rados.0.smithi070.stdout:11: finishing write tid 3 to smithi07085154-11 2024-09-14T03:48:10.000 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 11 v 2 (ObjNum 10 snap 0 seq_num 10) dirty exists 2024-09-14T03:48:10.000 INFO:tasks.rados.rados.0.smithi070.stdout:11: left oid 11 (ObjNum 10 snap 0 seq_num 10) 2024-09-14T03:48:10.001 INFO:tasks.rados.rados.0.smithi070.stdout:11: done (5 left) 2024-09-14T03:48:10.001 INFO:tasks.rados.rados.0.smithi070.stdout:12: done (4 left) 2024-09-14T03:48:10.001 INFO:tasks.rados.rados.0.smithi070.stdout:13: done (3 left) 2024-09-14T03:48:10.001 INFO:tasks.rados.rados.0.smithi070.stdout:17: write initial oid 17 2024-09-14T03:48:10.001 INFO:tasks.rados.rados.0.smithi070.stdout:17: seq_num 16 ranges {0=270336} 2024-09-14T03:48:10.001 INFO:tasks.rados.rados.0.smithi070.stdout:17: writing smithi07085154-17 from 0 to 270336 tid 1 2024-09-14T03:48:10.002 INFO:tasks.rados.rados.0.smithi070.stdout:18: write initial oid 18 2024-09-14T03:48:10.002 INFO:tasks.rados.rados.0.smithi070.stdout:18: seq_num 17 ranges {0=286720} 2024-09-14T03:48:10.003 INFO:tasks.rados.rados.0.smithi070.stdout:18: writing smithi07085154-18 from 0 to 286720 tid 1 2024-09-14T03:48:10.003 INFO:tasks.rados.rados.0.smithi070.stdout:19: write initial oid 19 2024-09-14T03:48:10.003 INFO:tasks.rados.rados.0.smithi070.stdout:19: seq_num 18 ranges {0=303104} 2024-09-14T03:48:10.006 INFO:tasks.rados.rados.0.smithi070.stdout:19: writing smithi07085154-19 from 0 to 303104 tid 1 2024-09-14T03:48:10.006 INFO:tasks.rados.rados.0.smithi070.stdout:20: write initial oid 20 2024-09-14T03:48:10.006 INFO:tasks.rados.rados.0.smithi070.stdout:15: finishing write tid 1 to smithi07085154-15 2024-09-14T03:48:10.006 INFO:tasks.rados.rados.0.smithi070.stdout:15: finishing write tid 2 to smithi07085154-15 2024-09-14T03:48:10.006 INFO:tasks.rados.rados.0.smithi070.stdout:14: finishing write tid 1 to smithi07085154-14 2024-09-14T03:48:10.006 INFO:tasks.rados.rados.0.smithi070.stdout:14: finishing write tid 2 to smithi07085154-14 2024-09-14T03:48:10.006 INFO:tasks.rados.rados.0.smithi070.stdout:15: finishing write tid 3 to smithi07085154-15 2024-09-14T03:48:10.006 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 15 v 4 (ObjNum 14 snap 0 seq_num 14) dirty exists 2024-09-14T03:48:10.006 INFO:tasks.rados.rados.0.smithi070.stdout:15: left oid 15 (ObjNum 14 snap 0 seq_num 14) 2024-09-14T03:48:10.006 INFO:tasks.rados.rados.0.smithi070.stdout:16: finishing write tid 1 to smithi07085154-16 2024-09-14T03:48:10.006 INFO:tasks.rados.rados.0.smithi070.stdout:14: finishing write tid 3 to smithi07085154-14 2024-09-14T03:48:10.006 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 2 (ObjNum 13 snap 0 seq_num 13) dirty exists 2024-09-14T03:48:10.007 INFO:tasks.rados.rados.0.smithi070.stdout:14: left oid 14 (ObjNum 13 snap 0 seq_num 13) 2024-09-14T03:48:10.007 INFO:tasks.rados.rados.0.smithi070.stdout:16: finishing write tid 2 to smithi07085154-16 2024-09-14T03:48:10.007 INFO:tasks.rados.rados.0.smithi070.stdout:16: finishing write tid 3 to smithi07085154-16 2024-09-14T03:48:10.007 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 8 (ObjNum 15 snap 0 seq_num 15) dirty exists 2024-09-14T03:48:10.007 INFO:tasks.rados.rados.0.smithi070.stdout:16: left oid 16 (ObjNum 15 snap 0 seq_num 15) 2024-09-14T03:48:10.007 INFO:tasks.rados.rados.0.smithi070.stdout:17: finishing write tid 1 to smithi07085154-17 2024-09-14T03:48:10.007 INFO:tasks.rados.rados.0.smithi070.stdout:17: finishing write tid 2 to smithi07085154-17 2024-09-14T03:48:10.007 INFO:tasks.rados.rados.0.smithi070.stdout:17: finishing write tid 3 to smithi07085154-17 2024-09-14T03:48:10.007 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 4 (ObjNum 16 snap 0 seq_num 16) dirty exists 2024-09-14T03:48:10.007 INFO:tasks.rados.rados.0.smithi070.stdout:17: left oid 17 (ObjNum 16 snap 0 seq_num 16) 2024-09-14T03:48:10.007 INFO:tasks.rados.rados.0.smithi070.stdout:18: finishing write tid 1 to smithi07085154-18 2024-09-14T03:48:10.007 INFO:tasks.rados.rados.0.smithi070.stdout:18: finishing write tid 2 to smithi07085154-18 2024-09-14T03:48:10.007 INFO:tasks.rados.rados.0.smithi070.stdout:20: seq_num 19 ranges {0=319488} 2024-09-14T03:48:10.008 INFO:tasks.rados.rados.0.smithi070.stdout:20: writing smithi07085154-20 from 0 to 319488 tid 1 2024-09-14T03:48:10.008 INFO:tasks.rados.rados.0.smithi070.stdout:14: done (6 left) 2024-09-14T03:48:10.008 INFO:tasks.rados.rados.0.smithi070.stdout:15: done (5 left) 2024-09-14T03:48:10.008 INFO:tasks.rados.rados.0.smithi070.stdout:16: done (4 left) 2024-09-14T03:48:10.009 INFO:tasks.rados.rados.0.smithi070.stdout:17: done (3 left) 2024-09-14T03:48:10.009 INFO:tasks.rados.rados.0.smithi070.stdout:21: write initial oid 21 2024-09-14T03:48:10.009 INFO:tasks.rados.rados.0.smithi070.stdout:21: seq_num 20 ranges {0=344064} 2024-09-14T03:48:10.010 INFO:tasks.rados.rados.0.smithi070.stdout:21: writing smithi07085154-21 from 0 to 344064 tid 1 2024-09-14T03:48:10.010 INFO:tasks.rados.rados.0.smithi070.stdout:18: finishing write tid 3 to smithi07085154-18 2024-09-14T03:48:10.010 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 6 (ObjNum 17 snap 0 seq_num 17) dirty exists 2024-09-14T03:48:10.010 INFO:tasks.rados.rados.0.smithi070.stdout:18: left oid 18 (ObjNum 17 snap 0 seq_num 17) 2024-09-14T03:48:10.010 INFO:tasks.rados.rados.0.smithi070.stdout:19: finishing write tid 1 to smithi07085154-19 2024-09-14T03:48:10.011 INFO:tasks.rados.rados.0.smithi070.stdout:19: finishing write tid 2 to smithi07085154-19 2024-09-14T03:48:10.011 INFO:tasks.rados.rados.0.smithi070.stdout:19: finishing write tid 3 to smithi07085154-19 2024-09-14T03:48:10.011 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 19 v 2 (ObjNum 18 snap 0 seq_num 18) dirty exists 2024-09-14T03:48:10.011 INFO:tasks.rados.rados.0.smithi070.stdout:19: left oid 19 (ObjNum 18 snap 0 seq_num 18) 2024-09-14T03:48:10.011 INFO:tasks.rados.rados.0.smithi070.stdout:20: finishing write tid 1 to smithi07085154-20 2024-09-14T03:48:10.011 INFO:tasks.rados.rados.0.smithi070.stdout:20: finishing write tid 2 to smithi07085154-20 2024-09-14T03:48:10.011 INFO:tasks.rados.rados.0.smithi070.stdout:18: done (3 left) 2024-09-14T03:48:10.011 INFO:tasks.rados.rados.0.smithi070.stdout:19: done (2 left) 2024-09-14T03:48:10.011 INFO:tasks.rados.rados.0.smithi070.stdout:22: write initial oid 22 2024-09-14T03:48:10.011 INFO:tasks.rados.rados.0.smithi070.stdout:22: seq_num 21 ranges {0=360448} 2024-09-14T03:48:10.012 INFO:tasks.rados.rados.0.smithi070.stdout:22: writing smithi07085154-22 from 0 to 360448 tid 1 2024-09-14T03:48:10.013 INFO:tasks.rados.rados.0.smithi070.stdout:23: write initial oid 23 2024-09-14T03:48:10.013 INFO:tasks.rados.rados.0.smithi070.stdout:23: seq_num 22 ranges {0=376832} 2024-09-14T03:48:10.015 INFO:tasks.rados.rados.0.smithi070.stdout:23: writing smithi07085154-23 from 0 to 376832 tid 1 2024-09-14T03:48:10.015 INFO:tasks.rados.rados.0.smithi070.stdout:24: write initial oid 24 2024-09-14T03:48:10.015 INFO:tasks.rados.rados.0.smithi070.stdout:24: seq_num 23 ranges {0=393216} 2024-09-14T03:48:10.017 INFO:tasks.rados.rados.0.smithi070.stdout:24: writing smithi07085154-24 from 0 to 393216 tid 1 2024-09-14T03:48:10.017 INFO:tasks.rados.rados.0.smithi070.stdout:25: write initial oid 25 2024-09-14T03:48:10.017 INFO:tasks.rados.rados.0.smithi070.stdout:25: seq_num 24 ranges {0=409600} 2024-09-14T03:48:10.019 INFO:tasks.rados.rados.0.smithi070.stdout:25: writing smithi07085154-25 from 0 to 409600 tid 1 2024-09-14T03:48:10.020 INFO:tasks.rados.rados.0.smithi070.stdout:26: write initial oid 26 2024-09-14T03:48:10.020 INFO:tasks.rados.rados.0.smithi070.stdout:26: seq_num 25 ranges {0=425984} 2024-09-14T03:48:10.022 INFO:tasks.rados.rados.0.smithi070.stdout:26: writing smithi07085154-26 from 0 to 425984 tid 1 2024-09-14T03:48:10.022 INFO:tasks.rados.rados.0.smithi070.stdout:27: write initial oid 27 2024-09-14T03:48:10.022 INFO:tasks.rados.rados.0.smithi070.stdout:27: seq_num 26 ranges {0=442368} 2024-09-14T03:48:10.025 INFO:tasks.rados.rados.0.smithi070.stdout:27: writing smithi07085154-27 from 0 to 442368 tid 1 2024-09-14T03:48:10.026 INFO:tasks.rados.rados.0.smithi070.stdout:20: finishing write tid 3 to smithi07085154-20 2024-09-14T03:48:10.026 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 20 v 6 (ObjNum 19 snap 0 seq_num 19) dirty exists 2024-09-14T03:48:10.026 INFO:tasks.rados.rados.0.smithi070.stdout:20: left oid 20 (ObjNum 19 snap 0 seq_num 19) 2024-09-14T03:48:10.026 INFO:tasks.rados.rados.0.smithi070.stdout:20: done (7 left) 2024-09-14T03:48:10.026 INFO:tasks.rados.rados.0.smithi070.stdout:28: write initial oid 28 2024-09-14T03:48:10.026 INFO:tasks.rados.rados.0.smithi070.stdout:28: seq_num 27 ranges {0=458752} 2024-09-14T03:48:10.028 INFO:tasks.rados.rados.0.smithi070.stdout:28: writing smithi07085154-28 from 0 to 458752 tid 1 2024-09-14T03:48:10.029 INFO:tasks.rados.rados.0.smithi070.stdout:29: write initial oid 29 2024-09-14T03:48:10.029 INFO:tasks.rados.rados.0.smithi070.stdout:29: seq_num 28 ranges {0=475136} 2024-09-14T03:48:10.032 INFO:tasks.rados.rados.0.smithi070.stdout:29: writing smithi07085154-29 from 0 to 475136 tid 1 2024-09-14T03:48:10.032 INFO:tasks.rados.rados.0.smithi070.stdout:30: write initial oid 30 2024-09-14T03:48:10.032 INFO:tasks.rados.rados.0.smithi070.stdout:30: seq_num 29 ranges {0=491520} 2024-09-14T03:48:10.035 INFO:tasks.rados.rados.0.smithi070.stdout:30: writing smithi07085154-30 from 0 to 491520 tid 1 2024-09-14T03:48:10.035 INFO:tasks.rados.rados.0.smithi070.stdout:31: write initial oid 31 2024-09-14T03:48:10.035 INFO:tasks.rados.rados.0.smithi070.stdout:31: seq_num 30 ranges {0=507904} 2024-09-14T03:48:10.038 INFO:tasks.rados.rados.0.smithi070.stdout:31: writing smithi07085154-31 from 0 to 507904 tid 1 2024-09-14T03:48:10.038 INFO:tasks.rados.rados.0.smithi070.stdout:32: write initial oid 32 2024-09-14T03:48:10.038 INFO:tasks.rados.rados.0.smithi070.stdout:32: seq_num 31 ranges {0=524288} 2024-09-14T03:48:10.041 INFO:tasks.rados.rados.0.smithi070.stdout:32: writing smithi07085154-32 from 0 to 524288 tid 1 2024-09-14T03:48:10.041 INFO:tasks.rados.rados.0.smithi070.stdout:33: write initial oid 33 2024-09-14T03:48:10.041 INFO:tasks.rados.rados.0.smithi070.stdout:33: seq_num 32 ranges {0=540672} 2024-09-14T03:48:10.044 INFO:tasks.rados.rados.0.smithi070.stdout:33: writing smithi07085154-33 from 0 to 540672 tid 1 2024-09-14T03:48:10.044 INFO:tasks.rados.rados.0.smithi070.stdout:34: write initial oid 34 2024-09-14T03:48:10.044 INFO:tasks.rados.rados.0.smithi070.stdout:34: seq_num 33 ranges {0=557056} 2024-09-14T03:48:10.048 INFO:tasks.rados.rados.0.smithi070.stdout:34: writing smithi07085154-34 from 0 to 557056 tid 1 2024-09-14T03:48:10.048 INFO:tasks.rados.rados.0.smithi070.stdout:35: write initial oid 35 2024-09-14T03:48:10.048 INFO:tasks.rados.rados.0.smithi070.stdout:35: seq_num 34 ranges {0=573440} 2024-09-14T03:48:10.051 INFO:tasks.rados.rados.0.smithi070.stdout:35: writing smithi07085154-35 from 0 to 573440 tid 1 2024-09-14T03:48:10.051 INFO:tasks.rados.rados.0.smithi070.stdout:36: write initial oid 36 2024-09-14T03:48:10.051 INFO:tasks.rados.rados.0.smithi070.stdout:36: seq_num 35 ranges {0=589824} 2024-09-14T03:48:10.055 INFO:tasks.rados.rados.0.smithi070.stdout:36: writing smithi07085154-36 from 0 to 589824 tid 1 2024-09-14T03:48:10.055 INFO:tasks.rados.rados.0.smithi070.stdout: waiting on 16 2024-09-14T03:48:10.055 INFO:tasks.rados.rados.0.smithi070.stdout:21: finishing write tid 1 to smithi07085154-21 2024-09-14T03:48:10.056 INFO:tasks.rados.rados.0.smithi070.stdout:21: finishing write tid 2 to smithi07085154-21 2024-09-14T03:48:10.056 INFO:tasks.rados.rados.0.smithi070.stdout:21: finishing write tid 3 to smithi07085154-21 2024-09-14T03:48:10.056 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 2 (ObjNum 20 snap 0 seq_num 20) dirty exists 2024-09-14T03:48:10.056 INFO:tasks.rados.rados.0.smithi070.stdout:21: left oid 21 (ObjNum 20 snap 0 seq_num 20) 2024-09-14T03:48:10.056 INFO:tasks.rados.rados.0.smithi070.stdout:22: finishing write tid 1 to smithi07085154-22 2024-09-14T03:48:10.056 INFO:tasks.rados.rados.0.smithi070.stdout:22: finishing write tid 2 to smithi07085154-22 2024-09-14T03:48:10.056 INFO:tasks.rados.rados.0.smithi070.stdout:21: done (15 left) 2024-09-14T03:48:10.056 INFO:tasks.rados.rados.0.smithi070.stdout:37: write initial oid 37 2024-09-14T03:48:10.056 INFO:tasks.rados.rados.0.smithi070.stdout:37: seq_num 36 ranges {0=606208} 2024-09-14T03:48:10.060 INFO:tasks.rados.rados.0.smithi070.stdout:37: writing smithi07085154-37 from 0 to 606208 tid 1 2024-09-14T03:48:10.060 INFO:tasks.rados.rados.0.smithi070.stdout: waiting on 16 2024-09-14T03:48:10.060 INFO:tasks.rados.rados.0.smithi070.stdout:22: finishing write tid 3 to smithi07085154-22 2024-09-14T03:48:10.060 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 4 (ObjNum 21 snap 0 seq_num 21) dirty exists 2024-09-14T03:48:10.060 INFO:tasks.rados.rados.0.smithi070.stdout:22: left oid 22 (ObjNum 21 snap 0 seq_num 21) 2024-09-14T03:48:10.060 INFO:tasks.rados.rados.0.smithi070.stdout:23: finishing write tid 1 to smithi07085154-23 2024-09-14T03:48:10.060 INFO:tasks.rados.rados.0.smithi070.stdout:23: finishing write tid 2 to smithi07085154-23 2024-09-14T03:48:10.060 INFO:tasks.rados.rados.0.smithi070.stdout:23: finishing write tid 3 to smithi07085154-23 2024-09-14T03:48:10.060 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 4 (ObjNum 22 snap 0 seq_num 22) dirty exists 2024-09-14T03:48:10.060 INFO:tasks.rados.rados.0.smithi070.stdout:23: left oid 23 (ObjNum 22 snap 0 seq_num 22) 2024-09-14T03:48:10.060 INFO:tasks.rados.rados.0.smithi070.stdout:24: finishing write tid 1 to smithi07085154-24 2024-09-14T03:48:10.061 INFO:tasks.rados.rados.0.smithi070.stdout:24: finishing write tid 2 to smithi07085154-24 2024-09-14T03:48:10.061 INFO:tasks.rados.rados.0.smithi070.stdout:24: finishing write tid 3 to smithi07085154-24 2024-09-14T03:48:10.061 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 6 (ObjNum 23 snap 0 seq_num 23) dirty exists 2024-09-14T03:48:10.061 INFO:tasks.rados.rados.0.smithi070.stdout:24: left oid 24 (ObjNum 23 snap 0 seq_num 23) 2024-09-14T03:48:10.061 INFO:tasks.rados.rados.0.smithi070.stdout:25: finishing write tid 1 to smithi07085154-25 2024-09-14T03:48:10.061 INFO:tasks.rados.rados.0.smithi070.stdout:25: finishing write tid 2 to smithi07085154-25 2024-09-14T03:48:10.061 INFO:tasks.rados.rados.0.smithi070.stdout:25: finishing write tid 3 to smithi07085154-25 2024-09-14T03:48:10.061 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 6 (ObjNum 24 snap 0 seq_num 24) dirty exists 2024-09-14T03:48:10.061 INFO:tasks.rados.rados.0.smithi070.stdout:25: left oid 25 (ObjNum 24 snap 0 seq_num 24) 2024-09-14T03:48:10.061 INFO:tasks.rados.rados.0.smithi070.stdout:26: finishing write tid 1 to smithi07085154-26 2024-09-14T03:48:10.061 INFO:tasks.rados.rados.0.smithi070.stdout:26: finishing write tid 2 to smithi07085154-26 2024-09-14T03:48:10.061 INFO:tasks.rados.rados.0.smithi070.stdout:26: finishing write tid 3 to smithi07085154-26 2024-09-14T03:48:10.061 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 10 (ObjNum 25 snap 0 seq_num 25) dirty exists 2024-09-14T03:48:10.062 INFO:tasks.rados.rados.0.smithi070.stdout:26: left oid 26 (ObjNum 25 snap 0 seq_num 25) 2024-09-14T03:48:10.062 INFO:tasks.rados.rados.0.smithi070.stdout:27: finishing write tid 1 to smithi07085154-27 2024-09-14T03:48:10.062 INFO:tasks.rados.rados.0.smithi070.stdout:27: finishing write tid 2 to smithi07085154-27 2024-09-14T03:48:10.062 INFO:tasks.rados.rados.0.smithi070.stdout:27: finishing write tid 3 to smithi07085154-27 2024-09-14T03:48:10.062 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 12 (ObjNum 26 snap 0 seq_num 26) dirty exists 2024-09-14T03:48:10.062 INFO:tasks.rados.rados.0.smithi070.stdout:27: left oid 27 (ObjNum 26 snap 0 seq_num 26) 2024-09-14T03:48:10.062 INFO:tasks.rados.rados.0.smithi070.stdout:28: finishing write tid 1 to smithi07085154-28 2024-09-14T03:48:10.062 INFO:tasks.rados.rados.0.smithi070.stdout:28: finishing write tid 2 to smithi07085154-28 2024-09-14T03:48:10.062 INFO:tasks.rados.rados.0.smithi070.stdout:22: done (15 left) 2024-09-14T03:48:10.062 INFO:tasks.rados.rados.0.smithi070.stdout:23: done (14 left) 2024-09-14T03:48:10.062 INFO:tasks.rados.rados.0.smithi070.stdout:24: done (13 left) 2024-09-14T03:48:10.063 INFO:tasks.rados.rados.0.smithi070.stdout:25: done (12 left) 2024-09-14T03:48:10.063 INFO:tasks.rados.rados.0.smithi070.stdout:26: done (11 left) 2024-09-14T03:48:10.063 INFO:tasks.rados.rados.0.smithi070.stdout:27: done (10 left) 2024-09-14T03:48:10.063 INFO:tasks.rados.rados.0.smithi070.stdout:38: write initial oid 38 2024-09-14T03:48:10.063 INFO:tasks.rados.rados.0.smithi070.stdout:38: seq_num 37 ranges {0=622592} 2024-09-14T03:48:10.063 INFO:tasks.rados.rados.0.smithi070.stdout:38: writing smithi07085154-38 from 0 to 622592 tid 1 2024-09-14T03:48:10.063 INFO:tasks.rados.rados.0.smithi070.stdout:39: write initial oid 39 2024-09-14T03:48:10.064 INFO:tasks.rados.rados.0.smithi070.stdout:39: seq_num 38 ranges {0=638976} 2024-09-14T03:48:10.067 INFO:tasks.rados.rados.0.smithi070.stdout:39: writing smithi07085154-39 from 0 to 638976 tid 1 2024-09-14T03:48:10.067 INFO:tasks.rados.rados.0.smithi070.stdout:40: write initial oid 40 2024-09-14T03:48:10.067 INFO:tasks.rados.rados.0.smithi070.stdout:40: seq_num 39 ranges {0=663552} 2024-09-14T03:48:10.070 INFO:tasks.rados.rados.0.smithi070.stdout:40: writing smithi07085154-40 from 0 to 663552 tid 1 2024-09-14T03:48:10.071 INFO:tasks.rados.rados.0.smithi070.stdout:41: write initial oid 41 2024-09-14T03:48:10.071 INFO:tasks.rados.rados.0.smithi070.stdout:41: seq_num 40 ranges {0=679936} 2024-09-14T03:48:10.074 INFO:tasks.rados.rados.0.smithi070.stdout:41: writing smithi07085154-41 from 0 to 679936 tid 1 2024-09-14T03:48:10.075 INFO:tasks.rados.rados.0.smithi070.stdout:42: write initial oid 42 2024-09-14T03:48:10.075 INFO:tasks.rados.rados.0.smithi070.stdout:42: seq_num 41 ranges {0=696320} 2024-09-14T03:48:10.079 INFO:tasks.rados.rados.0.smithi070.stdout:42: writing smithi07085154-42 from 0 to 696320 tid 1 2024-09-14T03:48:10.079 INFO:tasks.rados.rados.0.smithi070.stdout:43: write initial oid 43 2024-09-14T03:48:10.079 INFO:tasks.rados.rados.0.smithi070.stdout:43: seq_num 42 ranges {0=712704} 2024-09-14T03:48:10.084 INFO:tasks.rados.rados.0.smithi070.stdout:43: writing smithi07085154-43 from 0 to 712704 tid 1 2024-09-14T03:48:10.084 INFO:tasks.rados.rados.0.smithi070.stdout: waiting on 16 2024-09-14T03:48:10.084 INFO:tasks.rados.rados.0.smithi070.stdout:28: finishing write tid 3 to smithi07085154-28 2024-09-14T03:48:10.084 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 28 v 8 (ObjNum 27 snap 0 seq_num 27) dirty exists 2024-09-14T03:48:10.084 INFO:tasks.rados.rados.0.smithi070.stdout:28: left oid 28 (ObjNum 27 snap 0 seq_num 27) 2024-09-14T03:48:10.084 INFO:tasks.rados.rados.0.smithi070.stdout:29: finishing write tid 1 to smithi07085154-29 2024-09-14T03:48:10.084 INFO:tasks.rados.rados.0.smithi070.stdout:29: finishing write tid 2 to smithi07085154-29 2024-09-14T03:48:10.084 INFO:tasks.rados.rados.0.smithi070.stdout:28: done (15 left) 2024-09-14T03:48:10.085 INFO:tasks.rados.rados.0.smithi070.stdout:44: write initial oid 44 2024-09-14T03:48:10.085 INFO:tasks.rados.rados.0.smithi070.stdout:44: seq_num 43 ranges {0=729088} 2024-09-14T03:48:10.089 INFO:tasks.rados.rados.0.smithi070.stdout:44: writing smithi07085154-44 from 0 to 729088 tid 1 2024-09-14T03:48:10.089 INFO:tasks.rados.rados.0.smithi070.stdout: waiting on 16 2024-09-14T03:48:10.089 INFO:tasks.rados.rados.0.smithi070.stdout:29: finishing write tid 3 to smithi07085154-29 2024-09-14T03:48:10.089 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 6 (ObjNum 28 snap 0 seq_num 28) dirty exists 2024-09-14T03:48:10.089 INFO:tasks.rados.rados.0.smithi070.stdout:29: left oid 29 (ObjNum 28 snap 0 seq_num 28) 2024-09-14T03:48:10.090 INFO:tasks.rados.rados.0.smithi070.stdout:30: finishing write tid 1 to smithi07085154-30 2024-09-14T03:48:10.090 INFO:tasks.rados.rados.0.smithi070.stdout:30: finishing write tid 2 to smithi07085154-30 2024-09-14T03:48:10.090 INFO:tasks.rados.rados.0.smithi070.stdout:30: finishing write tid 3 to smithi07085154-30 2024-09-14T03:48:10.090 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 30 v 4 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-09-14T03:48:10.090 INFO:tasks.rados.rados.0.smithi070.stdout:30: left oid 30 (ObjNum 29 snap 0 seq_num 29) 2024-09-14T03:48:10.090 INFO:tasks.rados.rados.0.smithi070.stdout:31: finishing write tid 1 to smithi07085154-31 2024-09-14T03:48:10.090 INFO:tasks.rados.rados.0.smithi070.stdout:31: finishing write tid 2 to smithi07085154-31 2024-09-14T03:48:10.090 INFO:tasks.rados.rados.0.smithi070.stdout:29: done (15 left) 2024-09-14T03:48:10.090 INFO:tasks.rados.rados.0.smithi070.stdout:30: done (14 left) 2024-09-14T03:48:10.090 INFO:tasks.rados.rados.0.smithi070.stdout:45: write initial oid 45 2024-09-14T03:48:10.090 INFO:tasks.rados.rados.0.smithi070.stdout:45: seq_num 44 ranges {0=745472} 2024-09-14T03:48:10.093 INFO:tasks.rados.rados.0.smithi070.stdout:45: writing smithi07085154-45 from 0 to 745472 tid 1 2024-09-14T03:48:10.093 INFO:tasks.rados.rados.0.smithi070.stdout:46: write initial oid 46 2024-09-14T03:48:10.093 INFO:tasks.rados.rados.0.smithi070.stdout:46: seq_num 45 ranges {0=761856} 2024-09-14T03:48:10.099 INFO:tasks.rados.rados.0.smithi070.stdout:46: writing smithi07085154-46 from 0 to 761856 tid 1 2024-09-14T03:48:10.099 INFO:tasks.rados.rados.0.smithi070.stdout: waiting on 16 2024-09-14T03:48:10.099 INFO:tasks.rados.rados.0.smithi070.stdout:31: finishing write tid 3 to smithi07085154-31 2024-09-14T03:48:10.099 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 8 (ObjNum 30 snap 0 seq_num 30) dirty exists 2024-09-14T03:48:10.099 INFO:tasks.rados.rados.0.smithi070.stdout:31: left oid 31 (ObjNum 30 snap 0 seq_num 30) 2024-09-14T03:48:10.099 INFO:tasks.rados.rados.0.smithi070.stdout:32: finishing write tid 1 to smithi07085154-32 2024-09-14T03:48:10.099 INFO:tasks.rados.rados.0.smithi070.stdout:32: finishing write tid 2 to smithi07085154-32 2024-09-14T03:48:10.099 INFO:tasks.rados.rados.0.smithi070.stdout:32: finishing write tid 3 to smithi07085154-32 2024-09-14T03:48:10.099 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 6 (ObjNum 31 snap 0 seq_num 31) dirty exists 2024-09-14T03:48:10.099 INFO:tasks.rados.rados.0.smithi070.stdout:32: left oid 32 (ObjNum 31 snap 0 seq_num 31) 2024-09-14T03:48:10.099 INFO:tasks.rados.rados.0.smithi070.stdout:33: finishing write tid 1 to smithi07085154-33 2024-09-14T03:48:10.099 INFO:tasks.rados.rados.0.smithi070.stdout:31: done (15 left) 2024-09-14T03:48:10.099 INFO:tasks.rados.rados.0.smithi070.stdout:32: done (14 left) 2024-09-14T03:48:10.100 INFO:tasks.rados.rados.0.smithi070.stdout:47: write initial oid 47 2024-09-14T03:48:10.100 INFO:tasks.rados.rados.0.smithi070.stdout:47: seq_num 46 ranges {0=778240} 2024-09-14T03:48:10.103 INFO:tasks.rados.rados.0.smithi070.stdout:47: writing smithi07085154-47 from 0 to 778240 tid 1 2024-09-14T03:48:10.103 INFO:tasks.rados.rados.0.smithi070.stdout:48: write initial oid 48 2024-09-14T03:48:10.104 INFO:tasks.rados.rados.0.smithi070.stdout:48: seq_num 47 ranges {0=794624} 2024-09-14T03:48:10.109 INFO:tasks.rados.rados.0.smithi070.stdout:48: writing smithi07085154-48 from 0 to 794624 tid 1 2024-09-14T03:48:10.109 INFO:tasks.rados.rados.0.smithi070.stdout: waiting on 16 2024-09-14T03:48:10.109 INFO:tasks.rados.rados.0.smithi070.stdout:33: finishing write tid 2 to smithi07085154-33 2024-09-14T03:48:10.109 INFO:tasks.rados.rados.0.smithi070.stdout:33: finishing write tid 3 to smithi07085154-33 2024-09-14T03:48:10.109 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 33 v 2 (ObjNum 32 snap 0 seq_num 32) dirty exists 2024-09-14T03:48:10.109 INFO:tasks.rados.rados.0.smithi070.stdout:33: left oid 33 (ObjNum 32 snap 0 seq_num 32) 2024-09-14T03:48:10.110 INFO:tasks.rados.rados.0.smithi070.stdout:34: finishing write tid 1 to smithi07085154-34 2024-09-14T03:48:10.110 INFO:tasks.rados.rados.0.smithi070.stdout:34: finishing write tid 2 to smithi07085154-34 2024-09-14T03:48:10.110 INFO:tasks.rados.rados.0.smithi070.stdout:34: finishing write tid 3 to smithi07085154-34 2024-09-14T03:48:10.110 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 8 (ObjNum 33 snap 0 seq_num 33) dirty exists 2024-09-14T03:48:10.110 INFO:tasks.rados.rados.0.smithi070.stdout:34: left oid 34 (ObjNum 33 snap 0 seq_num 33) 2024-09-14T03:48:10.110 INFO:tasks.rados.rados.0.smithi070.stdout:35: finishing write tid 1 to smithi07085154-35 2024-09-14T03:48:10.110 INFO:tasks.rados.rados.0.smithi070.stdout:35: finishing write tid 2 to smithi07085154-35 2024-09-14T03:48:10.110 INFO:tasks.rados.rados.0.smithi070.stdout:35: finishing write tid 3 to smithi07085154-35 2024-09-14T03:48:10.111 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 35 v 2 (ObjNum 34 snap 0 seq_num 34) dirty exists 2024-09-14T03:48:10.111 INFO:tasks.rados.rados.0.smithi070.stdout:35: left oid 35 (ObjNum 34 snap 0 seq_num 34) 2024-09-14T03:48:10.111 INFO:tasks.rados.rados.0.smithi070.stdout:36: finishing write tid 1 to smithi07085154-36 2024-09-14T03:48:10.111 INFO:tasks.rados.rados.0.smithi070.stdout:36: finishing write tid 2 to smithi07085154-36 2024-09-14T03:48:10.111 INFO:tasks.rados.rados.0.smithi070.stdout:36: finishing write tid 3 to smithi07085154-36 2024-09-14T03:48:10.111 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 36 v 4 (ObjNum 35 snap 0 seq_num 35) dirty exists 2024-09-14T03:48:10.111 INFO:tasks.rados.rados.0.smithi070.stdout:36: left oid 36 (ObjNum 35 snap 0 seq_num 35) 2024-09-14T03:48:10.111 INFO:tasks.rados.rados.0.smithi070.stdout:33: done (15 left) 2024-09-14T03:48:10.111 INFO:tasks.rados.rados.0.smithi070.stdout:34: done (14 left) 2024-09-14T03:48:10.111 INFO:tasks.rados.rados.0.smithi070.stdout:35: done (13 left) 2024-09-14T03:48:10.112 INFO:tasks.rados.rados.0.smithi070.stdout:36: done (12 left) 2024-09-14T03:48:10.112 INFO:tasks.rados.rados.0.smithi070.stdout:49: write initial oid 49 2024-09-14T03:48:10.112 INFO:tasks.rados.rados.0.smithi070.stdout:49: seq_num 48 ranges {0=409600,409600=401408} 2024-09-14T03:48:10.112 INFO:tasks.rados.rados.0.smithi070.stdout:49: writing smithi07085154-49 from 0 to 409600 tid 1 2024-09-14T03:48:10.113 INFO:tasks.rados.rados.0.smithi070.stdout:49: writing smithi07085154-49 from 409600 to 811008 tid 2 2024-09-14T03:48:10.114 INFO:tasks.rados.rados.0.smithi070.stdout:50: write initial oid 50 2024-09-14T03:48:10.114 INFO:tasks.rados.rados.0.smithi070.stdout:50: seq_num 49 ranges {0=425984,425984=401408} 2024-09-14T03:48:10.116 INFO:tasks.rados.rados.0.smithi070.stdout:50: writing smithi07085154-50 from 0 to 425984 tid 1 2024-09-14T03:48:10.120 INFO:tasks.rados.rados.0.smithi070.stdout:50: writing smithi07085154-50 from 425984 to 827392 tid 2 2024-09-14T03:48:10.120 INFO:tasks.rados.rados.0.smithi070.stdout:51: copy_from oid 8 from oid 18 current snap is 0 2024-09-14T03:48:10.120 INFO:tasks.rados.rados.0.smithi070.stdout:37: finishing write tid 1 to smithi07085154-37 2024-09-14T03:48:10.120 INFO:tasks.rados.rados.0.smithi070.stdout:37: finishing write tid 2 to smithi07085154-37 2024-09-14T03:48:10.120 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 24 current snap is 0 2024-09-14T03:48:10.120 INFO:tasks.rados.rados.0.smithi070.stdout:37: finishing write tid 3 to smithi07085154-37 2024-09-14T03:48:10.120 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 37 v 8 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-14T03:48:10.120 INFO:tasks.rados.rados.0.smithi070.stdout:37: left oid 37 (ObjNum 36 snap 0 seq_num 36) 2024-09-14T03:48:10.120 INFO:tasks.rados.rados.0.smithi070.stdout:38: finishing write tid 1 to smithi07085154-38 2024-09-14T03:48:10.120 INFO:tasks.rados.rados.0.smithi070.stdout:52: seq_num 50 ranges {393216=442368,835584=401408} 2024-09-14T03:48:10.124 INFO:tasks.rados.rados.0.smithi070.stdout:52: writing smithi07085154-24 from 393216 to 835584 tid 1 2024-09-14T03:48:10.130 INFO:tasks.rados.rados.0.smithi070.stdout:52: writing smithi07085154-24 from 835584 to 1236992 tid 2 2024-09-14T03:48:10.130 INFO:tasks.rados.rados.0.smithi070.stdout:37: done (15 left) 2024-09-14T03:48:10.130 INFO:tasks.rados.rados.0.smithi070.stdout:53: rmattr oid 3 current snap is 0 2024-09-14T03:48:10.130 INFO:tasks.rados.rados.0.smithi070.stdout:53: done (15 left) 2024-09-14T03:48:10.130 INFO:tasks.rados.rados.0.smithi070.stdout:54: delete oid 29 current snap is 0 2024-09-14T03:48:10.130 INFO:tasks.rados.rados.0.smithi070.stdout:38: finishing write tid 2 to smithi07085154-38 2024-09-14T03:48:10.130 INFO:tasks.rados.rados.0.smithi070.stdout:38: finishing write tid 3 to smithi07085154-38 2024-09-14T03:48:10.130 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 38 v 6 (ObjNum 37 snap 0 seq_num 37) dirty exists 2024-09-14T03:48:10.130 INFO:tasks.rados.rados.0.smithi070.stdout:38: left oid 38 (ObjNum 37 snap 0 seq_num 37) 2024-09-14T03:48:10.130 INFO:tasks.rados.rados.0.smithi070.stdout:39: finishing write tid 1 to smithi07085154-39 2024-09-14T03:48:10.130 INFO:tasks.rados.rados.0.smithi070.stdout:39: finishing write tid 2 to smithi07085154-39 2024-09-14T03:48:10.130 INFO:tasks.rados.rados.0.smithi070.stdout:39: finishing write tid 3 to smithi07085154-39 2024-09-14T03:48:10.131 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 39 v 2 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-14T03:48:10.131 INFO:tasks.rados.rados.0.smithi070.stdout:39: left oid 39 (ObjNum 38 snap 0 seq_num 38) 2024-09-14T03:48:10.131 INFO:tasks.rados.rados.0.smithi070.stdout:40: finishing write tid 1 to smithi07085154-40 2024-09-14T03:48:10.131 INFO:tasks.rados.rados.0.smithi070.stdout:40: finishing write tid 2 to smithi07085154-40 2024-09-14T03:48:10.131 INFO:tasks.rados.rados.0.smithi070.stdout:40: finishing write tid 3 to smithi07085154-40 2024-09-14T03:48:10.131 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 6 (ObjNum 39 snap 0 seq_num 39) dirty exists 2024-09-14T03:48:10.131 INFO:tasks.rados.rados.0.smithi070.stdout:40: left oid 40 (ObjNum 39 snap 0 seq_num 39) 2024-09-14T03:48:10.131 INFO:tasks.rados.rados.0.smithi070.stdout:41: finishing write tid 1 to smithi07085154-41 2024-09-14T03:48:10.131 INFO:tasks.rados.rados.0.smithi070.stdout:41: finishing write tid 2 to smithi07085154-41 2024-09-14T03:48:10.131 INFO:tasks.rados.rados.0.smithi070.stdout:41: finishing write tid 3 to smithi07085154-41 2024-09-14T03:48:10.131 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 10 (ObjNum 40 snap 0 seq_num 40) dirty exists 2024-09-14T03:48:10.131 INFO:tasks.rados.rados.0.smithi070.stdout:41: left oid 41 (ObjNum 40 snap 0 seq_num 40) 2024-09-14T03:48:10.131 INFO:tasks.rados.rados.0.smithi070.stdout:42: finishing write tid 1 to smithi07085154-42 2024-09-14T03:48:10.131 INFO:tasks.rados.rados.0.smithi070.stdout:42: finishing write tid 2 to smithi07085154-42 2024-09-14T03:48:10.132 INFO:tasks.rados.rados.0.smithi070.stdout:42: finishing write tid 3 to smithi07085154-42 2024-09-14T03:48:10.132 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 14 (ObjNum 41 snap 0 seq_num 41) dirty exists 2024-09-14T03:48:10.132 INFO:tasks.rados.rados.0.smithi070.stdout:42: left oid 42 (ObjNum 41 snap 0 seq_num 41) 2024-09-14T03:48:10.132 INFO:tasks.rados.rados.0.smithi070.stdout:43: finishing write tid 1 to smithi07085154-43 2024-09-14T03:48:10.132 INFO:tasks.rados.rados.0.smithi070.stdout:43: finishing write tid 2 to smithi07085154-43 2024-09-14T03:48:10.132 INFO:tasks.rados.rados.0.smithi070.stdout:43: finishing write tid 3 to smithi07085154-43 2024-09-14T03:48:10.132 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 12 (ObjNum 42 snap 0 seq_num 42) dirty exists 2024-09-14T03:48:10.132 INFO:tasks.rados.rados.0.smithi070.stdout:43: left oid 43 (ObjNum 42 snap 0 seq_num 42) 2024-09-14T03:48:10.132 INFO:tasks.rados.rados.0.smithi070.stdout:44: finishing write tid 1 to smithi07085154-44 2024-09-14T03:48:10.132 INFO:tasks.rados.rados.0.smithi070.stdout:44: finishing write tid 2 to smithi07085154-44 2024-09-14T03:48:10.132 INFO:tasks.rados.rados.0.smithi070.stdout:44: finishing write tid 3 to smithi07085154-44 2024-09-14T03:48:10.132 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 8 (ObjNum 43 snap 0 seq_num 43) dirty exists 2024-09-14T03:48:10.133 INFO:tasks.rados.rados.0.smithi070.stdout:44: left oid 44 (ObjNum 43 snap 0 seq_num 43) 2024-09-14T03:48:10.133 INFO:tasks.rados.rados.0.smithi070.stdout:45: finishing write tid 1 to smithi07085154-45 2024-09-14T03:48:10.133 INFO:tasks.rados.rados.0.smithi070.stdout:45: finishing write tid 2 to smithi07085154-45 2024-09-14T03:48:10.133 INFO:tasks.rados.rados.0.smithi070.stdout:45: finishing write tid 3 to smithi07085154-45 2024-09-14T03:48:10.133 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 45 v 4 (ObjNum 44 snap 0 seq_num 44) dirty exists 2024-09-14T03:48:10.133 INFO:tasks.rados.rados.0.smithi070.stdout:45: left oid 45 (ObjNum 44 snap 0 seq_num 44) 2024-09-14T03:48:10.133 INFO:tasks.rados.rados.0.smithi070.stdout:46: finishing write tid 1 to smithi07085154-46 2024-09-14T03:48:10.133 INFO:tasks.rados.rados.0.smithi070.stdout:46: finishing write tid 2 to smithi07085154-46 2024-09-14T03:48:10.133 INFO:tasks.rados.rados.0.smithi070.stdout:46: finishing write tid 3 to smithi07085154-46 2024-09-14T03:48:10.133 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 46 v 16 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-14T03:48:10.133 INFO:tasks.rados.rados.0.smithi070.stdout:46: left oid 46 (ObjNum 45 snap 0 seq_num 45) 2024-09-14T03:48:10.134 INFO:tasks.rados.rados.0.smithi070.stdout:47: finishing write tid 1 to smithi07085154-47 2024-09-14T03:48:10.134 INFO:tasks.rados.rados.0.smithi070.stdout:47: finishing write tid 2 to smithi07085154-47 2024-09-14T03:48:10.134 INFO:tasks.rados.rados.0.smithi070.stdout:47: finishing write tid 3 to smithi07085154-47 2024-09-14T03:48:10.134 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 47 v 8 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-14T03:48:10.134 INFO:tasks.rados.rados.0.smithi070.stdout:47: left oid 47 (ObjNum 46 snap 0 seq_num 46) 2024-09-14T03:48:10.134 INFO:tasks.rados.rados.0.smithi070.stdout:48: finishing write tid 1 to smithi07085154-48 2024-09-14T03:48:10.134 INFO:tasks.rados.rados.0.smithi070.stdout:48: finishing write tid 2 to smithi07085154-48 2024-09-14T03:48:10.134 INFO:tasks.rados.rados.0.smithi070.stdout:48: finishing write tid 3 to smithi07085154-48 2024-09-14T03:48:10.134 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 4 (ObjNum 47 snap 0 seq_num 47) dirty exists 2024-09-14T03:48:10.135 INFO:tasks.rados.rados.0.smithi070.stdout:48: left oid 48 (ObjNum 47 snap 0 seq_num 47) 2024-09-14T03:48:10.135 INFO:tasks.rados.rados.0.smithi070.stdout:49: finishing write tid 1 to smithi07085154-49 2024-09-14T03:48:10.135 INFO:tasks.rados.rados.0.smithi070.stdout:49: finishing write tid 2 to smithi07085154-49 2024-09-14T03:48:10.135 INFO:tasks.rados.rados.0.smithi070.stdout:49: finishing write tid 3 to smithi07085154-49 2024-09-14T03:48:10.135 INFO:tasks.rados.rados.0.smithi070.stdout:49: finishing write tid 4 to smithi07085154-49 2024-09-14T03:48:10.135 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 49 v 7 (ObjNum 48 snap 0 seq_num 48) dirty exists 2024-09-14T03:48:10.135 INFO:tasks.rados.rados.0.smithi070.stdout:49: left oid 49 (ObjNum 48 snap 0 seq_num 48) 2024-09-14T03:48:10.135 INFO:tasks.rados.rados.0.smithi070.stdout:50: finishing write tid 1 to smithi07085154-50 2024-09-14T03:48:10.135 INFO:tasks.rados.rados.0.smithi070.stdout:50: finishing write tid 2 to smithi07085154-50 2024-09-14T03:48:10.135 INFO:tasks.rados.rados.0.smithi070.stdout:50: finishing write tid 3 to smithi07085154-50 2024-09-14T03:48:10.135 INFO:tasks.rados.rados.0.smithi070.stdout:50: finishing write tid 4 to smithi07085154-50 2024-09-14T03:48:10.135 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 11 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-14T03:48:10.135 INFO:tasks.rados.rados.0.smithi070.stdout:50: left oid 50 (ObjNum 49 snap 0 seq_num 49) 2024-09-14T03:48:10.135 INFO:tasks.rados.rados.0.smithi070.stdout:51: finishing copy_from racing read to smithi07085154-8 2024-09-14T03:48:10.136 INFO:tasks.rados.rados.0.smithi070.stdout:51: finishing copy_from to smithi07085154-8 2024-09-14T03:48:10.136 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 8 v 6 (ObjNum 17 snap 0 seq_num 17) dirty exists 2024-09-14T03:48:10.136 INFO:tasks.rados.rados.0.smithi070.stdout:52: finishing write tid 1 to smithi07085154-24 2024-09-14T03:48:10.136 INFO:tasks.rados.rados.0.smithi070.stdout:38: done (15 left) 2024-09-14T03:48:10.136 INFO:tasks.rados.rados.0.smithi070.stdout:39: done (14 left) 2024-09-14T03:48:10.136 INFO:tasks.rados.rados.0.smithi070.stdout:40: done (13 left) 2024-09-14T03:48:10.136 INFO:tasks.rados.rados.0.smithi070.stdout:41: done (12 left) 2024-09-14T03:48:10.136 INFO:tasks.rados.rados.0.smithi070.stdout:42: done (11 left) 2024-09-14T03:48:10.136 INFO:tasks.rados.rados.0.smithi070.stdout:43: done (10 left) 2024-09-14T03:48:10.136 INFO:tasks.rados.rados.0.smithi070.stdout:44: done (9 left) 2024-09-14T03:48:10.136 INFO:tasks.rados.rados.0.smithi070.stdout:45: done (8 left) 2024-09-14T03:48:10.136 INFO:tasks.rados.rados.0.smithi070.stdout:46: done (7 left) 2024-09-14T03:48:10.136 INFO:tasks.rados.rados.0.smithi070.stdout:47: done (6 left) 2024-09-14T03:48:10.136 INFO:tasks.rados.rados.0.smithi070.stdout:48: done (5 left) 2024-09-14T03:48:10.137 INFO:tasks.rados.rados.0.smithi070.stdout:49: done (4 left) 2024-09-14T03:48:10.137 INFO:tasks.rados.rados.0.smithi070.stdout:50: done (3 left) 2024-09-14T03:48:10.137 INFO:tasks.rados.rados.0.smithi070.stdout:51: done (2 left) 2024-09-14T03:48:10.137 INFO:tasks.rados.rados.0.smithi070.stdout:54: done (1 left) 2024-09-14T03:48:10.137 INFO:tasks.rados.rados.0.smithi070.stdout:55: read oid 15 snap -1 2024-09-14T03:48:10.137 INFO:tasks.rados.rados.0.smithi070.stdout:55: expect (ObjNum 14 snap 0 seq_num 14) 2024-09-14T03:48:10.137 INFO:tasks.rados.rados.0.smithi070.stdout:56: rollback oid 31 current snap is 0 2024-09-14T03:48:10.137 INFO:tasks.rados.rados.0.smithi070.stdout:56: done (2 left) 2024-09-14T03:48:10.137 INFO:tasks.rados.rados.0.smithi070.stdout:57: read oid 28 snap -1 2024-09-14T03:48:10.137 INFO:tasks.rados.rados.0.smithi070.stdout:57: expect (ObjNum 27 snap 0 seq_num 27) 2024-09-14T03:48:10.138 INFO:tasks.rados.rados.0.smithi070.stdout:58: copy_from oid 29 from oid 47 current snap is 0 2024-09-14T03:48:10.138 INFO:tasks.rados.rados.0.smithi070.stdout:59: rollback oid 45 current snap is 0 2024-09-14T03:48:10.138 INFO:tasks.rados.rados.0.smithi070.stdout:59: done (4 left) 2024-09-14T03:48:10.138 INFO:tasks.rados.rados.0.smithi070.stdout:60: copy_from oid 22 from oid 48 current snap is 0 2024-09-14T03:48:10.138 INFO:tasks.rados.rados.0.smithi070.stdout:61: delete oid 1 current snap is 0 2024-09-14T03:48:10.138 INFO:tasks.rados.rados.0.smithi070.stdout:52: finishing write tid 2 to smithi07085154-24 2024-09-14T03:48:10.138 INFO:tasks.rados.rados.0.smithi070.stdout:52: finishing write tid 3 to smithi07085154-24 2024-09-14T03:48:10.138 INFO:tasks.rados.rados.0.smithi070.stdout:61: done (5 left) 2024-09-14T03:48:10.138 INFO:tasks.rados.rados.0.smithi070.stdout:62: read oid 17 snap -1 2024-09-14T03:48:10.138 INFO:tasks.rados.rados.0.smithi070.stdout:62: expect (ObjNum 16 snap 0 seq_num 16) 2024-09-14T03:48:10.138 INFO:tasks.rados.rados.0.smithi070.stdout:63: snap_create 2024-09-14T03:48:10.139 INFO:tasks.rados.rados.0.smithi070.stdout:55: expect (ObjNum 14 snap 0 seq_num 14) 2024-09-14T03:48:10.139 INFO:tasks.rados.rados.0.smithi070.stdout:52: finishing write tid 4 to smithi07085154-24 2024-09-14T03:48:10.139 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 9 (ObjNum 50 snap 0 seq_num 50) dirty exists 2024-09-14T03:48:10.139 INFO:tasks.rados.rados.0.smithi070.stdout:52: left oid 24 (ObjNum 50 snap 0 seq_num 50) 2024-09-14T03:48:10.139 INFO:tasks.rados.rados.0.smithi070.stdout:57: expect (ObjNum 27 snap 0 seq_num 27) 2024-09-14T03:48:10.151 INFO:tasks.rados.rados.0.smithi070.stdout:62: expect (ObjNum 16 snap 0 seq_num 16) 2024-09-14T03:48:10.159 INFO:tasks.rados.rados.0.smithi070.stdout:60: finishing copy_from racing read to smithi07085154-22 2024-09-14T03:48:10.159 INFO:tasks.rados.rados.0.smithi070.stdout:58: finishing copy_from racing read to smithi07085154-29 2024-09-14T03:48:10.159 INFO:tasks.rados.rados.0.smithi070.stdout:58: finishing copy_from to smithi07085154-29 2024-09-14T03:48:10.159 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 8 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-14T03:48:10.159 INFO:tasks.rados.rados.0.smithi070.stdout:60: finishing copy_from to smithi07085154-22 2024-09-14T03:48:10.159 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 4 (ObjNum 47 snap 0 seq_num 47) dirty exists 2024-09-14T03:48:10.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:09 smithi094 ceph-mon[31053]: pgmap v107: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:48:10.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:09 smithi094 ceph-mon[31053]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:48:10.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:09 smithi094 ceph-mon[31053]: Cluster is now healthy 2024-09-14T03:48:10.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:09 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/4186131977' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-09-14T03:48:10.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:09 smithi070 ceph-mon[27495]: pgmap v107: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:48:10.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:09 smithi070 ceph-mon[27495]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:48:10.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:09 smithi070 ceph-mon[27495]: Cluster is now healthy 2024-09-14T03:48:10.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:09 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/4186131977' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-09-14T03:48:10.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:09 smithi070 ceph-mon[39288]: pgmap v107: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:48:10.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:09 smithi070 ceph-mon[39288]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:48:10.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:09 smithi070 ceph-mon[39288]: Cluster is now healthy 2024-09-14T03:48:10.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:09 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/4186131977' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-09-14T03:48:10.985 INFO:tasks.rados.rados.0.smithi070.stdout:52: done (6 left) 2024-09-14T03:48:10.985 INFO:tasks.rados.rados.0.smithi070.stdout:55: done (5 left) 2024-09-14T03:48:10.985 INFO:tasks.rados.rados.0.smithi070.stdout:57: done (4 left) 2024-09-14T03:48:10.985 INFO:tasks.rados.rados.0.smithi070.stdout:58: done (3 left) 2024-09-14T03:48:10.985 INFO:tasks.rados.rados.0.smithi070.stdout:60: done (2 left) 2024-09-14T03:48:10.985 INFO:tasks.rados.rados.0.smithi070.stdout:62: done (1 left) 2024-09-14T03:48:10.985 INFO:tasks.rados.rados.0.smithi070.stdout:63: done (0 left) 2024-09-14T03:48:10.985 INFO:tasks.rados.rados.0.smithi070.stdout:64: copy_from oid 15 from oid 2 current snap is 1 2024-09-14T03:48:10.985 INFO:tasks.rados.rados.0.smithi070.stdout:65: read oid 30 snap -1 2024-09-14T03:48:10.985 INFO:tasks.rados.rados.0.smithi070.stdout:65: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-14T03:48:10.985 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 23 current snap is 1 2024-09-14T03:48:10.986 INFO:tasks.rados.rados.0.smithi070.stdout:66: seq_num 53 ranges {376832=491520,868352=401408} 2024-09-14T03:48:10.989 INFO:tasks.rados.rados.0.smithi070.stdout:66: writing smithi07085154-23 from 376832 to 868352 tid 1 2024-09-14T03:48:10.992 INFO:tasks.rados.rados.0.smithi070.stdout:66: writing smithi07085154-23 from 868352 to 1269760 tid 2 2024-09-14T03:48:10.992 INFO:tasks.rados.rados.0.smithi070.stdout:67: rollback oid 10 current snap is 1 2024-09-14T03:48:10.992 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 10 to 0 2024-09-14T03:48:10.993 INFO:tasks.rados.rados.0.smithi070.stdout:65: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-14T03:48:11.009 INFO:tasks.rados.rados.0.smithi070.stdout:64: finishing copy_from racing read to smithi07085154-15 2024-09-14T03:48:11.009 INFO:tasks.rados.rados.0.smithi070.stdout:64: finishing copy_from to smithi07085154-15 2024-09-14T03:48:11.010 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 15 v 2 (ObjNum 1 snap 0 seq_num 1) dirty exists 2024-09-14T03:48:11.010 INFO:tasks.rados.rados.0.smithi070.stdout:66: finishing write tid 1 to smithi07085154-23 2024-09-14T03:48:11.010 INFO:tasks.rados.rados.0.smithi070.stdout:64: done (3 left) 2024-09-14T03:48:11.010 INFO:tasks.rados.rados.0.smithi070.stdout:65: done (2 left) 2024-09-14T03:48:11.010 INFO:tasks.rados.rados.0.smithi070.stdout:68: setattr oid 20 current snap is 1 2024-09-14T03:48:11.010 INFO:tasks.rados.rados.0.smithi070.stdout:66: finishing write tid 2 to smithi07085154-23 2024-09-14T03:48:11.010 INFO:tasks.rados.rados.0.smithi070.stdout:67: finishing rollback tid 0 to smithi07085154-10 2024-09-14T03:48:11.010 INFO:tasks.rados.rados.0.smithi070.stdout:67: finishing rollback tid 1 to smithi07085154-10 2024-09-14T03:48:11.010 INFO:tasks.rados.rados.0.smithi070.stdout:66: finishing write tid 3 to smithi07085154-23 2024-09-14T03:48:11.010 INFO:tasks.rados.rados.0.smithi070.stdout:67: finishing rollback tid 2 to smithi07085154-10 2024-09-14T03:48:11.010 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 20 (ObjNum 9 snap 0 seq_num 9) dirty exists 2024-09-14T03:48:11.010 INFO:tasks.rados.rados.0.smithi070.stdout:66: finishing write tid 4 to smithi07085154-23 2024-09-14T03:48:11.010 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 13 (ObjNum 53 snap 1 seq_num 53) dirty exists 2024-09-14T03:48:11.010 INFO:tasks.rados.rados.0.smithi070.stdout:66: left oid 23 (ObjNum 53 snap 1 seq_num 53) 2024-09-14T03:48:11.012 INFO:tasks.rados.rados.0.smithi070.stdout:66: done (2 left) 2024-09-14T03:48:11.012 INFO:tasks.rados.rados.0.smithi070.stdout:67: done (1 left) 2024-09-14T03:48:11.012 INFO:tasks.rados.rados.0.smithi070.stdout:69: copy_from oid 45 from oid 34 current snap is 1 2024-09-14T03:48:11.012 INFO:tasks.rados.rados.0.smithi070.stdout:70: read oid 24 snap 0 2024-09-14T03:48:11.012 INFO:tasks.rados.rados.0.smithi070.stdout:70: expect (ObjNum 50 snap 0 seq_num 50) 2024-09-14T03:48:11.012 INFO:tasks.rados.rados.0.smithi070.stdout:71: copy_from oid 36 from oid 28 current snap is 1 2024-09-14T03:48:11.013 INFO:tasks.rados.rados.0.smithi070.stdout:72: rollback oid 49 current snap is 1 2024-09-14T03:48:11.013 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 49 to 0 2024-09-14T03:48:11.013 INFO:tasks.rados.rados.0.smithi070.stdout:73: read oid 4 snap 0 2024-09-14T03:48:11.013 INFO:tasks.rados.rados.0.smithi070.stdout:73: expect (ObjNum 3 snap 0 seq_num 3) 2024-09-14T03:48:11.013 INFO:tasks.rados.rados.0.smithi070.stdout:74: rmattr oid 7 current snap is 1 2024-09-14T03:48:11.013 INFO:tasks.rados.rados.0.smithi070.stdout:74: done (6 left) 2024-09-14T03:48:11.013 INFO:tasks.rados.rados.0.smithi070.stdout:75: copy_from oid 21 from oid 12 current snap is 1 2024-09-14T03:48:11.013 INFO:tasks.rados.rados.0.smithi070.stdout:76: read oid 39 snap -1 2024-09-14T03:48:11.013 INFO:tasks.rados.rados.0.smithi070.stdout:76: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-14T03:48:11.013 INFO:tasks.rados.rados.0.smithi070.stdout:77: copy_from oid 6 from oid 37 current snap is 1 2024-09-14T03:48:11.013 INFO:tasks.rados.rados.0.smithi070.stdout:78: read oid 2 snap -1 2024-09-14T03:48:11.013 INFO:tasks.rados.rados.0.smithi070.stdout:78: expect (ObjNum 1 snap 0 seq_num 1) 2024-09-14T03:48:11.013 INFO:tasks.rados.rados.0.smithi070.stdout:79: setattr oid 41 current snap is 1 2024-09-14T03:48:11.013 INFO:tasks.rados.rados.0.smithi070.stdout:80: snap_create 2024-09-14T03:48:11.015 INFO:tasks.rados.rados.0.smithi070.stdout:73: expect (ObjNum 3 snap 0 seq_num 3) 2024-09-14T03:48:11.018 INFO:tasks.rados.rados.0.smithi070.stdout:72: finishing rollback tid 0 to smithi07085154-49 2024-09-14T03:48:11.018 INFO:tasks.rados.rados.0.smithi070.stdout:78: expect (ObjNum 1 snap 0 seq_num 1) 2024-09-14T03:48:11.019 INFO:tasks.rados.rados.0.smithi070.stdout:72: finishing rollback tid 1 to smithi07085154-49 2024-09-14T03:48:11.019 INFO:tasks.rados.rados.0.smithi070.stdout:72: finishing rollback tid 2 to smithi07085154-49 2024-09-14T03:48:11.019 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 49 v 11 (ObjNum 48 snap 0 seq_num 48) dirty exists 2024-09-14T03:48:11.019 INFO:tasks.rados.rados.0.smithi070.stdout:76: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-14T03:48:11.039 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 20 v 10 (ObjNum 19 snap 0 seq_num 19) dirty exists 2024-09-14T03:48:11.040 INFO:tasks.rados.rados.0.smithi070.stdout:75: finishing copy_from racing read to smithi07085154-21 2024-09-14T03:48:11.040 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 17 (ObjNum 40 snap 0 seq_num 40) dirty exists 2024-09-14T03:48:11.040 INFO:tasks.rados.rados.0.smithi070.stdout:71: finishing copy_from racing read to smithi07085154-36 2024-09-14T03:48:11.040 INFO:tasks.rados.rados.0.smithi070.stdout:69: finishing copy_from racing read to smithi07085154-45 2024-09-14T03:48:11.040 INFO:tasks.rados.rados.0.smithi070.stdout:70: expect (ObjNum 50 snap 0 seq_num 50) 2024-09-14T03:48:11.083 INFO:tasks.rados.rados.0.smithi070.stdout:69: finishing copy_from to smithi07085154-45 2024-09-14T03:48:11.083 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 45 v 8 (ObjNum 33 snap 0 seq_num 33) dirty exists 2024-09-14T03:48:11.083 INFO:tasks.rados.rados.0.smithi070.stdout:75: finishing copy_from to smithi07085154-21 2024-09-14T03:48:11.083 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 2 (ObjNum 11 snap 0 seq_num 11) dirty exists 2024-09-14T03:48:11.083 INFO:tasks.rados.rados.0.smithi070.stdout:71: finishing copy_from to smithi07085154-36 2024-09-14T03:48:11.083 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 36 v 8 (ObjNum 27 snap 0 seq_num 27) dirty exists 2024-09-14T03:48:11.083 INFO:tasks.rados.rados.0.smithi070.stdout:77: finishing copy_from racing read to smithi07085154-6 2024-09-14T03:48:11.083 INFO:tasks.rados.rados.0.smithi070.stdout:77: finishing copy_from to smithi07085154-6 2024-09-14T03:48:11.083 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 8 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-14T03:48:11.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:10 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/4186131977' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2024-09-14T03:48:11.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:10 smithi094 ceph-mon[31053]: osdmap e75: 8 total, 8 up, 8 in 2024-09-14T03:48:11.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:10 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/4186131977' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2024-09-14T03:48:11.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:10 smithi070 ceph-mon[27495]: osdmap e75: 8 total, 8 up, 8 in 2024-09-14T03:48:11.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:10 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/4186131977' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2024-09-14T03:48:11.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:10 smithi070 ceph-mon[39288]: osdmap e75: 8 total, 8 up, 8 in 2024-09-14T03:48:11.776 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:48:11 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:48:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:48:11.988 INFO:tasks.rados.rados.0.smithi070.stdout:68: done (11 left) 2024-09-14T03:48:11.988 INFO:tasks.rados.rados.0.smithi070.stdout:69: done (10 left) 2024-09-14T03:48:11.988 INFO:tasks.rados.rados.0.smithi070.stdout:70: done (9 left) 2024-09-14T03:48:11.989 INFO:tasks.rados.rados.0.smithi070.stdout:71: done (8 left) 2024-09-14T03:48:11.989 INFO:tasks.rados.rados.0.smithi070.stdout:72: done (7 left) 2024-09-14T03:48:11.989 INFO:tasks.rados.rados.0.smithi070.stdout:73: done (6 left) 2024-09-14T03:48:11.989 INFO:tasks.rados.rados.0.smithi070.stdout:75: done (5 left) 2024-09-14T03:48:11.989 INFO:tasks.rados.rados.0.smithi070.stdout:76: done (4 left) 2024-09-14T03:48:11.989 INFO:tasks.rados.rados.0.smithi070.stdout:77: done (3 left) 2024-09-14T03:48:11.989 INFO:tasks.rados.rados.0.smithi070.stdout:78: done (2 left) 2024-09-14T03:48:11.989 INFO:tasks.rados.rados.0.smithi070.stdout:79: done (1 left) 2024-09-14T03:48:11.989 INFO:tasks.rados.rados.0.smithi070.stdout:80: done (0 left) 2024-09-14T03:48:11.989 INFO:tasks.rados.rados.0.smithi070.stdout:81: rollback oid 40 current snap is 2 2024-09-14T03:48:11.989 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 40 to 1 2024-09-14T03:48:11.989 INFO:tasks.rados.rados.0.smithi070.stdout:82: snap_create 2024-09-14T03:48:11.990 INFO:tasks.rados.rados.0.smithi070.stdout:81: finishing rollback tid 0 to smithi07085154-40 2024-09-14T03:48:11.990 INFO:tasks.rados.rados.0.smithi070.stdout:81: finishing rollback tid 1 to smithi07085154-40 2024-09-14T03:48:11.991 INFO:tasks.rados.rados.0.smithi070.stdout:81: finishing rollback tid 2 to smithi07085154-40 2024-09-14T03:48:11.991 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 15 (ObjNum 39 snap 0 seq_num 39) dirty exists 2024-09-14T03:48:12.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:11 smithi094 ceph-mon[31053]: pgmap v109: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:48:12.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:11 smithi094 ceph-mon[31053]: osdmap e76: 8 total, 8 up, 8 in 2024-09-14T03:48:12.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:11 smithi070 ceph-mon[39288]: pgmap v109: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:48:12.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:11 smithi070 ceph-mon[39288]: osdmap e76: 8 total, 8 up, 8 in 2024-09-14T03:48:12.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:11 smithi070 ceph-mon[27495]: pgmap v109: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:48:12.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:11 smithi070 ceph-mon[27495]: osdmap e76: 8 total, 8 up, 8 in 2024-09-14T03:48:12.993 INFO:tasks.rados.rados.0.smithi070.stdout:81: done (1 left) 2024-09-14T03:48:12.994 INFO:tasks.rados.rados.0.smithi070.stdout:82: done (0 left) 2024-09-14T03:48:12.994 INFO:tasks.rados.rados.0.smithi070.stdout:83: rollback oid 9 current snap is 3 2024-09-14T03:48:12.994 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 9 to 0 2024-09-14T03:48:12.994 INFO:tasks.rados.rados.0.smithi070.stdout:84: read oid 20 snap 1 2024-09-14T03:48:12.994 INFO:tasks.rados.rados.0.smithi070.stdout:84: expect (ObjNum 19 snap 0 seq_num 19) 2024-09-14T03:48:12.994 INFO:tasks.rados.rados.0.smithi070.stdout:85: read oid 25 snap -1 2024-09-14T03:48:12.994 INFO:tasks.rados.rados.0.smithi070.stdout:85: expect (ObjNum 24 snap 0 seq_num 24) 2024-09-14T03:48:12.994 INFO:tasks.rados.rados.0.smithi070.stdout:86: copy_from oid 34 from oid 28 current snap is 3 2024-09-14T03:48:12.994 INFO:tasks.rados.rados.0.smithi070.stdout:87: setattr oid 48 current snap is 3 2024-09-14T03:48:12.994 INFO:tasks.rados.rados.0.smithi070.stdout:88: snap_remove snap 2 2024-09-14T03:48:13.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:12 smithi094 ceph-mon[31053]: osdmap e77: 8 total, 8 up, 8 in 2024-09-14T03:48:13.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:12 smithi070 ceph-mon[27495]: osdmap e77: 8 total, 8 up, 8 in 2024-09-14T03:48:13.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:12 smithi070 ceph-mon[39288]: osdmap e77: 8 total, 8 up, 8 in 2024-09-14T03:48:13.994 INFO:tasks.rados.rados.0.smithi070.stdout:88: done (5 left) 2024-09-14T03:48:13.994 INFO:tasks.rados.rados.0.smithi070.stdout:89: read oid 38 snap 1 2024-09-14T03:48:13.994 INFO:tasks.rados.rados.0.smithi070.stdout:89: expect (ObjNum 37 snap 0 seq_num 37) 2024-09-14T03:48:13.994 INFO:tasks.rados.rados.0.smithi070.stdout:90: read oid 18 snap -1 2024-09-14T03:48:13.994 INFO:tasks.rados.rados.0.smithi070.stdout:90: expect (ObjNum 17 snap 0 seq_num 17) 2024-09-14T03:48:13.994 INFO:tasks.rados.rados.0.smithi070.stdout:91: copy_from oid 2 from oid 29 current snap is 3 2024-09-14T03:48:13.995 INFO:tasks.rados.rados.0.smithi070.stdout:84: expect (ObjNum 19 snap 0 seq_num 19) 2024-09-14T03:48:14.010 INFO:tasks.rados.rados.0.smithi070.stdout:83: finishing rollback tid 0 to smithi07085154-9 2024-09-14T03:48:14.011 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 6 (ObjNum 47 snap 0 seq_num 47) dirty exists 2024-09-14T03:48:14.011 INFO:tasks.rados.rados.0.smithi070.stdout:83: finishing rollback tid 1 to smithi07085154-9 2024-09-14T03:48:14.011 INFO:tasks.rados.rados.0.smithi070.stdout:83: finishing rollback tid 2 to smithi07085154-9 2024-09-14T03:48:14.011 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 24 (ObjNum 8 snap 0 seq_num 8) dirty exists 2024-09-14T03:48:14.011 INFO:tasks.rados.rados.0.smithi070.stdout:85: expect (ObjNum 24 snap 0 seq_num 24) 2024-09-14T03:48:14.022 INFO:tasks.rados.rados.0.smithi070.stdout:86: finishing copy_from racing read to smithi07085154-34 2024-09-14T03:48:14.023 INFO:tasks.rados.rados.0.smithi070.stdout:86: finishing copy_from to smithi07085154-34 2024-09-14T03:48:14.023 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 8 (ObjNum 27 snap 0 seq_num 27) dirty exists 2024-09-14T03:48:14.023 INFO:tasks.rados.rados.0.smithi070.stdout:83: done (7 left) 2024-09-14T03:48:14.023 INFO:tasks.rados.rados.0.smithi070.stdout:84: done (6 left) 2024-09-14T03:48:14.023 INFO:tasks.rados.rados.0.smithi070.stdout:85: done (5 left) 2024-09-14T03:48:14.023 INFO:tasks.rados.rados.0.smithi070.stdout:86: done (4 left) 2024-09-14T03:48:14.023 INFO:tasks.rados.rados.0.smithi070.stdout:87: done (3 left) 2024-09-14T03:48:14.023 INFO:tasks.rados.rados.0.smithi070.stdout:92: rmattr oid 49 current snap is 3 2024-09-14T03:48:14.023 INFO:tasks.rados.rados.0.smithi070.stdout:92: done (3 left) 2024-09-14T03:48:14.023 INFO:tasks.rados.rados.0.smithi070.stdout:93: snap_remove snap 0 2024-09-14T03:48:14.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:13 smithi094 ceph-mon[31053]: pgmap v112: 121 pgs: 121 active+clean; 25 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 5.4 MiB/s wr, 66 op/s 2024-09-14T03:48:14.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:13 smithi094 ceph-mon[31053]: osdmap e78: 8 total, 8 up, 8 in 2024-09-14T03:48:14.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:13 smithi070 ceph-mon[27495]: pgmap v112: 121 pgs: 121 active+clean; 25 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 5.4 MiB/s wr, 66 op/s 2024-09-14T03:48:14.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:13 smithi070 ceph-mon[27495]: osdmap e78: 8 total, 8 up, 8 in 2024-09-14T03:48:14.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:13 smithi070 ceph-mon[39288]: pgmap v112: 121 pgs: 121 active+clean; 25 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 5.4 MiB/s wr, 66 op/s 2024-09-14T03:48:14.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:13 smithi070 ceph-mon[39288]: osdmap e78: 8 total, 8 up, 8 in 2024-09-14T03:48:15.002 INFO:tasks.rados.rados.0.smithi070.stdout:93: done (3 left) 2024-09-14T03:48:15.003 INFO:tasks.rados.rados.0.smithi070.stdout:94: rmattr oid 31 current snap is 3 2024-09-14T03:48:15.003 INFO:tasks.rados.rados.0.smithi070.stdout:94: done (3 left) 2024-09-14T03:48:15.003 INFO:tasks.rados.rados.0.smithi070.stdout:95: read oid 43 snap 1 2024-09-14T03:48:15.003 INFO:tasks.rados.rados.0.smithi070.stdout:95: expect (ObjNum 42 snap 0 seq_num 42) 2024-09-14T03:48:15.003 INFO:tasks.rados.rados.0.smithi070.stdout:96: rmattr oid 14 current snap is 3 2024-09-14T03:48:15.003 INFO:tasks.rados.rados.0.smithi070.stdout:96: done (4 left) 2024-09-14T03:48:15.003 INFO:tasks.rados.rados.0.smithi070.stdout:97: read oid 9 snap 1 2024-09-14T03:48:15.003 INFO:tasks.rados.rados.0.smithi070.stdout:97: expect (ObjNum 8 snap 0 seq_num 8) 2024-09-14T03:48:15.003 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 25 current snap is 3 2024-09-14T03:48:15.003 INFO:tasks.rados.rados.0.smithi070.stdout:98: seq_num 54 ranges {409600=507904,917504=401408} 2024-09-14T03:48:15.008 INFO:tasks.rados.rados.0.smithi070.stdout:98: writing smithi07085154-25 from 409600 to 917504 tid 1 2024-09-14T03:48:15.012 INFO:tasks.rados.rados.0.smithi070.stdout:98: writing smithi07085154-25 from 917504 to 1318912 tid 2 2024-09-14T03:48:15.012 INFO:tasks.rados.rados.0.smithi070.stdout:89: expect (ObjNum 37 snap 0 seq_num 37) 2024-09-14T03:48:15.034 INFO:tasks.rados.rados.0.smithi070.stdout:90: expect (ObjNum 17 snap 0 seq_num 17) 2024-09-14T03:48:15.043 INFO:tasks.rados.rados.0.smithi070.stdout:91: finishing copy_from racing read to smithi07085154-2 2024-09-14T03:48:15.043 INFO:tasks.rados.rados.0.smithi070.stdout:91: finishing copy_from to smithi07085154-2 2024-09-14T03:48:15.043 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 2 v 8 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-14T03:48:15.043 INFO:tasks.rados.rados.0.smithi070.stdout:98: finishing write tid 1 to smithi07085154-25 2024-09-14T03:48:15.043 INFO:tasks.rados.rados.0.smithi070.stdout:89: done (5 left) 2024-09-14T03:48:15.043 INFO:tasks.rados.rados.0.smithi070.stdout:90: done (4 left) 2024-09-14T03:48:15.044 INFO:tasks.rados.rados.0.smithi070.stdout:91: done (3 left) 2024-09-14T03:48:15.044 INFO:tasks.rados.rados.0.smithi070.stdout:99: copy_from oid 34 from oid 40 current snap is 3 2024-09-14T03:48:15.044 INFO:tasks.rados.rados.0.smithi070.stdout:95: expect (ObjNum 42 snap 0 seq_num 42) 2024-09-14T03:48:15.063 INFO:tasks.rados.rados.0.smithi070.stdout:98: finishing write tid 2 to smithi07085154-25 2024-09-14T03:48:15.063 INFO:tasks.rados.rados.0.smithi070.stdout:98: finishing write tid 3 to smithi07085154-25 2024-09-14T03:48:15.063 INFO:tasks.rados.rados.0.smithi070.stdout:98: finishing write tid 4 to smithi07085154-25 2024-09-14T03:48:15.063 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 16 (ObjNum 54 snap 3 seq_num 54) dirty exists 2024-09-14T03:48:15.063 INFO:tasks.rados.rados.0.smithi070.stdout:98: left oid 25 (ObjNum 54 snap 3 seq_num 54) 2024-09-14T03:48:15.063 INFO:tasks.rados.rados.0.smithi070.stdout:97: expect (ObjNum 8 snap 0 seq_num 8) 2024-09-14T03:48:15.067 INFO:tasks.rados.rados.0.smithi070.stdout:99: finishing copy_from racing read to smithi07085154-34 2024-09-14T03:48:15.067 INFO:tasks.rados.rados.0.smithi070.stdout:99: finishing copy_from to smithi07085154-34 2024-09-14T03:48:15.067 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 15 (ObjNum 39 snap 0 seq_num 39) dirty exists 2024-09-14T03:48:15.067 INFO:tasks.rados.rados.0.smithi070.stdout:95: done (3 left) 2024-09-14T03:48:15.068 INFO:tasks.rados.rados.0.smithi070.stdout:97: done (2 left) 2024-09-14T03:48:15.068 INFO:tasks.rados.rados.0.smithi070.stdout:98: done (1 left) 2024-09-14T03:48:15.068 INFO:tasks.rados.rados.0.smithi070.stdout:99: done (0 left) 2024-09-14T03:48:15.068 INFO:tasks.rados.rados.0.smithi070.stdout:100: snap_remove snap 1 2024-09-14T03:48:15.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:14 smithi094 ceph-mon[31053]: osdmap e79: 8 total, 8 up, 8 in 2024-09-14T03:48:15.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:14 smithi070 ceph-mon[27495]: osdmap e79: 8 total, 8 up, 8 in 2024-09-14T03:48:15.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:14 smithi070 ceph-mon[39288]: osdmap e79: 8 total, 8 up, 8 in 2024-09-14T03:48:16.006 INFO:tasks.rados.rados.0.smithi070.stdout:100: done (0 left) 2024-09-14T03:48:16.007 INFO:tasks.rados.rados.0.smithi070.stdout:101: copy_from oid 42 from oid 39 current snap is 3 2024-09-14T03:48:16.007 INFO:tasks.rados.rados.0.smithi070.stdout:102: read oid 14 snap -1 2024-09-14T03:48:16.007 INFO:tasks.rados.rados.0.smithi070.stdout:102: expect (ObjNum 13 snap 0 seq_num 13) 2024-09-14T03:48:16.007 INFO:tasks.rados.rados.0.smithi070.stdout:103: rollback oid 20 current snap is 3 2024-09-14T03:48:16.007 INFO:tasks.rados.rados.0.smithi070.stdout:103: done (2 left) 2024-09-14T03:48:16.007 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 36 current snap is 3 2024-09-14T03:48:16.007 INFO:tasks.rados.rados.0.smithi070.stdout:104: seq_num 55 ranges {458752=524288,983040=401408} 2024-09-14T03:48:16.014 INFO:tasks.rados.rados.0.smithi070.stdout:104: writing smithi07085154-36 from 458752 to 983040 tid 1 2024-09-14T03:48:16.017 INFO:tasks.rados.rados.0.smithi070.stdout:104: writing smithi07085154-36 from 983040 to 1384448 tid 2 2024-09-14T03:48:16.017 INFO:tasks.rados.rados.0.smithi070.stdout:105: copy_from oid 22 from oid 49 current snap is 3 2024-09-14T03:48:16.017 INFO:tasks.rados.rados.0.smithi070.stdout:102: expect (ObjNum 13 snap 0 seq_num 13) 2024-09-14T03:48:16.025 INFO:tasks.rados.rados.0.smithi070.stdout:101: finishing copy_from racing read to smithi07085154-42 2024-09-14T03:48:16.025 INFO:tasks.rados.rados.0.smithi070.stdout:101: finishing copy_from to smithi07085154-42 2024-09-14T03:48:16.025 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 2 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-14T03:48:16.025 INFO:tasks.rados.rados.0.smithi070.stdout:104: finishing write tid 1 to smithi07085154-36 2024-09-14T03:48:16.025 INFO:tasks.rados.rados.0.smithi070.stdout:104: finishing write tid 2 to smithi07085154-36 2024-09-14T03:48:16.025 INFO:tasks.rados.rados.0.smithi070.stdout:101: done (3 left) 2024-09-14T03:48:16.025 INFO:tasks.rados.rados.0.smithi070.stdout:102: done (2 left) 2024-09-14T03:48:16.025 INFO:tasks.rados.rados.0.smithi070.stdout:106: rollback oid 4 current snap is 3 2024-09-14T03:48:16.025 INFO:tasks.rados.rados.0.smithi070.stdout:106: done (2 left) 2024-09-14T03:48:16.025 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 12 current snap is 3 2024-09-14T03:48:16.025 INFO:tasks.rados.rados.0.smithi070.stdout:104: finishing write tid 3 to smithi07085154-36 2024-09-14T03:48:16.026 INFO:tasks.rados.rados.0.smithi070.stdout:104: finishing write tid 4 to smithi07085154-36 2024-09-14T03:48:16.026 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 36 v 24 (ObjNum 55 snap 3 seq_num 55) dirty exists 2024-09-14T03:48:16.026 INFO:tasks.rados.rados.0.smithi070.stdout:104: left oid 36 (ObjNum 55 snap 3 seq_num 55) 2024-09-14T03:48:16.026 INFO:tasks.rados.rados.0.smithi070.stdout:105: finishing copy_from racing read to smithi07085154-22 2024-09-14T03:48:16.026 INFO:tasks.rados.rados.0.smithi070.stdout:105: finishing copy_from to smithi07085154-22 2024-09-14T03:48:16.026 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 11 (ObjNum 48 snap 0 seq_num 48) dirty exists 2024-09-14T03:48:16.026 INFO:tasks.rados.rados.0.smithi070.stdout:107: seq_num 56 ranges {188416=540672,729088=401408} 2024-09-14T03:48:16.028 INFO:tasks.rados.rados.0.smithi070.stdout:107: writing smithi07085154-12 from 188416 to 729088 tid 1 2024-09-14T03:48:16.031 INFO:tasks.rados.rados.0.smithi070.stdout:107: writing smithi07085154-12 from 729088 to 1130496 tid 2 2024-09-14T03:48:16.031 INFO:tasks.rados.rados.0.smithi070.stdout:104: done (2 left) 2024-09-14T03:48:16.031 INFO:tasks.rados.rados.0.smithi070.stdout:105: done (1 left) 2024-09-14T03:48:16.031 INFO:tasks.rados.rados.0.smithi070.stdout:108: read oid 46 snap -1 2024-09-14T03:48:16.032 INFO:tasks.rados.rados.0.smithi070.stdout:108: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-14T03:48:16.032 INFO:tasks.rados.rados.0.smithi070.stdout:109: rollback oid 14 current snap is 3 2024-09-14T03:48:16.032 INFO:tasks.rados.rados.0.smithi070.stdout:109: done (2 left) 2024-09-14T03:48:16.032 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 28 current snap is 3 2024-09-14T03:48:16.032 INFO:tasks.rados.rados.0.smithi070.stdout:110: seq_num 57 ranges {458752=557056,1015808=401408} 2024-09-14T03:48:16.037 INFO:tasks.rados.rados.0.smithi070.stdout:110: writing smithi07085154-28 from 458752 to 1015808 tid 1 2024-09-14T03:48:16.040 INFO:tasks.rados.rados.0.smithi070.stdout:110: writing smithi07085154-28 from 1015808 to 1417216 tid 2 2024-09-14T03:48:16.040 INFO:tasks.rados.rados.0.smithi070.stdout:111: delete oid 24 current snap is 3 2024-09-14T03:48:16.040 INFO:tasks.rados.rados.0.smithi070.stdout:107: finishing write tid 1 to smithi07085154-12 2024-09-14T03:48:16.040 INFO:tasks.rados.rados.0.smithi070.stdout:107: finishing write tid 2 to smithi07085154-12 2024-09-14T03:48:16.040 INFO:tasks.rados.rados.0.smithi070.stdout:107: finishing write tid 3 to smithi07085154-12 2024-09-14T03:48:16.040 INFO:tasks.rados.rados.0.smithi070.stdout:107: finishing write tid 4 to smithi07085154-12 2024-09-14T03:48:16.041 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 15 (ObjNum 56 snap 3 seq_num 56) dirty exists 2024-09-14T03:48:16.041 INFO:tasks.rados.rados.0.smithi070.stdout:107: left oid 12 (ObjNum 56 snap 3 seq_num 56) 2024-09-14T03:48:16.041 INFO:tasks.rados.rados.0.smithi070.stdout:108: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-14T03:48:16.065 INFO:tasks.rados.rados.0.smithi070.stdout:110: finishing write tid 1 to smithi07085154-28 2024-09-14T03:48:16.065 INFO:tasks.rados.rados.0.smithi070.stdout:107: done (3 left) 2024-09-14T03:48:16.065 INFO:tasks.rados.rados.0.smithi070.stdout:108: done (2 left) 2024-09-14T03:48:16.066 INFO:tasks.rados.rados.0.smithi070.stdout:111: done (1 left) 2024-09-14T03:48:16.066 INFO:tasks.rados.rados.0.smithi070.stdout:112: rmattr oid 7 current snap is 3 2024-09-14T03:48:16.066 INFO:tasks.rados.rados.0.smithi070.stdout:112: done (1 left) 2024-09-14T03:48:16.066 INFO:tasks.rados.rados.0.smithi070.stdout:113: delete oid 26 current snap is 3 2024-09-14T03:48:16.066 INFO:tasks.rados.rados.0.smithi070.stdout:110: finishing write tid 2 to smithi07085154-28 2024-09-14T03:48:16.066 INFO:tasks.rados.rados.0.smithi070.stdout:110: finishing write tid 3 to smithi07085154-28 2024-09-14T03:48:16.066 INFO:tasks.rados.rados.0.smithi070.stdout:110: finishing write tid 4 to smithi07085154-28 2024-09-14T03:48:16.066 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 28 v 15 (ObjNum 57 snap 3 seq_num 57) dirty exists 2024-09-14T03:48:16.066 INFO:tasks.rados.rados.0.smithi070.stdout:110: left oid 28 (ObjNum 57 snap 3 seq_num 57) 2024-09-14T03:48:16.067 INFO:tasks.rados.rados.0.smithi070.stdout:110: done (1 left) 2024-09-14T03:48:16.067 INFO:tasks.rados.rados.0.smithi070.stdout:113: done (0 left) 2024-09-14T03:48:16.067 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 25 current snap is 3 2024-09-14T03:48:16.067 INFO:tasks.rados.rados.0.smithi070.stdout:114: seq_num 60 ranges {1318912=614400,1933312=401408} 2024-09-14T03:48:16.077 INFO:tasks.rados.rados.0.smithi070.stdout:114: writing smithi07085154-25 from 1318912 to 1933312 tid 1 2024-09-14T03:48:16.080 INFO:tasks.rados.rados.0.smithi070.stdout:114: writing smithi07085154-25 from 1933312 to 2334720 tid 2 2024-09-14T03:48:16.080 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 3 current snap is 3 2024-09-14T03:48:16.080 INFO:tasks.rados.rados.0.smithi070.stdout:115: seq_num 61 ranges {40960=630784,671744=401408} 2024-09-14T03:48:16.083 INFO:tasks.rados.rados.0.smithi070.stdout:115: writing smithi07085154-3 from 40960 to 671744 tid 1 2024-09-14T03:48:16.086 INFO:tasks.rados.rados.0.smithi070.stdout:115: writing smithi07085154-3 from 671744 to 1073152 tid 2 2024-09-14T03:48:16.086 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 32 current snap is 3 2024-09-14T03:48:16.086 INFO:tasks.rados.rados.0.smithi070.stdout:114: finishing write tid 1 to smithi07085154-25 2024-09-14T03:48:16.086 INFO:tasks.rados.rados.0.smithi070.stdout:114: finishing write tid 2 to smithi07085154-25 2024-09-14T03:48:16.086 INFO:tasks.rados.rados.0.smithi070.stdout:114: finishing write tid 3 to smithi07085154-25 2024-09-14T03:48:16.087 INFO:tasks.rados.rados.0.smithi070.stdout:114: finishing write tid 4 to smithi07085154-25 2024-09-14T03:48:16.087 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 21 (ObjNum 60 snap 3 seq_num 60) dirty exists 2024-09-14T03:48:16.087 INFO:tasks.rados.rados.0.smithi070.stdout:114: left oid 25 (ObjNum 60 snap 3 seq_num 60) 2024-09-14T03:48:16.087 INFO:tasks.rados.rados.0.smithi070.stdout:116: seq_num 62 ranges {524288=647168,1171456=401408} 2024-09-14T03:48:16.092 INFO:tasks.rados.rados.0.smithi070.stdout:116: writing smithi07085154-32 from 524288 to 1171456 tid 1 2024-09-14T03:48:16.095 INFO:tasks.rados.rados.0.smithi070.stdout:116: writing smithi07085154-32 from 1171456 to 1572864 tid 2 2024-09-14T03:48:16.095 INFO:tasks.rados.rados.0.smithi070.stdout:114: done (2 left) 2024-09-14T03:48:16.095 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 2 current snap is 3 2024-09-14T03:48:16.095 INFO:tasks.rados.rados.0.smithi070.stdout:115: finishing write tid 1 to smithi07085154-3 2024-09-14T03:48:16.095 INFO:tasks.rados.rados.0.smithi070.stdout:117: seq_num 63 ranges {778240=663552,1441792=401408} 2024-09-14T03:48:16.103 INFO:tasks.rados.rados.0.smithi070.stdout:117: writing smithi07085154-2 from 778240 to 1441792 tid 1 2024-09-14T03:48:16.106 INFO:tasks.rados.rados.0.smithi070.stdout:117: writing smithi07085154-2 from 1441792 to 1843200 tid 2 2024-09-14T03:48:16.106 INFO:tasks.rados.rados.0.smithi070.stdout:118: rollback oid 15 current snap is 3 2024-09-14T03:48:16.106 INFO:tasks.rados.rados.0.smithi070.stdout:118: done (3 left) 2024-09-14T03:48:16.106 INFO:tasks.rados.rados.0.smithi070.stdout:119: delete oid 38 current snap is 3 2024-09-14T03:48:16.106 INFO:tasks.rados.rados.0.smithi070.stdout:115: finishing write tid 2 to smithi07085154-3 2024-09-14T03:48:16.106 INFO:tasks.rados.rados.0.smithi070.stdout:115: finishing write tid 3 to smithi07085154-3 2024-09-14T03:48:16.106 INFO:tasks.rados.rados.0.smithi070.stdout:115: finishing write tid 4 to smithi07085154-3 2024-09-14T03:48:16.106 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 3 v 11 (ObjNum 61 snap 3 seq_num 61) dirty exists 2024-09-14T03:48:16.106 INFO:tasks.rados.rados.0.smithi070.stdout:115: left oid 3 (ObjNum 61 snap 3 seq_num 61) 2024-09-14T03:48:16.106 INFO:tasks.rados.rados.0.smithi070.stdout:116: finishing write tid 1 to smithi07085154-32 2024-09-14T03:48:16.106 INFO:tasks.rados.rados.0.smithi070.stdout:116: finishing write tid 2 to smithi07085154-32 2024-09-14T03:48:16.106 INFO:tasks.rados.rados.0.smithi070.stdout:116: finishing write tid 3 to smithi07085154-32 2024-09-14T03:48:16.107 INFO:tasks.rados.rados.0.smithi070.stdout:116: finishing write tid 4 to smithi07085154-32 2024-09-14T03:48:16.107 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 9 (ObjNum 62 snap 3 seq_num 62) dirty exists 2024-09-14T03:48:16.107 INFO:tasks.rados.rados.0.smithi070.stdout:116: left oid 32 (ObjNum 62 snap 3 seq_num 62) 2024-09-14T03:48:16.107 INFO:tasks.rados.rados.0.smithi070.stdout:117: finishing write tid 1 to smithi07085154-2 2024-09-14T03:48:16.107 INFO:tasks.rados.rados.0.smithi070.stdout:115: done (3 left) 2024-09-14T03:48:16.107 INFO:tasks.rados.rados.0.smithi070.stdout:116: done (2 left) 2024-09-14T03:48:16.107 INFO:tasks.rados.rados.0.smithi070.stdout:119: done (1 left) 2024-09-14T03:48:16.107 INFO:tasks.rados.rados.0.smithi070.stdout:120: delete oid 37 current snap is 3 2024-09-14T03:48:16.108 INFO:tasks.rados.rados.0.smithi070.stdout:117: finishing write tid 2 to smithi07085154-2 2024-09-14T03:48:16.109 INFO:tasks.rados.rados.0.smithi070.stdout:120: done (1 left) 2024-09-14T03:48:16.109 INFO:tasks.rados.rados.0.smithi070.stdout:121: snap_create 2024-09-14T03:48:16.109 INFO:tasks.rados.rados.0.smithi070.stdout:117: finishing write tid 3 to smithi07085154-2 2024-09-14T03:48:16.109 INFO:tasks.rados.rados.0.smithi070.stdout:117: finishing write tid 4 to smithi07085154-2 2024-09-14T03:48:16.109 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 2 v 43 (ObjNum 63 snap 3 seq_num 63) dirty exists 2024-09-14T03:48:16.109 INFO:tasks.rados.rados.0.smithi070.stdout:117: left oid 2 (ObjNum 63 snap 3 seq_num 63) 2024-09-14T03:48:16.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:15 smithi094 ceph-mon[31053]: pgmap v115: 121 pgs: 121 active+clean; 25 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 6.3 MiB/s wr, 76 op/s 2024-09-14T03:48:16.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:15 smithi094 ceph-mon[31053]: osdmap e80: 8 total, 8 up, 8 in 2024-09-14T03:48:16.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:15 smithi070 ceph-mon[27495]: pgmap v115: 121 pgs: 121 active+clean; 25 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 6.3 MiB/s wr, 76 op/s 2024-09-14T03:48:16.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:15 smithi070 ceph-mon[27495]: osdmap e80: 8 total, 8 up, 8 in 2024-09-14T03:48:16.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:15 smithi070 ceph-mon[39288]: pgmap v115: 121 pgs: 121 active+clean; 25 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 6.3 MiB/s wr, 76 op/s 2024-09-14T03:48:16.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:15 smithi070 ceph-mon[39288]: osdmap e80: 8 total, 8 up, 8 in 2024-09-14T03:48:16.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:48:16 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:48:16] "GET /metrics HTTP/1.1" 200 238857 "" "Prometheus/2.43.0" 2024-09-14T03:48:17.011 INFO:tasks.rados.rados.0.smithi070.stdout:117: done (1 left) 2024-09-14T03:48:17.011 INFO:tasks.rados.rados.0.smithi070.stdout:121: done (0 left) 2024-09-14T03:48:17.012 INFO:tasks.rados.rados.0.smithi070.stdout:122: read oid 48 snap 3 2024-09-14T03:48:17.012 INFO:tasks.rados.rados.0.smithi070.stdout:122: expect (ObjNum 47 snap 0 seq_num 47) 2024-09-14T03:48:17.012 INFO:tasks.rados.rados.0.smithi070.stdout:123: rollback oid 22 current snap is 4 2024-09-14T03:48:17.012 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 22 to 3 2024-09-14T03:48:17.012 INFO:tasks.rados.rados.0.smithi070.stdout:124: read oid 17 snap 3 2024-09-14T03:48:17.012 INFO:tasks.rados.rados.0.smithi070.stdout:124: expect (ObjNum 16 snap 0 seq_num 16) 2024-09-14T03:48:17.012 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 32 current snap is 4 2024-09-14T03:48:17.012 INFO:tasks.rados.rados.0.smithi070.stdout:125: seq_num 66 ranges {1572864=712704,2285568=401408} 2024-09-14T03:48:17.023 INFO:tasks.rados.rados.0.smithi070.stdout:125: writing smithi07085154-32 from 1572864 to 2285568 tid 1 2024-09-14T03:48:17.026 INFO:tasks.rados.rados.0.smithi070.stdout:125: writing smithi07085154-32 from 2285568 to 2686976 tid 2 2024-09-14T03:48:17.027 INFO:tasks.rados.rados.0.smithi070.stdout:126: copy_from oid 1 from oid 24 current snap is 4 2024-09-14T03:48:17.027 INFO:tasks.rados.rados.0.smithi070.stdout:124: expect (ObjNum 16 snap 0 seq_num 16) 2024-09-14T03:48:17.036 INFO:tasks.rados.rados.0.smithi070.stdout:123: finishing rollback tid 0 to smithi07085154-22 2024-09-14T03:48:17.036 INFO:tasks.rados.rados.0.smithi070.stdout:124: done (4 left) 2024-09-14T03:48:17.036 INFO:tasks.rados.rados.0.smithi070.stdout:127: read oid 43 snap -1 2024-09-14T03:48:17.036 INFO:tasks.rados.rados.0.smithi070.stdout:127: expect (ObjNum 42 snap 0 seq_num 42) 2024-09-14T03:48:17.036 INFO:tasks.rados.rados.0.smithi070.stdout:128: copy_from oid 33 from oid 19 current snap is 4 2024-09-14T03:48:17.036 INFO:tasks.rados.rados.0.smithi070.stdout:123: finishing rollback tid 1 to smithi07085154-22 2024-09-14T03:48:17.036 INFO:tasks.rados.rados.0.smithi070.stdout:123: finishing rollback tid 2 to smithi07085154-22 2024-09-14T03:48:17.036 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 28 (ObjNum 48 snap 0 seq_num 48) dirty exists 2024-09-14T03:48:17.037 INFO:tasks.rados.rados.0.smithi070.stdout:122: expect (ObjNum 47 snap 0 seq_num 47) 2024-09-14T03:48:17.062 INFO:tasks.rados.rados.0.smithi070.stdout:126: finishing copy_from to smithi07085154-1 2024-09-14T03:48:17.062 INFO:tasks.rados.rados.0.smithi070.stdout:126: got expected ENOENT (src dne) 2024-09-14T03:48:17.062 INFO:tasks.rados.rados.0.smithi070.stdout:126: finishing copy_from racing read to smithi07085154-1 2024-09-14T03:48:17.062 INFO:tasks.rados.rados.0.smithi070.stdout:125: finishing write tid 1 to smithi07085154-32 2024-09-14T03:48:17.062 INFO:tasks.rados.rados.0.smithi070.stdout:125: finishing write tid 2 to smithi07085154-32 2024-09-14T03:48:17.062 INFO:tasks.rados.rados.0.smithi070.stdout:125: finishing write tid 3 to smithi07085154-32 2024-09-14T03:48:17.062 INFO:tasks.rados.rados.0.smithi070.stdout:125: finishing write tid 4 to smithi07085154-32 2024-09-14T03:48:17.062 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 13 (ObjNum 66 snap 4 seq_num 66) dirty exists 2024-09-14T03:48:17.062 INFO:tasks.rados.rados.0.smithi070.stdout:125: left oid 32 (ObjNum 66 snap 4 seq_num 66) 2024-09-14T03:48:17.062 INFO:tasks.rados.rados.0.smithi070.stdout:122: done (5 left) 2024-09-14T03:48:17.063 INFO:tasks.rados.rados.0.smithi070.stdout:123: done (4 left) 2024-09-14T03:48:17.063 INFO:tasks.rados.rados.0.smithi070.stdout:125: done (3 left) 2024-09-14T03:48:17.063 INFO:tasks.rados.rados.0.smithi070.stdout:126: done (2 left) 2024-09-14T03:48:17.063 INFO:tasks.rados.rados.0.smithi070.stdout:129: copy_from oid 29 from oid 10 current snap is 4 2024-09-14T03:48:17.063 INFO:tasks.rados.rados.0.smithi070.stdout:127: expect (ObjNum 42 snap 0 seq_num 42) 2024-09-14T03:48:17.083 INFO:tasks.rados.rados.0.smithi070.stdout:128: finishing copy_from racing read to smithi07085154-33 2024-09-14T03:48:17.083 INFO:tasks.rados.rados.0.smithi070.stdout:128: finishing copy_from to smithi07085154-33 2024-09-14T03:48:17.083 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 33 v 2 (ObjNum 18 snap 0 seq_num 18) dirty exists 2024-09-14T03:48:17.083 INFO:tasks.rados.rados.0.smithi070.stdout:129: finishing copy_from racing read to smithi07085154-29 2024-09-14T03:48:17.083 INFO:tasks.rados.rados.0.smithi070.stdout:129: finishing copy_from to smithi07085154-29 2024-09-14T03:48:17.083 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 20 (ObjNum 9 snap 0 seq_num 9) dirty exists 2024-09-14T03:48:17.083 INFO:tasks.rados.rados.0.smithi070.stdout:127: done (2 left) 2024-09-14T03:48:17.083 INFO:tasks.rados.rados.0.smithi070.stdout:128: done (1 left) 2024-09-14T03:48:17.083 INFO:tasks.rados.rados.0.smithi070.stdout:129: done (0 left) 2024-09-14T03:48:17.083 INFO:tasks.rados.rados.0.smithi070.stdout:130: snap_create 2024-09-14T03:48:17.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:17 smithi094 ceph-mon[31053]: osdmap e81: 8 total, 8 up, 8 in 2024-09-14T03:48:17.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:17 smithi070 ceph-mon[27495]: osdmap e81: 8 total, 8 up, 8 in 2024-09-14T03:48:17.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:17 smithi070 ceph-mon[39288]: osdmap e81: 8 total, 8 up, 8 in 2024-09-14T03:48:18.015 INFO:tasks.rados.rados.0.smithi070.stdout:130: done (0 left) 2024-09-14T03:48:18.015 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 9 current snap is 5 2024-09-14T03:48:18.015 INFO:tasks.rados.rados.0.smithi070.stdout:131: seq_num 67 ranges {139264=729088,868352=401408} 2024-09-14T03:48:18.020 INFO:tasks.rados.rados.0.smithi070.stdout:131: writing smithi07085154-9 from 139264 to 868352 tid 1 2024-09-14T03:48:18.022 INFO:tasks.rados.rados.0.smithi070.stdout:131: writing smithi07085154-9 from 868352 to 1269760 tid 2 2024-09-14T03:48:18.023 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 12 current snap is 5 2024-09-14T03:48:18.023 INFO:tasks.rados.rados.0.smithi070.stdout:132: seq_num 68 ranges {1130496=745472,1875968=401408} 2024-09-14T03:48:18.033 INFO:tasks.rados.rados.0.smithi070.stdout:132: writing smithi07085154-12 from 1130496 to 1875968 tid 1 2024-09-14T03:48:18.036 INFO:tasks.rados.rados.0.smithi070.stdout:132: writing smithi07085154-12 from 1875968 to 2277376 tid 2 2024-09-14T03:48:18.036 INFO:tasks.rados.rados.0.smithi070.stdout:133: rollback oid 22 current snap is 5 2024-09-14T03:48:18.036 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 22 to 3 2024-09-14T03:48:18.036 INFO:tasks.rados.rados.0.smithi070.stdout:131: finishing write tid 1 to smithi07085154-9 2024-09-14T03:48:18.036 INFO:tasks.rados.rados.0.smithi070.stdout:131: finishing write tid 2 to smithi07085154-9 2024-09-14T03:48:18.036 INFO:tasks.rados.rados.0.smithi070.stdout:131: finishing write tid 3 to smithi07085154-9 2024-09-14T03:48:18.036 INFO:tasks.rados.rados.0.smithi070.stdout:131: finishing write tid 4 to smithi07085154-9 2024-09-14T03:48:18.036 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 47 (ObjNum 67 snap 5 seq_num 67) dirty exists 2024-09-14T03:48:18.037 INFO:tasks.rados.rados.0.smithi070.stdout:131: left oid 9 (ObjNum 67 snap 5 seq_num 67) 2024-09-14T03:48:18.037 INFO:tasks.rados.rados.0.smithi070.stdout:131: done (2 left) 2024-09-14T03:48:18.037 INFO:tasks.rados.rados.0.smithi070.stdout:134: snap_create 2024-09-14T03:48:18.039 INFO:tasks.rados.rados.0.smithi070.stdout:133: finishing rollback tid 0 to smithi07085154-22 2024-09-14T03:48:18.040 INFO:tasks.rados.rados.0.smithi070.stdout:133: finishing rollback tid 1 to smithi07085154-22 2024-09-14T03:48:18.040 INFO:tasks.rados.rados.0.smithi070.stdout:132: finishing write tid 1 to smithi07085154-12 2024-09-14T03:48:18.040 INFO:tasks.rados.rados.0.smithi070.stdout:133: finishing rollback tid 2 to smithi07085154-22 2024-09-14T03:48:18.040 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 34 (ObjNum 48 snap 0 seq_num 48) dirty exists 2024-09-14T03:48:18.040 INFO:tasks.rados.rados.0.smithi070.stdout:132: finishing write tid 2 to smithi07085154-12 2024-09-14T03:48:18.040 INFO:tasks.rados.rados.0.smithi070.stdout:132: finishing write tid 3 to smithi07085154-12 2024-09-14T03:48:18.040 INFO:tasks.rados.rados.0.smithi070.stdout:132: finishing write tid 4 to smithi07085154-12 2024-09-14T03:48:18.040 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 19 (ObjNum 68 snap 5 seq_num 68) dirty exists 2024-09-14T03:48:18.041 INFO:tasks.rados.rados.0.smithi070.stdout:132: left oid 12 (ObjNum 68 snap 5 seq_num 68) 2024-09-14T03:48:18.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:18 smithi094 ceph-mon[31053]: pgmap v118: 121 pgs: 121 active+clean; 25 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:48:18.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:18 smithi094 ceph-mon[31053]: osdmap e82: 8 total, 8 up, 8 in 2024-09-14T03:48:18.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:18 smithi070 ceph-mon[27495]: pgmap v118: 121 pgs: 121 active+clean; 25 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:48:18.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:18 smithi070 ceph-mon[27495]: osdmap e82: 8 total, 8 up, 8 in 2024-09-14T03:48:18.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:18 smithi070 ceph-mon[39288]: pgmap v118: 121 pgs: 121 active+clean; 25 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:48:18.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:18 smithi070 ceph-mon[39288]: osdmap e82: 8 total, 8 up, 8 in 2024-09-14T03:48:19.020 INFO:tasks.rados.rados.0.smithi070.stdout:132: done (2 left) 2024-09-14T03:48:19.021 INFO:tasks.rados.rados.0.smithi070.stdout:133: done (1 left) 2024-09-14T03:48:19.021 INFO:tasks.rados.rados.0.smithi070.stdout:134: done (0 left) 2024-09-14T03:48:19.021 INFO:tasks.rados.rados.0.smithi070.stdout:135: read oid 33 snap -1 2024-09-14T03:48:19.021 INFO:tasks.rados.rados.0.smithi070.stdout:135: expect (ObjNum 18 snap 0 seq_num 18) 2024-09-14T03:48:19.021 INFO:tasks.rados.rados.0.smithi070.stdout:136: read oid 45 snap -1 2024-09-14T03:48:19.021 INFO:tasks.rados.rados.0.smithi070.stdout:136: expect (ObjNum 33 snap 0 seq_num 33) 2024-09-14T03:48:19.021 INFO:tasks.rados.rados.0.smithi070.stdout:137: snap_create 2024-09-14T03:48:19.025 INFO:tasks.rados.rados.0.smithi070.stdout:135: expect (ObjNum 18 snap 0 seq_num 18) 2024-09-14T03:48:19.034 INFO:tasks.rados.rados.0.smithi070.stdout:136: expect (ObjNum 33 snap 0 seq_num 33) 2024-09-14T03:48:19.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:19 smithi094 ceph-mon[31053]: osdmap e83: 8 total, 8 up, 8 in 2024-09-14T03:48:19.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:19 smithi070 ceph-mon[27495]: osdmap e83: 8 total, 8 up, 8 in 2024-09-14T03:48:19.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:19 smithi070 ceph-mon[39288]: osdmap e83: 8 total, 8 up, 8 in 2024-09-14T03:48:20.021 INFO:tasks.rados.rados.0.smithi070.stdout:135: done (2 left) 2024-09-14T03:48:20.022 INFO:tasks.rados.rados.0.smithi070.stdout:136: done (1 left) 2024-09-14T03:48:20.022 INFO:tasks.rados.rados.0.smithi070.stdout:137: done (0 left) 2024-09-14T03:48:20.022 INFO:tasks.rados.rados.0.smithi070.stdout:138: copy_from oid 48 from oid 42 current snap is 7 2024-09-14T03:48:20.022 INFO:tasks.rados.rados.0.smithi070.stdout:139: delete oid 29 current snap is 7 2024-09-14T03:48:20.026 INFO:tasks.rados.rados.0.smithi070.stdout:139: done (1 left) 2024-09-14T03:48:20.026 INFO:tasks.rados.rados.0.smithi070.stdout:140: snap_create 2024-09-14T03:48:20.032 INFO:tasks.rados.rados.0.smithi070.stdout:138: finishing copy_from racing read to smithi07085154-48 2024-09-14T03:48:20.034 INFO:tasks.rados.rados.0.smithi070.stdout:138: finishing copy_from to smithi07085154-48 2024-09-14T03:48:20.034 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 2 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-14T03:48:20.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:20 smithi094 ceph-mon[31053]: pgmap v121: 121 pgs: 121 active+clean; 30 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 3.1 MiB/s wr, 30 op/s 2024-09-14T03:48:20.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:20 smithi094 ceph-mon[31053]: osdmap e84: 8 total, 8 up, 8 in 2024-09-14T03:48:20.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:20 smithi070 ceph-mon[27495]: pgmap v121: 121 pgs: 121 active+clean; 30 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 3.1 MiB/s wr, 30 op/s 2024-09-14T03:48:20.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:20 smithi070 ceph-mon[27495]: osdmap e84: 8 total, 8 up, 8 in 2024-09-14T03:48:20.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:20 smithi070 ceph-mon[39288]: pgmap v121: 121 pgs: 121 active+clean; 30 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 3.1 MiB/s wr, 30 op/s 2024-09-14T03:48:20.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:20 smithi070 ceph-mon[39288]: osdmap e84: 8 total, 8 up, 8 in 2024-09-14T03:48:20.493 INFO:tasks.rados.rados.0.smithi070.stdout:138: done (1 left) 2024-09-14T03:48:20.494 INFO:tasks.rados.rados.0.smithi070.stdout:140: done (0 left) 2024-09-14T03:48:20.494 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 43 current snap is 8 2024-09-14T03:48:20.494 INFO:tasks.rados.rados.0.smithi070.stdout:141: seq_num 70 ranges {712704=778240,1490944=401408} 2024-09-14T03:48:20.501 INFO:tasks.rados.rados.0.smithi070.stdout:141: writing smithi07085154-43 from 712704 to 1490944 tid 1 2024-09-14T03:48:20.504 INFO:tasks.rados.rados.0.smithi070.stdout:141: writing smithi07085154-43 from 1490944 to 1892352 tid 2 2024-09-14T03:48:20.504 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 10 current snap is 8 2024-09-14T03:48:20.504 INFO:tasks.rados.rados.0.smithi070.stdout:142: seq_num 71 ranges {155648=794624,950272=401408} 2024-09-14T03:48:20.510 INFO:tasks.rados.rados.0.smithi070.stdout:142: writing smithi07085154-10 from 155648 to 950272 tid 1 2024-09-14T03:48:20.513 INFO:tasks.rados.rados.0.smithi070.stdout:142: writing smithi07085154-10 from 950272 to 1351680 tid 2 2024-09-14T03:48:20.513 INFO:tasks.rados.rados.0.smithi070.stdout:143: snap_create 2024-09-14T03:48:20.513 INFO:tasks.rados.rados.0.smithi070.stdout:141: finishing write tid 1 to smithi07085154-43 2024-09-14T03:48:20.514 INFO:tasks.rados.rados.0.smithi070.stdout:141: finishing write tid 2 to smithi07085154-43 2024-09-14T03:48:20.514 INFO:tasks.rados.rados.0.smithi070.stdout:141: finishing write tid 3 to smithi07085154-43 2024-09-14T03:48:20.514 INFO:tasks.rados.rados.0.smithi070.stdout:141: finishing write tid 4 to smithi07085154-43 2024-09-14T03:48:20.514 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 26 (ObjNum 70 snap 8 seq_num 70) dirty exists 2024-09-14T03:48:20.514 INFO:tasks.rados.rados.0.smithi070.stdout:141: left oid 43 (ObjNum 70 snap 8 seq_num 70) 2024-09-14T03:48:20.520 INFO:tasks.rados.rados.0.smithi070.stdout:142: finishing write tid 1 to smithi07085154-10 2024-09-14T03:48:20.520 INFO:tasks.rados.rados.0.smithi070.stdout:142: finishing write tid 2 to smithi07085154-10 2024-09-14T03:48:20.520 INFO:tasks.rados.rados.0.smithi070.stdout:142: finishing write tid 3 to smithi07085154-10 2024-09-14T03:48:20.521 INFO:tasks.rados.rados.0.smithi070.stdout:142: finishing write tid 4 to smithi07085154-10 2024-09-14T03:48:20.521 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 51 (ObjNum 71 snap 8 seq_num 71) dirty exists 2024-09-14T03:48:20.521 INFO:tasks.rados.rados.0.smithi070.stdout:142: left oid 10 (ObjNum 71 snap 8 seq_num 71) 2024-09-14T03:48:21.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:21 smithi094 ceph-mon[31053]: osdmap e85: 8 total, 8 up, 8 in 2024-09-14T03:48:21.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:21 smithi094 ceph-mon[31053]: osdmap e86: 8 total, 8 up, 8 in 2024-09-14T03:48:21.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:21 smithi070 ceph-mon[27495]: osdmap e85: 8 total, 8 up, 8 in 2024-09-14T03:48:21.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:21 smithi070 ceph-mon[27495]: osdmap e86: 8 total, 8 up, 8 in 2024-09-14T03:48:21.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:21 smithi070 ceph-mon[39288]: osdmap e85: 8 total, 8 up, 8 in 2024-09-14T03:48:21.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:21 smithi070 ceph-mon[39288]: osdmap e86: 8 total, 8 up, 8 in 2024-09-14T03:48:21.497 INFO:tasks.rados.rados.0.smithi070.stdout:141: done (2 left) 2024-09-14T03:48:21.497 INFO:tasks.rados.rados.0.smithi070.stdout:142: done (1 left) 2024-09-14T03:48:21.497 INFO:tasks.rados.rados.0.smithi070.stdout:143: done (0 left) 2024-09-14T03:48:21.497 INFO:tasks.rados.rados.0.smithi070.stdout:144: snap_create 2024-09-14T03:48:21.776 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:48:21 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:48:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:48:22.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:22 smithi094 ceph-mon[31053]: pgmap v125: 121 pgs: 121 active+clean; 30 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 3.4 MiB/s wr, 34 op/s 2024-09-14T03:48:22.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:22 smithi094 ceph-mon[31053]: osdmap e87: 8 total, 8 up, 8 in 2024-09-14T03:48:22.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:22 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:48:22.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:22 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:48:22.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:22 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:48:22.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:22 smithi070 ceph-mon[27495]: pgmap v125: 121 pgs: 121 active+clean; 30 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 3.4 MiB/s wr, 34 op/s 2024-09-14T03:48:22.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:22 smithi070 ceph-mon[27495]: osdmap e87: 8 total, 8 up, 8 in 2024-09-14T03:48:22.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:22 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:48:22.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:22 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:48:22.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:22 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:48:22.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:22 smithi070 ceph-mon[39288]: pgmap v125: 121 pgs: 121 active+clean; 30 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 3.4 MiB/s wr, 34 op/s 2024-09-14T03:48:22.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:22 smithi070 ceph-mon[39288]: osdmap e87: 8 total, 8 up, 8 in 2024-09-14T03:48:22.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:22 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:48:22.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:22 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:48:22.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:22 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:48:22.500 INFO:tasks.rados.rados.0.smithi070.stdout:144: done (0 left) 2024-09-14T03:48:22.501 INFO:tasks.rados.rados.0.smithi070.stdout:145: read oid 42 snap -1 2024-09-14T03:48:22.501 INFO:tasks.rados.rados.0.smithi070.stdout:145: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-14T03:48:22.501 INFO:tasks.rados.rados.0.smithi070.stdout:146: rmattr oid 17 current snap is 10 2024-09-14T03:48:22.501 INFO:tasks.rados.rados.0.smithi070.stdout:146: done (1 left) 2024-09-14T03:48:22.501 INFO:tasks.rados.rados.0.smithi070.stdout:147: setattr oid 28 current snap is 10 2024-09-14T03:48:22.504 INFO:tasks.rados.rados.0.smithi070.stdout:148: copy_from oid 33 from oid 16 current snap is 10 2024-09-14T03:48:22.504 INFO:tasks.rados.rados.0.smithi070.stdout:149: setattr oid 32 current snap is 10 2024-09-14T03:48:22.507 INFO:tasks.rados.rados.0.smithi070.stdout:150: snap_remove snap 5 2024-09-14T03:48:23.505 INFO:tasks.rados.rados.0.smithi070.stdout:150: done (4 left) 2024-09-14T03:48:23.506 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 25 current snap is 10 2024-09-14T03:48:23.506 INFO:tasks.rados.rados.0.smithi070.stdout:151: seq_num 72 ranges {2334720=409600,2744320=524288,3268608=278528} 2024-09-14T03:48:23.521 INFO:tasks.rados.rados.0.smithi070.stdout:151: writing smithi07085154-25 from 2334720 to 2744320 tid 1 2024-09-14T03:48:23.524 INFO:tasks.rados.rados.0.smithi070.stdout:151: writing smithi07085154-25 from 2744320 to 3268608 tid 2 2024-09-14T03:48:23.526 INFO:tasks.rados.rados.0.smithi070.stdout:151: writing smithi07085154-25 from 3268608 to 3547136 tid 3 2024-09-14T03:48:23.526 INFO:tasks.rados.rados.0.smithi070.stdout:145: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-14T03:48:23.546 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 28 v 17 (ObjNum 57 snap 3 seq_num 57) dirty exists 2024-09-14T03:48:23.546 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 15 (ObjNum 66 snap 4 seq_num 66) dirty exists 2024-09-14T03:48:23.546 INFO:tasks.rados.rados.0.smithi070.stdout:148: finishing copy_from racing read to smithi07085154-33 2024-09-14T03:48:23.546 INFO:tasks.rados.rados.0.smithi070.stdout:148: finishing copy_from to smithi07085154-33 2024-09-14T03:48:23.546 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 33 v 8 (ObjNum 15 snap 0 seq_num 15) dirty exists 2024-09-14T03:48:23.547 INFO:tasks.rados.rados.0.smithi070.stdout:145: done (4 left) 2024-09-14T03:48:23.547 INFO:tasks.rados.rados.0.smithi070.stdout:147: done (3 left) 2024-09-14T03:48:23.547 INFO:tasks.rados.rados.0.smithi070.stdout:148: done (2 left) 2024-09-14T03:48:23.547 INFO:tasks.rados.rados.0.smithi070.stdout:149: done (1 left) 2024-09-14T03:48:23.547 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 45 current snap is 10 2024-09-14T03:48:23.547 INFO:tasks.rados.rados.0.smithi070.stdout:151: finishing write tid 1 to smithi07085154-25 2024-09-14T03:48:23.547 INFO:tasks.rados.rados.0.smithi070.stdout:152: seq_num 73 ranges {557056=425984,983040=409600,1392640=393216} 2024-09-14T03:48:23.552 INFO:tasks.rados.rados.0.smithi070.stdout:152: writing smithi07085154-45 from 557056 to 983040 tid 1 2024-09-14T03:48:23.554 INFO:tasks.rados.rados.0.smithi070.stdout:152: writing smithi07085154-45 from 983040 to 1392640 tid 2 2024-09-14T03:48:23.557 INFO:tasks.rados.rados.0.smithi070.stdout:152: writing smithi07085154-45 from 1392640 to 1785856 tid 3 2024-09-14T03:48:23.557 INFO:tasks.rados.rados.0.smithi070.stdout:153: snap_remove snap 3 2024-09-14T03:48:23.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:23 smithi094 ceph-mon[31053]: osdmap e88: 8 total, 8 up, 8 in 2024-09-14T03:48:23.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:23 smithi094 ceph-mon[31053]: pgmap v128: 121 pgs: 121 active+clean; 35 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.5 MiB/s wr, 21 op/s 2024-09-14T03:48:23.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:23 smithi070 ceph-mon[27495]: osdmap e88: 8 total, 8 up, 8 in 2024-09-14T03:48:23.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:23 smithi070 ceph-mon[27495]: pgmap v128: 121 pgs: 121 active+clean; 35 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.5 MiB/s wr, 21 op/s 2024-09-14T03:48:23.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:23 smithi070 ceph-mon[39288]: osdmap e88: 8 total, 8 up, 8 in 2024-09-14T03:48:23.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:23 smithi070 ceph-mon[39288]: pgmap v128: 121 pgs: 121 active+clean; 35 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.5 MiB/s wr, 21 op/s 2024-09-14T03:48:24.513 INFO:tasks.rados.rados.0.smithi070.stdout:153: done (2 left) 2024-09-14T03:48:24.513 INFO:tasks.rados.rados.0.smithi070.stdout:154: delete oid 22 current snap is 10 2024-09-14T03:48:24.513 INFO:tasks.rados.rados.0.smithi070.stdout:151: finishing write tid 2 to smithi07085154-25 2024-09-14T03:48:24.514 INFO:tasks.rados.rados.0.smithi070.stdout:151: finishing write tid 3 to smithi07085154-25 2024-09-14T03:48:24.514 INFO:tasks.rados.rados.0.smithi070.stdout:151: finishing write tid 4 to smithi07085154-25 2024-09-14T03:48:24.514 INFO:tasks.rados.rados.0.smithi070.stdout:151: finishing write tid 5 to smithi07085154-25 2024-09-14T03:48:24.514 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 26 (ObjNum 72 snap 10 seq_num 72) dirty exists 2024-09-14T03:48:24.514 INFO:tasks.rados.rados.0.smithi070.stdout:151: left oid 25 (ObjNum 72 snap 10 seq_num 72) 2024-09-14T03:48:24.514 INFO:tasks.rados.rados.0.smithi070.stdout:152: finishing write tid 1 to smithi07085154-45 2024-09-14T03:48:24.514 INFO:tasks.rados.rados.0.smithi070.stdout:152: finishing write tid 2 to smithi07085154-45 2024-09-14T03:48:24.514 INFO:tasks.rados.rados.0.smithi070.stdout:152: finishing write tid 3 to smithi07085154-45 2024-09-14T03:48:24.514 INFO:tasks.rados.rados.0.smithi070.stdout:152: finishing write tid 4 to smithi07085154-45 2024-09-14T03:48:24.514 INFO:tasks.rados.rados.0.smithi070.stdout:152: finishing write tid 5 to smithi07085154-45 2024-09-14T03:48:24.514 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 45 v 28 (ObjNum 73 snap 10 seq_num 73) dirty exists 2024-09-14T03:48:24.514 INFO:tasks.rados.rados.0.smithi070.stdout:152: left oid 45 (ObjNum 73 snap 10 seq_num 73) 2024-09-14T03:48:24.517 INFO:tasks.rados.rados.0.smithi070.stdout:151: done (2 left) 2024-09-14T03:48:24.517 INFO:tasks.rados.rados.0.smithi070.stdout:152: done (1 left) 2024-09-14T03:48:24.517 INFO:tasks.rados.rados.0.smithi070.stdout:154: done (0 left) 2024-09-14T03:48:24.518 INFO:tasks.rados.rados.0.smithi070.stdout:155: snap_create 2024-09-14T03:48:24.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:24 smithi094 ceph-mon[31053]: osdmap e89: 8 total, 8 up, 8 in 2024-09-14T03:48:24.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:24 smithi070 ceph-mon[27495]: osdmap e89: 8 total, 8 up, 8 in 2024-09-14T03:48:24.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:24 smithi070 ceph-mon[39288]: osdmap e89: 8 total, 8 up, 8 in 2024-09-14T03:48:25.519 INFO:tasks.rados.rados.0.smithi070.stdout:155: done (0 left) 2024-09-14T03:48:25.519 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 31 current snap is 11 2024-09-14T03:48:25.519 INFO:tasks.rados.rados.0.smithi070.stdout:156: seq_num 75 ranges {507904=458752,966656=581632,1548288=221184} 2024-09-14T03:48:25.524 INFO:tasks.rados.rados.0.smithi070.stdout:156: writing smithi07085154-31 from 507904 to 966656 tid 1 2024-09-14T03:48:25.528 INFO:tasks.rados.rados.0.smithi070.stdout:156: writing smithi07085154-31 from 966656 to 1548288 tid 2 2024-09-14T03:48:25.530 INFO:tasks.rados.rados.0.smithi070.stdout:156: writing smithi07085154-31 from 1548288 to 1769472 tid 3 2024-09-14T03:48:25.530 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 16 current snap is 11 2024-09-14T03:48:25.530 INFO:tasks.rados.rados.0.smithi070.stdout:157: seq_num 76 ranges {253952=475136,729088=458752,1187840=344064} 2024-09-14T03:48:25.534 INFO:tasks.rados.rados.0.smithi070.stdout:157: writing smithi07085154-16 from 253952 to 729088 tid 1 2024-09-14T03:48:25.538 INFO:tasks.rados.rados.0.smithi070.stdout:157: writing smithi07085154-16 from 729088 to 1187840 tid 2 2024-09-14T03:48:25.541 INFO:tasks.rados.rados.0.smithi070.stdout:157: writing smithi07085154-16 from 1187840 to 1531904 tid 3 2024-09-14T03:48:25.541 INFO:tasks.rados.rados.0.smithi070.stdout:158: copy_from oid 10 from oid 39 current snap is 11 2024-09-14T03:48:25.541 INFO:tasks.rados.rados.0.smithi070.stdout:156: finishing write tid 1 to smithi07085154-31 2024-09-14T03:48:25.541 INFO:tasks.rados.rados.0.smithi070.stdout:159: read oid 48 snap 7 2024-09-14T03:48:25.541 INFO:tasks.rados.rados.0.smithi070.stdout:159: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-14T03:48:25.541 INFO:tasks.rados.rados.0.smithi070.stdout:160: copy_from oid 50 from oid 1 current snap is 11 2024-09-14T03:48:25.542 INFO:tasks.rados.rados.0.smithi070.stdout:156: finishing write tid 2 to smithi07085154-31 2024-09-14T03:48:25.542 INFO:tasks.rados.rados.0.smithi070.stdout:156: finishing write tid 3 to smithi07085154-31 2024-09-14T03:48:25.542 INFO:tasks.rados.rados.0.smithi070.stdout:156: finishing write tid 4 to smithi07085154-31 2024-09-14T03:48:25.542 INFO:tasks.rados.rados.0.smithi070.stdout:156: finishing write tid 5 to smithi07085154-31 2024-09-14T03:48:25.542 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 33 (ObjNum 75 snap 11 seq_num 75) dirty exists 2024-09-14T03:48:25.542 INFO:tasks.rados.rados.0.smithi070.stdout:156: left oid 31 (ObjNum 75 snap 11 seq_num 75) 2024-09-14T03:48:25.542 INFO:tasks.rados.rados.0.smithi070.stdout:157: finishing write tid 1 to smithi07085154-16 2024-09-14T03:48:25.542 INFO:tasks.rados.rados.0.smithi070.stdout:157: finishing write tid 2 to smithi07085154-16 2024-09-14T03:48:25.542 INFO:tasks.rados.rados.0.smithi070.stdout:156: done (4 left) 2024-09-14T03:48:25.542 INFO:tasks.rados.rados.0.smithi070.stdout:161: snap_remove snap 6 2024-09-14T03:48:25.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:25 smithi094 ceph-mon[31053]: osdmap e90: 8 total, 8 up, 8 in 2024-09-14T03:48:25.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:25 smithi094 ceph-mon[31053]: pgmap v131: 121 pgs: 121 active+clean; 35 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.3 MiB/s wr, 18 op/s 2024-09-14T03:48:25.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:25 smithi070 ceph-mon[27495]: osdmap e90: 8 total, 8 up, 8 in 2024-09-14T03:48:25.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:25 smithi070 ceph-mon[27495]: pgmap v131: 121 pgs: 121 active+clean; 35 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.3 MiB/s wr, 18 op/s 2024-09-14T03:48:25.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:25 smithi070 ceph-mon[39288]: osdmap e90: 8 total, 8 up, 8 in 2024-09-14T03:48:25.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:25 smithi070 ceph-mon[39288]: pgmap v131: 121 pgs: 121 active+clean; 35 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.3 MiB/s wr, 18 op/s 2024-09-14T03:48:26.523 INFO:tasks.rados.rados.0.smithi070.stdout:161: done (4 left) 2024-09-14T03:48:26.523 INFO:tasks.rados.rados.0.smithi070.stdout:162: delete oid 8 current snap is 11 2024-09-14T03:48:26.523 INFO:tasks.rados.rados.0.smithi070.stdout:160: finishing copy_from to smithi07085154-50 2024-09-14T03:48:26.524 INFO:tasks.rados.rados.0.smithi070.stdout:160: got expected ENOENT (src dne) 2024-09-14T03:48:26.524 INFO:tasks.rados.rados.0.smithi070.stdout:160: finishing copy_from racing read to smithi07085154-50 2024-09-14T03:48:26.524 INFO:tasks.rados.rados.0.smithi070.stdout:157: finishing write tid 3 to smithi07085154-16 2024-09-14T03:48:26.524 INFO:tasks.rados.rados.0.smithi070.stdout:157: finishing write tid 4 to smithi07085154-16 2024-09-14T03:48:26.524 INFO:tasks.rados.rados.0.smithi070.stdout:157: finishing write tid 5 to smithi07085154-16 2024-09-14T03:48:26.524 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 62 (ObjNum 76 snap 11 seq_num 76) dirty exists 2024-09-14T03:48:26.524 INFO:tasks.rados.rados.0.smithi070.stdout:157: left oid 16 (ObjNum 76 snap 11 seq_num 76) 2024-09-14T03:48:26.524 INFO:tasks.rados.rados.0.smithi070.stdout:159: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-14T03:48:26.546 INFO:tasks.rados.rados.0.smithi070.stdout:158: finishing copy_from racing read to smithi07085154-10 2024-09-14T03:48:26.546 INFO:tasks.rados.rados.0.smithi070.stdout:158: finishing copy_from to smithi07085154-10 2024-09-14T03:48:26.546 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 2 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-14T03:48:26.546 INFO:tasks.rados.rados.0.smithi070.stdout:157: done (4 left) 2024-09-14T03:48:26.547 INFO:tasks.rados.rados.0.smithi070.stdout:158: done (3 left) 2024-09-14T03:48:26.547 INFO:tasks.rados.rados.0.smithi070.stdout:159: done (2 left) 2024-09-14T03:48:26.547 INFO:tasks.rados.rados.0.smithi070.stdout:160: done (1 left) 2024-09-14T03:48:26.547 INFO:tasks.rados.rados.0.smithi070.stdout:162: done (0 left) 2024-09-14T03:48:26.547 INFO:tasks.rados.rados.0.smithi070.stdout:163: snap_remove snap 9 2024-09-14T03:48:26.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:26 smithi094 ceph-mon[31053]: osdmap e91: 8 total, 8 up, 8 in 2024-09-14T03:48:26.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:48:26 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:48:26] "GET /metrics HTTP/1.1" 200 239822 "" "Prometheus/2.43.0" 2024-09-14T03:48:26.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:26 smithi070 ceph-mon[27495]: osdmap e91: 8 total, 8 up, 8 in 2024-09-14T03:48:26.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:26 smithi070 ceph-mon[39288]: osdmap e91: 8 total, 8 up, 8 in 2024-09-14T03:48:27.530 INFO:tasks.rados.rados.0.smithi070.stdout:163: done (0 left) 2024-09-14T03:48:27.530 INFO:tasks.rados.rados.0.smithi070.stdout:164: snap_create 2024-09-14T03:48:27.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:27 smithi094 ceph-mon[31053]: osdmap e92: 8 total, 8 up, 8 in 2024-09-14T03:48:27.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:27 smithi094 ceph-mon[31053]: pgmap v134: 121 pgs: 121 active+clean; 35 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:48:27.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:27 smithi070 ceph-mon[27495]: osdmap e92: 8 total, 8 up, 8 in 2024-09-14T03:48:27.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:27 smithi070 ceph-mon[27495]: pgmap v134: 121 pgs: 121 active+clean; 35 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:48:27.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:27 smithi070 ceph-mon[39288]: osdmap e92: 8 total, 8 up, 8 in 2024-09-14T03:48:27.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:27 smithi070 ceph-mon[39288]: pgmap v134: 121 pgs: 121 active+clean; 35 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:48:28.532 INFO:tasks.rados.rados.0.smithi070.stdout:164: done (0 left) 2024-09-14T03:48:28.532 INFO:tasks.rados.rados.0.smithi070.stdout:165: read oid 46 snap -1 2024-09-14T03:48:28.532 INFO:tasks.rados.rados.0.smithi070.stdout:165: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-14T03:48:28.532 INFO:tasks.rados.rados.0.smithi070.stdout:166: read oid 7 snap 8 2024-09-14T03:48:28.533 INFO:tasks.rados.rados.0.smithi070.stdout:166: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-14T03:48:28.533 INFO:tasks.rados.rados.0.smithi070.stdout:167: snap_create 2024-09-14T03:48:28.536 INFO:tasks.rados.rados.0.smithi070.stdout:166: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-14T03:48:28.540 INFO:tasks.rados.rados.0.smithi070.stdout:165: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-14T03:48:28.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:28 smithi094 ceph-mon[31053]: osdmap e93: 8 total, 8 up, 8 in 2024-09-14T03:48:28.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:28 smithi070 ceph-mon[27495]: osdmap e93: 8 total, 8 up, 8 in 2024-09-14T03:48:28.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:28 smithi070 ceph-mon[39288]: osdmap e93: 8 total, 8 up, 8 in 2024-09-14T03:48:29.537 INFO:tasks.rados.rados.0.smithi070.stdout:165: done (2 left) 2024-09-14T03:48:29.538 INFO:tasks.rados.rados.0.smithi070.stdout:166: done (1 left) 2024-09-14T03:48:29.538 INFO:tasks.rados.rados.0.smithi070.stdout:167: done (0 left) 2024-09-14T03:48:29.538 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 13 current snap is 13 2024-09-14T03:48:29.538 INFO:tasks.rados.rados.0.smithi070.stdout:168: seq_num 78 ranges {204800=516096,720896=679936,1400832=122880} 2024-09-14T03:48:29.541 INFO:tasks.rados.rados.0.smithi070.stdout:168: writing smithi07085154-13 from 204800 to 720896 tid 1 2024-09-14T03:48:29.546 INFO:tasks.rados.rados.0.smithi070.stdout:168: writing smithi07085154-13 from 720896 to 1400832 tid 2 2024-09-14T03:48:29.547 INFO:tasks.rados.rados.0.smithi070.stdout:168: writing smithi07085154-13 from 1400832 to 1523712 tid 3 2024-09-14T03:48:29.548 INFO:tasks.rados.rados.0.smithi070.stdout:169: copy_from oid 15 from oid 5 current snap is 13 2024-09-14T03:48:29.548 INFO:tasks.rados.rados.0.smithi070.stdout:170: rollback oid 19 current snap is 13 2024-09-14T03:48:29.548 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 19 to 7 2024-09-14T03:48:29.548 INFO:tasks.rados.rados.0.smithi070.stdout:171: copy_from oid 21 from oid 24 current snap is 13 2024-09-14T03:48:29.548 INFO:tasks.rados.rados.0.smithi070.stdout:172: copy_from oid 3 from oid 7 current snap is 13 2024-09-14T03:48:29.548 INFO:tasks.rados.rados.0.smithi070.stdout:173: read oid 33 snap -1 2024-09-14T03:48:29.548 INFO:tasks.rados.rados.0.smithi070.stdout:173: expect (ObjNum 15 snap 0 seq_num 15) 2024-09-14T03:48:29.548 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 16 current snap is 13 2024-09-14T03:48:29.548 INFO:tasks.rados.rados.0.smithi070.stdout:174: seq_num 79 ranges {1531904=532480,2064384=565248,2629632=237568} 2024-09-14T03:48:29.563 INFO:tasks.rados.rados.0.smithi070.stdout:174: writing smithi07085154-16 from 1531904 to 2064384 tid 1 2024-09-14T03:48:29.566 INFO:tasks.rados.rados.0.smithi070.stdout:174: writing smithi07085154-16 from 2064384 to 2629632 tid 2 2024-09-14T03:48:29.568 INFO:tasks.rados.rados.0.smithi070.stdout:174: writing smithi07085154-16 from 2629632 to 2867200 tid 3 2024-09-14T03:48:29.568 INFO:tasks.rados.rados.0.smithi070.stdout:175: rmattr oid 34 current snap is 13 2024-09-14T03:48:29.569 INFO:tasks.rados.rados.0.smithi070.stdout:175: done (7 left) 2024-09-14T03:48:29.569 INFO:tasks.rados.rados.0.smithi070.stdout:176: snap_remove snap 11 2024-09-14T03:48:29.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:29 smithi070 ceph-mon[27495]: osdmap e94: 8 total, 8 up, 8 in 2024-09-14T03:48:29.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:29 smithi070 ceph-mon[27495]: pgmap v137: 121 pgs: 121 active+clean; 39 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 469 KiB/s rd, 1.3 MiB/s wr, 7 op/s 2024-09-14T03:48:29.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:29 smithi070 ceph-mon[39288]: osdmap e94: 8 total, 8 up, 8 in 2024-09-14T03:48:29.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:29 smithi070 ceph-mon[39288]: pgmap v137: 121 pgs: 121 active+clean; 39 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 469 KiB/s rd, 1.3 MiB/s wr, 7 op/s 2024-09-14T03:48:30.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:29 smithi094 ceph-mon[31053]: osdmap e94: 8 total, 8 up, 8 in 2024-09-14T03:48:30.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:29 smithi094 ceph-mon[31053]: pgmap v137: 121 pgs: 121 active+clean; 39 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 469 KiB/s rd, 1.3 MiB/s wr, 7 op/s 2024-09-14T03:48:30.498 INFO:tasks.rados.rados.0.smithi070.stdout:176: done (7 left) 2024-09-14T03:48:30.498 INFO:tasks.rados.rados.0.smithi070.stdout:177: read oid 39 snap -1 2024-09-14T03:48:30.498 INFO:tasks.rados.rados.0.smithi070.stdout:177: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-14T03:48:30.498 INFO:tasks.rados.rados.0.smithi070.stdout:178: rmattr oid 4 current snap is 13 2024-09-14T03:48:30.498 INFO:tasks.rados.rados.0.smithi070.stdout:178: done (8 left) 2024-09-14T03:48:30.498 INFO:tasks.rados.rados.0.smithi070.stdout:179: snap_create 2024-09-14T03:48:30.499 INFO:tasks.rados.rados.0.smithi070.stdout:168: finishing write tid 1 to smithi07085154-13 2024-09-14T03:48:30.499 INFO:tasks.rados.rados.0.smithi070.stdout:168: finishing write tid 2 to smithi07085154-13 2024-09-14T03:48:30.499 INFO:tasks.rados.rados.0.smithi070.stdout:168: finishing write tid 3 to smithi07085154-13 2024-09-14T03:48:30.499 INFO:tasks.rados.rados.0.smithi070.stdout:168: finishing write tid 4 to smithi07085154-13 2024-09-14T03:48:30.499 INFO:tasks.rados.rados.0.smithi070.stdout:170: finishing rollback tid 0 to smithi07085154-19 2024-09-14T03:48:30.499 INFO:tasks.rados.rados.0.smithi070.stdout:170: finishing rollback tid 1 to smithi07085154-19 2024-09-14T03:48:30.499 INFO:tasks.rados.rados.0.smithi070.stdout:170: finishing rollback tid 2 to smithi07085154-19 2024-09-14T03:48:30.499 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 19 v 54 (ObjNum 18 snap 0 seq_num 18) dirty exists 2024-09-14T03:48:30.499 INFO:tasks.rados.rados.0.smithi070.stdout:173: expect (ObjNum 15 snap 0 seq_num 15) 2024-09-14T03:48:30.506 INFO:tasks.rados.rados.0.smithi070.stdout:171: finishing copy_from to smithi07085154-21 2024-09-14T03:48:30.506 INFO:tasks.rados.rados.0.smithi070.stdout:171: got expected ENOENT (src dne) 2024-09-14T03:48:30.506 INFO:tasks.rados.rados.0.smithi070.stdout:171: finishing copy_from racing read to smithi07085154-21 2024-09-14T03:48:30.506 INFO:tasks.rados.rados.0.smithi070.stdout:168: finishing write tid 5 to smithi07085154-13 2024-09-14T03:48:30.506 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 28 (ObjNum 78 snap 13 seq_num 78) dirty exists 2024-09-14T03:48:30.506 INFO:tasks.rados.rados.0.smithi070.stdout:168: left oid 13 (ObjNum 78 snap 13 seq_num 78) 2024-09-14T03:48:30.506 INFO:tasks.rados.rados.0.smithi070.stdout:172: finishing copy_from racing read to smithi07085154-3 2024-09-14T03:48:30.506 INFO:tasks.rados.rados.0.smithi070.stdout:169: finishing copy_from racing read to smithi07085154-15 2024-09-14T03:48:30.506 INFO:tasks.rados.rados.0.smithi070.stdout:169: finishing copy_from to smithi07085154-15 2024-09-14T03:48:30.507 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 15 v 2 (ObjNum 4 snap 0 seq_num 4) dirty exists 2024-09-14T03:48:30.507 INFO:tasks.rados.rados.0.smithi070.stdout:172: finishing copy_from to smithi07085154-3 2024-09-14T03:48:30.507 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 3 v 4 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-09-14T03:48:30.507 INFO:tasks.rados.rados.0.smithi070.stdout:174: finishing write tid 1 to smithi07085154-16 2024-09-14T03:48:30.507 INFO:tasks.rados.rados.0.smithi070.stdout:174: finishing write tid 2 to smithi07085154-16 2024-09-14T03:48:30.507 INFO:tasks.rados.rados.0.smithi070.stdout:174: finishing write tid 3 to smithi07085154-16 2024-09-14T03:48:30.507 INFO:tasks.rados.rados.0.smithi070.stdout:174: finishing write tid 4 to smithi07085154-16 2024-09-14T03:48:30.507 INFO:tasks.rados.rados.0.smithi070.stdout:174: finishing write tid 5 to smithi07085154-16 2024-09-14T03:48:30.507 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 77 (ObjNum 79 snap 13 seq_num 79) dirty exists 2024-09-14T03:48:30.507 INFO:tasks.rados.rados.0.smithi070.stdout:174: left oid 16 (ObjNum 79 snap 13 seq_num 79) 2024-09-14T03:48:30.508 INFO:tasks.rados.rados.0.smithi070.stdout:177: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-14T03:48:30.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:30 smithi070 ceph-mon[27495]: osdmap e95: 8 total, 8 up, 8 in 2024-09-14T03:48:30.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:30 smithi070 ceph-mon[27495]: osdmap e96: 8 total, 8 up, 8 in 2024-09-14T03:48:30.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:30 smithi070 ceph-mon[39288]: osdmap e95: 8 total, 8 up, 8 in 2024-09-14T03:48:30.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:30 smithi070 ceph-mon[39288]: osdmap e96: 8 total, 8 up, 8 in 2024-09-14T03:48:31.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:30 smithi094 ceph-mon[31053]: osdmap e95: 8 total, 8 up, 8 in 2024-09-14T03:48:31.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:30 smithi094 ceph-mon[31053]: osdmap e96: 8 total, 8 up, 8 in 2024-09-14T03:48:31.500 INFO:tasks.rados.rados.0.smithi070.stdout:168: done (8 left) 2024-09-14T03:48:31.500 INFO:tasks.rados.rados.0.smithi070.stdout:169: done (7 left) 2024-09-14T03:48:31.500 INFO:tasks.rados.rados.0.smithi070.stdout:170: done (6 left) 2024-09-14T03:48:31.500 INFO:tasks.rados.rados.0.smithi070.stdout:171: done (5 left) 2024-09-14T03:48:31.500 INFO:tasks.rados.rados.0.smithi070.stdout:172: done (4 left) 2024-09-14T03:48:31.500 INFO:tasks.rados.rados.0.smithi070.stdout:173: done (3 left) 2024-09-14T03:48:31.500 INFO:tasks.rados.rados.0.smithi070.stdout:174: done (2 left) 2024-09-14T03:48:31.500 INFO:tasks.rados.rados.0.smithi070.stdout:177: done (1 left) 2024-09-14T03:48:31.501 INFO:tasks.rados.rados.0.smithi070.stdout:179: done (0 left) 2024-09-14T03:48:31.501 INFO:tasks.rados.rados.0.smithi070.stdout:180: copy_from oid 17 from oid 8 current snap is 14 2024-09-14T03:48:31.501 INFO:tasks.rados.rados.0.smithi070.stdout:181: read oid 32 snap -1 2024-09-14T03:48:31.501 INFO:tasks.rados.rados.0.smithi070.stdout:181: expect (ObjNum 66 snap 4 seq_num 66) 2024-09-14T03:48:31.501 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 43 current snap is 14 2024-09-14T03:48:31.501 INFO:tasks.rados.rados.0.smithi070.stdout:182: seq_num 80 ranges {1892352=548864,2441216=450560,2891776=352256} 2024-09-14T03:48:31.512 INFO:tasks.rados.rados.0.smithi070.stdout:182: writing smithi07085154-43 from 1892352 to 2441216 tid 1 2024-09-14T03:48:31.515 INFO:tasks.rados.rados.0.smithi070.stdout:182: writing smithi07085154-43 from 2441216 to 2891776 tid 2 2024-09-14T03:48:31.518 INFO:tasks.rados.rados.0.smithi070.stdout:182: writing smithi07085154-43 from 2891776 to 3244032 tid 3 2024-09-14T03:48:31.518 INFO:tasks.rados.rados.0.smithi070.stdout:183: setattr oid 39 current snap is 14 2024-09-14T03:48:31.518 INFO:tasks.rados.rados.0.smithi070.stdout:180: finishing copy_from to smithi07085154-17 2024-09-14T03:48:31.518 INFO:tasks.rados.rados.0.smithi070.stdout:180: got expected ENOENT (src dne) 2024-09-14T03:48:31.518 INFO:tasks.rados.rados.0.smithi070.stdout:180: finishing copy_from racing read to smithi07085154-17 2024-09-14T03:48:31.518 INFO:tasks.rados.rados.0.smithi070.stdout:181: expect (ObjNum 66 snap 4 seq_num 66) 2024-09-14T03:48:31.627 INFO:tasks.rados.rados.0.smithi070.stdout:182: finishing write tid 1 to smithi07085154-43 2024-09-14T03:48:31.627 INFO:tasks.rados.rados.0.smithi070.stdout:182: finishing write tid 2 to smithi07085154-43 2024-09-14T03:48:31.627 INFO:tasks.rados.rados.0.smithi070.stdout:182: finishing write tid 3 to smithi07085154-43 2024-09-14T03:48:31.627 INFO:tasks.rados.rados.0.smithi070.stdout:182: finishing write tid 4 to smithi07085154-43 2024-09-14T03:48:31.627 INFO:tasks.rados.rados.0.smithi070.stdout:182: finishing write tid 5 to smithi07085154-43 2024-09-14T03:48:31.627 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 45 (ObjNum 80 snap 14 seq_num 80) dirty exists 2024-09-14T03:48:31.627 INFO:tasks.rados.rados.0.smithi070.stdout:182: left oid 43 (ObjNum 80 snap 14 seq_num 80) 2024-09-14T03:48:31.627 INFO:tasks.rados.rados.0.smithi070.stdout:180: done (3 left) 2024-09-14T03:48:31.627 INFO:tasks.rados.rados.0.smithi070.stdout:181: done (2 left) 2024-09-14T03:48:31.627 INFO:tasks.rados.rados.0.smithi070.stdout:182: done (1 left) 2024-09-14T03:48:31.627 INFO:tasks.rados.rados.0.smithi070.stdout:184: read oid 21 snap 12 2024-09-14T03:48:31.628 INFO:tasks.rados.rados.0.smithi070.stdout:184: expect (ObjNum 11 snap 0 seq_num 11) 2024-09-14T03:48:31.628 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 34 current snap is 14 2024-09-14T03:48:31.628 INFO:tasks.rados.rados.0.smithi070.stdout:185: seq_num 81 ranges {663552=565248,1228800=737280,1966080=65536} 2024-09-14T03:48:31.633 INFO:tasks.rados.rados.0.smithi070.stdout:185: writing smithi07085154-34 from 663552 to 1228800 tid 1 2024-09-14T03:48:31.637 INFO:tasks.rados.rados.0.smithi070.stdout:185: writing smithi07085154-34 from 1228800 to 1966080 tid 2 2024-09-14T03:48:31.638 INFO:tasks.rados.rados.0.smithi070.stdout:185: writing smithi07085154-34 from 1966080 to 2031616 tid 3 2024-09-14T03:48:31.638 INFO:tasks.rados.rados.0.smithi070.stdout:186: snap_create 2024-09-14T03:48:31.638 INFO:tasks.rados.rados.0.smithi070.stdout:184: expect (ObjNum 11 snap 0 seq_num 11) 2024-09-14T03:48:31.643 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 39 v 4 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-14T03:48:31.643 INFO:tasks.rados.rados.0.smithi070.stdout:185: finishing write tid 1 to smithi07085154-34 2024-09-14T03:48:31.644 INFO:tasks.rados.rados.0.smithi070.stdout:185: finishing write tid 2 to smithi07085154-34 2024-09-14T03:48:31.644 INFO:tasks.rados.rados.0.smithi070.stdout:185: finishing write tid 3 to smithi07085154-34 2024-09-14T03:48:31.644 INFO:tasks.rados.rados.0.smithi070.stdout:185: finishing write tid 4 to smithi07085154-34 2024-09-14T03:48:31.644 INFO:tasks.rados.rados.0.smithi070.stdout:185: finishing write tid 5 to smithi07085154-34 2024-09-14T03:48:31.644 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 61 (ObjNum 81 snap 14 seq_num 81) dirty exists 2024-09-14T03:48:31.644 INFO:tasks.rados.rados.0.smithi070.stdout:185: left oid 34 (ObjNum 81 snap 14 seq_num 81) 2024-09-14T03:48:31.731 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:31 smithi070 ceph-mon[39288]: pgmap v140: 121 pgs: 121 active+clean; 39 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 469 KiB/s rd, 1.3 MiB/s wr, 7 op/s 2024-09-14T03:48:31.732 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:31 smithi070 ceph-mon[39288]: osdmap e97: 8 total, 8 up, 8 in 2024-09-14T03:48:31.732 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:31 smithi070 ceph-mon[27495]: pgmap v140: 121 pgs: 121 active+clean; 39 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 469 KiB/s rd, 1.3 MiB/s wr, 7 op/s 2024-09-14T03:48:31.732 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:31 smithi070 ceph-mon[27495]: osdmap e97: 8 total, 8 up, 8 in 2024-09-14T03:48:32.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:31 smithi094 ceph-mon[31053]: pgmap v140: 121 pgs: 121 active+clean; 39 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 469 KiB/s rd, 1.3 MiB/s wr, 7 op/s 2024-09-14T03:48:32.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:31 smithi094 ceph-mon[31053]: osdmap e97: 8 total, 8 up, 8 in 2024-09-14T03:48:32.027 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:48:31 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:48:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:48:32.545 INFO:tasks.rados.rados.0.smithi070.stdout:183: done (3 left) 2024-09-14T03:48:32.545 INFO:tasks.rados.rados.0.smithi070.stdout:184: done (2 left) 2024-09-14T03:48:32.545 INFO:tasks.rados.rados.0.smithi070.stdout:185: done (1 left) 2024-09-14T03:48:32.545 INFO:tasks.rados.rados.0.smithi070.stdout:186: done (0 left) 2024-09-14T03:48:32.545 INFO:tasks.rados.rados.0.smithi070.stdout:187: snap_create 2024-09-14T03:48:33.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:32 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:48:33.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:32 smithi094 ceph-mon[31053]: Upgrade: Target is version 19.3.0-4907-ga2782315 (unknown) 2024-09-14T03:48:33.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:32 smithi094 ceph-mon[31053]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T03:48:33.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:32 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:48:33.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:32 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:48:33.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:32 smithi094 ceph-mon[31053]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-14T03:48:33.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:32 smithi094 ceph-mon[31053]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 on smithi094 2024-09-14T03:48:33.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:32 smithi094 ceph-mon[31053]: osdmap e98: 8 total, 8 up, 8 in 2024-09-14T03:48:33.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:32 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:48:33.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:32 smithi070 ceph-mon[27495]: Upgrade: Target is version 19.3.0-4907-ga2782315 (unknown) 2024-09-14T03:48:33.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:32 smithi070 ceph-mon[27495]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T03:48:33.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:32 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:48:33.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:32 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:48:33.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:32 smithi070 ceph-mon[27495]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-14T03:48:33.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:32 smithi070 ceph-mon[27495]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 on smithi094 2024-09-14T03:48:33.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:32 smithi070 ceph-mon[27495]: osdmap e98: 8 total, 8 up, 8 in 2024-09-14T03:48:33.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:32 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:48:33.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:32 smithi070 ceph-mon[39288]: Upgrade: Target is version 19.3.0-4907-ga2782315 (unknown) 2024-09-14T03:48:33.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:32 smithi070 ceph-mon[39288]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T03:48:33.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:32 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:48:33.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:32 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:48:33.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:32 smithi070 ceph-mon[39288]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-14T03:48:33.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:32 smithi070 ceph-mon[39288]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 on smithi094 2024-09-14T03:48:33.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:32 smithi070 ceph-mon[39288]: osdmap e98: 8 total, 8 up, 8 in 2024-09-14T03:48:33.549 INFO:tasks.rados.rados.0.smithi070.stdout:187: done (0 left) 2024-09-14T03:48:33.550 INFO:tasks.rados.rados.0.smithi070.stdout:188: read oid 45 snap -1 2024-09-14T03:48:33.550 INFO:tasks.rados.rados.0.smithi070.stdout:188: expect (ObjNum 73 snap 10 seq_num 73) 2024-09-14T03:48:33.550 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 31 current snap is 16 2024-09-14T03:48:33.550 INFO:tasks.rados.rados.0.smithi070.stdout:189: seq_num 82 ranges {1769472=581632,2351104=622592,2973696=180224} 2024-09-14T03:48:33.562 INFO:tasks.rados.rados.0.smithi070.stdout:189: writing smithi07085154-31 from 1769472 to 2351104 tid 1 2024-09-14T03:48:33.566 INFO:tasks.rados.rados.0.smithi070.stdout:189: writing smithi07085154-31 from 2351104 to 2973696 tid 2 2024-09-14T03:48:33.568 INFO:tasks.rados.rados.0.smithi070.stdout:189: writing smithi07085154-31 from 2973696 to 3153920 tid 3 2024-09-14T03:48:33.568 INFO:tasks.rados.rados.0.smithi070.stdout:190: read oid 49 snap -1 2024-09-14T03:48:33.568 INFO:tasks.rados.rados.0.smithi070.stdout:190: expect (ObjNum 48 snap 0 seq_num 48) 2024-09-14T03:48:33.568 INFO:tasks.rados.rados.0.smithi070.stdout:191: read oid 38 snap -1 2024-09-14T03:48:33.568 INFO:tasks.rados.rados.0.smithi070.stdout:191: expect deleted 2024-09-14T03:48:33.569 INFO:tasks.rados.rados.0.smithi070.stdout:192: read oid 40 snap -1 2024-09-14T03:48:33.569 INFO:tasks.rados.rados.0.smithi070.stdout:192: expect (ObjNum 39 snap 0 seq_num 39) 2024-09-14T03:48:33.569 INFO:tasks.rados.rados.0.smithi070.stdout:193: snap_create 2024-09-14T03:48:33.569 INFO:tasks.rados.rados.0.smithi070.stdout:188: expect (ObjNum 73 snap 10 seq_num 73) 2024-09-14T03:48:33.628 INFO:tasks.rados.rados.0.smithi070.stdout:189: finishing write tid 1 to smithi07085154-31 2024-09-14T03:48:33.628 INFO:tasks.rados.rados.0.smithi070.stdout:189: finishing write tid 2 to smithi07085154-31 2024-09-14T03:48:33.628 INFO:tasks.rados.rados.0.smithi070.stdout:189: finishing write tid 3 to smithi07085154-31 2024-09-14T03:48:33.629 INFO:tasks.rados.rados.0.smithi070.stdout:189: finishing write tid 4 to smithi07085154-31 2024-09-14T03:48:33.629 INFO:tasks.rados.rados.0.smithi070.stdout:192: expect (ObjNum 39 snap 0 seq_num 39) 2024-09-14T03:48:33.647 INFO:tasks.rados.rados.0.smithi070.stdout:189: finishing write tid 5 to smithi07085154-31 2024-09-14T03:48:33.648 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 46 (ObjNum 82 snap 16 seq_num 82) dirty exists 2024-09-14T03:48:33.648 INFO:tasks.rados.rados.0.smithi070.stdout:189: left oid 31 (ObjNum 82 snap 16 seq_num 82) 2024-09-14T03:48:33.648 INFO:tasks.rados.rados.0.smithi070.stdout:190: expect (ObjNum 48 snap 0 seq_num 48) 2024-09-14T03:48:34.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:33 smithi094 ceph-mon[31053]: pgmap v143: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.3 MiB/s wr, 17 op/s 2024-09-14T03:48:34.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:33 smithi094 ceph-mon[31053]: osdmap e99: 8 total, 8 up, 8 in 2024-09-14T03:48:34.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:33 smithi070 ceph-mon[27495]: pgmap v143: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.3 MiB/s wr, 17 op/s 2024-09-14T03:48:34.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:33 smithi070 ceph-mon[27495]: osdmap e99: 8 total, 8 up, 8 in 2024-09-14T03:48:34.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:33 smithi070 ceph-mon[39288]: pgmap v143: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.3 MiB/s wr, 17 op/s 2024-09-14T03:48:34.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:33 smithi070 ceph-mon[39288]: osdmap e99: 8 total, 8 up, 8 in 2024-09-14T03:48:34.551 INFO:tasks.rados.rados.0.smithi070.stdout:188: done (5 left) 2024-09-14T03:48:34.551 INFO:tasks.rados.rados.0.smithi070.stdout:189: done (4 left) 2024-09-14T03:48:34.551 INFO:tasks.rados.rados.0.smithi070.stdout:190: done (3 left) 2024-09-14T03:48:34.551 INFO:tasks.rados.rados.0.smithi070.stdout:191: done (2 left) 2024-09-14T03:48:34.551 INFO:tasks.rados.rados.0.smithi070.stdout:192: done (1 left) 2024-09-14T03:48:34.551 INFO:tasks.rados.rados.0.smithi070.stdout:193: done (0 left) 2024-09-14T03:48:34.551 INFO:tasks.rados.rados.0.smithi070.stdout:194: snap_remove snap 4 2024-09-14T03:48:35.500 INFO:tasks.rados.rados.0.smithi070.stdout:194: done (0 left) 2024-09-14T03:48:35.500 INFO:tasks.rados.rados.0.smithi070.stdout:195: read oid 36 snap -1 2024-09-14T03:48:35.500 INFO:tasks.rados.rados.0.smithi070.stdout:195: expect (ObjNum 55 snap 3 seq_num 55) 2024-09-14T03:48:35.500 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 1 current snap is 17 2024-09-14T03:48:35.500 INFO:tasks.rados.rados.0.smithi070.stdout:196: seq_num 83 ranges {0=598016,598016=499712,1097728=303104} 2024-09-14T03:48:35.504 INFO:tasks.rados.rados.0.smithi070.stdout:196: writing smithi07085154-1 from 0 to 598016 tid 1 2024-09-14T03:48:35.507 INFO:tasks.rados.rados.0.smithi070.stdout:196: writing smithi07085154-1 from 598016 to 1097728 tid 2 2024-09-14T03:48:35.510 INFO:tasks.rados.rados.0.smithi070.stdout:196: writing smithi07085154-1 from 1097728 to 1400832 tid 3 2024-09-14T03:48:35.510 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 9 current snap is 17 2024-09-14T03:48:35.510 INFO:tasks.rados.rados.0.smithi070.stdout:197: seq_num 84 ranges {1269760=614400,1884160=434176,2318336=368640} 2024-09-14T03:48:35.521 INFO:tasks.rados.rados.0.smithi070.stdout:197: writing smithi07085154-9 from 1269760 to 1884160 tid 1 2024-09-14T03:48:35.523 INFO:tasks.rados.rados.0.smithi070.stdout:197: writing smithi07085154-9 from 1884160 to 2318336 tid 2 2024-09-14T03:48:35.526 INFO:tasks.rados.rados.0.smithi070.stdout:197: writing smithi07085154-9 from 2318336 to 2686976 tid 3 2024-09-14T03:48:35.526 INFO:tasks.rados.rados.0.smithi070.stdout:198: rmattr oid 11 current snap is 17 2024-09-14T03:48:35.526 INFO:tasks.rados.rados.0.smithi070.stdout:198: done (3 left) 2024-09-14T03:48:35.526 INFO:tasks.rados.rados.0.smithi070.stdout:199: snap_remove snap 12 2024-09-14T03:48:36.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:35 smithi094 ceph-mon[31053]: osdmap e100: 8 total, 8 up, 8 in 2024-09-14T03:48:36.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:35 smithi094 ceph-mon[31053]: pgmap v146: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.3 MiB/s wr, 17 op/s 2024-09-14T03:48:36.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:35 smithi094 ceph-mon[31053]: osdmap e101: 8 total, 8 up, 8 in 2024-09-14T03:48:36.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:35 smithi070 ceph-mon[27495]: osdmap e100: 8 total, 8 up, 8 in 2024-09-14T03:48:36.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:35 smithi070 ceph-mon[27495]: pgmap v146: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.3 MiB/s wr, 17 op/s 2024-09-14T03:48:36.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:35 smithi070 ceph-mon[27495]: osdmap e101: 8 total, 8 up, 8 in 2024-09-14T03:48:36.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:35 smithi070 ceph-mon[39288]: osdmap e100: 8 total, 8 up, 8 in 2024-09-14T03:48:36.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:35 smithi070 ceph-mon[39288]: pgmap v146: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.3 MiB/s wr, 17 op/s 2024-09-14T03:48:36.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:35 smithi070 ceph-mon[39288]: osdmap e101: 8 total, 8 up, 8 in 2024-09-14T03:48:36.500 INFO:tasks.rados.rados.0.smithi070.stdout:199: done (3 left) 2024-09-14T03:48:36.501 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 2 current snap is 17 2024-09-14T03:48:36.501 INFO:tasks.rados.rados.0.smithi070.stdout:200: seq_num 85 ranges {1843200=630784,2473984=720896,3194880=81920} 2024-09-14T03:48:36.514 INFO:tasks.rados.rados.0.smithi070.stdout:200: writing smithi07085154-2 from 1843200 to 2473984 tid 1 2024-09-14T03:48:36.519 INFO:tasks.rados.rados.0.smithi070.stdout:200: writing smithi07085154-2 from 2473984 to 3194880 tid 2 2024-09-14T03:48:36.519 INFO:tasks.rados.rados.0.smithi070.stdout:200: writing smithi07085154-2 from 3194880 to 3276800 tid 3 2024-09-14T03:48:36.519 INFO:tasks.rados.rados.0.smithi070.stdout:201: copy_from oid 38 from oid 46 current snap is 17 2024-09-14T03:48:36.519 INFO:tasks.rados.rados.0.smithi070.stdout:195: expect (ObjNum 55 snap 3 seq_num 55) 2024-09-14T03:48:36.564 INFO:tasks.rados.rados.0.smithi070.stdout:196: finishing write tid 1 to smithi07085154-1 2024-09-14T03:48:36.564 INFO:tasks.rados.rados.0.smithi070.stdout:196: finishing write tid 2 to smithi07085154-1 2024-09-14T03:48:36.564 INFO:tasks.rados.rados.0.smithi070.stdout:196: finishing write tid 3 to smithi07085154-1 2024-09-14T03:48:36.564 INFO:tasks.rados.rados.0.smithi070.stdout:196: finishing write tid 4 to smithi07085154-1 2024-09-14T03:48:36.564 INFO:tasks.rados.rados.0.smithi070.stdout:196: finishing write tid 5 to smithi07085154-1 2024-09-14T03:48:36.564 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 1 v 28 (ObjNum 83 snap 17 seq_num 83) dirty exists 2024-09-14T03:48:36.565 INFO:tasks.rados.rados.0.smithi070.stdout:196: left oid 1 (ObjNum 83 snap 17 seq_num 83) 2024-09-14T03:48:36.565 INFO:tasks.rados.rados.0.smithi070.stdout:197: finishing write tid 1 to smithi07085154-9 2024-09-14T03:48:36.565 INFO:tasks.rados.rados.0.smithi070.stdout:197: finishing write tid 2 to smithi07085154-9 2024-09-14T03:48:36.565 INFO:tasks.rados.rados.0.smithi070.stdout:197: finishing write tid 3 to smithi07085154-9 2024-09-14T03:48:36.565 INFO:tasks.rados.rados.0.smithi070.stdout:197: finishing write tid 4 to smithi07085154-9 2024-09-14T03:48:36.565 INFO:tasks.rados.rados.0.smithi070.stdout:197: finishing write tid 5 to smithi07085154-9 2024-09-14T03:48:36.565 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 90 (ObjNum 84 snap 17 seq_num 84) dirty exists 2024-09-14T03:48:36.565 INFO:tasks.rados.rados.0.smithi070.stdout:197: left oid 9 (ObjNum 84 snap 17 seq_num 84) 2024-09-14T03:48:36.565 INFO:tasks.rados.rados.0.smithi070.stdout:200: finishing write tid 1 to smithi07085154-2 2024-09-14T03:48:36.565 INFO:tasks.rados.rados.0.smithi070.stdout:200: finishing write tid 2 to smithi07085154-2 2024-09-14T03:48:36.565 INFO:tasks.rados.rados.0.smithi070.stdout:200: finishing write tid 3 to smithi07085154-2 2024-09-14T03:48:36.565 INFO:tasks.rados.rados.0.smithi070.stdout:200: finishing write tid 4 to smithi07085154-2 2024-09-14T03:48:36.566 INFO:tasks.rados.rados.0.smithi070.stdout:200: finishing write tid 5 to smithi07085154-2 2024-09-14T03:48:36.566 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 2 v 99 (ObjNum 85 snap 17 seq_num 85) dirty exists 2024-09-14T03:48:36.566 INFO:tasks.rados.rados.0.smithi070.stdout:200: left oid 2 (ObjNum 85 snap 17 seq_num 85) 2024-09-14T03:48:36.566 INFO:tasks.rados.rados.0.smithi070.stdout:201: finishing copy_from racing read to smithi07085154-38 2024-09-14T03:48:36.566 INFO:tasks.rados.rados.0.smithi070.stdout:201: finishing copy_from to smithi07085154-38 2024-09-14T03:48:36.566 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 38 v 16 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-14T03:48:36.566 INFO:tasks.rados.rados.0.smithi070.stdout:195: done (4 left) 2024-09-14T03:48:36.566 INFO:tasks.rados.rados.0.smithi070.stdout:196: done (3 left) 2024-09-14T03:48:36.566 INFO:tasks.rados.rados.0.smithi070.stdout:197: done (2 left) 2024-09-14T03:48:36.566 INFO:tasks.rados.rados.0.smithi070.stdout:200: done (1 left) 2024-09-14T03:48:36.566 INFO:tasks.rados.rados.0.smithi070.stdout:201: done (0 left) 2024-09-14T03:48:36.566 INFO:tasks.rados.rados.0.smithi070.stdout:202: snap_remove snap 15 2024-09-14T03:48:36.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:48:36 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:48:36] "GET /metrics HTTP/1.1" 200 239822 "" "Prometheus/2.43.0" 2024-09-14T03:48:36.990 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-09-14T03:48:37.494 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:48:37.494 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (2m) 116s ago 3m 15.1M - 0.25.0 c8568f914cd2 fef584d3d5c6 2024-09-14T03:48:37.494 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi094 *:3000 running (3m) 118s ago 3m 85.0M - 9.4.7 954c08fa6188 846770f48257 2024-09-14T03:48:37.494 INFO:teuthology.orchestra.run.smithi070.stdout:mds.a smithi070 running (4m) 116s ago 4m 16.7M - 17.2.7-1629-geb16b97b ab51d1bebd12 52e0dbb7be50 2024-09-14T03:48:37.494 INFO:teuthology.orchestra.run.smithi070.stdout:mds.b smithi094 running (4m) 118s ago 4m 13.6M - 17.2.7-1629-geb16b97b ab51d1bebd12 e3b5e3e00189 2024-09-14T03:48:37.495 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi094 *:8443 running (8m) 118s ago 8m 399M - 17.2.7-1629-geb16b97b ab51d1bebd12 8a5c729a2096 2024-09-14T03:48:37.495 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:9283,8765 running (9m) 116s ago 9m 447M - 17.2.7-1629-geb16b97b ab51d1bebd12 dc71810017b9 2024-09-14T03:48:37.495 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (10m) 116s ago 10m 44.3M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 12d19db91f54 2024-09-14T03:48:37.495 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi094 running (8m) 118s ago 8m 37.9M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 90ea2e164909 2024-09-14T03:48:37.495 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (8m) 116s ago 8m 38.1M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 44ee80c29ee0 2024-09-14T03:48:37.495 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (2m) 116s ago 2m 13.3M - 1.5.0 0da6a335fe13 315c3f32399f 2024-09-14T03:48:37.495 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi094 *:9100 running (2m) 118s ago 2m 3577k - 1.5.0 0da6a335fe13 3dcf1723ea35 2024-09-14T03:48:37.495 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (7m) 116s ago 7m 44.0M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 741c06b6adbf 2024-09-14T03:48:37.495 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (7m) 116s ago 7m 44.6M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 af459ea7ef68 2024-09-14T03:48:37.495 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (7m) 116s ago 7m 42.4M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 109b66e4e60f 2024-09-14T03:48:37.495 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (6m) 116s ago 6m 43.7M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 2e4199b58392 2024-09-14T03:48:37.495 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi094 running (6m) 118s ago 6m 42.0M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 52b64527e24c 2024-09-14T03:48:37.495 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi094 running (5m) 118s ago 5m 42.4M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 b543f152ba87 2024-09-14T03:48:37.495 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi094 running (5m) 118s ago 5m 42.0M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 e66b592ac940 2024-09-14T03:48:37.496 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi094 running (5m) 118s ago 5m 44.9M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 037c6208e9ce 2024-09-14T03:48:37.496 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi094 *:9095 running (2m) 118s ago 3m 24.5M - 2.43.0 a07b618ecd1d 9c88e2923e9d 2024-09-14T03:48:37.504 INFO:tasks.rados.rados.0.smithi070.stdout:202: done (0 left) 2024-09-14T03:48:37.505 INFO:tasks.rados.rados.0.smithi070.stdout:203: rmattr oid 46 current snap is 17 2024-09-14T03:48:37.505 INFO:tasks.rados.rados.0.smithi070.stdout:203: done (0 left) 2024-09-14T03:48:37.505 INFO:tasks.rados.rados.0.smithi070.stdout:204: rmattr oid 47 current snap is 17 2024-09-14T03:48:37.505 INFO:tasks.rados.rados.0.smithi070.stdout:204: done (0 left) 2024-09-14T03:48:37.505 INFO:tasks.rados.rados.0.smithi070.stdout:205: rmattr oid 39 current snap is 17 2024-09-14T03:48:37.505 INFO:tasks.rados.rados.0.smithi070.stdout:206: read oid 5 snap -1 2024-09-14T03:48:37.505 INFO:tasks.rados.rados.0.smithi070.stdout:206: expect (ObjNum 4 snap 0 seq_num 4) 2024-09-14T03:48:37.505 INFO:tasks.rados.rados.0.smithi070.stdout:207: read oid 48 snap -1 2024-09-14T03:48:37.505 INFO:tasks.rados.rados.0.smithi070.stdout:207: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-14T03:48:37.505 INFO:tasks.rados.rados.0.smithi070.stdout:208: setattr oid 38 current snap is 17 2024-09-14T03:48:37.506 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 37 current snap is 17 2024-09-14T03:48:37.506 INFO:tasks.rados.rados.0.smithi070.stdout:209: seq_num 86 ranges {0=647168,647168=606208,1253376=196608} 2024-09-14T03:48:37.510 INFO:tasks.rados.rados.0.smithi070.stdout:209: writing smithi07085154-37 from 0 to 647168 tid 1 2024-09-14T03:48:37.514 INFO:tasks.rados.rados.0.smithi070.stdout:209: writing smithi07085154-37 from 647168 to 1253376 tid 2 2024-09-14T03:48:37.516 INFO:tasks.rados.rados.0.smithi070.stdout:209: writing smithi07085154-37 from 1253376 to 1449984 tid 3 2024-09-14T03:48:37.516 INFO:tasks.rados.rados.0.smithi070.stdout:210: copy_from oid 41 from oid 31 current snap is 17 2024-09-14T03:48:37.516 INFO:tasks.rados.rados.0.smithi070.stdout:206: expect (ObjNum 4 snap 0 seq_num 4) 2024-09-14T03:48:37.519 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 38 v 22 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-14T03:48:37.520 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 39 v 10 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-14T03:48:37.520 INFO:tasks.rados.rados.0.smithi070.stdout:207: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-14T03:48:37.550 INFO:tasks.rados.rados.0.smithi070.stdout:209: finishing write tid 1 to smithi07085154-37 2024-09-14T03:48:37.551 INFO:tasks.rados.rados.0.smithi070.stdout:205: done (5 left) 2024-09-14T03:48:37.551 INFO:tasks.rados.rados.0.smithi070.stdout:206: done (4 left) 2024-09-14T03:48:37.551 INFO:tasks.rados.rados.0.smithi070.stdout:207: done (3 left) 2024-09-14T03:48:37.551 INFO:tasks.rados.rados.0.smithi070.stdout:208: done (2 left) 2024-09-14T03:48:37.551 INFO:tasks.rados.rados.0.smithi070.stdout:211: snap_create 2024-09-14T03:48:37.551 INFO:tasks.rados.rados.0.smithi070.stdout:209: finishing write tid 2 to smithi07085154-37 2024-09-14T03:48:37.551 INFO:tasks.rados.rados.0.smithi070.stdout:209: finishing write tid 3 to smithi07085154-37 2024-09-14T03:48:37.551 INFO:tasks.rados.rados.0.smithi070.stdout:209: finishing write tid 4 to smithi07085154-37 2024-09-14T03:48:37.551 INFO:tasks.rados.rados.0.smithi070.stdout:209: finishing write tid 5 to smithi07085154-37 2024-09-14T03:48:37.551 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 37 v 59 (ObjNum 86 snap 17 seq_num 86) dirty exists 2024-09-14T03:48:37.551 INFO:tasks.rados.rados.0.smithi070.stdout:209: left oid 37 (ObjNum 86 snap 17 seq_num 86) 2024-09-14T03:48:37.552 INFO:tasks.rados.rados.0.smithi070.stdout:210: finishing copy_from racing read to smithi07085154-41 2024-09-14T03:48:37.552 INFO:tasks.rados.rados.0.smithi070.stdout:210: finishing copy_from to smithi07085154-41 2024-09-14T03:48:37.552 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 46 (ObjNum 82 snap 16 seq_num 82) dirty exists 2024-09-14T03:48:37.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:37 smithi094 ceph-mon[31053]: osdmap e102: 8 total, 8 up, 8 in 2024-09-14T03:48:37.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:37 smithi094 ceph-mon[31053]: pgmap v149: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:48:37.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:37 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:48:37.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:37 smithi094 ceph-mon[31053]: from='client.14970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:37.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:37 smithi094 ceph-mon[31053]: from='client.14976 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:37.789 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:37 smithi070 ceph-mon[27495]: osdmap e102: 8 total, 8 up, 8 in 2024-09-14T03:48:37.789 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:37 smithi070 ceph-mon[27495]: pgmap v149: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:48:37.789 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:37 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:48:37.789 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:37 smithi070 ceph-mon[27495]: from='client.14970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:37.789 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:37 smithi070 ceph-mon[27495]: from='client.14976 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:37.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:37 smithi070 ceph-mon[39288]: osdmap e102: 8 total, 8 up, 8 in 2024-09-14T03:48:37.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:37 smithi070 ceph-mon[39288]: pgmap v149: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:48:37.790 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:37 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:48:37.790 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:37 smithi070 ceph-mon[39288]: from='client.14970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:37.790 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:37 smithi070 ceph-mon[39288]: from='client.14976 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:37.817 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:48:37.817 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-09-14T03:48:37.817 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 3 2024-09-14T03:48:37.817 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:48:37.817 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-09-14T03:48:37.817 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:48:37.817 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:48:37.817 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-09-14T03:48:37.817 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 8 2024-09-14T03:48:37.818 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:48:37.818 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": { 2024-09-14T03:48:37.818 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:48:37.818 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:48:37.818 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-09-14T03:48:37.818 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 15 2024-09-14T03:48:37.818 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-09-14T03:48:37.818 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:48:38.077 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:48:38.077 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:48:38.077 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-09-14T03:48:38.077 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:48:38.077 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [], 2024-09-14T03:48:38.077 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "0/20 daemons upgraded", 2024-09-14T03:48:38.077 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 image on host smithi094", 2024-09-14T03:48:38.077 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-09-14T03:48:38.077 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:48:38.507 INFO:tasks.rados.rados.0.smithi070.stdout:209: done (2 left) 2024-09-14T03:48:38.507 INFO:tasks.rados.rados.0.smithi070.stdout:210: done (1 left) 2024-09-14T03:48:38.508 INFO:tasks.rados.rados.0.smithi070.stdout:211: done (0 left) 2024-09-14T03:48:38.508 INFO:tasks.rados.rados.0.smithi070.stdout:212: delete oid 31 current snap is 18 2024-09-14T03:48:38.509 INFO:tasks.rados.rados.0.smithi070.stdout:212: done (0 left) 2024-09-14T03:48:38.509 INFO:tasks.rados.rados.0.smithi070.stdout:213: delete oid 26 current snap is 18 2024-09-14T03:48:38.511 INFO:tasks.rados.rados.0.smithi070.stdout:213: done (0 left) 2024-09-14T03:48:38.511 INFO:tasks.rados.rados.0.smithi070.stdout:214: copy_from oid 20 from oid 42 current snap is 18 2024-09-14T03:48:38.511 INFO:tasks.rados.rados.0.smithi070.stdout:215: delete oid 5 current snap is 18 2024-09-14T03:48:38.512 INFO:tasks.rados.rados.0.smithi070.stdout:215: done (1 left) 2024-09-14T03:48:38.512 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 13 current snap is 18 2024-09-14T03:48:38.512 INFO:tasks.rados.rados.0.smithi070.stdout:216: seq_num 90 ranges {1523712=712704,2236416=540672,2777088=262144} 2024-09-14T03:48:38.524 INFO:tasks.rados.rados.0.smithi070.stdout:216: writing smithi07085154-13 from 1523712 to 2236416 tid 1 2024-09-14T03:48:38.527 INFO:tasks.rados.rados.0.smithi070.stdout:216: writing smithi07085154-13 from 2236416 to 2777088 tid 2 2024-09-14T03:48:38.530 INFO:tasks.rados.rados.0.smithi070.stdout:216: writing smithi07085154-13 from 2777088 to 3039232 tid 3 2024-09-14T03:48:38.530 INFO:tasks.rados.rados.0.smithi070.stdout:217: read oid 19 snap -1 2024-09-14T03:48:38.530 INFO:tasks.rados.rados.0.smithi070.stdout:217: expect (ObjNum 18 snap 0 seq_num 18) 2024-09-14T03:48:38.530 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 21 current snap is 18 2024-09-14T03:48:38.530 INFO:tasks.rados.rados.0.smithi070.stdout:214: finishing copy_from racing read to smithi07085154-20 2024-09-14T03:48:38.530 INFO:tasks.rados.rados.0.smithi070.stdout:218: seq_num 91 ranges {188416=729088,917504=425984,1343488=376832} 2024-09-14T03:48:38.535 INFO:tasks.rados.rados.0.smithi070.stdout:218: writing smithi07085154-21 from 188416 to 917504 tid 1 2024-09-14T03:48:38.537 INFO:tasks.rados.rados.0.smithi070.stdout:218: writing smithi07085154-21 from 917504 to 1343488 tid 2 2024-09-14T03:48:38.541 INFO:tasks.rados.rados.0.smithi070.stdout:218: writing smithi07085154-21 from 1343488 to 1720320 tid 3 2024-09-14T03:48:38.541 INFO:tasks.rados.rados.0.smithi070.stdout:219: delete oid 22 current snap is 18 2024-09-14T03:48:38.541 INFO:tasks.rados.rados.0.smithi070.stdout:214: finishing copy_from to smithi07085154-20 2024-09-14T03:48:38.541 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 20 v 2 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-14T03:48:38.541 INFO:tasks.rados.rados.0.smithi070.stdout:216: finishing write tid 1 to smithi07085154-13 2024-09-14T03:48:38.541 INFO:tasks.rados.rados.0.smithi070.stdout:216: finishing write tid 2 to smithi07085154-13 2024-09-14T03:48:38.541 INFO:tasks.rados.rados.0.smithi070.stdout:216: finishing write tid 3 to smithi07085154-13 2024-09-14T03:48:38.541 INFO:tasks.rados.rados.0.smithi070.stdout:216: finishing write tid 4 to smithi07085154-13 2024-09-14T03:48:38.541 INFO:tasks.rados.rados.0.smithi070.stdout:217: expect (ObjNum 18 snap 0 seq_num 18) 2024-09-14T03:48:38.550 INFO:tasks.rados.rados.0.smithi070.stdout:216: finishing write tid 5 to smithi07085154-13 2024-09-14T03:48:38.550 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 41 (ObjNum 90 snap 18 seq_num 90) dirty exists 2024-09-14T03:48:38.550 INFO:tasks.rados.rados.0.smithi070.stdout:216: left oid 13 (ObjNum 90 snap 18 seq_num 90) 2024-09-14T03:48:38.550 INFO:tasks.rados.rados.0.smithi070.stdout:218: finishing write tid 1 to smithi07085154-21 2024-09-14T03:48:38.550 INFO:tasks.rados.rados.0.smithi070.stdout:218: finishing write tid 2 to smithi07085154-21 2024-09-14T03:48:38.550 INFO:tasks.rados.rados.0.smithi070.stdout:214: done (4 left) 2024-09-14T03:48:38.550 INFO:tasks.rados.rados.0.smithi070.stdout:216: done (3 left) 2024-09-14T03:48:38.550 INFO:tasks.rados.rados.0.smithi070.stdout:217: done (2 left) 2024-09-14T03:48:38.550 INFO:tasks.rados.rados.0.smithi070.stdout:219: done (1 left) 2024-09-14T03:48:38.550 INFO:tasks.rados.rados.0.smithi070.stdout:220: read oid 32 snap -1 2024-09-14T03:48:38.550 INFO:tasks.rados.rados.0.smithi070.stdout:220: expect (ObjNum 66 snap 4 seq_num 66) 2024-09-14T03:48:38.550 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 49 current snap is 18 2024-09-14T03:48:38.551 INFO:tasks.rados.rados.0.smithi070.stdout:218: finishing write tid 3 to smithi07085154-21 2024-09-14T03:48:38.551 INFO:tasks.rados.rados.0.smithi070.stdout:218: finishing write tid 4 to smithi07085154-21 2024-09-14T03:48:38.551 INFO:tasks.rados.rados.0.smithi070.stdout:221: seq_num 93 ranges {811008=761856,1572864=647168,2220032=155648} 2024-09-14T03:48:38.559 INFO:tasks.rados.rados.0.smithi070.stdout:221: writing smithi07085154-49 from 811008 to 1572864 tid 1 2024-09-14T03:48:38.563 INFO:tasks.rados.rados.0.smithi070.stdout:221: writing smithi07085154-49 from 1572864 to 2220032 tid 2 2024-09-14T03:48:38.564 INFO:tasks.rados.rados.0.smithi070.stdout:221: writing smithi07085154-49 from 2220032 to 2375680 tid 3 2024-09-14T03:48:38.564 INFO:tasks.rados.rados.0.smithi070.stdout:222: delete oid 15 current snap is 18 2024-09-14T03:48:38.564 INFO:tasks.rados.rados.0.smithi070.stdout:218: finishing write tid 5 to smithi07085154-21 2024-09-14T03:48:38.564 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 11 (ObjNum 91 snap 18 seq_num 91) dirty exists 2024-09-14T03:48:38.564 INFO:tasks.rados.rados.0.smithi070.stdout:218: left oid 21 (ObjNum 91 snap 18 seq_num 91) 2024-09-14T03:48:38.564 INFO:tasks.rados.rados.0.smithi070.stdout:220: expect (ObjNum 66 snap 4 seq_num 66) 2024-09-14T03:48:38.670 INFO:tasks.rados.rados.0.smithi070.stdout:221: finishing write tid 1 to smithi07085154-49 2024-09-14T03:48:38.670 INFO:tasks.rados.rados.0.smithi070.stdout:221: finishing write tid 2 to smithi07085154-49 2024-09-14T03:48:38.670 INFO:tasks.rados.rados.0.smithi070.stdout:221: finishing write tid 3 to smithi07085154-49 2024-09-14T03:48:38.670 INFO:tasks.rados.rados.0.smithi070.stdout:221: finishing write tid 4 to smithi07085154-49 2024-09-14T03:48:38.670 INFO:tasks.rados.rados.0.smithi070.stdout:221: finishing write tid 5 to smithi07085154-49 2024-09-14T03:48:38.671 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 49 v 49 (ObjNum 93 snap 18 seq_num 93) dirty exists 2024-09-14T03:48:38.671 INFO:tasks.rados.rados.0.smithi070.stdout:221: left oid 49 (ObjNum 93 snap 18 seq_num 93) 2024-09-14T03:48:38.671 INFO:tasks.rados.rados.0.smithi070.stdout:218: done (3 left) 2024-09-14T03:48:38.671 INFO:tasks.rados.rados.0.smithi070.stdout:220: done (2 left) 2024-09-14T03:48:38.671 INFO:tasks.rados.rados.0.smithi070.stdout:221: done (1 left) 2024-09-14T03:48:38.671 INFO:tasks.rados.rados.0.smithi070.stdout:222: done (0 left) 2024-09-14T03:48:38.671 INFO:tasks.rados.rados.0.smithi070.stdout:223: delete oid 35 current snap is 18 2024-09-14T03:48:38.672 INFO:tasks.rados.rados.0.smithi070.stdout:223: done (0 left) 2024-09-14T03:48:38.672 INFO:tasks.rados.rados.0.smithi070.stdout:224: setattr oid 29 current snap is 18 2024-09-14T03:48:38.674 INFO:tasks.rados.rados.0.smithi070.stdout:225: snap_remove snap 7 2024-09-14T03:48:38.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:38 smithi094 ceph-mon[31053]: from='client.14982 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:38.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:38 smithi094 ceph-mon[31053]: osdmap e103: 8 total, 8 up, 8 in 2024-09-14T03:48:38.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:38 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/1045304449' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:38.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:38 smithi094 ceph-mon[31053]: from='client.14994 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:38.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:38 smithi070 ceph-mon[39288]: from='client.14982 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:38.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:38 smithi070 ceph-mon[39288]: osdmap e103: 8 total, 8 up, 8 in 2024-09-14T03:48:38.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:38 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/1045304449' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:38.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:38 smithi070 ceph-mon[39288]: from='client.14994 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:38.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:38 smithi070 ceph-mon[27495]: from='client.14982 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:38.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:38 smithi070 ceph-mon[27495]: osdmap e103: 8 total, 8 up, 8 in 2024-09-14T03:48:38.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:38 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/1045304449' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:48:38.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:38 smithi070 ceph-mon[27495]: from='client.14994 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:39.516 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 75 (ObjNum 1 snap 0 seq_num 304) dirty exists 2024-09-14T03:48:39.516 INFO:tasks.rados.rados.0.smithi070.stdout:224: done (1 left) 2024-09-14T03:48:39.516 INFO:tasks.rados.rados.0.smithi070.stdout:225: done (0 left) 2024-09-14T03:48:39.516 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 26 current snap is 18 2024-09-14T03:48:39.516 INFO:tasks.rados.rados.0.smithi070.stdout:226: seq_num 96 ranges {0=409600,409600=696320,1105920=507904} 2024-09-14T03:48:39.518 INFO:tasks.rados.rados.0.smithi070.stdout:226: writing smithi07085154-26 from 0 to 409600 tid 1 2024-09-14T03:48:39.523 INFO:tasks.rados.rados.0.smithi070.stdout:226: writing smithi07085154-26 from 409600 to 1105920 tid 2 2024-09-14T03:48:39.527 INFO:tasks.rados.rados.0.smithi070.stdout:226: writing smithi07085154-26 from 1105920 to 1613824 tid 3 2024-09-14T03:48:39.527 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 2 current snap is 18 2024-09-14T03:48:39.527 INFO:tasks.rados.rados.0.smithi070.stdout:227: seq_num 97 ranges {3276800=434176,3710976=581632,4292608=540672,4833280=81920} 2024-09-14T03:48:39.547 INFO:tasks.rados.rados.0.smithi070.stdout:227: writing smithi07085154-2 from 3276800 to 3710976 tid 1 2024-09-14T03:48:39.550 INFO:tasks.rados.rados.0.smithi070.stdout:227: writing smithi07085154-2 from 3710976 to 4292608 tid 2 2024-09-14T03:48:39.553 INFO:tasks.rados.rados.0.smithi070.stdout:227: writing smithi07085154-2 from 4292608 to 4833280 tid 3 2024-09-14T03:48:39.554 INFO:tasks.rados.rados.0.smithi070.stdout:227: writing smithi07085154-2 from 4833280 to 4915200 tid 4 2024-09-14T03:48:39.555 INFO:tasks.rados.rados.0.smithi070.stdout:228: copy_from oid 28 from oid 27 current snap is 18 2024-09-14T03:48:39.555 INFO:tasks.rados.rados.0.smithi070.stdout:226: finishing write tid 1 to smithi07085154-26 2024-09-14T03:48:39.555 INFO:tasks.rados.rados.0.smithi070.stdout:226: finishing write tid 2 to smithi07085154-26 2024-09-14T03:48:39.555 INFO:tasks.rados.rados.0.smithi070.stdout:226: finishing write tid 3 to smithi07085154-26 2024-09-14T03:48:39.555 INFO:tasks.rados.rados.0.smithi070.stdout:226: finishing write tid 4 to smithi07085154-26 2024-09-14T03:48:39.555 INFO:tasks.rados.rados.0.smithi070.stdout:226: finishing write tid 5 to smithi07085154-26 2024-09-14T03:48:39.555 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 112 (ObjNum 96 snap 18 seq_num 96) dirty exists 2024-09-14T03:48:39.555 INFO:tasks.rados.rados.0.smithi070.stdout:226: left oid 26 (ObjNum 96 snap 18 seq_num 96) 2024-09-14T03:48:39.555 INFO:tasks.rados.rados.0.smithi070.stdout:227: finishing write tid 1 to smithi07085154-2 2024-09-14T03:48:39.555 INFO:tasks.rados.rados.0.smithi070.stdout:227: finishing write tid 2 to smithi07085154-2 2024-09-14T03:48:39.555 INFO:tasks.rados.rados.0.smithi070.stdout:226: done (2 left) 2024-09-14T03:48:39.555 INFO:tasks.rados.rados.0.smithi070.stdout:229: snap_create 2024-09-14T03:48:39.556 INFO:tasks.rados.rados.0.smithi070.stdout:227: finishing write tid 3 to smithi07085154-2 2024-09-14T03:48:39.557 INFO:tasks.rados.rados.0.smithi070.stdout:227: finishing write tid 4 to smithi07085154-2 2024-09-14T03:48:39.557 INFO:tasks.rados.rados.0.smithi070.stdout:227: finishing write tid 5 to smithi07085154-2 2024-09-14T03:48:39.557 INFO:tasks.rados.rados.0.smithi070.stdout:227: finishing write tid 6 to smithi07085154-2 2024-09-14T03:48:39.557 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 2 v 122 (ObjNum 97 snap 18 seq_num 97) dirty exists 2024-09-14T03:48:39.558 INFO:tasks.rados.rados.0.smithi070.stdout:227: left oid 2 (ObjNum 97 snap 18 seq_num 97) 2024-09-14T03:48:39.559 INFO:tasks.rados.rados.0.smithi070.stdout:228: finishing copy_from racing read to smithi07085154-28 2024-09-14T03:48:39.560 INFO:tasks.rados.rados.0.smithi070.stdout:228: finishing copy_from to smithi07085154-28 2024-09-14T03:48:39.560 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 28 v 12 (ObjNum 26 snap 0 seq_num 26) dirty exists 2024-09-14T03:48:39.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:39 smithi094 ceph-mon[31053]: osdmap e104: 8 total, 8 up, 8 in 2024-09-14T03:48:39.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:39 smithi094 ceph-mon[31053]: pgmap v152: 121 pgs: 121 active+clean; 54 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 2.6 MiB/s wr, 19 op/s 2024-09-14T03:48:39.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:39 smithi070 ceph-mon[27495]: osdmap e104: 8 total, 8 up, 8 in 2024-09-14T03:48:39.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:39 smithi070 ceph-mon[27495]: pgmap v152: 121 pgs: 121 active+clean; 54 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 2.6 MiB/s wr, 19 op/s 2024-09-14T03:48:39.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:39 smithi070 ceph-mon[39288]: osdmap e104: 8 total, 8 up, 8 in 2024-09-14T03:48:39.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:39 smithi070 ceph-mon[39288]: pgmap v152: 121 pgs: 121 active+clean; 54 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 2.6 MiB/s wr, 19 op/s 2024-09-14T03:48:40.500 INFO:tasks.rados.rados.0.smithi070.stdout:227: done (2 left) 2024-09-14T03:48:40.501 INFO:tasks.rados.rados.0.smithi070.stdout:228: done (1 left) 2024-09-14T03:48:40.501 INFO:tasks.rados.rados.0.smithi070.stdout:229: done (0 left) 2024-09-14T03:48:40.501 INFO:tasks.rados.rados.0.smithi070.stdout:230: snap_create 2024-09-14T03:48:40.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:40 smithi094 ceph-mon[31053]: osdmap e105: 8 total, 8 up, 8 in 2024-09-14T03:48:40.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:40 smithi094 ceph-mon[31053]: osdmap e106: 8 total, 8 up, 8 in 2024-09-14T03:48:40.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:40 smithi070 ceph-mon[27495]: osdmap e105: 8 total, 8 up, 8 in 2024-09-14T03:48:40.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:40 smithi070 ceph-mon[27495]: osdmap e106: 8 total, 8 up, 8 in 2024-09-14T03:48:40.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:40 smithi070 ceph-mon[39288]: osdmap e105: 8 total, 8 up, 8 in 2024-09-14T03:48:40.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:40 smithi070 ceph-mon[39288]: osdmap e106: 8 total, 8 up, 8 in 2024-09-14T03:48:41.502 INFO:tasks.rados.rados.0.smithi070.stdout:230: done (0 left) 2024-09-14T03:48:41.502 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 17 current snap is 20 2024-09-14T03:48:41.502 INFO:tasks.rados.rados.0.smithi070.stdout:231: seq_num 98 ranges {270336=450560,720896=466944,1187840=737280} 2024-09-14T03:48:41.505 INFO:tasks.rados.rados.0.smithi070.stdout:231: writing smithi07085154-17 from 270336 to 720896 tid 1 2024-09-14T03:48:41.508 INFO:tasks.rados.rados.0.smithi070.stdout:231: writing smithi07085154-17 from 720896 to 1187840 tid 2 2024-09-14T03:48:41.513 INFO:tasks.rados.rados.0.smithi070.stdout:231: writing smithi07085154-17 from 1187840 to 1925120 tid 3 2024-09-14T03:48:41.513 INFO:tasks.rados.rados.0.smithi070.stdout:232: rollback oid 8 current snap is 20 2024-09-14T03:48:41.513 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 8 to 10 2024-09-14T03:48:41.513 INFO:tasks.rados.rados.0.smithi070.stdout:231: finishing write tid 1 to smithi07085154-17 2024-09-14T03:48:41.513 INFO:tasks.rados.rados.0.smithi070.stdout:231: finishing write tid 2 to smithi07085154-17 2024-09-14T03:48:41.514 INFO:tasks.rados.rados.0.smithi070.stdout:233: rmattr oid 26 current snap is 20 2024-09-14T03:48:41.514 INFO:tasks.rados.rados.0.smithi070.stdout:233: done (2 left) 2024-09-14T03:48:41.514 INFO:tasks.rados.rados.0.smithi070.stdout:234: delete oid 2 current snap is 20 2024-09-14T03:48:41.515 INFO:tasks.rados.rados.0.smithi070.stdout:232: finishing rollback tid 1 to smithi07085154-8 2024-09-14T03:48:41.516 INFO:tasks.rados.rados.0.smithi070.stdout:231: finishing write tid 3 to smithi07085154-17 2024-09-14T03:48:41.516 INFO:tasks.rados.rados.0.smithi070.stdout:232: finishing rollback tid 2 to smithi07085154-8 2024-09-14T03:48:41.516 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 8 v 73 (ObjNum 17 snap 0 seq_num 17) dirty exists 2024-09-14T03:48:41.516 INFO:tasks.rados.rados.0.smithi070.stdout:231: finishing write tid 4 to smithi07085154-17 2024-09-14T03:48:41.517 INFO:tasks.rados.rados.0.smithi070.stdout:231: finishing write tid 5 to smithi07085154-17 2024-09-14T03:48:41.517 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 65 (ObjNum 98 snap 20 seq_num 98) dirty exists 2024-09-14T03:48:41.517 INFO:tasks.rados.rados.0.smithi070.stdout:231: left oid 17 (ObjNum 98 snap 20 seq_num 98) 2024-09-14T03:48:41.520 INFO:tasks.rados.rados.0.smithi070.stdout:231: done (2 left) 2024-09-14T03:48:41.520 INFO:tasks.rados.rados.0.smithi070.stdout:232: done (1 left) 2024-09-14T03:48:41.520 INFO:tasks.rados.rados.0.smithi070.stdout:234: done (0 left) 2024-09-14T03:48:41.520 INFO:tasks.rados.rados.0.smithi070.stdout:235: rmattr oid 38 current snap is 20 2024-09-14T03:48:41.520 INFO:tasks.rados.rados.0.smithi070.stdout:236: snap_create 2024-09-14T03:48:41.522 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 38 v 26 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-14T03:48:41.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:41 smithi094 ceph-mon[31053]: pgmap v155: 121 pgs: 121 active+clean; 54 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 2.6 MiB/s wr, 19 op/s 2024-09-14T03:48:41.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:41 smithi094 ceph-mon[31053]: osdmap e107: 8 total, 8 up, 8 in 2024-09-14T03:48:41.777 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:48:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:48:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:48:41.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:41 smithi070 ceph-mon[27495]: pgmap v155: 121 pgs: 121 active+clean; 54 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 2.6 MiB/s wr, 19 op/s 2024-09-14T03:48:41.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:41 smithi070 ceph-mon[27495]: osdmap e107: 8 total, 8 up, 8 in 2024-09-14T03:48:41.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:41 smithi070 ceph-mon[39288]: pgmap v155: 121 pgs: 121 active+clean; 54 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 2.6 MiB/s wr, 19 op/s 2024-09-14T03:48:41.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:41 smithi070 ceph-mon[39288]: osdmap e107: 8 total, 8 up, 8 in 2024-09-14T03:48:42.523 INFO:tasks.rados.rados.0.smithi070.stdout:235: done (1 left) 2024-09-14T03:48:42.523 INFO:tasks.rados.rados.0.smithi070.stdout:236: done (0 left) 2024-09-14T03:48:42.523 INFO:tasks.rados.rados.0.smithi070.stdout:237: copy_from oid 48 from oid 47 current snap is 21 2024-09-14T03:48:42.523 INFO:tasks.rados.rados.0.smithi070.stdout:238: read oid 34 snap -1 2024-09-14T03:48:42.523 INFO:tasks.rados.rados.0.smithi070.stdout:238: expect (ObjNum 81 snap 14 seq_num 81) 2024-09-14T03:48:42.523 INFO:tasks.rados.rados.0.smithi070.stdout:239: snap_create 2024-09-14T03:48:42.533 INFO:tasks.rados.rados.0.smithi070.stdout:237: finishing copy_from racing read to smithi07085154-48 2024-09-14T03:48:42.534 INFO:tasks.rados.rados.0.smithi070.stdout:238: expect (ObjNum 81 snap 14 seq_num 81) 2024-09-14T03:48:42.597 INFO:tasks.rados.rados.0.smithi070.stdout:237: finishing copy_from to smithi07085154-48 2024-09-14T03:48:42.597 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 8 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-14T03:48:43.528 INFO:tasks.rados.rados.0.smithi070.stdout:237: done (2 left) 2024-09-14T03:48:43.528 INFO:tasks.rados.rados.0.smithi070.stdout:238: done (1 left) 2024-09-14T03:48:43.528 INFO:tasks.rados.rados.0.smithi070.stdout:239: done (0 left) 2024-09-14T03:48:43.529 INFO:tasks.rados.rados.0.smithi070.stdout:240: read oid 47 snap -1 2024-09-14T03:48:43.529 INFO:tasks.rados.rados.0.smithi070.stdout:240: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-14T03:48:43.529 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 45 current snap is 22 2024-09-14T03:48:43.529 INFO:tasks.rados.rados.0.smithi070.stdout:241: seq_num 100 ranges {1785856=483328,2269184=688128,2957312=516096} 2024-09-14T03:48:43.540 INFO:tasks.rados.rados.0.smithi070.stdout:241: writing smithi07085154-45 from 1785856 to 2269184 tid 1 2024-09-14T03:48:43.544 INFO:tasks.rados.rados.0.smithi070.stdout:241: writing smithi07085154-45 from 2269184 to 2957312 tid 2 2024-09-14T03:48:43.548 INFO:tasks.rados.rados.0.smithi070.stdout:241: writing smithi07085154-45 from 2957312 to 3473408 tid 3 2024-09-14T03:48:43.548 INFO:tasks.rados.rados.0.smithi070.stdout:242: read oid 49 snap 16 2024-09-14T03:48:43.548 INFO:tasks.rados.rados.0.smithi070.stdout:242: expect (ObjNum 48 snap 0 seq_num 48) 2024-09-14T03:48:43.549 INFO:tasks.rados.rados.0.smithi070.stdout:243: read oid 28 snap -1 2024-09-14T03:48:43.549 INFO:tasks.rados.rados.0.smithi070.stdout:243: expect (ObjNum 26 snap 0 seq_num 26) 2024-09-14T03:48:43.549 INFO:tasks.rados.rados.0.smithi070.stdout:244: read oid 46 snap 13 2024-09-14T03:48:43.549 INFO:tasks.rados.rados.0.smithi070.stdout:244: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-14T03:48:43.549 INFO:tasks.rados.rados.0.smithi070.stdout:245: snap_remove snap 21 2024-09-14T03:48:43.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:43 smithi094 ceph-mon[31053]: osdmap e108: 8 total, 8 up, 8 in 2024-09-14T03:48:43.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:43 smithi094 ceph-mon[31053]: pgmap v158: 121 pgs: 121 active+clean; 65 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.7 MiB/s wr, 22 op/s 2024-09-14T03:48:43.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:43 smithi070 ceph-mon[27495]: osdmap e108: 8 total, 8 up, 8 in 2024-09-14T03:48:43.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:43 smithi070 ceph-mon[27495]: pgmap v158: 121 pgs: 121 active+clean; 65 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.7 MiB/s wr, 22 op/s 2024-09-14T03:48:43.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:43 smithi070 ceph-mon[39288]: osdmap e108: 8 total, 8 up, 8 in 2024-09-14T03:48:43.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:43 smithi070 ceph-mon[39288]: pgmap v158: 121 pgs: 121 active+clean; 65 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.7 MiB/s wr, 22 op/s 2024-09-14T03:48:44.532 INFO:tasks.rados.rados.0.smithi070.stdout:245: done (5 left) 2024-09-14T03:48:44.532 INFO:tasks.rados.rados.0.smithi070.stdout:246: rollback oid 13 current snap is 22 2024-09-14T03:48:44.532 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 13 to 13 2024-09-14T03:48:44.532 INFO:tasks.rados.rados.0.smithi070.stdout:240: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-14T03:48:44.554 INFO:tasks.rados.rados.0.smithi070.stdout:241: finishing write tid 1 to smithi07085154-45 2024-09-14T03:48:44.554 INFO:tasks.rados.rados.0.smithi070.stdout:241: finishing write tid 2 to smithi07085154-45 2024-09-14T03:48:44.554 INFO:tasks.rados.rados.0.smithi070.stdout:241: finishing write tid 3 to smithi07085154-45 2024-09-14T03:48:44.554 INFO:tasks.rados.rados.0.smithi070.stdout:240: done (5 left) 2024-09-14T03:48:44.554 INFO:tasks.rados.rados.0.smithi070.stdout:247: read oid 12 snap -1 2024-09-14T03:48:44.555 INFO:tasks.rados.rados.0.smithi070.stdout:247: expect (ObjNum 68 snap 5 seq_num 68) 2024-09-14T03:48:44.555 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 24 current snap is 22 2024-09-14T03:48:44.555 INFO:tasks.rados.rados.0.smithi070.stdout:241: finishing write tid 4 to smithi07085154-45 2024-09-14T03:48:44.555 INFO:tasks.rados.rados.0.smithi070.stdout:248: seq_num 101 ranges {0=499712,499712=573440,1073152=524288,1597440=106496} 2024-09-14T03:48:44.557 INFO:tasks.rados.rados.0.smithi070.stdout:248: writing smithi07085154-24 from 0 to 499712 tid 1 2024-09-14T03:48:44.560 INFO:tasks.rados.rados.0.smithi070.stdout:248: writing smithi07085154-24 from 499712 to 1073152 tid 2 2024-09-14T03:48:44.563 INFO:tasks.rados.rados.0.smithi070.stdout:248: writing smithi07085154-24 from 1073152 to 1597440 tid 3 2024-09-14T03:48:44.564 INFO:tasks.rados.rados.0.smithi070.stdout:248: writing smithi07085154-24 from 1597440 to 1703936 tid 4 2024-09-14T03:48:44.564 INFO:tasks.rados.rados.0.smithi070.stdout:249: setattr oid 6 current snap is 22 2024-09-14T03:48:44.564 INFO:tasks.rados.rados.0.smithi070.stdout:244: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-14T03:48:44.586 INFO:tasks.rados.rados.0.smithi070.stdout:243: expect (ObjNum 26 snap 0 seq_num 26) 2024-09-14T03:48:44.597 INFO:tasks.rados.rados.0.smithi070.stdout:241: finishing write tid 5 to smithi07085154-45 2024-09-14T03:48:44.597 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 45 v 60 (ObjNum 100 snap 22 seq_num 100) dirty exists 2024-09-14T03:48:44.597 INFO:tasks.rados.rados.0.smithi070.stdout:241: left oid 45 (ObjNum 100 snap 22 seq_num 100) 2024-09-14T03:48:44.597 INFO:tasks.rados.rados.0.smithi070.stdout:242: expect (ObjNum 48 snap 0 seq_num 48) 2024-09-14T03:48:44.618 INFO:tasks.rados.rados.0.smithi070.stdout:246: finishing rollback tid 0 to smithi07085154-13 2024-09-14T03:48:44.619 INFO:tasks.rados.rados.0.smithi070.stdout:246: finishing rollback tid 1 to smithi07085154-13 2024-09-14T03:48:44.619 INFO:tasks.rados.rados.0.smithi070.stdout:246: finishing rollback tid 2 to smithi07085154-13 2024-09-14T03:48:44.619 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 49 (ObjNum 78 snap 13 seq_num 78) dirty exists 2024-09-14T03:48:44.619 INFO:tasks.rados.rados.0.smithi070.stdout:248: finishing write tid 1 to smithi07085154-24 2024-09-14T03:48:44.619 INFO:tasks.rados.rados.0.smithi070.stdout:248: finishing write tid 2 to smithi07085154-24 2024-09-14T03:48:44.619 INFO:tasks.rados.rados.0.smithi070.stdout:248: finishing write tid 3 to smithi07085154-24 2024-09-14T03:48:44.619 INFO:tasks.rados.rados.0.smithi070.stdout:248: finishing write tid 4 to smithi07085154-24 2024-09-14T03:48:44.619 INFO:tasks.rados.rados.0.smithi070.stdout:248: finishing write tid 5 to smithi07085154-24 2024-09-14T03:48:44.619 INFO:tasks.rados.rados.0.smithi070.stdout:247: expect (ObjNum 68 snap 5 seq_num 68) 2024-09-14T03:48:44.697 INFO:tasks.rados.rados.0.smithi070.stdout:248: finishing write tid 6 to smithi07085154-24 2024-09-14T03:48:44.697 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 21 (ObjNum 101 snap 22 seq_num 101) dirty exists 2024-09-14T03:48:44.697 INFO:tasks.rados.rados.0.smithi070.stdout:248: left oid 24 (ObjNum 101 snap 22 seq_num 101) 2024-09-14T03:48:44.698 INFO:tasks.rados.rados.0.smithi070.stdout:241: done (7 left) 2024-09-14T03:48:44.698 INFO:tasks.rados.rados.0.smithi070.stdout:242: done (6 left) 2024-09-14T03:48:44.698 INFO:tasks.rados.rados.0.smithi070.stdout:243: done (5 left) 2024-09-14T03:48:44.698 INFO:tasks.rados.rados.0.smithi070.stdout:244: done (4 left) 2024-09-14T03:48:44.698 INFO:tasks.rados.rados.0.smithi070.stdout:246: done (3 left) 2024-09-14T03:48:44.698 INFO:tasks.rados.rados.0.smithi070.stdout:247: done (2 left) 2024-09-14T03:48:44.698 INFO:tasks.rados.rados.0.smithi070.stdout:248: done (1 left) 2024-09-14T03:48:44.698 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 19 current snap is 22 2024-09-14T03:48:44.698 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 67 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-14T03:48:44.698 INFO:tasks.rados.rados.0.smithi070.stdout:250: seq_num 102 ranges {303104=516096,819200=450560,1269760=729088,1998848=24576} 2024-09-14T03:48:44.701 INFO:tasks.rados.rados.0.smithi070.stdout:250: writing smithi07085154-19 from 303104 to 819200 tid 1 2024-09-14T03:48:44.704 INFO:tasks.rados.rados.0.smithi070.stdout:250: writing smithi07085154-19 from 819200 to 1269760 tid 2 2024-09-14T03:48:44.708 INFO:tasks.rados.rados.0.smithi070.stdout:250: writing smithi07085154-19 from 1269760 to 1998848 tid 3 2024-09-14T03:48:44.708 INFO:tasks.rados.rados.0.smithi070.stdout:250: writing smithi07085154-19 from 1998848 to 2023424 tid 4 2024-09-14T03:48:44.709 INFO:tasks.rados.rados.0.smithi070.stdout:249: done (1 left) 2024-09-14T03:48:44.709 INFO:tasks.rados.rados.0.smithi070.stdout:251: read oid 7 snap 14 2024-09-14T03:48:44.709 INFO:tasks.rados.rados.0.smithi070.stdout:251: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-14T03:48:44.709 INFO:tasks.rados.rados.0.smithi070.stdout:252: snap_remove snap 19 2024-09-14T03:48:44.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:44 smithi094 ceph-mon[31053]: osdmap e109: 8 total, 8 up, 8 in 2024-09-14T03:48:44.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:44 smithi070 ceph-mon[27495]: osdmap e109: 8 total, 8 up, 8 in 2024-09-14T03:48:44.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:44 smithi070 ceph-mon[39288]: osdmap e109: 8 total, 8 up, 8 in 2024-09-14T03:48:45.502 INFO:tasks.rados.rados.0.smithi070.stdout:252: done (2 left) 2024-09-14T03:48:45.502 INFO:tasks.rados.rados.0.smithi070.stdout:253: copy_from oid 25 from oid 35 current snap is 22 2024-09-14T03:48:45.502 INFO:tasks.rados.rados.0.smithi070.stdout:250: finishing write tid 1 to smithi07085154-19 2024-09-14T03:48:45.502 INFO:tasks.rados.rados.0.smithi070.stdout:250: finishing write tid 2 to smithi07085154-19 2024-09-14T03:48:45.502 INFO:tasks.rados.rados.0.smithi070.stdout:251: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-14T03:48:45.507 INFO:tasks.rados.rados.0.smithi070.stdout:250: finishing write tid 3 to smithi07085154-19 2024-09-14T03:48:45.507 INFO:tasks.rados.rados.0.smithi070.stdout:250: finishing write tid 4 to smithi07085154-19 2024-09-14T03:48:45.507 INFO:tasks.rados.rados.0.smithi070.stdout:250: finishing write tid 5 to smithi07085154-19 2024-09-14T03:48:45.507 INFO:tasks.rados.rados.0.smithi070.stdout:250: finishing write tid 6 to smithi07085154-19 2024-09-14T03:48:45.507 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 19 v 87 (ObjNum 102 snap 22 seq_num 102) dirty exists 2024-09-14T03:48:45.507 INFO:tasks.rados.rados.0.smithi070.stdout:250: left oid 19 (ObjNum 102 snap 22 seq_num 102) 2024-09-14T03:48:45.507 INFO:tasks.rados.rados.0.smithi070.stdout:253: finishing copy_from to smithi07085154-25 2024-09-14T03:48:45.507 INFO:tasks.rados.rados.0.smithi070.stdout:253: got expected ENOENT (src dne) 2024-09-14T03:48:45.508 INFO:tasks.rados.rados.0.smithi070.stdout:253: finishing copy_from racing read to smithi07085154-25 2024-09-14T03:48:45.508 INFO:tasks.rados.rados.0.smithi070.stdout:250: done (2 left) 2024-09-14T03:48:45.508 INFO:tasks.rados.rados.0.smithi070.stdout:251: done (1 left) 2024-09-14T03:48:45.508 INFO:tasks.rados.rados.0.smithi070.stdout:253: done (0 left) 2024-09-14T03:48:45.508 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 34 current snap is 22 2024-09-14T03:48:45.508 INFO:tasks.rados.rados.0.smithi070.stdout:254: seq_num 103 ranges {2031616=532480,2564096=737280,3301376=466944} 2024-09-14T03:48:45.521 INFO:tasks.rados.rados.0.smithi070.stdout:254: writing smithi07085154-34 from 2031616 to 2564096 tid 1 2024-09-14T03:48:45.525 INFO:tasks.rados.rados.0.smithi070.stdout:254: writing smithi07085154-34 from 2564096 to 3301376 tid 2 2024-09-14T03:48:45.528 INFO:tasks.rados.rados.0.smithi070.stdout:254: writing smithi07085154-34 from 3301376 to 3768320 tid 3 2024-09-14T03:48:45.528 INFO:tasks.rados.rados.0.smithi070.stdout:254: finishing write tid 1 to smithi07085154-34 2024-09-14T03:48:45.528 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 49 current snap is 22 2024-09-14T03:48:45.528 INFO:tasks.rados.rados.0.smithi070.stdout:254: finishing write tid 2 to smithi07085154-34 2024-09-14T03:48:45.528 INFO:tasks.rados.rados.0.smithi070.stdout:255: seq_num 104 ranges {2375680=548864,2924544=622592,3547136=581632} 2024-09-14T03:48:45.544 INFO:tasks.rados.rados.0.smithi070.stdout:255: writing smithi07085154-49 from 2375680 to 2924544 tid 1 2024-09-14T03:48:45.547 INFO:tasks.rados.rados.0.smithi070.stdout:255: writing smithi07085154-49 from 2924544 to 3547136 tid 2 2024-09-14T03:48:45.552 INFO:tasks.rados.rados.0.smithi070.stdout:255: writing smithi07085154-49 from 3547136 to 4128768 tid 3 2024-09-14T03:48:45.552 INFO:tasks.rados.rados.0.smithi070.stdout:256: read oid 5 snap -1 2024-09-14T03:48:45.552 INFO:tasks.rados.rados.0.smithi070.stdout:256: expect deleted 2024-09-14T03:48:45.552 INFO:tasks.rados.rados.0.smithi070.stdout:257: read oid 1 snap 20 2024-09-14T03:48:45.552 INFO:tasks.rados.rados.0.smithi070.stdout:257: expect (ObjNum 83 snap 17 seq_num 83) 2024-09-14T03:48:45.552 INFO:tasks.rados.rados.0.smithi070.stdout:258: setattr oid 37 current snap is 22 2024-09-14T03:48:45.552 INFO:tasks.rados.rados.0.smithi070.stdout:254: finishing write tid 3 to smithi07085154-34 2024-09-14T03:48:45.552 INFO:tasks.rados.rados.0.smithi070.stdout:254: finishing write tid 4 to smithi07085154-34 2024-09-14T03:48:45.552 INFO:tasks.rados.rados.0.smithi070.stdout:254: finishing write tid 5 to smithi07085154-34 2024-09-14T03:48:45.552 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 94 (ObjNum 103 snap 22 seq_num 103) dirty exists 2024-09-14T03:48:45.552 INFO:tasks.rados.rados.0.smithi070.stdout:254: left oid 34 (ObjNum 103 snap 22 seq_num 103) 2024-09-14T03:48:45.552 INFO:tasks.rados.rados.0.smithi070.stdout:255: finishing write tid 1 to smithi07085154-49 2024-09-14T03:48:45.553 INFO:tasks.rados.rados.0.smithi070.stdout:255: finishing write tid 2 to smithi07085154-49 2024-09-14T03:48:45.554 INFO:tasks.rados.rados.0.smithi070.stdout:255: finishing write tid 3 to smithi07085154-49 2024-09-14T03:48:45.554 INFO:tasks.rados.rados.0.smithi070.stdout:255: finishing write tid 4 to smithi07085154-49 2024-09-14T03:48:45.554 INFO:tasks.rados.rados.0.smithi070.stdout:254: done (4 left) 2024-09-14T03:48:45.554 INFO:tasks.rados.rados.0.smithi070.stdout:256: done (3 left) 2024-09-14T03:48:45.554 INFO:tasks.rados.rados.0.smithi070.stdout:259: rollback oid 25 current snap is 22 2024-09-14T03:48:45.554 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 25 to 14 2024-09-14T03:48:45.554 INFO:tasks.rados.rados.0.smithi070.stdout:260: delete oid 30 current snap is 22 2024-09-14T03:48:45.555 INFO:tasks.rados.rados.0.smithi070.stdout:255: finishing write tid 5 to smithi07085154-49 2024-09-14T03:48:45.555 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 49 v 69 (ObjNum 104 snap 22 seq_num 104) dirty exists 2024-09-14T03:48:45.555 INFO:tasks.rados.rados.0.smithi070.stdout:255: left oid 49 (ObjNum 104 snap 22 seq_num 104) 2024-09-14T03:48:45.557 INFO:tasks.rados.rados.0.smithi070.stdout:255: done (4 left) 2024-09-14T03:48:45.557 INFO:tasks.rados.rados.0.smithi070.stdout:260: done (3 left) 2024-09-14T03:48:45.557 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 40 current snap is 22 2024-09-14T03:48:45.557 INFO:tasks.rados.rados.0.smithi070.stdout:261: seq_num 106 ranges {663552=581632,1245184=794624,2039808=409600} 2024-09-14T03:48:45.563 INFO:tasks.rados.rados.0.smithi070.stdout:261: writing smithi07085154-40 from 663552 to 1245184 tid 1 2024-09-14T03:48:45.568 INFO:tasks.rados.rados.0.smithi070.stdout:261: writing smithi07085154-40 from 1245184 to 2039808 tid 2 2024-09-14T03:48:45.571 INFO:tasks.rados.rados.0.smithi070.stdout:261: writing smithi07085154-40 from 2039808 to 2449408 tid 3 2024-09-14T03:48:45.571 INFO:tasks.rados.rados.0.smithi070.stdout:262: setattr oid 10 current snap is 22 2024-09-14T03:48:45.571 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 37 v 75 (ObjNum 86 snap 17 seq_num 86) dirty exists 2024-09-14T03:48:45.571 INFO:tasks.rados.rados.0.smithi070.stdout:257: expect (ObjNum 83 snap 17 seq_num 83) 2024-09-14T03:48:45.607 INFO:tasks.rados.rados.0.smithi070.stdout:259: finishing rollback tid 0 to smithi07085154-25 2024-09-14T03:48:45.607 INFO:tasks.rados.rados.0.smithi070.stdout:259: finishing rollback tid 1 to smithi07085154-25 2024-09-14T03:48:45.607 INFO:tasks.rados.rados.0.smithi070.stdout:259: finishing rollback tid 2 to smithi07085154-25 2024-09-14T03:48:45.607 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 75 (ObjNum 72 snap 10 seq_num 72) dirty exists 2024-09-14T03:48:45.607 INFO:tasks.rados.rados.0.smithi070.stdout:261: finishing write tid 1 to smithi07085154-40 2024-09-14T03:48:45.607 INFO:tasks.rados.rados.0.smithi070.stdout:261: finishing write tid 2 to smithi07085154-40 2024-09-14T03:48:45.607 INFO:tasks.rados.rados.0.smithi070.stdout:261: finishing write tid 3 to smithi07085154-40 2024-09-14T03:48:45.607 INFO:tasks.rados.rados.0.smithi070.stdout:261: finishing write tid 4 to smithi07085154-40 2024-09-14T03:48:45.607 INFO:tasks.rados.rados.0.smithi070.stdout:261: finishing write tid 5 to smithi07085154-40 2024-09-14T03:48:45.607 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 33 (ObjNum 106 snap 22 seq_num 106) dirty exists 2024-09-14T03:48:45.608 INFO:tasks.rados.rados.0.smithi070.stdout:261: left oid 40 (ObjNum 106 snap 22 seq_num 106) 2024-09-14T03:48:45.608 INFO:tasks.rados.rados.0.smithi070.stdout:257: done (4 left) 2024-09-14T03:48:45.608 INFO:tasks.rados.rados.0.smithi070.stdout:258: done (3 left) 2024-09-14T03:48:45.608 INFO:tasks.rados.rados.0.smithi070.stdout:259: done (2 left) 2024-09-14T03:48:45.608 INFO:tasks.rados.rados.0.smithi070.stdout:261: done (1 left) 2024-09-14T03:48:45.608 INFO:tasks.rados.rados.0.smithi070.stdout:263: setattr oid 37 current snap is 22 2024-09-14T03:48:45.608 INFO:tasks.rados.rados.0.smithi070.stdout:264: snap_remove snap 18 2024-09-14T03:48:45.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:45 smithi070 ceph-mon[27495]: osdmap e110: 8 total, 8 up, 8 in 2024-09-14T03:48:45.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:45 smithi070 ceph-mon[27495]: pgmap v161: 121 pgs: 121 active+clean; 65 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.7 MiB/s wr, 22 op/s 2024-09-14T03:48:45.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:45 smithi070 ceph-mon[27495]: osdmap e111: 8 total, 8 up, 8 in 2024-09-14T03:48:45.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:45 smithi070 ceph-mon[39288]: osdmap e110: 8 total, 8 up, 8 in 2024-09-14T03:48:45.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:45 smithi070 ceph-mon[39288]: pgmap v161: 121 pgs: 121 active+clean; 65 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.7 MiB/s wr, 22 op/s 2024-09-14T03:48:45.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:45 smithi070 ceph-mon[39288]: osdmap e111: 8 total, 8 up, 8 in 2024-09-14T03:48:46.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:45 smithi094 ceph-mon[31053]: osdmap e110: 8 total, 8 up, 8 in 2024-09-14T03:48:46.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:45 smithi094 ceph-mon[31053]: pgmap v161: 121 pgs: 121 active+clean; 65 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.7 MiB/s wr, 22 op/s 2024-09-14T03:48:46.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:45 smithi094 ceph-mon[31053]: osdmap e111: 8 total, 8 up, 8 in 2024-09-14T03:48:46.539 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 128 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-14T03:48:46.540 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 37 v 76 (ObjNum 86 snap 17 seq_num 86) dirty exists 2024-09-14T03:48:46.540 INFO:tasks.rados.rados.0.smithi070.stdout:262: done (2 left) 2024-09-14T03:48:46.540 INFO:tasks.rados.rados.0.smithi070.stdout:263: done (1 left) 2024-09-14T03:48:46.540 INFO:tasks.rados.rados.0.smithi070.stdout:264: done (0 left) 2024-09-14T03:48:46.540 INFO:tasks.rados.rados.0.smithi070.stdout:265: read oid 31 snap -1 2024-09-14T03:48:46.540 INFO:tasks.rados.rados.0.smithi070.stdout:265: expect deleted 2024-09-14T03:48:46.540 INFO:tasks.rados.rados.0.smithi070.stdout:266: read oid 6 snap -1 2024-09-14T03:48:46.540 INFO:tasks.rados.rados.0.smithi070.stdout:266: expect (ObjNum 36 snap 0 seq_num 36) 2024-09-14T03:48:46.540 INFO:tasks.rados.rados.0.smithi070.stdout:267: snap_create 2024-09-14T03:48:46.546 INFO:tasks.rados.rados.0.smithi070.stdout:266: expect (ObjNum 36 snap 0 seq_num 36) 2024-09-14T03:48:46.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:48:46 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:48:46] "GET /metrics HTTP/1.1" 200 239890 "" "Prometheus/2.43.0" 2024-09-14T03:48:47.543 INFO:tasks.rados.rados.0.smithi070.stdout:265: done (2 left) 2024-09-14T03:48:47.543 INFO:tasks.rados.rados.0.smithi070.stdout:266: done (1 left) 2024-09-14T03:48:47.543 INFO:tasks.rados.rados.0.smithi070.stdout:267: done (0 left) 2024-09-14T03:48:47.543 INFO:tasks.rados.rados.0.smithi070.stdout:268: rollback oid 30 current snap is 23 2024-09-14T03:48:47.543 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 30 to 16 2024-09-14T03:48:47.543 INFO:tasks.rados.rados.0.smithi070.stdout:269: copy_from oid 9 from oid 25 current snap is 23 2024-09-14T03:48:47.544 INFO:tasks.rados.rados.0.smithi070.stdout:270: copy_from oid 12 from oid 28 current snap is 23 2024-09-14T03:48:47.544 INFO:tasks.rados.rados.0.smithi070.stdout:271: read oid 1 snap -1 2024-09-14T03:48:47.544 INFO:tasks.rados.rados.0.smithi070.stdout:271: expect (ObjNum 83 snap 17 seq_num 83) 2024-09-14T03:48:47.544 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 32 current snap is 23 2024-09-14T03:48:47.544 INFO:tasks.rados.rados.0.smithi070.stdout:272: seq_num 107 ranges {2686976=598016,3284992=729088,4014080=466944,4481024=8192} 2024-09-14T03:48:47.561 INFO:tasks.rados.rados.0.smithi070.stdout:272: writing smithi07085154-32 from 2686976 to 3284992 tid 1 2024-09-14T03:48:47.565 INFO:tasks.rados.rados.0.smithi070.stdout:272: writing smithi07085154-32 from 3284992 to 4014080 tid 2 2024-09-14T03:48:47.569 INFO:tasks.rados.rados.0.smithi070.stdout:272: writing smithi07085154-32 from 4014080 to 4481024 tid 3 2024-09-14T03:48:47.569 INFO:tasks.rados.rados.0.smithi070.stdout:272: writing smithi07085154-32 from 4481024 to 4489216 tid 4 2024-09-14T03:48:47.569 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 36 current snap is 23 2024-09-14T03:48:47.569 INFO:tasks.rados.rados.0.smithi070.stdout:268: finishing rollback tid 1 to smithi07085154-30 2024-09-14T03:48:47.569 INFO:tasks.rados.rados.0.smithi070.stdout:273: seq_num 108 ranges {1384448=614400,1998848=606208,2605056=598016} 2024-09-14T03:48:47.579 INFO:tasks.rados.rados.0.smithi070.stdout:273: writing smithi07085154-36 from 1384448 to 1998848 tid 1 2024-09-14T03:48:47.583 INFO:tasks.rados.rados.0.smithi070.stdout:273: writing smithi07085154-36 from 1998848 to 2605056 tid 2 2024-09-14T03:48:47.587 INFO:tasks.rados.rados.0.smithi070.stdout:273: writing smithi07085154-36 from 2605056 to 3203072 tid 3 2024-09-14T03:48:47.587 INFO:tasks.rados.rados.0.smithi070.stdout:274: rollback oid 2 current snap is 23 2024-09-14T03:48:47.587 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 2 to 16 2024-09-14T03:48:47.587 INFO:tasks.rados.rados.0.smithi070.stdout:268: finishing rollback tid 2 to smithi07085154-30 2024-09-14T03:48:47.587 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 30 v 36 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-09-14T03:48:47.587 INFO:tasks.rados.rados.0.smithi070.stdout:270: finishing copy_from racing read to smithi07085154-12 2024-09-14T03:48:47.587 INFO:tasks.rados.rados.0.smithi070.stdout:270: finishing copy_from to smithi07085154-12 2024-09-14T03:48:47.587 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 12 (ObjNum 26 snap 0 seq_num 26) dirty exists 2024-09-14T03:48:47.587 INFO:tasks.rados.rados.0.smithi070.stdout:271: expect (ObjNum 83 snap 17 seq_num 83) 2024-09-14T03:48:47.625 INFO:tasks.rados.rados.0.smithi070.stdout:269: finishing copy_from racing read to smithi07085154-9 2024-09-14T03:48:47.625 INFO:tasks.rados.rados.0.smithi070.stdout:272: finishing write tid 1 to smithi07085154-32 2024-09-14T03:48:47.625 INFO:tasks.rados.rados.0.smithi070.stdout:269: finishing copy_from to smithi07085154-9 2024-09-14T03:48:47.625 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 75 (ObjNum 72 snap 10 seq_num 72) dirty exists 2024-09-14T03:48:47.625 INFO:tasks.rados.rados.0.smithi070.stdout:272: finishing write tid 2 to smithi07085154-32 2024-09-14T03:48:47.625 INFO:tasks.rados.rados.0.smithi070.stdout:272: finishing write tid 3 to smithi07085154-32 2024-09-14T03:48:47.625 INFO:tasks.rados.rados.0.smithi070.stdout:272: finishing write tid 4 to smithi07085154-32 2024-09-14T03:48:47.625 INFO:tasks.rados.rados.0.smithi070.stdout:268: done (6 left) 2024-09-14T03:48:47.625 INFO:tasks.rados.rados.0.smithi070.stdout:269: done (5 left) 2024-09-14T03:48:47.626 INFO:tasks.rados.rados.0.smithi070.stdout:270: done (4 left) 2024-09-14T03:48:47.626 INFO:tasks.rados.rados.0.smithi070.stdout:271: done (3 left) 2024-09-14T03:48:47.626 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 25 current snap is 23 2024-09-14T03:48:47.626 INFO:tasks.rados.rados.0.smithi070.stdout:272: finishing write tid 5 to smithi07085154-32 2024-09-14T03:48:47.626 INFO:tasks.rados.rados.0.smithi070.stdout:275: seq_num 109 ranges {3547136=630784,4177920=491520,4669440=483328,5152768=229376} 2024-09-14T03:48:47.646 INFO:tasks.rados.rados.0.smithi070.stdout:275: writing smithi07085154-25 from 3547136 to 4177920 tid 1 2024-09-14T03:48:47.649 INFO:tasks.rados.rados.0.smithi070.stdout:275: writing smithi07085154-25 from 4177920 to 4669440 tid 2 2024-09-14T03:48:47.652 INFO:tasks.rados.rados.0.smithi070.stdout:275: writing smithi07085154-25 from 4669440 to 5152768 tid 3 2024-09-14T03:48:47.654 INFO:tasks.rados.rados.0.smithi070.stdout:275: writing smithi07085154-25 from 5152768 to 5382144 tid 4 2024-09-14T03:48:47.655 INFO:tasks.rados.rados.0.smithi070.stdout:276: read oid 5 snap -1 2024-09-14T03:48:47.655 INFO:tasks.rados.rados.0.smithi070.stdout:276: expect deleted 2024-09-14T03:48:47.655 INFO:tasks.rados.rados.0.smithi070.stdout:277: rollback oid 28 current snap is 23 2024-09-14T03:48:47.655 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 28 to 14 2024-09-14T03:48:47.655 INFO:tasks.rados.rados.0.smithi070.stdout:272: finishing write tid 6 to smithi07085154-32 2024-09-14T03:48:47.655 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 59 (ObjNum 107 snap 23 seq_num 107) dirty exists 2024-09-14T03:48:47.655 INFO:tasks.rados.rados.0.smithi070.stdout:272: left oid 32 (ObjNum 107 snap 23 seq_num 107) 2024-09-14T03:48:47.655 INFO:tasks.rados.rados.0.smithi070.stdout:273: finishing write tid 1 to smithi07085154-36 2024-09-14T03:48:47.655 INFO:tasks.rados.rados.0.smithi070.stdout:273: finishing write tid 2 to smithi07085154-36 2024-09-14T03:48:47.655 INFO:tasks.rados.rados.0.smithi070.stdout:274: finishing rollback tid 1 to smithi07085154-2 2024-09-14T03:48:47.655 INFO:tasks.rados.rados.0.smithi070.stdout:274: finishing rollback tid 2 to smithi07085154-2 2024-09-14T03:48:47.655 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 2 v 136 (ObjNum 63 snap 3 seq_num 63) dirty exists 2024-09-14T03:48:47.656 INFO:tasks.rados.rados.0.smithi070.stdout:273: finishing write tid 3 to smithi07085154-36 2024-09-14T03:48:47.656 INFO:tasks.rados.rados.0.smithi070.stdout:273: finishing write tid 4 to smithi07085154-36 2024-09-14T03:48:47.656 INFO:tasks.rados.rados.0.smithi070.stdout:273: finishing write tid 5 to smithi07085154-36 2024-09-14T03:48:47.656 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 36 v 40 (ObjNum 108 snap 23 seq_num 108) dirty exists 2024-09-14T03:48:47.656 INFO:tasks.rados.rados.0.smithi070.stdout:273: left oid 36 (ObjNum 108 snap 23 seq_num 108) 2024-09-14T03:48:47.656 INFO:tasks.rados.rados.0.smithi070.stdout:275: finishing write tid 1 to smithi07085154-25 2024-09-14T03:48:47.656 INFO:tasks.rados.rados.0.smithi070.stdout:275: finishing write tid 2 to smithi07085154-25 2024-09-14T03:48:47.656 INFO:tasks.rados.rados.0.smithi070.stdout:272: done (5 left) 2024-09-14T03:48:47.656 INFO:tasks.rados.rados.0.smithi070.stdout:273: done (4 left) 2024-09-14T03:48:47.656 INFO:tasks.rados.rados.0.smithi070.stdout:274: done (3 left) 2024-09-14T03:48:47.656 INFO:tasks.rados.rados.0.smithi070.stdout:278: rmattr oid 22 current snap is 23 2024-09-14T03:48:47.656 INFO:tasks.rados.rados.0.smithi070.stdout:278: done (3 left) 2024-09-14T03:48:47.657 INFO:tasks.rados.rados.0.smithi070.stdout:279: copy_from oid 8 from oid 7 current snap is 23 2024-09-14T03:48:47.657 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 31 current snap is 23 2024-09-14T03:48:47.657 INFO:tasks.rados.rados.0.smithi070.stdout:280: seq_num 110 ranges {0=647168,647168=778240,1425408=425984} 2024-09-14T03:48:47.658 INFO:tasks.rados.rados.0.smithi070.stdout:280: writing smithi07085154-31 from 0 to 647168 tid 1 2024-09-14T03:48:47.663 INFO:tasks.rados.rados.0.smithi070.stdout:280: writing smithi07085154-31 from 647168 to 1425408 tid 2 2024-09-14T03:48:47.665 INFO:tasks.rados.rados.0.smithi070.stdout:280: writing smithi07085154-31 from 1425408 to 1851392 tid 3 2024-09-14T03:48:47.665 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 11 current snap is 23 2024-09-14T03:48:47.665 INFO:tasks.rados.rados.0.smithi070.stdout:281: seq_num 111 ranges {172032=663552,835584=663552,1499136=450560,1949696=90112} 2024-09-14T03:48:47.669 INFO:tasks.rados.rados.0.smithi070.stdout:281: writing smithi07085154-11 from 172032 to 835584 tid 1 2024-09-14T03:48:47.673 INFO:tasks.rados.rados.0.smithi070.stdout:281: writing smithi07085154-11 from 835584 to 1499136 tid 2 2024-09-14T03:48:47.676 INFO:tasks.rados.rados.0.smithi070.stdout:281: writing smithi07085154-11 from 1499136 to 1949696 tid 3 2024-09-14T03:48:47.677 INFO:tasks.rados.rados.0.smithi070.stdout:281: writing smithi07085154-11 from 1949696 to 2039808 tid 4 2024-09-14T03:48:47.677 INFO:tasks.rados.rados.0.smithi070.stdout:282: read oid 44 snap 20 2024-09-14T03:48:47.677 INFO:tasks.rados.rados.0.smithi070.stdout:282: expect (ObjNum 43 snap 0 seq_num 43) 2024-09-14T03:48:47.677 INFO:tasks.rados.rados.0.smithi070.stdout:283: rmattr oid 15 current snap is 23 2024-09-14T03:48:47.677 INFO:tasks.rados.rados.0.smithi070.stdout:283: done (7 left) 2024-09-14T03:48:47.677 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 47 current snap is 23 2024-09-14T03:48:47.677 INFO:tasks.rados.rados.0.smithi070.stdout:284: seq_num 112 ranges {778240=679936,1458176=548864,2007040=655360} 2024-09-14T03:48:47.684 INFO:tasks.rados.rados.0.smithi070.stdout:284: writing smithi07085154-47 from 778240 to 1458176 tid 1 2024-09-14T03:48:47.688 INFO:tasks.rados.rados.0.smithi070.stdout:284: writing smithi07085154-47 from 1458176 to 2007040 tid 2 2024-09-14T03:48:47.692 INFO:tasks.rados.rados.0.smithi070.stdout:284: writing smithi07085154-47 from 2007040 to 2662400 tid 3 2024-09-14T03:48:47.692 INFO:tasks.rados.rados.0.smithi070.stdout:285: read oid 46 snap -1 2024-09-14T03:48:47.692 INFO:tasks.rados.rados.0.smithi070.stdout:285: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-14T03:48:47.692 INFO:tasks.rados.rados.0.smithi070.stdout:286: read oid 17 snap 13 2024-09-14T03:48:47.692 INFO:tasks.rados.rados.0.smithi070.stdout:286: expect (ObjNum 16 snap 0 seq_num 16) 2024-09-14T03:48:47.692 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 22 current snap is 23 2024-09-14T03:48:47.692 INFO:tasks.rados.rados.0.smithi070.stdout:275: finishing write tid 3 to smithi07085154-25 2024-09-14T03:48:47.692 INFO:tasks.rados.rados.0.smithi070.stdout:275: finishing write tid 4 to smithi07085154-25 2024-09-14T03:48:47.692 INFO:tasks.rados.rados.0.smithi070.stdout:287: seq_num 113 ranges {0=696320,696320=434176,1130496=466944,1597440=303104} 2024-09-14T03:48:47.695 INFO:tasks.rados.rados.0.smithi070.stdout:287: writing smithi07085154-22 from 0 to 696320 tid 1 2024-09-14T03:48:47.698 INFO:tasks.rados.rados.0.smithi070.stdout:287: writing smithi07085154-22 from 696320 to 1130496 tid 2 2024-09-14T03:48:47.700 INFO:tasks.rados.rados.0.smithi070.stdout:287: writing smithi07085154-22 from 1130496 to 1597440 tid 3 2024-09-14T03:48:47.702 INFO:tasks.rados.rados.0.smithi070.stdout:287: writing smithi07085154-22 from 1597440 to 1900544 tid 4 2024-09-14T03:48:47.703 INFO:tasks.rados.rados.0.smithi070.stdout:276: done (10 left) 2024-09-14T03:48:47.703 INFO:tasks.rados.rados.0.smithi070.stdout:288: snap_remove snap 8 2024-09-14T03:48:48.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:47 smithi094 ceph-mon[31053]: osdmap e112: 8 total, 8 up, 8 in 2024-09-14T03:48:48.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:47 smithi094 ceph-mon[31053]: pgmap v164: 121 pgs: 121 active+clean; 65 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:48:48.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:47 smithi070 ceph-mon[27495]: osdmap e112: 8 total, 8 up, 8 in 2024-09-14T03:48:48.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:47 smithi070 ceph-mon[27495]: pgmap v164: 121 pgs: 121 active+clean; 65 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:48:48.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:47 smithi070 ceph-mon[39288]: osdmap e112: 8 total, 8 up, 8 in 2024-09-14T03:48:48.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:47 smithi070 ceph-mon[39288]: pgmap v164: 121 pgs: 121 active+clean; 65 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:48:48.549 INFO:tasks.rados.rados.0.smithi070.stdout:288: done (10 left) 2024-09-14T03:48:48.549 INFO:tasks.rados.rados.0.smithi070.stdout:289: rollback oid 24 current snap is 23 2024-09-14T03:48:48.549 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 24 to 10 2024-09-14T03:48:48.549 INFO:tasks.rados.rados.0.smithi070.stdout:275: finishing write tid 5 to smithi07085154-25 2024-09-14T03:48:48.549 INFO:tasks.rados.rados.0.smithi070.stdout:277: finishing rollback tid 0 to smithi07085154-28 2024-09-14T03:48:48.549 INFO:tasks.rados.rados.0.smithi070.stdout:275: finishing write tid 6 to smithi07085154-25 2024-09-14T03:48:48.549 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 87 (ObjNum 109 snap 23 seq_num 109) dirty exists 2024-09-14T03:48:48.549 INFO:tasks.rados.rados.0.smithi070.stdout:275: left oid 25 (ObjNum 109 snap 23 seq_num 109) 2024-09-14T03:48:48.549 INFO:tasks.rados.rados.0.smithi070.stdout:277: finishing rollback tid 1 to smithi07085154-28 2024-09-14T03:48:48.550 INFO:tasks.rados.rados.0.smithi070.stdout:277: finishing rollback tid 2 to smithi07085154-28 2024-09-14T03:48:48.550 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 28 v 39 (ObjNum 57 snap 3 seq_num 57) dirty exists 2024-09-14T03:48:48.550 INFO:tasks.rados.rados.0.smithi070.stdout:279: finishing copy_from racing read to smithi07085154-8 2024-09-14T03:48:48.550 INFO:tasks.rados.rados.0.smithi070.stdout:279: finishing copy_from to smithi07085154-8 2024-09-14T03:48:48.550 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 8 v 4 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-09-14T03:48:48.550 INFO:tasks.rados.rados.0.smithi070.stdout:280: finishing write tid 1 to smithi07085154-31 2024-09-14T03:48:48.550 INFO:tasks.rados.rados.0.smithi070.stdout:275: done (10 left) 2024-09-14T03:48:48.550 INFO:tasks.rados.rados.0.smithi070.stdout:277: done (9 left) 2024-09-14T03:48:48.550 INFO:tasks.rados.rados.0.smithi070.stdout:279: done (8 left) 2024-09-14T03:48:48.550 INFO:tasks.rados.rados.0.smithi070.stdout:290: rollback oid 2 current snap is 23 2024-09-14T03:48:48.550 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 2 to 13 2024-09-14T03:48:48.550 INFO:tasks.rados.rados.0.smithi070.stdout:280: finishing write tid 2 to smithi07085154-31 2024-09-14T03:48:48.550 INFO:tasks.rados.rados.0.smithi070.stdout:280: finishing write tid 3 to smithi07085154-31 2024-09-14T03:48:48.550 INFO:tasks.rados.rados.0.smithi070.stdout:280: finishing write tid 4 to smithi07085154-31 2024-09-14T03:48:48.551 INFO:tasks.rados.rados.0.smithi070.stdout:280: finishing write tid 5 to smithi07085154-31 2024-09-14T03:48:48.551 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 91 (ObjNum 110 snap 23 seq_num 110) dirty exists 2024-09-14T03:48:48.551 INFO:tasks.rados.rados.0.smithi070.stdout:280: left oid 31 (ObjNum 110 snap 23 seq_num 110) 2024-09-14T03:48:48.551 INFO:tasks.rados.rados.0.smithi070.stdout:281: finishing write tid 1 to smithi07085154-11 2024-09-14T03:48:48.551 INFO:tasks.rados.rados.0.smithi070.stdout:281: finishing write tid 2 to smithi07085154-11 2024-09-14T03:48:48.551 INFO:tasks.rados.rados.0.smithi070.stdout:281: finishing write tid 3 to smithi07085154-11 2024-09-14T03:48:48.551 INFO:tasks.rados.rados.0.smithi070.stdout:281: finishing write tid 4 to smithi07085154-11 2024-09-14T03:48:48.551 INFO:tasks.rados.rados.0.smithi070.stdout:281: finishing write tid 5 to smithi07085154-11 2024-09-14T03:48:48.551 INFO:tasks.rados.rados.0.smithi070.stdout:281: finishing write tid 6 to smithi07085154-11 2024-09-14T03:48:48.551 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 11 v 65 (ObjNum 111 snap 23 seq_num 111) dirty exists 2024-09-14T03:48:48.551 INFO:tasks.rados.rados.0.smithi070.stdout:281: left oid 11 (ObjNum 111 snap 23 seq_num 111) 2024-09-14T03:48:48.551 INFO:tasks.rados.rados.0.smithi070.stdout:282: expect (ObjNum 43 snap 0 seq_num 43) 2024-09-14T03:48:48.575 INFO:tasks.rados.rados.0.smithi070.stdout:284: finishing write tid 1 to smithi07085154-47 2024-09-14T03:48:48.575 INFO:tasks.rados.rados.0.smithi070.stdout:284: finishing write tid 2 to smithi07085154-47 2024-09-14T03:48:48.575 INFO:tasks.rados.rados.0.smithi070.stdout:285: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-14T03:48:48.594 INFO:tasks.rados.rados.0.smithi070.stdout:286: expect (ObjNum 16 snap 0 seq_num 16) 2024-09-14T03:48:48.602 INFO:tasks.rados.rados.0.smithi070.stdout:284: finishing write tid 3 to smithi07085154-47 2024-09-14T03:48:48.602 INFO:tasks.rados.rados.0.smithi070.stdout:284: finishing write tid 4 to smithi07085154-47 2024-09-14T03:48:48.603 INFO:tasks.rados.rados.0.smithi070.stdout:284: finishing write tid 5 to smithi07085154-47 2024-09-14T03:48:48.603 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 47 v 41 (ObjNum 112 snap 23 seq_num 112) dirty exists 2024-09-14T03:48:48.603 INFO:tasks.rados.rados.0.smithi070.stdout:284: left oid 47 (ObjNum 112 snap 23 seq_num 112) 2024-09-14T03:48:48.603 INFO:tasks.rados.rados.0.smithi070.stdout:287: finishing write tid 1 to smithi07085154-22 2024-09-14T03:48:48.603 INFO:tasks.rados.rados.0.smithi070.stdout:287: finishing write tid 2 to smithi07085154-22 2024-09-14T03:48:48.603 INFO:tasks.rados.rados.0.smithi070.stdout:287: finishing write tid 3 to smithi07085154-22 2024-09-14T03:48:48.603 INFO:tasks.rados.rados.0.smithi070.stdout:287: finishing write tid 4 to smithi07085154-22 2024-09-14T03:48:48.603 INFO:tasks.rados.rados.0.smithi070.stdout:280: done (8 left) 2024-09-14T03:48:48.603 INFO:tasks.rados.rados.0.smithi070.stdout:281: done (7 left) 2024-09-14T03:48:48.603 INFO:tasks.rados.rados.0.smithi070.stdout:282: done (6 left) 2024-09-14T03:48:48.603 INFO:tasks.rados.rados.0.smithi070.stdout:284: done (5 left) 2024-09-14T03:48:48.603 INFO:tasks.rados.rados.0.smithi070.stdout:285: done (4 left) 2024-09-14T03:48:48.604 INFO:tasks.rados.rados.0.smithi070.stdout:286: done (3 left) 2024-09-14T03:48:48.604 INFO:tasks.rados.rados.0.smithi070.stdout:291: rmattr oid 46 current snap is 23 2024-09-14T03:48:48.604 INFO:tasks.rados.rados.0.smithi070.stdout:291: done (3 left) 2024-09-14T03:48:48.604 INFO:tasks.rados.rados.0.smithi070.stdout:292: rollback oid 18 current snap is 23 2024-09-14T03:48:48.604 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 18 to 16 2024-09-14T03:48:48.604 INFO:tasks.rados.rados.0.smithi070.stdout:287: finishing write tid 5 to smithi07085154-22 2024-09-14T03:48:48.604 INFO:tasks.rados.rados.0.smithi070.stdout:287: finishing write tid 6 to smithi07085154-22 2024-09-14T03:48:48.604 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 103 (ObjNum 113 snap 23 seq_num 113) dirty exists 2024-09-14T03:48:48.604 INFO:tasks.rados.rados.0.smithi070.stdout:287: left oid 22 (ObjNum 113 snap 23 seq_num 113) 2024-09-14T03:48:48.604 INFO:tasks.rados.rados.0.smithi070.stdout:290: finishing rollback tid 0 to smithi07085154-2 2024-09-14T03:48:48.604 INFO:tasks.rados.rados.0.smithi070.stdout:290: finishing rollback tid 1 to smithi07085154-2 2024-09-14T03:48:48.604 INFO:tasks.rados.rados.0.smithi070.stdout:290: finishing rollback tid 2 to smithi07085154-2 2024-09-14T03:48:48.605 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 2 v 139 (ObjNum 63 snap 3 seq_num 63) dirty exists 2024-09-14T03:48:48.605 INFO:tasks.rados.rados.0.smithi070.stdout:289: finishing rollback tid 0 to smithi07085154-24 2024-09-14T03:48:48.605 INFO:tasks.rados.rados.0.smithi070.stdout:289: finishing rollback tid 1 to smithi07085154-24 2024-09-14T03:48:48.605 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 24 (ObjNum 129 snap 0 seq_num 2929998728) dirty dne 2024-09-14T03:48:48.605 INFO:tasks.rados.rados.0.smithi070.stdout:287: done (3 left) 2024-09-14T03:48:48.605 INFO:tasks.rados.rados.0.smithi070.stdout:289: done (2 left) 2024-09-14T03:48:48.605 INFO:tasks.rados.rados.0.smithi070.stdout:290: done (1 left) 2024-09-14T03:48:48.605 INFO:tasks.rados.rados.0.smithi070.stdout:293: rmattr oid 47 current snap is 23 2024-09-14T03:48:48.605 INFO:tasks.rados.rados.0.smithi070.stdout:293: done (1 left) 2024-09-14T03:48:48.605 INFO:tasks.rados.rados.0.smithi070.stdout:294: copy_from oid 19 from oid 3 current snap is 23 2024-09-14T03:48:48.605 INFO:tasks.rados.rados.0.smithi070.stdout:295: rollback oid 38 current snap is 23 2024-09-14T03:48:48.605 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 38 to 13 2024-09-14T03:48:48.605 INFO:tasks.rados.rados.0.smithi070.stdout:296: snap_remove snap 14 2024-09-14T03:48:49.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:48 smithi094 ceph-mon[31053]: osdmap e113: 8 total, 8 up, 8 in 2024-09-14T03:48:49.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:48 smithi070 ceph-mon[39288]: osdmap e113: 8 total, 8 up, 8 in 2024-09-14T03:48:49.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:48 smithi070 ceph-mon[27495]: osdmap e113: 8 total, 8 up, 8 in 2024-09-14T03:48:49.553 INFO:tasks.rados.rados.0.smithi070.stdout:296: done (3 left) 2024-09-14T03:48:49.553 INFO:tasks.rados.rados.0.smithi070.stdout:297: snap_create 2024-09-14T03:48:49.553 INFO:tasks.rados.rados.0.smithi070.stdout:295: finishing rollback tid 0 to smithi07085154-38 2024-09-14T03:48:49.553 INFO:tasks.rados.rados.0.smithi070.stdout:295: finishing rollback tid 1 to smithi07085154-38 2024-09-14T03:48:49.554 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 38 v 50 (ObjNum 129 snap 0 seq_num 2930000160) dirty dne 2024-09-14T03:48:49.554 INFO:tasks.rados.rados.0.smithi070.stdout:292: finishing rollback tid 0 to smithi07085154-18 2024-09-14T03:48:49.554 INFO:tasks.rados.rados.0.smithi070.stdout:292: finishing rollback tid 1 to smithi07085154-18 2024-09-14T03:48:49.554 INFO:tasks.rados.rados.0.smithi070.stdout:292: finishing rollback tid 2 to smithi07085154-18 2024-09-14T03:48:49.554 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 92 (ObjNum 17 snap 0 seq_num 17) dirty exists 2024-09-14T03:48:49.554 INFO:tasks.rados.rados.0.smithi070.stdout:294: finishing copy_from racing read to smithi07085154-19 2024-09-14T03:48:49.554 INFO:tasks.rados.rados.0.smithi070.stdout:294: finishing copy_from to smithi07085154-19 2024-09-14T03:48:49.554 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 19 v 4 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-09-14T03:48:50.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:49 smithi094 ceph-mon[31053]: osdmap e114: 8 total, 8 up, 8 in 2024-09-14T03:48:50.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:49 smithi094 ceph-mon[31053]: pgmap v167: 121 pgs: 121 active+clean; 74 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 2.6 MiB/s wr, 39 op/s 2024-09-14T03:48:50.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:49 smithi070 ceph-mon[39288]: osdmap e114: 8 total, 8 up, 8 in 2024-09-14T03:48:50.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:49 smithi070 ceph-mon[39288]: pgmap v167: 121 pgs: 121 active+clean; 74 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 2.6 MiB/s wr, 39 op/s 2024-09-14T03:48:50.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:49 smithi070 ceph-mon[27495]: osdmap e114: 8 total, 8 up, 8 in 2024-09-14T03:48:50.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:49 smithi070 ceph-mon[27495]: pgmap v167: 121 pgs: 121 active+clean; 74 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 2.6 MiB/s wr, 39 op/s 2024-09-14T03:48:50.503 INFO:tasks.rados.rados.0.smithi070.stdout:292: done (3 left) 2024-09-14T03:48:50.503 INFO:tasks.rados.rados.0.smithi070.stdout:294: done (2 left) 2024-09-14T03:48:50.503 INFO:tasks.rados.rados.0.smithi070.stdout:295: done (1 left) 2024-09-14T03:48:50.503 INFO:tasks.rados.rados.0.smithi070.stdout:297: done (0 left) 2024-09-14T03:48:50.504 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 16 current snap is 24 2024-09-14T03:48:50.504 INFO:tasks.rados.rados.0.smithi070.stdout:298: seq_num 114 ranges {2867200=712704,3579904=712704,4292608=491520} 2024-09-14T03:48:50.522 INFO:tasks.rados.rados.0.smithi070.stdout:298: writing smithi07085154-16 from 2867200 to 3579904 tid 1 2024-09-14T03:48:50.526 INFO:tasks.rados.rados.0.smithi070.stdout:298: writing smithi07085154-16 from 3579904 to 4292608 tid 2 2024-09-14T03:48:50.530 INFO:tasks.rados.rados.0.smithi070.stdout:298: writing smithi07085154-16 from 4292608 to 4784128 tid 3 2024-09-14T03:48:50.530 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 18 current snap is 24 2024-09-14T03:48:50.530 INFO:tasks.rados.rados.0.smithi070.stdout:299: seq_num 115 ranges {286720=729088,1015808=647168,1662976=434176,2097152=122880} 2024-09-14T03:48:50.536 INFO:tasks.rados.rados.0.smithi070.stdout:299: writing smithi07085154-18 from 286720 to 1015808 tid 1 2024-09-14T03:48:50.540 INFO:tasks.rados.rados.0.smithi070.stdout:299: writing smithi07085154-18 from 1015808 to 1662976 tid 2 2024-09-14T03:48:50.543 INFO:tasks.rados.rados.0.smithi070.stdout:299: writing smithi07085154-18 from 1662976 to 2097152 tid 3 2024-09-14T03:48:50.544 INFO:tasks.rados.rados.0.smithi070.stdout:299: writing smithi07085154-18 from 2097152 to 2220032 tid 4 2024-09-14T03:48:50.544 INFO:tasks.rados.rados.0.smithi070.stdout:300: setattr oid 45 current snap is 24 2024-09-14T03:48:50.544 INFO:tasks.rados.rados.0.smithi070.stdout:298: finishing write tid 1 to smithi07085154-16 2024-09-14T03:48:50.544 INFO:tasks.rados.rados.0.smithi070.stdout:298: finishing write tid 2 to smithi07085154-16 2024-09-14T03:48:50.544 INFO:tasks.rados.rados.0.smithi070.stdout:298: finishing write tid 3 to smithi07085154-16 2024-09-14T03:48:50.544 INFO:tasks.rados.rados.0.smithi070.stdout:298: finishing write tid 4 to smithi07085154-16 2024-09-14T03:48:50.544 INFO:tasks.rados.rados.0.smithi070.stdout:298: finishing write tid 5 to smithi07085154-16 2024-09-14T03:48:50.544 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 158 (ObjNum 114 snap 24 seq_num 114) dirty exists 2024-09-14T03:48:50.545 INFO:tasks.rados.rados.0.smithi070.stdout:298: left oid 16 (ObjNum 114 snap 24 seq_num 114) 2024-09-14T03:48:50.545 INFO:tasks.rados.rados.0.smithi070.stdout:299: finishing write tid 1 to smithi07085154-18 2024-09-14T03:48:50.545 INFO:tasks.rados.rados.0.smithi070.stdout:299: finishing write tid 2 to smithi07085154-18 2024-09-14T03:48:50.546 INFO:tasks.rados.rados.0.smithi070.stdout:299: finishing write tid 3 to smithi07085154-18 2024-09-14T03:48:50.546 INFO:tasks.rados.rados.0.smithi070.stdout:299: finishing write tid 4 to smithi07085154-18 2024-09-14T03:48:50.546 INFO:tasks.rados.rados.0.smithi070.stdout:299: finishing write tid 5 to smithi07085154-18 2024-09-14T03:48:50.546 INFO:tasks.rados.rados.0.smithi070.stdout:298: done (2 left) 2024-09-14T03:48:50.546 INFO:tasks.rados.rados.0.smithi070.stdout:301: snap_remove snap 10 2024-09-14T03:48:51.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:50 smithi094 ceph-mon[31053]: osdmap e115: 8 total, 8 up, 8 in 2024-09-14T03:48:51.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:50 smithi094 ceph-mon[31053]: osdmap e116: 8 total, 8 up, 8 in 2024-09-14T03:48:51.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:50 smithi070 ceph-mon[39288]: osdmap e115: 8 total, 8 up, 8 in 2024-09-14T03:48:51.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:50 smithi070 ceph-mon[39288]: osdmap e116: 8 total, 8 up, 8 in 2024-09-14T03:48:51.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:50 smithi070 ceph-mon[27495]: osdmap e115: 8 total, 8 up, 8 in 2024-09-14T03:48:51.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:50 smithi070 ceph-mon[27495]: osdmap e116: 8 total, 8 up, 8 in 2024-09-14T03:48:51.508 INFO:tasks.rados.rados.0.smithi070.stdout:301: done (2 left) 2024-09-14T03:48:51.508 INFO:tasks.rados.rados.0.smithi070.stdout:302: setattr oid 15 current snap is 24 2024-09-14T03:48:51.509 INFO:tasks.rados.rados.0.smithi070.stdout:299: finishing write tid 6 to smithi07085154-18 2024-09-14T03:48:51.509 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 104 (ObjNum 115 snap 24 seq_num 115) dirty exists 2024-09-14T03:48:51.509 INFO:tasks.rados.rados.0.smithi070.stdout:299: left oid 18 (ObjNum 115 snap 24 seq_num 115) 2024-09-14T03:48:51.509 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 45 v 85 (ObjNum 100 snap 22 seq_num 100) dirty exists 2024-09-14T03:48:51.510 INFO:tasks.rados.rados.0.smithi070.stdout:299: done (2 left) 2024-09-14T03:48:51.510 INFO:tasks.rados.rados.0.smithi070.stdout:300: done (1 left) 2024-09-14T03:48:51.510 INFO:tasks.rados.rados.0.smithi070.stdout:303: snap_create 2024-09-14T03:48:51.517 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 15 v 105 (ObjNum 129 snap 0 seq_num 4215193477) dirty exists 2024-09-14T03:48:52.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:51 smithi094 ceph-mon[31053]: pgmap v170: 121 pgs: 121 active+clean; 74 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 2.6 MiB/s wr, 39 op/s 2024-09-14T03:48:52.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:51 smithi094 ceph-mon[31053]: osdmap e117: 8 total, 8 up, 8 in 2024-09-14T03:48:52.027 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:48:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:48:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:48:52.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:51 smithi070 ceph-mon[27495]: pgmap v170: 121 pgs: 121 active+clean; 74 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 2.6 MiB/s wr, 39 op/s 2024-09-14T03:48:52.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:51 smithi070 ceph-mon[27495]: osdmap e117: 8 total, 8 up, 8 in 2024-09-14T03:48:52.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:51 smithi070 ceph-mon[39288]: pgmap v170: 121 pgs: 121 active+clean; 74 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 2.6 MiB/s wr, 39 op/s 2024-09-14T03:48:52.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:51 smithi070 ceph-mon[39288]: osdmap e117: 8 total, 8 up, 8 in 2024-09-14T03:48:52.507 INFO:tasks.rados.rados.0.smithi070.stdout:302: done (1 left) 2024-09-14T03:48:52.507 INFO:tasks.rados.rados.0.smithi070.stdout:303: done (0 left) 2024-09-14T03:48:52.507 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 33 current snap is 25 2024-09-14T03:48:52.507 INFO:tasks.rados.rados.0.smithi070.stdout:304: seq_num 116 ranges {253952=745472,999424=532480,1531904=638976,2170880=32768} 2024-09-14T03:48:52.511 INFO:tasks.rados.rados.0.smithi070.stdout:304: writing smithi07085154-33 from 253952 to 999424 tid 1 2024-09-14T03:48:52.515 INFO:tasks.rados.rados.0.smithi070.stdout:304: writing smithi07085154-33 from 999424 to 1531904 tid 2 2024-09-14T03:48:52.519 INFO:tasks.rados.rados.0.smithi070.stdout:304: writing smithi07085154-33 from 1531904 to 2170880 tid 3 2024-09-14T03:48:52.519 INFO:tasks.rados.rados.0.smithi070.stdout:304: writing smithi07085154-33 from 2170880 to 2203648 tid 4 2024-09-14T03:48:52.519 INFO:tasks.rados.rados.0.smithi070.stdout:305: copy_from oid 18 from oid 13 current snap is 25 2024-09-14T03:48:52.519 INFO:tasks.rados.rados.0.smithi070.stdout:304: finishing write tid 1 to smithi07085154-33 2024-09-14T03:48:52.519 INFO:tasks.rados.rados.0.smithi070.stdout:304: finishing write tid 2 to smithi07085154-33 2024-09-14T03:48:52.519 INFO:tasks.rados.rados.0.smithi070.stdout:306: snap_create 2024-09-14T03:48:52.522 INFO:tasks.rados.rados.0.smithi070.stdout:304: finishing write tid 3 to smithi07085154-33 2024-09-14T03:48:52.522 INFO:tasks.rados.rados.0.smithi070.stdout:304: finishing write tid 4 to smithi07085154-33 2024-09-14T03:48:52.522 INFO:tasks.rados.rados.0.smithi070.stdout:304: finishing write tid 5 to smithi07085154-33 2024-09-14T03:48:52.524 INFO:tasks.rados.rados.0.smithi070.stdout:304: finishing write tid 6 to smithi07085154-33 2024-09-14T03:48:52.524 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 33 v 95 (ObjNum 116 snap 25 seq_num 116) dirty exists 2024-09-14T03:48:52.524 INFO:tasks.rados.rados.0.smithi070.stdout:304: left oid 33 (ObjNum 116 snap 25 seq_num 116) 2024-09-14T03:48:52.529 INFO:tasks.rados.rados.0.smithi070.stdout:305: finishing copy_from racing read to smithi07085154-18 2024-09-14T03:48:52.530 INFO:tasks.rados.rados.0.smithi070.stdout:305: finishing copy_from to smithi07085154-18 2024-09-14T03:48:52.530 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 49 (ObjNum 78 snap 13 seq_num 78) dirty exists 2024-09-14T03:48:52.936 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:52 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:48:52.937 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:52 smithi070 ceph-mon[39288]: osdmap e118: 8 total, 8 up, 8 in 2024-09-14T03:48:52.937 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:52 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:48:52.937 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:52 smithi070 ceph-mon[27495]: osdmap e118: 8 total, 8 up, 8 in 2024-09-14T03:48:53.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:52 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:48:53.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:52 smithi094 ceph-mon[31053]: osdmap e118: 8 total, 8 up, 8 in 2024-09-14T03:48:53.507 INFO:tasks.rados.rados.0.smithi070.stdout:304: done (2 left) 2024-09-14T03:48:53.508 INFO:tasks.rados.rados.0.smithi070.stdout:305: done (1 left) 2024-09-14T03:48:53.508 INFO:tasks.rados.rados.0.smithi070.stdout:306: done (0 left) 2024-09-14T03:48:53.508 INFO:tasks.rados.rados.0.smithi070.stdout:307: snap_create 2024-09-14T03:48:54.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:53 smithi094 ceph-mon[31053]: pgmap v173: 121 pgs: 121 active+clean; 97 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 5.7 MiB/s wr, 31 op/s 2024-09-14T03:48:54.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:53 smithi094 ceph-mon[31053]: osdmap e119: 8 total, 8 up, 8 in 2024-09-14T03:48:54.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:53 smithi070 ceph-mon[27495]: pgmap v173: 121 pgs: 121 active+clean; 97 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 5.7 MiB/s wr, 31 op/s 2024-09-14T03:48:54.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:53 smithi070 ceph-mon[27495]: osdmap e119: 8 total, 8 up, 8 in 2024-09-14T03:48:54.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:53 smithi070 ceph-mon[39288]: pgmap v173: 121 pgs: 121 active+clean; 97 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 5.7 MiB/s wr, 31 op/s 2024-09-14T03:48:54.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:53 smithi070 ceph-mon[39288]: osdmap e119: 8 total, 8 up, 8 in 2024-09-14T03:48:54.509 INFO:tasks.rados.rados.0.smithi070.stdout:307: done (0 left) 2024-09-14T03:48:54.558 INFO:tasks.rados.rados.0.smithi070.stdout:308: read oid 15 snap -1 2024-09-14T03:48:54.558 INFO:tasks.rados.rados.0.smithi070.stdout:308: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:48:54.558 INFO:tasks.rados.rados.0.smithi070.stdout:309: snap_create 2024-09-14T03:48:54.559 INFO:tasks.rados.rados.0.smithi070.stdout:308: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:48:55.504 INFO:tasks.rados.rados.0.smithi070.stdout:308: done (1 left) 2024-09-14T03:48:55.504 INFO:tasks.rados.rados.0.smithi070.stdout:309: done (0 left) 2024-09-14T03:48:55.505 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 23 current snap is 28 2024-09-14T03:48:55.505 INFO:tasks.rados.rados.0.smithi070.stdout:310: seq_num 117 ranges {1269760=770048,2039808=417792,2457600=442368,2899968=344064} 2024-09-14T03:48:55.516 INFO:tasks.rados.rados.0.smithi070.stdout:310: writing smithi07085154-23 from 1269760 to 2039808 tid 1 2024-09-14T03:48:55.518 INFO:tasks.rados.rados.0.smithi070.stdout:310: writing smithi07085154-23 from 2039808 to 2457600 tid 2 2024-09-14T03:48:55.521 INFO:tasks.rados.rados.0.smithi070.stdout:310: writing smithi07085154-23 from 2457600 to 2899968 tid 3 2024-09-14T03:48:55.523 INFO:tasks.rados.rados.0.smithi070.stdout:310: writing smithi07085154-23 from 2899968 to 3244032 tid 4 2024-09-14T03:48:55.523 INFO:tasks.rados.rados.0.smithi070.stdout:311: rollback oid 9 current snap is 28 2024-09-14T03:48:55.523 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 9 to 22 2024-09-14T03:48:55.523 INFO:tasks.rados.rados.0.smithi070.stdout:310: finishing write tid 1 to smithi07085154-23 2024-09-14T03:48:55.524 INFO:tasks.rados.rados.0.smithi070.stdout:310: finishing write tid 2 to smithi07085154-23 2024-09-14T03:48:55.524 INFO:tasks.rados.rados.0.smithi070.stdout:312: snap_remove snap 17 2024-09-14T03:48:55.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:55 smithi094 ceph-mon[31053]: osdmap e120: 8 total, 8 up, 8 in 2024-09-14T03:48:55.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:55 smithi094 ceph-mon[31053]: pgmap v176: 121 pgs: 121 active+clean; 97 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 5.7 MiB/s wr, 31 op/s 2024-09-14T03:48:55.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:55 smithi094 ceph-mon[31053]: osdmap e121: 8 total, 8 up, 8 in 2024-09-14T03:48:55.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:55 smithi070 ceph-mon[27495]: osdmap e120: 8 total, 8 up, 8 in 2024-09-14T03:48:55.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:55 smithi070 ceph-mon[27495]: pgmap v176: 121 pgs: 121 active+clean; 97 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 5.7 MiB/s wr, 31 op/s 2024-09-14T03:48:55.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:55 smithi070 ceph-mon[27495]: osdmap e121: 8 total, 8 up, 8 in 2024-09-14T03:48:55.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:55 smithi070 ceph-mon[39288]: osdmap e120: 8 total, 8 up, 8 in 2024-09-14T03:48:55.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:55 smithi070 ceph-mon[39288]: pgmap v176: 121 pgs: 121 active+clean; 97 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 5.7 MiB/s wr, 31 op/s 2024-09-14T03:48:55.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:55 smithi070 ceph-mon[39288]: osdmap e121: 8 total, 8 up, 8 in 2024-09-14T03:48:56.516 INFO:tasks.rados.rados.0.smithi070.stdout:312: done (2 left) 2024-09-14T03:48:56.516 INFO:tasks.rados.rados.0.smithi070.stdout:313: read oid 18 snap 22 2024-09-14T03:48:56.516 INFO:tasks.rados.rados.0.smithi070.stdout:313: expect (ObjNum 17 snap 0 seq_num 17) 2024-09-14T03:48:56.517 INFO:tasks.rados.rados.0.smithi070.stdout:310: finishing write tid 3 to smithi07085154-23 2024-09-14T03:48:56.517 INFO:tasks.rados.rados.0.smithi070.stdout:310: finishing write tid 4 to smithi07085154-23 2024-09-14T03:48:56.517 INFO:tasks.rados.rados.0.smithi070.stdout:311: finishing rollback tid 0 to smithi07085154-9 2024-09-14T03:48:56.517 INFO:tasks.rados.rados.0.smithi070.stdout:310: finishing write tid 5 to smithi07085154-23 2024-09-14T03:48:56.517 INFO:tasks.rados.rados.0.smithi070.stdout:311: finishing rollback tid 1 to smithi07085154-9 2024-09-14T03:48:56.517 INFO:tasks.rados.rados.0.smithi070.stdout:311: finishing rollback tid 2 to smithi07085154-9 2024-09-14T03:48:56.517 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 170 (ObjNum 84 snap 17 seq_num 84) dirty exists 2024-09-14T03:48:56.517 INFO:tasks.rados.rados.0.smithi070.stdout:310: finishing write tid 6 to smithi07085154-23 2024-09-14T03:48:56.517 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 30 (ObjNum 117 snap 28 seq_num 117) dirty exists 2024-09-14T03:48:56.517 INFO:tasks.rados.rados.0.smithi070.stdout:310: left oid 23 (ObjNum 117 snap 28 seq_num 117) 2024-09-14T03:48:56.517 INFO:tasks.rados.rados.0.smithi070.stdout:310: done (2 left) 2024-09-14T03:48:56.517 INFO:tasks.rados.rados.0.smithi070.stdout:311: done (1 left) 2024-09-14T03:48:56.517 INFO:tasks.rados.rados.0.smithi070.stdout:314: snap_create 2024-09-14T03:48:56.526 INFO:tasks.rados.rados.0.smithi070.stdout:313: expect (ObjNum 17 snap 0 seq_num 17) 2024-09-14T03:48:56.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:48:56 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:48:56] "GET /metrics HTTP/1.1" 200 240010 "" "Prometheus/2.43.0" 2024-09-14T03:48:57.521 INFO:tasks.rados.rados.0.smithi070.stdout:313: done (1 left) 2024-09-14T03:48:57.522 INFO:tasks.rados.rados.0.smithi070.stdout:314: done (0 left) 2024-09-14T03:48:57.522 INFO:tasks.rados.rados.0.smithi070.stdout:315: read oid 23 snap -1 2024-09-14T03:48:57.522 INFO:tasks.rados.rados.0.smithi070.stdout:315: expect (ObjNum 117 snap 28 seq_num 117) 2024-09-14T03:48:57.522 INFO:tasks.rados.rados.0.smithi070.stdout:316: snap_remove snap 24 2024-09-14T03:48:57.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:57 smithi094 ceph-mon[31053]: osdmap e122: 8 total, 8 up, 8 in 2024-09-14T03:48:57.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:57 smithi094 ceph-mon[31053]: pgmap v179: 121 pgs: 121 active+clean; 97 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:48:57.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:57 smithi070 ceph-mon[27495]: osdmap e122: 8 total, 8 up, 8 in 2024-09-14T03:48:57.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:57 smithi070 ceph-mon[27495]: pgmap v179: 121 pgs: 121 active+clean; 97 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:48:57.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:57 smithi070 ceph-mon[39288]: osdmap e122: 8 total, 8 up, 8 in 2024-09-14T03:48:57.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:57 smithi070 ceph-mon[39288]: pgmap v179: 121 pgs: 121 active+clean; 97 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:48:58.523 INFO:tasks.rados.rados.0.smithi070.stdout:316: done (1 left) 2024-09-14T03:48:58.523 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 45 current snap is 29 2024-09-14T03:48:58.524 INFO:tasks.rados.rados.0.smithi070.stdout:317: seq_num 118 ranges {3473408=786432,4259840=704512,4964352=499712} 2024-09-14T03:48:58.546 INFO:tasks.rados.rados.0.smithi070.stdout:317: writing smithi07085154-45 from 3473408 to 4259840 tid 1 2024-09-14T03:48:58.550 INFO:tasks.rados.rados.0.smithi070.stdout:317: writing smithi07085154-45 from 4259840 to 4964352 tid 2 2024-09-14T03:48:58.553 INFO:tasks.rados.rados.0.smithi070.stdout:317: writing smithi07085154-45 from 4964352 to 5464064 tid 3 2024-09-14T03:48:58.553 INFO:tasks.rados.rados.0.smithi070.stdout:318: rollback oid 35 current snap is 29 2024-09-14T03:48:58.554 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 35 to 23 2024-09-14T03:48:58.554 INFO:tasks.rados.rados.0.smithi070.stdout:315: expect (ObjNum 117 snap 28 seq_num 117) 2024-09-14T03:48:58.663 INFO:tasks.rados.rados.0.smithi070.stdout:317: finishing write tid 1 to smithi07085154-45 2024-09-14T03:48:58.663 INFO:tasks.rados.rados.0.smithi070.stdout:317: finishing write tid 2 to smithi07085154-45 2024-09-14T03:48:58.663 INFO:tasks.rados.rados.0.smithi070.stdout:315: done (2 left) 2024-09-14T03:48:58.664 INFO:tasks.rados.rados.0.smithi070.stdout:319: snap_create 2024-09-14T03:48:58.664 INFO:tasks.rados.rados.0.smithi070.stdout:317: finishing write tid 3 to smithi07085154-45 2024-09-14T03:48:58.664 INFO:tasks.rados.rados.0.smithi070.stdout:317: finishing write tid 4 to smithi07085154-45 2024-09-14T03:48:58.664 INFO:tasks.rados.rados.0.smithi070.stdout:317: finishing write tid 5 to smithi07085154-45 2024-09-14T03:48:58.664 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 45 v 108 (ObjNum 118 snap 29 seq_num 118) dirty exists 2024-09-14T03:48:58.664 INFO:tasks.rados.rados.0.smithi070.stdout:317: left oid 45 (ObjNum 118 snap 29 seq_num 118) 2024-09-14T03:48:58.664 INFO:tasks.rados.rados.0.smithi070.stdout:318: finishing rollback tid 1 to smithi07085154-35 2024-09-14T03:48:58.664 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 35 v 4 (ObjNum 129 snap 0 seq_num 2930186856) dirty dne 2024-09-14T03:48:58.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:58 smithi094 ceph-mon[31053]: osdmap e123: 8 total, 8 up, 8 in 2024-09-14T03:48:58.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:58 smithi070 ceph-mon[27495]: osdmap e123: 8 total, 8 up, 8 in 2024-09-14T03:48:58.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:58 smithi070 ceph-mon[39288]: osdmap e123: 8 total, 8 up, 8 in 2024-09-14T03:48:59.532 INFO:tasks.rados.rados.0.smithi070.stdout:317: done (2 left) 2024-09-14T03:48:59.533 INFO:tasks.rados.rados.0.smithi070.stdout:318: done (1 left) 2024-09-14T03:48:59.533 INFO:tasks.rados.rados.0.smithi070.stdout:319: done (0 left) 2024-09-14T03:48:59.533 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 18 current snap is 30 2024-09-14T03:48:59.533 INFO:tasks.rados.rados.0.smithi070.stdout:320: seq_num 119 ranges {1523712=802816,2326528=589824,2916352=409600,3325952=204800} 2024-09-14T03:48:59.545 INFO:tasks.rados.rados.0.smithi070.stdout:320: writing smithi07085154-18 from 1523712 to 2326528 tid 1 2024-09-14T03:48:59.548 INFO:tasks.rados.rados.0.smithi070.stdout:320: writing smithi07085154-18 from 2326528 to 2916352 tid 2 2024-09-14T03:48:59.551 INFO:tasks.rados.rados.0.smithi070.stdout:320: writing smithi07085154-18 from 2916352 to 3325952 tid 3 2024-09-14T03:48:59.553 INFO:tasks.rados.rados.0.smithi070.stdout:320: writing smithi07085154-18 from 3325952 to 3530752 tid 4 2024-09-14T03:48:59.553 INFO:tasks.rados.rados.0.smithi070.stdout:321: read oid 31 snap -1 2024-09-14T03:48:59.553 INFO:tasks.rados.rados.0.smithi070.stdout:321: expect (ObjNum 110 snap 23 seq_num 110) 2024-09-14T03:48:59.553 INFO:tasks.rados.rados.0.smithi070.stdout:322: read oid 5 snap -1 2024-09-14T03:48:59.553 INFO:tasks.rados.rados.0.smithi070.stdout:322: expect deleted 2024-09-14T03:48:59.553 INFO:tasks.rados.rados.0.smithi070.stdout:323: copy_from oid 21 from oid 32 current snap is 30 2024-09-14T03:48:59.553 INFO:tasks.rados.rados.0.smithi070.stdout:320: finishing write tid 1 to smithi07085154-18 2024-09-14T03:48:59.553 INFO:tasks.rados.rados.0.smithi070.stdout:324: snap_create 2024-09-14T03:48:59.553 INFO:tasks.rados.rados.0.smithi070.stdout:320: finishing write tid 2 to smithi07085154-18 2024-09-14T03:48:59.555 INFO:tasks.rados.rados.0.smithi070.stdout:320: finishing write tid 3 to smithi07085154-18 2024-09-14T03:48:59.556 INFO:tasks.rados.rados.0.smithi070.stdout:320: finishing write tid 4 to smithi07085154-18 2024-09-14T03:48:59.557 INFO:tasks.rados.rados.0.smithi070.stdout:320: finishing write tid 5 to smithi07085154-18 2024-09-14T03:48:59.558 INFO:tasks.rados.rados.0.smithi070.stdout:320: finishing write tid 6 to smithi07085154-18 2024-09-14T03:48:59.558 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 131 (ObjNum 119 snap 30 seq_num 119) dirty exists 2024-09-14T03:48:59.558 INFO:tasks.rados.rados.0.smithi070.stdout:320: left oid 18 (ObjNum 119 snap 30 seq_num 119) 2024-09-14T03:48:59.563 INFO:tasks.rados.rados.0.smithi070.stdout:321: expect (ObjNum 110 snap 23 seq_num 110) 2024-09-14T03:48:59.611 INFO:tasks.rados.rados.0.smithi070.stdout:323: finishing copy_from racing read to smithi07085154-21 2024-09-14T03:48:59.612 INFO:tasks.rados.rados.0.smithi070.stdout:323: finishing copy_from to smithi07085154-21 2024-09-14T03:48:59.612 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 59 (ObjNum 107 snap 23 seq_num 107) dirty exists 2024-09-14T03:48:59.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:59 smithi070 ceph-mon[27495]: osdmap e124: 8 total, 8 up, 8 in 2024-09-14T03:48:59.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:48:59 smithi070 ceph-mon[27495]: pgmap v182: 121 pgs: 121 active+clean; 100 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 482 KiB/s wr, 4 op/s 2024-09-14T03:48:59.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:59 smithi070 ceph-mon[39288]: osdmap e124: 8 total, 8 up, 8 in 2024-09-14T03:48:59.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:48:59 smithi070 ceph-mon[39288]: pgmap v182: 121 pgs: 121 active+clean; 100 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 482 KiB/s wr, 4 op/s 2024-09-14T03:49:00.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:59 smithi094 ceph-mon[31053]: osdmap e124: 8 total, 8 up, 8 in 2024-09-14T03:49:00.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:48:59 smithi094 ceph-mon[31053]: pgmap v182: 121 pgs: 121 active+clean; 100 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 482 KiB/s wr, 4 op/s 2024-09-14T03:49:00.506 INFO:tasks.rados.rados.0.smithi070.stdout:320: done (4 left) 2024-09-14T03:49:00.506 INFO:tasks.rados.rados.0.smithi070.stdout:321: done (3 left) 2024-09-14T03:49:00.506 INFO:tasks.rados.rados.0.smithi070.stdout:322: done (2 left) 2024-09-14T03:49:00.506 INFO:tasks.rados.rados.0.smithi070.stdout:323: done (1 left) 2024-09-14T03:49:00.506 INFO:tasks.rados.rados.0.smithi070.stdout:324: done (0 left) 2024-09-14T03:49:00.506 INFO:tasks.rados.rados.0.smithi070.stdout:325: read oid 41 snap -1 2024-09-14T03:49:00.507 INFO:tasks.rados.rados.0.smithi070.stdout:325: expect (ObjNum 82 snap 16 seq_num 82) 2024-09-14T03:49:00.507 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 16 current snap is 31 2024-09-14T03:49:00.507 INFO:tasks.rados.rados.0.smithi070.stdout:326: seq_num 120 ranges {4784128=417792,5201920=475136,5677056=622592,6299648=507904} 2024-09-14T03:49:00.534 INFO:tasks.rados.rados.0.smithi070.stdout:326: writing smithi07085154-16 from 4784128 to 5201920 tid 1 2024-09-14T03:49:00.537 INFO:tasks.rados.rados.0.smithi070.stdout:326: writing smithi07085154-16 from 5201920 to 5677056 tid 2 2024-09-14T03:49:00.541 INFO:tasks.rados.rados.0.smithi070.stdout:326: writing smithi07085154-16 from 5677056 to 6299648 tid 3 2024-09-14T03:49:00.544 INFO:tasks.rados.rados.0.smithi070.stdout:326: writing smithi07085154-16 from 6299648 to 6807552 tid 4 2024-09-14T03:49:00.544 INFO:tasks.rados.rados.0.smithi070.stdout:327: setattr oid 13 current snap is 31 2024-09-14T03:49:00.545 INFO:tasks.rados.rados.0.smithi070.stdout:325: expect (ObjNum 82 snap 16 seq_num 82) 2024-09-14T03:49:00.661 INFO:tasks.rados.rados.0.smithi070.stdout:326: finishing write tid 1 to smithi07085154-16 2024-09-14T03:49:00.661 INFO:tasks.rados.rados.0.smithi070.stdout:326: finishing write tid 2 to smithi07085154-16 2024-09-14T03:49:00.661 INFO:tasks.rados.rados.0.smithi070.stdout:326: finishing write tid 3 to smithi07085154-16 2024-09-14T03:49:00.661 INFO:tasks.rados.rados.0.smithi070.stdout:326: finishing write tid 4 to smithi07085154-16 2024-09-14T03:49:00.661 INFO:tasks.rados.rados.0.smithi070.stdout:326: finishing write tid 5 to smithi07085154-16 2024-09-14T03:49:00.661 INFO:tasks.rados.rados.0.smithi070.stdout:326: finishing write tid 6 to smithi07085154-16 2024-09-14T03:49:00.661 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 186 (ObjNum 120 snap 31 seq_num 120) dirty exists 2024-09-14T03:49:00.661 INFO:tasks.rados.rados.0.smithi070.stdout:326: left oid 16 (ObjNum 120 snap 31 seq_num 120) 2024-09-14T03:49:00.661 INFO:tasks.rados.rados.0.smithi070.stdout:325: done (2 left) 2024-09-14T03:49:00.661 INFO:tasks.rados.rados.0.smithi070.stdout:326: done (1 left) 2024-09-14T03:49:00.661 INFO:tasks.rados.rados.0.smithi070.stdout:328: delete oid 31 current snap is 31 2024-09-14T03:49:00.664 INFO:tasks.rados.rados.0.smithi070.stdout:328: done (1 left) 2024-09-14T03:49:00.665 INFO:tasks.rados.rados.0.smithi070.stdout:329: read oid 21 snap -1 2024-09-14T03:49:00.665 INFO:tasks.rados.rados.0.smithi070.stdout:329: expect (ObjNum 107 snap 23 seq_num 107) 2024-09-14T03:49:00.665 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 91 (ObjNum 78 snap 13 seq_num 78) dirty exists 2024-09-14T03:49:00.665 INFO:tasks.rados.rados.0.smithi070.stdout:327: done (1 left) 2024-09-14T03:49:00.665 INFO:tasks.rados.rados.0.smithi070.stdout:330: copy_from oid 13 from oid 14 current snap is 31 2024-09-14T03:49:00.665 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 6 current snap is 31 2024-09-14T03:49:00.665 INFO:tasks.rados.rados.0.smithi070.stdout:331: seq_num 122 ranges {606208=450560,1056768=688128,1744896=630784,2375680=286720} 2024-09-14T03:49:00.671 INFO:tasks.rados.rados.0.smithi070.stdout:331: writing smithi07085154-6 from 606208 to 1056768 tid 1 2024-09-14T03:49:00.676 INFO:tasks.rados.rados.0.smithi070.stdout:331: writing smithi07085154-6 from 1056768 to 1744896 tid 2 2024-09-14T03:49:00.679 INFO:tasks.rados.rados.0.smithi070.stdout:331: writing smithi07085154-6 from 1744896 to 2375680 tid 3 2024-09-14T03:49:00.682 INFO:tasks.rados.rados.0.smithi070.stdout:331: writing smithi07085154-6 from 2375680 to 2662400 tid 4 2024-09-14T03:49:00.683 INFO:tasks.rados.rados.0.smithi070.stdout:332: read oid 37 snap -1 2024-09-14T03:49:00.683 INFO:tasks.rados.rados.0.smithi070.stdout:332: expect (ObjNum 86 snap 17 seq_num 86) 2024-09-14T03:49:00.683 INFO:tasks.rados.rados.0.smithi070.stdout:333: read oid 50 snap -1 2024-09-14T03:49:00.683 INFO:tasks.rados.rados.0.smithi070.stdout:333: expect (ObjNum 49 snap 0 seq_num 49) 2024-09-14T03:49:00.683 INFO:tasks.rados.rados.0.smithi070.stdout:334: copy_from oid 43 from oid 18 current snap is 31 2024-09-14T03:49:00.683 INFO:tasks.rados.rados.0.smithi070.stdout:330: finishing copy_from racing read to smithi07085154-13 2024-09-14T03:49:00.683 INFO:tasks.rados.rados.0.smithi070.stdout:330: finishing copy_from to smithi07085154-13 2024-09-14T03:49:00.683 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 2 (ObjNum 13 snap 0 seq_num 13) dirty exists 2024-09-14T03:49:00.683 INFO:tasks.rados.rados.0.smithi070.stdout:331: finishing write tid 1 to smithi07085154-6 2024-09-14T03:49:00.683 INFO:tasks.rados.rados.0.smithi070.stdout:331: finishing write tid 2 to smithi07085154-6 2024-09-14T03:49:00.684 INFO:tasks.rados.rados.0.smithi070.stdout:329: expect (ObjNum 107 snap 23 seq_num 107) 2024-09-14T03:49:00.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:00 smithi070 ceph-mon[39288]: osdmap e125: 8 total, 8 up, 8 in 2024-09-14T03:49:00.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:00 smithi070 ceph-mon[39288]: osdmap e126: 8 total, 8 up, 8 in 2024-09-14T03:49:00.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:00 smithi070 ceph-mon[27495]: osdmap e125: 8 total, 8 up, 8 in 2024-09-14T03:49:00.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:00 smithi070 ceph-mon[27495]: osdmap e126: 8 total, 8 up, 8 in 2024-09-14T03:49:00.876 INFO:tasks.rados.rados.0.smithi070.stdout:331: finishing write tid 3 to smithi07085154-6 2024-09-14T03:49:00.876 INFO:tasks.rados.rados.0.smithi070.stdout:331: finishing write tid 4 to smithi07085154-6 2024-09-14T03:49:00.876 INFO:tasks.rados.rados.0.smithi070.stdout:331: finishing write tid 5 to smithi07085154-6 2024-09-14T03:49:00.876 INFO:tasks.rados.rados.0.smithi070.stdout:331: finishing write tid 6 to smithi07085154-6 2024-09-14T03:49:00.877 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 131 (ObjNum 122 snap 31 seq_num 122) dirty exists 2024-09-14T03:49:00.877 INFO:tasks.rados.rados.0.smithi070.stdout:331: left oid 6 (ObjNum 122 snap 31 seq_num 122) 2024-09-14T03:49:00.877 INFO:tasks.rados.rados.0.smithi070.stdout:332: expect (ObjNum 86 snap 17 seq_num 86) 2024-09-14T03:49:00.918 INFO:tasks.rados.rados.0.smithi070.stdout:333: expect (ObjNum 49 snap 0 seq_num 49) 2024-09-14T03:49:00.939 INFO:tasks.rados.rados.0.smithi070.stdout:334: finishing copy_from racing read to smithi07085154-43 2024-09-14T03:49:00.939 INFO:tasks.rados.rados.0.smithi070.stdout:334: finishing copy_from to smithi07085154-43 2024-09-14T03:49:00.939 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 131 (ObjNum 119 snap 30 seq_num 119) dirty exists 2024-09-14T03:49:00.939 INFO:tasks.rados.rados.0.smithi070.stdout:329: done (5 left) 2024-09-14T03:49:00.940 INFO:tasks.rados.rados.0.smithi070.stdout:330: done (4 left) 2024-09-14T03:49:00.940 INFO:tasks.rados.rados.0.smithi070.stdout:331: done (3 left) 2024-09-14T03:49:00.941 INFO:tasks.rados.rados.0.smithi070.stdout:332: done (2 left) 2024-09-14T03:49:00.941 INFO:tasks.rados.rados.0.smithi070.stdout:333: done (1 left) 2024-09-14T03:49:00.941 INFO:tasks.rados.rados.0.smithi070.stdout:334: done (0 left) 2024-09-14T03:49:00.941 INFO:tasks.rados.rados.0.smithi070.stdout:335: read oid 25 snap -1 2024-09-14T03:49:00.941 INFO:tasks.rados.rados.0.smithi070.stdout:335: expect (ObjNum 109 snap 23 seq_num 109) 2024-09-14T03:49:00.941 INFO:tasks.rados.rados.0.smithi070.stdout:336: read oid 13 snap 23 2024-09-14T03:49:00.941 INFO:tasks.rados.rados.0.smithi070.stdout:336: expect (ObjNum 78 snap 13 seq_num 78) 2024-09-14T03:49:00.941 INFO:tasks.rados.rados.0.smithi070.stdout:337: read oid 16 snap -1 2024-09-14T03:49:00.941 INFO:tasks.rados.rados.0.smithi070.stdout:337: expect (ObjNum 120 snap 31 seq_num 120) 2024-09-14T03:49:00.941 INFO:tasks.rados.rados.0.smithi070.stdout:338: copy_from oid 48 from oid 5 current snap is 31 2024-09-14T03:49:00.941 INFO:tasks.rados.rados.0.smithi070.stdout:339: snap_create 2024-09-14T03:49:00.941 INFO:tasks.rados.rados.0.smithi070.stdout:338: finishing copy_from to smithi07085154-48 2024-09-14T03:49:00.942 INFO:tasks.rados.rados.0.smithi070.stdout:338: got expected ENOENT (src dne) 2024-09-14T03:49:00.942 INFO:tasks.rados.rados.0.smithi070.stdout:338: finishing copy_from racing read to smithi07085154-48 2024-09-14T03:49:00.944 INFO:tasks.rados.rados.0.smithi070.stdout:336: expect (ObjNum 78 snap 13 seq_num 78) 2024-09-14T03:49:00.991 INFO:tasks.rados.rados.0.smithi070.stdout:337: expect (ObjNum 120 snap 31 seq_num 120) 2024-09-14T03:49:01.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:00 smithi094 ceph-mon[31053]: osdmap e125: 8 total, 8 up, 8 in 2024-09-14T03:49:01.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:00 smithi094 ceph-mon[31053]: osdmap e126: 8 total, 8 up, 8 in 2024-09-14T03:49:01.310 INFO:tasks.rados.rados.0.smithi070.stdout:335: expect (ObjNum 109 snap 23 seq_num 109) 2024-09-14T03:49:01.564 INFO:tasks.rados.rados.0.smithi070.stdout:335: done (4 left) 2024-09-14T03:49:01.565 INFO:tasks.rados.rados.0.smithi070.stdout:336: done (3 left) 2024-09-14T03:49:01.565 INFO:tasks.rados.rados.0.smithi070.stdout:337: done (2 left) 2024-09-14T03:49:01.565 INFO:tasks.rados.rados.0.smithi070.stdout:338: done (1 left) 2024-09-14T03:49:01.565 INFO:tasks.rados.rados.0.smithi070.stdout:339: done (0 left) 2024-09-14T03:49:01.565 INFO:tasks.rados.rados.0.smithi070.stdout:340: delete oid 32 current snap is 32 2024-09-14T03:49:01.567 INFO:tasks.rados.rados.0.smithi070.stdout:340: done (0 left) 2024-09-14T03:49:01.567 INFO:tasks.rados.rados.0.smithi070.stdout:341: delete oid 20 current snap is 32 2024-09-14T03:49:01.570 INFO:tasks.rados.rados.0.smithi070.stdout:341: done (0 left) 2024-09-14T03:49:01.570 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 42 current snap is 32 2024-09-14T03:49:01.570 INFO:tasks.rados.rados.0.smithi070.stdout:342: seq_num 125 ranges {638976=499712,1138688=745472,1884160=409600,2293760=450560} 2024-09-14T03:49:01.576 INFO:tasks.rados.rados.0.smithi070.stdout:342: writing smithi07085154-42 from 638976 to 1138688 tid 1 2024-09-14T03:49:01.580 INFO:tasks.rados.rados.0.smithi070.stdout:342: writing smithi07085154-42 from 1138688 to 1884160 tid 2 2024-09-14T03:49:01.583 INFO:tasks.rados.rados.0.smithi070.stdout:342: writing smithi07085154-42 from 1884160 to 2293760 tid 3 2024-09-14T03:49:01.586 INFO:tasks.rados.rados.0.smithi070.stdout:342: writing smithi07085154-42 from 2293760 to 2744320 tid 4 2024-09-14T03:49:01.586 INFO:tasks.rados.rados.0.smithi070.stdout:343: snap_create 2024-09-14T03:49:01.586 INFO:tasks.rados.rados.0.smithi070.stdout:342: finishing write tid 1 to smithi07085154-42 2024-09-14T03:49:01.586 INFO:tasks.rados.rados.0.smithi070.stdout:342: finishing write tid 2 to smithi07085154-42 2024-09-14T03:49:01.586 INFO:tasks.rados.rados.0.smithi070.stdout:342: finishing write tid 3 to smithi07085154-42 2024-09-14T03:49:01.588 INFO:tasks.rados.rados.0.smithi070.stdout:342: finishing write tid 4 to smithi07085154-42 2024-09-14T03:49:01.588 INFO:tasks.rados.rados.0.smithi070.stdout:342: finishing write tid 5 to smithi07085154-42 2024-09-14T03:49:01.589 INFO:tasks.rados.rados.0.smithi070.stdout:342: finishing write tid 6 to smithi07085154-42 2024-09-14T03:49:01.589 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 192 (ObjNum 125 snap 32 seq_num 125) dirty exists 2024-09-14T03:49:01.589 INFO:tasks.rados.rados.0.smithi070.stdout:342: left oid 42 (ObjNum 125 snap 32 seq_num 125) 2024-09-14T03:49:01.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:01 smithi070 ceph-mon[27495]: pgmap v185: 121 pgs: 121 active+clean; 100 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 482 KiB/s wr, 4 op/s 2024-09-14T03:49:01.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:01 smithi070 ceph-mon[39288]: pgmap v185: 121 pgs: 121 active+clean; 100 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 482 KiB/s wr, 4 op/s 2024-09-14T03:49:02.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:01 smithi094 ceph-mon[31053]: pgmap v185: 121 pgs: 121 active+clean; 100 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 482 KiB/s wr, 4 op/s 2024-09-14T03:49:02.027 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:01 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:49:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:49:02.542 INFO:tasks.rados.rados.0.smithi070.stdout:342: done (1 left) 2024-09-14T03:49:02.542 INFO:tasks.rados.rados.0.smithi070.stdout:343: done (0 left) 2024-09-14T03:49:02.542 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 5 current snap is 33 2024-09-14T03:49:02.542 INFO:tasks.rados.rados.0.smithi070.stdout:344: seq_num 126 ranges {0=516096,516096=630784,1146880=614400,1761280=360448} 2024-09-14T03:49:02.544 INFO:tasks.rados.rados.0.smithi070.stdout:344: writing smithi07085154-5 from 0 to 516096 tid 1 2024-09-14T03:49:02.549 INFO:tasks.rados.rados.0.smithi070.stdout:344: writing smithi07085154-5 from 516096 to 1146880 tid 2 2024-09-14T03:49:02.553 INFO:tasks.rados.rados.0.smithi070.stdout:344: writing smithi07085154-5 from 1146880 to 1761280 tid 3 2024-09-14T03:49:02.556 INFO:tasks.rados.rados.0.smithi070.stdout:344: writing smithi07085154-5 from 1761280 to 2121728 tid 4 2024-09-14T03:49:02.556 INFO:tasks.rados.rados.0.smithi070.stdout:345: rollback oid 31 current snap is 33 2024-09-14T03:49:02.556 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 31 to 22 2024-09-14T03:49:02.556 INFO:tasks.rados.rados.0.smithi070.stdout:344: finishing write tid 1 to smithi07085154-5 2024-09-14T03:49:02.556 INFO:tasks.rados.rados.0.smithi070.stdout:344: finishing write tid 2 to smithi07085154-5 2024-09-14T03:49:02.556 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 49 current snap is 33 2024-09-14T03:49:02.556 INFO:tasks.rados.rados.0.smithi070.stdout:346: seq_num 127 ranges {4128768=532480,4661248=507904,5169152=778240,5947392=319488} 2024-09-14T03:49:02.581 INFO:tasks.rados.rados.0.smithi070.stdout:346: writing smithi07085154-49 from 4128768 to 4661248 tid 1 2024-09-14T03:49:02.584 INFO:tasks.rados.rados.0.smithi070.stdout:346: writing smithi07085154-49 from 4661248 to 5169152 tid 2 2024-09-14T03:49:02.589 INFO:tasks.rados.rados.0.smithi070.stdout:346: writing smithi07085154-49 from 5169152 to 5947392 tid 3 2024-09-14T03:49:02.591 INFO:tasks.rados.rados.0.smithi070.stdout:346: writing smithi07085154-49 from 5947392 to 6266880 tid 4 2024-09-14T03:49:02.591 INFO:tasks.rados.rados.0.smithi070.stdout:347: rmattr oid 41 current snap is 33 2024-09-14T03:49:02.591 INFO:tasks.rados.rados.0.smithi070.stdout:347: done (3 left) 2024-09-14T03:49:02.592 INFO:tasks.rados.rados.0.smithi070.stdout:348: read oid 7 snap -1 2024-09-14T03:49:02.592 INFO:tasks.rados.rados.0.smithi070.stdout:348: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-14T03:49:02.592 INFO:tasks.rados.rados.0.smithi070.stdout:349: snap_create 2024-09-14T03:49:02.592 INFO:tasks.rados.rados.0.smithi070.stdout:344: finishing write tid 3 to smithi07085154-5 2024-09-14T03:49:02.592 INFO:tasks.rados.rados.0.smithi070.stdout:345: finishing rollback tid 1 to smithi07085154-31 2024-09-14T03:49:02.592 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 125 (ObjNum 33 snap 0 seq_num 2929761056) dirty dne 2024-09-14T03:49:02.592 INFO:tasks.rados.rados.0.smithi070.stdout:344: finishing write tid 4 to smithi07085154-5 2024-09-14T03:49:02.592 INFO:tasks.rados.rados.0.smithi070.stdout:344: finishing write tid 5 to smithi07085154-5 2024-09-14T03:49:02.592 INFO:tasks.rados.rados.0.smithi070.stdout:344: finishing write tid 6 to smithi07085154-5 2024-09-14T03:49:02.592 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 19 (ObjNum 126 snap 33 seq_num 126) dirty exists 2024-09-14T03:49:02.592 INFO:tasks.rados.rados.0.smithi070.stdout:344: left oid 5 (ObjNum 126 snap 33 seq_num 126) 2024-09-14T03:49:02.592 INFO:tasks.rados.rados.0.smithi070.stdout:346: finishing write tid 1 to smithi07085154-49 2024-09-14T03:49:02.592 INFO:tasks.rados.rados.0.smithi070.stdout:346: finishing write tid 2 to smithi07085154-49 2024-09-14T03:49:02.593 INFO:tasks.rados.rados.0.smithi070.stdout:346: finishing write tid 3 to smithi07085154-49 2024-09-14T03:49:02.593 INFO:tasks.rados.rados.0.smithi070.stdout:346: finishing write tid 4 to smithi07085154-49 2024-09-14T03:49:02.593 INFO:tasks.rados.rados.0.smithi070.stdout:348: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-14T03:49:02.596 INFO:tasks.rados.rados.0.smithi070.stdout:346: finishing write tid 5 to smithi07085154-49 2024-09-14T03:49:02.597 INFO:tasks.rados.rados.0.smithi070.stdout:346: finishing write tid 6 to smithi07085154-49 2024-09-14T03:49:02.597 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 49 v 114 (ObjNum 127 snap 33 seq_num 127) dirty exists 2024-09-14T03:49:02.597 INFO:tasks.rados.rados.0.smithi070.stdout:346: left oid 49 (ObjNum 127 snap 33 seq_num 127) 2024-09-14T03:49:02.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:02 smithi070 ceph-mon[27495]: osdmap e127: 8 total, 8 up, 8 in 2024-09-14T03:49:02.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:02 smithi070 ceph-mon[39288]: osdmap e127: 8 total, 8 up, 8 in 2024-09-14T03:49:03.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:02 smithi094 ceph-mon[31053]: osdmap e127: 8 total, 8 up, 8 in 2024-09-14T03:49:03.547 INFO:tasks.rados.rados.0.smithi070.stdout:344: done (4 left) 2024-09-14T03:49:03.547 INFO:tasks.rados.rados.0.smithi070.stdout:345: done (3 left) 2024-09-14T03:49:03.547 INFO:tasks.rados.rados.0.smithi070.stdout:346: done (2 left) 2024-09-14T03:49:03.547 INFO:tasks.rados.rados.0.smithi070.stdout:348: done (1 left) 2024-09-14T03:49:03.547 INFO:tasks.rados.rados.0.smithi070.stdout:349: done (0 left) 2024-09-14T03:49:03.548 INFO:tasks.rados.rados.0.smithi070.stdout:350: delete oid 49 current snap is 34 2024-09-14T03:49:03.552 INFO:tasks.rados.rados.0.smithi070.stdout:350: done (0 left) 2024-09-14T03:49:03.552 INFO:tasks.rados.rados.0.smithi070.stdout:351: rollback oid 18 current snap is 34 2024-09-14T03:49:03.552 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 18 to 33 2024-09-14T03:49:03.552 INFO:tasks.rados.rados.0.smithi070.stdout:352: snap_create 2024-09-14T03:49:03.554 INFO:tasks.rados.rados.0.smithi070.stdout:351: finishing rollback tid 0 to smithi07085154-18 2024-09-14T03:49:03.554 INFO:tasks.rados.rados.0.smithi070.stdout:351: finishing rollback tid 1 to smithi07085154-18 2024-09-14T03:49:03.555 INFO:tasks.rados.rados.0.smithi070.stdout:351: finishing rollback tid 2 to smithi07085154-18 2024-09-14T03:49:03.555 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 137 (ObjNum 119 snap 30 seq_num 119) dirty exists 2024-09-14T03:49:04.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:03 smithi094 ceph-mon[31053]: osdmap e128: 8 total, 8 up, 8 in 2024-09-14T03:49:04.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:03 smithi094 ceph-mon[31053]: pgmap v188: 121 pgs: 121 active+clean; 118 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 4.9 MiB/s wr, 24 op/s 2024-09-14T03:49:04.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:03 smithi070 ceph-mon[27495]: osdmap e128: 8 total, 8 up, 8 in 2024-09-14T03:49:04.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:03 smithi070 ceph-mon[27495]: pgmap v188: 121 pgs: 121 active+clean; 118 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 4.9 MiB/s wr, 24 op/s 2024-09-14T03:49:04.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:03 smithi070 ceph-mon[39288]: osdmap e128: 8 total, 8 up, 8 in 2024-09-14T03:49:04.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:03 smithi070 ceph-mon[39288]: pgmap v188: 121 pgs: 121 active+clean; 118 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 4.9 MiB/s wr, 24 op/s 2024-09-14T03:49:04.549 INFO:tasks.rados.rados.0.smithi070.stdout:351: done (1 left) 2024-09-14T03:49:04.549 INFO:tasks.rados.rados.0.smithi070.stdout:352: done (0 left) 2024-09-14T03:49:04.549 INFO:tasks.rados.rados.0.smithi070.stdout:353: rollback oid 41 current snap is 35 2024-09-14T03:49:04.549 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 41 to 33 2024-09-14T03:49:04.549 INFO:tasks.rados.rados.0.smithi070.stdout:354: rollback oid 18 current snap is 35 2024-09-14T03:49:04.549 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 18 to 28 2024-09-14T03:49:04.549 INFO:tasks.rados.rados.0.smithi070.stdout:355: rmattr oid 23 current snap is 35 2024-09-14T03:49:04.549 INFO:tasks.rados.rados.0.smithi070.stdout:355: done (2 left) 2024-09-14T03:49:04.549 INFO:tasks.rados.rados.0.smithi070.stdout:356: copy_from oid 43 from oid 10 current snap is 35 2024-09-14T03:49:04.549 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 19 current snap is 35 2024-09-14T03:49:04.549 INFO:tasks.rados.rados.0.smithi070.stdout:357: seq_num 129 ranges {106496=565248,671744=679936,1351680=786432,2138112=139264} 2024-09-14T03:49:04.552 INFO:tasks.rados.rados.0.smithi070.stdout:357: writing smithi07085154-19 from 106496 to 671744 tid 1 2024-09-14T03:49:04.556 INFO:tasks.rados.rados.0.smithi070.stdout:357: writing smithi07085154-19 from 671744 to 1351680 tid 2 2024-09-14T03:49:04.562 INFO:tasks.rados.rados.0.smithi070.stdout:357: writing smithi07085154-19 from 1351680 to 2138112 tid 3 2024-09-14T03:49:04.563 INFO:tasks.rados.rados.0.smithi070.stdout:357: writing smithi07085154-19 from 2138112 to 2277376 tid 4 2024-09-14T03:49:04.563 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 13 current snap is 35 2024-09-14T03:49:04.563 INFO:tasks.rados.rados.0.smithi070.stdout:353: finishing rollback tid 0 to smithi07085154-41 2024-09-14T03:49:04.563 INFO:tasks.rados.rados.0.smithi070.stdout:353: finishing rollback tid 1 to smithi07085154-41 2024-09-14T03:49:04.563 INFO:tasks.rados.rados.0.smithi070.stdout:358: seq_num 130 ranges {221184=581632,802816=614400,1417216=598016,2015232=393216} 2024-09-14T03:49:04.567 INFO:tasks.rados.rados.0.smithi070.stdout:358: writing smithi07085154-13 from 221184 to 802816 tid 1 2024-09-14T03:49:04.572 INFO:tasks.rados.rados.0.smithi070.stdout:358: writing smithi07085154-13 from 802816 to 1417216 tid 2 2024-09-14T03:49:04.575 INFO:tasks.rados.rados.0.smithi070.stdout:358: writing smithi07085154-13 from 1417216 to 2015232 tid 3 2024-09-14T03:49:04.578 INFO:tasks.rados.rados.0.smithi070.stdout:358: writing smithi07085154-13 from 2015232 to 2408448 tid 4 2024-09-14T03:49:04.578 INFO:tasks.rados.rados.0.smithi070.stdout:359: setattr oid 26 current snap is 35 2024-09-14T03:49:04.578 INFO:tasks.rados.rados.0.smithi070.stdout:353: finishing rollback tid 2 to smithi07085154-41 2024-09-14T03:49:04.578 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 141 (ObjNum 82 snap 16 seq_num 82) dirty exists 2024-09-14T03:49:04.578 INFO:tasks.rados.rados.0.smithi070.stdout:354: finishing rollback tid 0 to smithi07085154-18 2024-09-14T03:49:04.578 INFO:tasks.rados.rados.0.smithi070.stdout:354: finishing rollback tid 1 to smithi07085154-18 2024-09-14T03:49:04.579 INFO:tasks.rados.rados.0.smithi070.stdout:354: finishing rollback tid 2 to smithi07085154-18 2024-09-14T03:49:04.579 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 145 (ObjNum 78 snap 13 seq_num 78) dirty exists 2024-09-14T03:49:04.579 INFO:tasks.rados.rados.0.smithi070.stdout:357: finishing write tid 1 to smithi07085154-19 2024-09-14T03:49:04.579 INFO:tasks.rados.rados.0.smithi070.stdout:357: finishing write tid 2 to smithi07085154-19 2024-09-14T03:49:04.579 INFO:tasks.rados.rados.0.smithi070.stdout:356: finishing copy_from racing read to smithi07085154-43 2024-09-14T03:49:04.579 INFO:tasks.rados.rados.0.smithi070.stdout:357: finishing write tid 3 to smithi07085154-19 2024-09-14T03:49:04.579 INFO:tasks.rados.rados.0.smithi070.stdout:357: finishing write tid 4 to smithi07085154-19 2024-09-14T03:49:04.579 INFO:tasks.rados.rados.0.smithi070.stdout:357: finishing write tid 5 to smithi07085154-19 2024-09-14T03:49:04.579 INFO:tasks.rados.rados.0.smithi070.stdout:356: finishing copy_from to smithi07085154-43 2024-09-14T03:49:04.579 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 128 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-14T03:49:04.579 INFO:tasks.rados.rados.0.smithi070.stdout:357: finishing write tid 6 to smithi07085154-19 2024-09-14T03:49:04.579 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 19 v 129 (ObjNum 129 snap 35 seq_num 129) dirty exists 2024-09-14T03:49:04.580 INFO:tasks.rados.rados.0.smithi070.stdout:357: left oid 19 (ObjNum 129 snap 35 seq_num 129) 2024-09-14T03:49:04.580 INFO:tasks.rados.rados.0.smithi070.stdout:358: finishing write tid 1 to smithi07085154-13 2024-09-14T03:49:04.580 INFO:tasks.rados.rados.0.smithi070.stdout:358: finishing write tid 2 to smithi07085154-13 2024-09-14T03:49:04.580 INFO:tasks.rados.rados.0.smithi070.stdout:358: finishing write tid 3 to smithi07085154-13 2024-09-14T03:49:04.580 INFO:tasks.rados.rados.0.smithi070.stdout:358: finishing write tid 4 to smithi07085154-13 2024-09-14T03:49:04.581 INFO:tasks.rados.rados.0.smithi070.stdout:358: finishing write tid 5 to smithi07085154-13 2024-09-14T03:49:04.581 INFO:tasks.rados.rados.0.smithi070.stdout:353: done (5 left) 2024-09-14T03:49:04.581 INFO:tasks.rados.rados.0.smithi070.stdout:354: done (4 left) 2024-09-14T03:49:04.581 INFO:tasks.rados.rados.0.smithi070.stdout:356: done (3 left) 2024-09-14T03:49:04.581 INFO:tasks.rados.rados.0.smithi070.stdout:357: done (2 left) 2024-09-14T03:49:04.581 INFO:tasks.rados.rados.0.smithi070.stdout:360: snap_create 2024-09-14T03:49:04.581 INFO:tasks.rados.rados.0.smithi070.stdout:358: finishing write tid 6 to smithi07085154-13 2024-09-14T03:49:04.581 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 100 (ObjNum 130 snap 35 seq_num 130) dirty exists 2024-09-14T03:49:04.581 INFO:tasks.rados.rados.0.smithi070.stdout:358: left oid 13 (ObjNum 130 snap 35 seq_num 130) 2024-09-14T03:49:04.582 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 194 (ObjNum 96 snap 18 seq_num 96) dirty exists 2024-09-14T03:49:05.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:04 smithi094 ceph-mon[31053]: osdmap e129: 8 total, 8 up, 8 in 2024-09-14T03:49:05.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:04 smithi070 ceph-mon[27495]: osdmap e129: 8 total, 8 up, 8 in 2024-09-14T03:49:05.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:04 smithi070 ceph-mon[39288]: osdmap e129: 8 total, 8 up, 8 in 2024-09-14T03:49:05.507 INFO:tasks.rados.rados.0.smithi070.stdout:358: done (2 left) 2024-09-14T03:49:05.508 INFO:tasks.rados.rados.0.smithi070.stdout:359: done (1 left) 2024-09-14T03:49:05.508 INFO:tasks.rados.rados.0.smithi070.stdout:360: done (0 left) 2024-09-14T03:49:05.508 INFO:tasks.rados.rados.0.smithi070.stdout:361: copy_from oid 34 from oid 46 current snap is 36 2024-09-14T03:49:05.508 INFO:tasks.rados.rados.0.smithi070.stdout:362: snap_remove snap 25 2024-09-14T03:49:06.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:05 smithi094 ceph-mon[31053]: osdmap e130: 8 total, 8 up, 8 in 2024-09-14T03:49:06.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:05 smithi094 ceph-mon[31053]: pgmap v191: 121 pgs: 121 active+clean; 118 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 4.9 MiB/s wr, 24 op/s 2024-09-14T03:49:06.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:05 smithi094 ceph-mon[31053]: osdmap e131: 8 total, 8 up, 8 in 2024-09-14T03:49:06.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:05 smithi070 ceph-mon[39288]: osdmap e130: 8 total, 8 up, 8 in 2024-09-14T03:49:06.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:05 smithi070 ceph-mon[39288]: pgmap v191: 121 pgs: 121 active+clean; 118 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 4.9 MiB/s wr, 24 op/s 2024-09-14T03:49:06.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:05 smithi070 ceph-mon[39288]: osdmap e131: 8 total, 8 up, 8 in 2024-09-14T03:49:06.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:05 smithi070 ceph-mon[27495]: osdmap e130: 8 total, 8 up, 8 in 2024-09-14T03:49:06.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:05 smithi070 ceph-mon[27495]: pgmap v191: 121 pgs: 121 active+clean; 118 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 4.9 MiB/s wr, 24 op/s 2024-09-14T03:49:06.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:05 smithi070 ceph-mon[27495]: osdmap e131: 8 total, 8 up, 8 in 2024-09-14T03:49:06.510 INFO:tasks.rados.rados.0.smithi070.stdout:362: done (1 left) 2024-09-14T03:49:06.510 INFO:tasks.rados.rados.0.smithi070.stdout:363: rollback oid 45 current snap is 36 2024-09-14T03:49:06.511 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 45 to 34 2024-09-14T03:49:06.511 INFO:tasks.rados.rados.0.smithi070.stdout:364: read oid 20 snap -1 2024-09-14T03:49:06.511 INFO:tasks.rados.rados.0.smithi070.stdout:364: expect deleted 2024-09-14T03:49:06.511 INFO:tasks.rados.rados.0.smithi070.stdout:365: read oid 37 snap -1 2024-09-14T03:49:06.511 INFO:tasks.rados.rados.0.smithi070.stdout:365: expect (ObjNum 86 snap 17 seq_num 86) 2024-09-14T03:49:06.511 INFO:tasks.rados.rados.0.smithi070.stdout:366: read oid 17 snap -1 2024-09-14T03:49:06.511 INFO:tasks.rados.rados.0.smithi070.stdout:366: expect (ObjNum 98 snap 20 seq_num 98) 2024-09-14T03:49:06.511 INFO:tasks.rados.rados.0.smithi070.stdout:367: snap_remove snap 26 2024-09-14T03:49:06.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:49:06] "GET /metrics HTTP/1.1" 200 240010 "" "Prometheus/2.43.0" 2024-09-14T03:49:07.517 INFO:tasks.rados.rados.0.smithi070.stdout:367: done (5 left) 2024-09-14T03:49:07.517 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 40 current snap is 36 2024-09-14T03:49:07.517 INFO:tasks.rados.rados.0.smithi070.stdout:368: seq_num 131 ranges {2449408=598016,3047424=499712,3547136=753664,4300800=352256} 2024-09-14T03:49:07.532 INFO:tasks.rados.rados.0.smithi070.stdout:368: writing smithi07085154-40 from 2449408 to 3047424 tid 1 2024-09-14T03:49:07.535 INFO:tasks.rados.rados.0.smithi070.stdout:368: writing smithi07085154-40 from 3047424 to 3547136 tid 2 2024-09-14T03:49:07.539 INFO:tasks.rados.rados.0.smithi070.stdout:368: writing smithi07085154-40 from 3547136 to 4300800 tid 3 2024-09-14T03:49:07.542 INFO:tasks.rados.rados.0.smithi070.stdout:368: writing smithi07085154-40 from 4300800 to 4653056 tid 4 2024-09-14T03:49:07.542 INFO:tasks.rados.rados.0.smithi070.stdout:369: delete oid 28 current snap is 36 2024-09-14T03:49:07.542 INFO:tasks.rados.rados.0.smithi070.stdout:361: finishing copy_from racing read to smithi07085154-34 2024-09-14T03:49:07.542 INFO:tasks.rados.rados.0.smithi070.stdout:361: finishing copy_from to smithi07085154-34 2024-09-14T03:49:07.542 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 16 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-14T03:49:07.542 INFO:tasks.rados.rados.0.smithi070.stdout:366: expect (ObjNum 98 snap 20 seq_num 98) 2024-09-14T03:49:07.597 INFO:tasks.rados.rados.0.smithi070.stdout:363: finishing rollback tid 0 to smithi07085154-45 2024-09-14T03:49:07.597 INFO:tasks.rados.rados.0.smithi070.stdout:361: done (6 left) 2024-09-14T03:49:07.597 INFO:tasks.rados.rados.0.smithi070.stdout:364: done (5 left) 2024-09-14T03:49:07.597 INFO:tasks.rados.rados.0.smithi070.stdout:366: done (4 left) 2024-09-14T03:49:07.597 INFO:tasks.rados.rados.0.smithi070.stdout:369: done (3 left) 2024-09-14T03:49:07.597 INFO:tasks.rados.rados.0.smithi070.stdout:370: rollback oid 23 current snap is 36 2024-09-14T03:49:07.598 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 23 to 29 2024-09-14T03:49:07.598 INFO:tasks.rados.rados.0.smithi070.stdout:363: finishing rollback tid 1 to smithi07085154-45 2024-09-14T03:49:07.598 INFO:tasks.rados.rados.0.smithi070.stdout:365: expect (ObjNum 86 snap 17 seq_num 86) 2024-09-14T03:49:07.639 INFO:tasks.rados.rados.0.smithi070.stdout:363: finishing rollback tid 2 to smithi07085154-45 2024-09-14T03:49:07.639 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 45 v 120 (ObjNum 118 snap 29 seq_num 118) dirty exists 2024-09-14T03:49:07.640 INFO:tasks.rados.rados.0.smithi070.stdout:363: done (3 left) 2024-09-14T03:49:07.640 INFO:tasks.rados.rados.0.smithi070.stdout:365: done (2 left) 2024-09-14T03:49:07.640 INFO:tasks.rados.rados.0.smithi070.stdout:371: rmattr oid 9 current snap is 36 2024-09-14T03:49:07.640 INFO:tasks.rados.rados.0.smithi070.stdout:368: finishing write tid 1 to smithi07085154-40 2024-09-14T03:49:07.640 INFO:tasks.rados.rados.0.smithi070.stdout:368: finishing write tid 2 to smithi07085154-40 2024-09-14T03:49:07.640 INFO:tasks.rados.rados.0.smithi070.stdout:368: finishing write tid 3 to smithi07085154-40 2024-09-14T03:49:07.640 INFO:tasks.rados.rados.0.smithi070.stdout:368: finishing write tid 4 to smithi07085154-40 2024-09-14T03:49:07.640 INFO:tasks.rados.rados.0.smithi070.stdout:368: finishing write tid 5 to smithi07085154-40 2024-09-14T03:49:07.640 INFO:tasks.rados.rados.0.smithi070.stdout:368: finishing write tid 6 to smithi07085154-40 2024-09-14T03:49:07.640 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 62 (ObjNum 131 snap 36 seq_num 131) dirty exists 2024-09-14T03:49:07.640 INFO:tasks.rados.rados.0.smithi070.stdout:368: left oid 40 (ObjNum 131 snap 36 seq_num 131) 2024-09-14T03:49:07.640 INFO:tasks.rados.rados.0.smithi070.stdout:370: finishing rollback tid 0 to smithi07085154-23 2024-09-14T03:49:07.640 INFO:tasks.rados.rados.0.smithi070.stdout:370: finishing rollback tid 1 to smithi07085154-23 2024-09-14T03:49:07.641 INFO:tasks.rados.rados.0.smithi070.stdout:370: finishing rollback tid 2 to smithi07085154-23 2024-09-14T03:49:07.641 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 42 (ObjNum 117 snap 28 seq_num 117) dirty exists 2024-09-14T03:49:07.641 INFO:tasks.rados.rados.0.smithi070.stdout:368: done (2 left) 2024-09-14T03:49:07.641 INFO:tasks.rados.rados.0.smithi070.stdout:370: done (1 left) 2024-09-14T03:49:07.641 INFO:tasks.rados.rados.0.smithi070.stdout:372: snap_remove snap 29 2024-09-14T03:49:07.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:07 smithi094 ceph-mon[31053]: osdmap e132: 8 total, 8 up, 8 in 2024-09-14T03:49:07.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:07 smithi094 ceph-mon[31053]: pgmap v194: 121 pgs: 121 active+clean; 118 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:49:07.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:07 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:49:07.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:07 smithi070 ceph-mon[27495]: osdmap e132: 8 total, 8 up, 8 in 2024-09-14T03:49:07.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:07 smithi070 ceph-mon[27495]: pgmap v194: 121 pgs: 121 active+clean; 118 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:49:07.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:07 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:49:07.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:07 smithi070 ceph-mon[39288]: osdmap e132: 8 total, 8 up, 8 in 2024-09-14T03:49:07.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:07 smithi070 ceph-mon[39288]: pgmap v194: 121 pgs: 121 active+clean; 118 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:49:07.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:07 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:49:08.344 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-09-14T03:49:08.518 INFO:tasks.rados.rados.0.smithi070.stdout:372: done (1 left) 2024-09-14T03:49:08.518 INFO:tasks.rados.rados.0.smithi070.stdout:373: copy_from oid 29 from oid 46 current snap is 36 2024-09-14T03:49:08.518 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 0 (ObjNum 84 snap 17 seq_num 84) dirty exists 2024-09-14T03:49:08.518 INFO:tasks.rados.rados.0.smithi070.stdout:371: done (1 left) 2024-09-14T03:49:08.518 INFO:tasks.rados.rados.0.smithi070.stdout:374: snap_create 2024-09-14T03:49:08.531 INFO:tasks.rados.rados.0.smithi070.stdout:373: finishing copy_from racing read to smithi07085154-29 2024-09-14T03:49:08.532 INFO:tasks.rados.rados.0.smithi070.stdout:373: finishing copy_from to smithi07085154-29 2024-09-14T03:49:08.532 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 16 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-14T03:49:08.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:08 smithi094 ceph-mon[31053]: osdmap e133: 8 total, 8 up, 8 in 2024-09-14T03:49:08.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:08 smithi070 ceph-mon[27495]: osdmap e133: 8 total, 8 up, 8 in 2024-09-14T03:49:08.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:08 smithi070 ceph-mon[39288]: osdmap e133: 8 total, 8 up, 8 in 2024-09-14T03:49:08.857 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:49:08.857 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (2m) 2m ago 3m 15.1M - 0.25.0 c8568f914cd2 fef584d3d5c6 2024-09-14T03:49:08.857 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi094 *:3000 running (3m) 2m ago 3m 85.0M - 9.4.7 954c08fa6188 846770f48257 2024-09-14T03:49:08.857 INFO:teuthology.orchestra.run.smithi070.stdout:mds.a smithi070 running (5m) 2m ago 5m 16.7M - 17.2.7-1629-geb16b97b ab51d1bebd12 52e0dbb7be50 2024-09-14T03:49:08.857 INFO:teuthology.orchestra.run.smithi070.stdout:mds.b smithi094 running (5m) 2m ago 5m 13.6M - 17.2.7-1629-geb16b97b ab51d1bebd12 e3b5e3e00189 2024-09-14T03:49:08.857 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi094 *:8443 running (8m) 2m ago 8m 399M - 17.2.7-1629-geb16b97b ab51d1bebd12 8a5c729a2096 2024-09-14T03:49:08.857 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:9283,8765 running (10m) 2m ago 10m 447M - 17.2.7-1629-geb16b97b ab51d1bebd12 dc71810017b9 2024-09-14T03:49:08.858 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (10m) 2m ago 10m 44.3M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 12d19db91f54 2024-09-14T03:49:08.858 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi094 running (9m) 2m ago 9m 37.9M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 90ea2e164909 2024-09-14T03:49:08.858 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (9m) 2m ago 8m 38.1M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 44ee80c29ee0 2024-09-14T03:49:08.858 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (3m) 2m ago 3m 13.3M - 1.5.0 0da6a335fe13 315c3f32399f 2024-09-14T03:49:08.858 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi094 *:9100 running (3m) 2m ago 3m 3577k - 1.5.0 0da6a335fe13 3dcf1723ea35 2024-09-14T03:49:08.858 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (8m) 2m ago 8m 44.0M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 741c06b6adbf 2024-09-14T03:49:08.858 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (8m) 2m ago 8m 44.6M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 af459ea7ef68 2024-09-14T03:49:08.858 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (7m) 2m ago 7m 42.4M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 109b66e4e60f 2024-09-14T03:49:08.858 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (7m) 2m ago 7m 43.7M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 2e4199b58392 2024-09-14T03:49:08.858 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi094 running (6m) 2m ago 6m 42.0M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 52b64527e24c 2024-09-14T03:49:08.858 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi094 running (6m) 2m ago 6m 42.4M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 b543f152ba87 2024-09-14T03:49:08.858 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi094 running (6m) 2m ago 6m 42.0M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 e66b592ac940 2024-09-14T03:49:08.858 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi094 running (5m) 2m ago 5m 44.9M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 037c6208e9ce 2024-09-14T03:49:08.858 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi094 *:9095 running (2m) 2m ago 4m 24.5M - 2.43.0 a07b618ecd1d 9c88e2923e9d 2024-09-14T03:49:09.164 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:49:09.164 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-09-14T03:49:09.164 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 3 2024-09-14T03:49:09.164 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:49:09.164 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-09-14T03:49:09.164 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:49:09.164 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:49:09.165 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-09-14T03:49:09.165 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 8 2024-09-14T03:49:09.165 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:49:09.165 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": { 2024-09-14T03:49:09.165 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:49:09.165 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:49:09.165 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-09-14T03:49:09.165 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 15 2024-09-14T03:49:09.165 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-09-14T03:49:09.165 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:49:09.418 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:49:09.418 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:49:09.418 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-09-14T03:49:09.418 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:49:09.418 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [], 2024-09-14T03:49:09.418 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "0/20 daemons upgraded", 2024-09-14T03:49:09.418 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 image on host smithi094", 2024-09-14T03:49:09.418 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-09-14T03:49:09.418 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:49:09.522 INFO:tasks.rados.rados.0.smithi070.stdout:373: done (1 left) 2024-09-14T03:49:09.522 INFO:tasks.rados.rados.0.smithi070.stdout:374: done (0 left) 2024-09-14T03:49:09.522 INFO:tasks.rados.rados.0.smithi070.stdout:375: copy_from oid 27 from oid 39 current snap is 37 2024-09-14T03:49:09.523 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 8 current snap is 37 2024-09-14T03:49:09.523 INFO:tasks.rados.rados.0.smithi070.stdout:376: seq_num 133 ranges {106496=630784,737280=663552,1400832=770048,2170880=172032} 2024-09-14T03:49:09.526 INFO:tasks.rados.rados.0.smithi070.stdout:376: writing smithi07085154-8 from 106496 to 737280 tid 1 2024-09-14T03:49:09.530 INFO:tasks.rados.rados.0.smithi070.stdout:376: writing smithi07085154-8 from 737280 to 1400832 tid 2 2024-09-14T03:49:09.535 INFO:tasks.rados.rados.0.smithi070.stdout:376: writing smithi07085154-8 from 1400832 to 2170880 tid 3 2024-09-14T03:49:09.536 INFO:tasks.rados.rados.0.smithi070.stdout:376: writing smithi07085154-8 from 2170880 to 2342912 tid 4 2024-09-14T03:49:09.536 INFO:tasks.rados.rados.0.smithi070.stdout:377: copy_from oid 45 from oid 50 current snap is 37 2024-09-14T03:49:09.536 INFO:tasks.rados.rados.0.smithi070.stdout:375: finishing copy_from racing read to smithi07085154-27 2024-09-14T03:49:09.536 INFO:tasks.rados.rados.0.smithi070.stdout:375: finishing copy_from to smithi07085154-27 2024-09-14T03:49:09.537 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 10 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-14T03:49:09.537 INFO:tasks.rados.rados.0.smithi070.stdout:376: finishing write tid 1 to smithi07085154-8 2024-09-14T03:49:09.537 INFO:tasks.rados.rados.0.smithi070.stdout:376: finishing write tid 2 to smithi07085154-8 2024-09-14T03:49:09.537 INFO:tasks.rados.rados.0.smithi070.stdout:375: done (2 left) 2024-09-14T03:49:09.537 INFO:tasks.rados.rados.0.smithi070.stdout:378: snap_create 2024-09-14T03:49:09.538 INFO:tasks.rados.rados.0.smithi070.stdout:376: finishing write tid 3 to smithi07085154-8 2024-09-14T03:49:09.538 INFO:tasks.rados.rados.0.smithi070.stdout:376: finishing write tid 4 to smithi07085154-8 2024-09-14T03:49:09.538 INFO:tasks.rados.rados.0.smithi070.stdout:376: finishing write tid 5 to smithi07085154-8 2024-09-14T03:49:09.539 INFO:tasks.rados.rados.0.smithi070.stdout:376: finishing write tid 6 to smithi07085154-8 2024-09-14T03:49:09.539 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 8 v 171 (ObjNum 133 snap 37 seq_num 133) dirty exists 2024-09-14T03:49:09.539 INFO:tasks.rados.rados.0.smithi070.stdout:376: left oid 8 (ObjNum 133 snap 37 seq_num 133) 2024-09-14T03:49:09.541 INFO:tasks.rados.rados.0.smithi070.stdout:377: finishing copy_from racing read to smithi07085154-45 2024-09-14T03:49:09.542 INFO:tasks.rados.rados.0.smithi070.stdout:377: finishing copy_from to smithi07085154-45 2024-09-14T03:49:09.542 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 45 v 11 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-14T03:49:09.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:09 smithi094 ceph-mon[31053]: from='client.15000 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:09.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:09 smithi094 ceph-mon[31053]: osdmap e134: 8 total, 8 up, 8 in 2024-09-14T03:49:09.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:09 smithi094 ceph-mon[31053]: pgmap v197: 121 pgs: 121 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.4 MiB/s wr, 17 op/s 2024-09-14T03:49:09.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:09 smithi094 ceph-mon[31053]: from='client.15006 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:09.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:09 smithi094 ceph-mon[31053]: from='client.15012 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:09.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:09 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/3235999642' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:09.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:09 smithi070 ceph-mon[27495]: from='client.15000 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:09.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:09 smithi070 ceph-mon[27495]: osdmap e134: 8 total, 8 up, 8 in 2024-09-14T03:49:09.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:09 smithi070 ceph-mon[27495]: pgmap v197: 121 pgs: 121 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.4 MiB/s wr, 17 op/s 2024-09-14T03:49:09.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:09 smithi070 ceph-mon[27495]: from='client.15006 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:09.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:09 smithi070 ceph-mon[27495]: from='client.15012 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:09.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:09 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/3235999642' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:09.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:09 smithi070 ceph-mon[39288]: from='client.15000 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:09.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:09 smithi070 ceph-mon[39288]: osdmap e134: 8 total, 8 up, 8 in 2024-09-14T03:49:09.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:09 smithi070 ceph-mon[39288]: pgmap v197: 121 pgs: 121 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.4 MiB/s wr, 17 op/s 2024-09-14T03:49:09.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:09 smithi070 ceph-mon[39288]: from='client.15006 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:09.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:09 smithi070 ceph-mon[39288]: from='client.15012 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:09.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:09 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/3235999642' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:10.509 INFO:tasks.rados.rados.0.smithi070.stdout:376: done (2 left) 2024-09-14T03:49:10.509 INFO:tasks.rados.rados.0.smithi070.stdout:377: done (1 left) 2024-09-14T03:49:10.509 INFO:tasks.rados.rados.0.smithi070.stdout:378: done (0 left) 2024-09-14T03:49:10.509 INFO:tasks.rados.rados.0.smithi070.stdout:379: read oid 36 snap -1 2024-09-14T03:49:10.509 INFO:tasks.rados.rados.0.smithi070.stdout:379: expect (ObjNum 108 snap 23 seq_num 108) 2024-09-14T03:49:10.509 INFO:tasks.rados.rados.0.smithi070.stdout:380: delete oid 21 current snap is 38 2024-09-14T03:49:10.514 INFO:tasks.rados.rados.0.smithi070.stdout:380: done (1 left) 2024-09-14T03:49:10.514 INFO:tasks.rados.rados.0.smithi070.stdout:381: rmattr oid 9 current snap is 38 2024-09-14T03:49:10.514 INFO:tasks.rados.rados.0.smithi070.stdout:381: done (1 left) 2024-09-14T03:49:10.514 INFO:tasks.rados.rados.0.smithi070.stdout:382: rmattr oid 32 current snap is 38 2024-09-14T03:49:10.514 INFO:tasks.rados.rados.0.smithi070.stdout:382: done (1 left) 2024-09-14T03:49:10.514 INFO:tasks.rados.rados.0.smithi070.stdout:383: rmattr oid 6 current snap is 38 2024-09-14T03:49:10.514 INFO:tasks.rados.rados.0.smithi070.stdout:384: copy_from oid 12 from oid 11 current snap is 38 2024-09-14T03:49:10.514 INFO:tasks.rados.rados.0.smithi070.stdout:385: delete oid 7 current snap is 38 2024-09-14T03:49:10.517 INFO:tasks.rados.rados.0.smithi070.stdout:385: done (3 left) 2024-09-14T03:49:10.517 INFO:tasks.rados.rados.0.smithi070.stdout:386: rollback oid 44 current snap is 38 2024-09-14T03:49:10.517 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 44 to 31 2024-09-14T03:49:10.518 INFO:tasks.rados.rados.0.smithi070.stdout:387: read oid 18 snap -1 2024-09-14T03:49:10.518 INFO:tasks.rados.rados.0.smithi070.stdout:387: expect (ObjNum 78 snap 13 seq_num 78) 2024-09-14T03:49:10.518 INFO:tasks.rados.rados.0.smithi070.stdout:388: snap_remove snap 37 2024-09-14T03:49:10.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:10 smithi094 ceph-mon[31053]: from='client.15024 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:10.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:10 smithi094 ceph-mon[31053]: osdmap e135: 8 total, 8 up, 8 in 2024-09-14T03:49:10.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:10 smithi094 ceph-mon[31053]: osdmap e136: 8 total, 8 up, 8 in 2024-09-14T03:49:10.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:10 smithi070 ceph-mon[27495]: from='client.15024 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:10.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:10 smithi070 ceph-mon[27495]: osdmap e135: 8 total, 8 up, 8 in 2024-09-14T03:49:10.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:10 smithi070 ceph-mon[27495]: osdmap e136: 8 total, 8 up, 8 in 2024-09-14T03:49:10.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:10 smithi070 ceph-mon[39288]: from='client.15024 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:10.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:10 smithi070 ceph-mon[39288]: osdmap e135: 8 total, 8 up, 8 in 2024-09-14T03:49:10.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:10 smithi070 ceph-mon[39288]: osdmap e136: 8 total, 8 up, 8 in 2024-09-14T03:49:11.511 INFO:tasks.rados.rados.0.smithi070.stdout:388: done (5 left) 2024-09-14T03:49:11.511 INFO:tasks.rados.rados.0.smithi070.stdout:389: delete oid 46 current snap is 38 2024-09-14T03:49:11.511 INFO:tasks.rados.rados.0.smithi070.stdout:387: expect (ObjNum 78 snap 13 seq_num 78) 2024-09-14T03:49:11.555 INFO:tasks.rados.rados.0.smithi070.stdout:379: expect (ObjNum 108 snap 23 seq_num 108) 2024-09-14T03:49:11.665 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 146 (ObjNum 122 snap 31 seq_num 122) dirty exists 2024-09-14T03:49:11.665 INFO:tasks.rados.rados.0.smithi070.stdout:386: finishing rollback tid 0 to smithi07085154-44 2024-09-14T03:49:11.665 INFO:tasks.rados.rados.0.smithi070.stdout:386: finishing rollback tid 1 to smithi07085154-44 2024-09-14T03:49:11.665 INFO:tasks.rados.rados.0.smithi070.stdout:386: finishing rollback tid 2 to smithi07085154-44 2024-09-14T03:49:11.665 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 68 (ObjNum 43 snap 0 seq_num 43) dirty exists 2024-09-14T03:49:11.665 INFO:tasks.rados.rados.0.smithi070.stdout:384: finishing copy_from racing read to smithi07085154-12 2024-09-14T03:49:11.665 INFO:tasks.rados.rados.0.smithi070.stdout:384: finishing copy_from to smithi07085154-12 2024-09-14T03:49:11.665 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 65 (ObjNum 111 snap 23 seq_num 111) dirty exists 2024-09-14T03:49:11.668 INFO:tasks.rados.rados.0.smithi070.stdout:379: done (5 left) 2024-09-14T03:49:11.669 INFO:tasks.rados.rados.0.smithi070.stdout:383: done (4 left) 2024-09-14T03:49:11.669 INFO:tasks.rados.rados.0.smithi070.stdout:384: done (3 left) 2024-09-14T03:49:11.669 INFO:tasks.rados.rados.0.smithi070.stdout:386: done (2 left) 2024-09-14T03:49:11.669 INFO:tasks.rados.rados.0.smithi070.stdout:387: done (1 left) 2024-09-14T03:49:11.669 INFO:tasks.rados.rados.0.smithi070.stdout:389: done (0 left) 2024-09-14T03:49:11.669 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 12 current snap is 38 2024-09-14T03:49:11.669 INFO:tasks.rados.rados.0.smithi070.stdout:390: seq_num 137 ranges {2039808=704512,2744320=655360,3399680=753664,4153344=196608} 2024-09-14T03:49:11.683 INFO:tasks.rados.rados.0.smithi070.stdout:390: writing smithi07085154-12 from 2039808 to 2744320 tid 1 2024-09-14T03:49:11.687 INFO:tasks.rados.rados.0.smithi070.stdout:390: writing smithi07085154-12 from 2744320 to 3399680 tid 2 2024-09-14T03:49:11.692 INFO:tasks.rados.rados.0.smithi070.stdout:390: writing smithi07085154-12 from 3399680 to 4153344 tid 3 2024-09-14T03:49:11.692 INFO:tasks.rados.rados.0.smithi070.stdout:390: writing smithi07085154-12 from 4153344 to 4349952 tid 4 2024-09-14T03:49:11.692 INFO:tasks.rados.rados.0.smithi070.stdout:391: snap_remove snap 16 2024-09-14T03:49:11.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:11 smithi070 ceph-mon[27495]: pgmap v200: 121 pgs: 121 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.4 MiB/s wr, 17 op/s 2024-09-14T03:49:11.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:11 smithi070 ceph-mon[27495]: osdmap e137: 8 total, 8 up, 8 in 2024-09-14T03:49:11.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:11 smithi070 ceph-mon[39288]: pgmap v200: 121 pgs: 121 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.4 MiB/s wr, 17 op/s 2024-09-14T03:49:11.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:11 smithi070 ceph-mon[39288]: osdmap e137: 8 total, 8 up, 8 in 2024-09-14T03:49:12.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:11 smithi094 ceph-mon[31053]: pgmap v200: 121 pgs: 121 active+clean; 127 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.4 MiB/s wr, 17 op/s 2024-09-14T03:49:12.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:11 smithi094 ceph-mon[31053]: osdmap e137: 8 total, 8 up, 8 in 2024-09-14T03:49:12.027 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:11 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:49:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:49:12.532 INFO:tasks.rados.rados.0.smithi070.stdout:390: finishing write tid 1 to smithi07085154-12 2024-09-14T03:49:12.532 INFO:tasks.rados.rados.0.smithi070.stdout:390: finishing write tid 2 to smithi07085154-12 2024-09-14T03:49:12.532 INFO:tasks.rados.rados.0.smithi070.stdout:390: finishing write tid 3 to smithi07085154-12 2024-09-14T03:49:12.532 INFO:tasks.rados.rados.0.smithi070.stdout:390: finishing write tid 4 to smithi07085154-12 2024-09-14T03:49:12.533 INFO:tasks.rados.rados.0.smithi070.stdout:390: finishing write tid 5 to smithi07085154-12 2024-09-14T03:49:12.533 INFO:tasks.rados.rados.0.smithi070.stdout:390: finishing write tid 6 to smithi07085154-12 2024-09-14T03:49:12.533 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 70 (ObjNum 137 snap 38 seq_num 137) dirty exists 2024-09-14T03:49:12.533 INFO:tasks.rados.rados.0.smithi070.stdout:390: left oid 12 (ObjNum 137 snap 38 seq_num 137) 2024-09-14T03:49:12.533 INFO:tasks.rados.rados.0.smithi070.stdout:390: done (1 left) 2024-09-14T03:49:12.533 INFO:tasks.rados.rados.0.smithi070.stdout:391: done (0 left) 2024-09-14T03:49:12.533 INFO:tasks.rados.rados.0.smithi070.stdout:392: snap_remove snap 32 2024-09-14T03:49:13.539 INFO:tasks.rados.rados.0.smithi070.stdout:392: done (0 left) 2024-09-14T03:49:13.539 INFO:tasks.rados.rados.0.smithi070.stdout:393: snap_remove snap 23 2024-09-14T03:49:13.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:13 smithi070 ceph-mon[27495]: osdmap e138: 8 total, 8 up, 8 in 2024-09-14T03:49:13.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:13 smithi070 ceph-mon[27495]: pgmap v203: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 2.1 MiB/s wr, 10 op/s 2024-09-14T03:49:13.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:13 smithi070 ceph-mon[39288]: osdmap e138: 8 total, 8 up, 8 in 2024-09-14T03:49:13.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:13 smithi070 ceph-mon[39288]: pgmap v203: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 2.1 MiB/s wr, 10 op/s 2024-09-14T03:49:14.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:13 smithi094 ceph-mon[31053]: osdmap e138: 8 total, 8 up, 8 in 2024-09-14T03:49:14.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:13 smithi094 ceph-mon[31053]: pgmap v203: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 2.1 MiB/s wr, 10 op/s 2024-09-14T03:49:14.538 INFO:tasks.rados.rados.0.smithi070.stdout:393: done (0 left) 2024-09-14T03:49:14.538 INFO:tasks.rados.rados.0.smithi070.stdout:394: snap_remove snap 31 2024-09-14T03:49:15.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:14 smithi094 ceph-mon[31053]: osdmap e139: 8 total, 8 up, 8 in 2024-09-14T03:49:15.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:14 smithi070 ceph-mon[27495]: osdmap e139: 8 total, 8 up, 8 in 2024-09-14T03:49:15.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:14 smithi070 ceph-mon[39288]: osdmap e139: 8 total, 8 up, 8 in 2024-09-14T03:49:15.511 INFO:tasks.rados.rados.0.smithi070.stdout:394: done (0 left) 2024-09-14T03:49:15.511 INFO:tasks.rados.rados.0.smithi070.stdout:395: setattr oid 41 current snap is 38 2024-09-14T03:49:15.512 INFO:tasks.rados.rados.0.smithi070.stdout:396: read oid 26 snap -1 2024-09-14T03:49:15.513 INFO:tasks.rados.rados.0.smithi070.stdout:396: expect (ObjNum 96 snap 18 seq_num 96) 2024-09-14T03:49:15.513 INFO:tasks.rados.rados.0.smithi070.stdout:397: read oid 5 snap -1 2024-09-14T03:49:15.513 INFO:tasks.rados.rados.0.smithi070.stdout:397: expect (ObjNum 126 snap 33 seq_num 126) 2024-09-14T03:49:15.513 INFO:tasks.rados.rados.0.smithi070.stdout:398: read oid 27 snap 20 2024-09-14T03:49:15.513 INFO:tasks.rados.rados.0.smithi070.stdout:398: expect (ObjNum 26 snap 0 seq_num 26) 2024-09-14T03:49:15.513 INFO:tasks.rados.rados.0.smithi070.stdout:399: setattr oid 6 current snap is 38 2024-09-14T03:49:15.515 INFO:tasks.rados.rados.0.smithi070.stdout:400: snap_create 2024-09-14T03:49:15.521 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 197 (ObjNum 82 snap 16 seq_num 82) dirty exists 2024-09-14T03:49:15.522 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 165 (ObjNum 122 snap 31 seq_num 122) dirty exists 2024-09-14T03:49:15.531 INFO:tasks.rados.rados.0.smithi070.stdout:396: expect (ObjNum 96 snap 18 seq_num 96) 2024-09-14T03:49:15.577 INFO:tasks.rados.rados.0.smithi070.stdout:397: expect (ObjNum 126 snap 33 seq_num 126) 2024-09-14T03:49:15.630 INFO:tasks.rados.rados.0.smithi070.stdout:398: expect (ObjNum 26 snap 0 seq_num 26) 2024-09-14T03:49:16.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:15 smithi094 ceph-mon[31053]: osdmap e140: 8 total, 8 up, 8 in 2024-09-14T03:49:16.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:15 smithi094 ceph-mon[31053]: pgmap v206: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 2.1 MiB/s wr, 10 op/s 2024-09-14T03:49:16.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:15 smithi094 ceph-mon[31053]: osdmap e141: 8 total, 8 up, 8 in 2024-09-14T03:49:16.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:15 smithi070 ceph-mon[27495]: osdmap e140: 8 total, 8 up, 8 in 2024-09-14T03:49:16.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:15 smithi070 ceph-mon[27495]: pgmap v206: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 2.1 MiB/s wr, 10 op/s 2024-09-14T03:49:16.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:15 smithi070 ceph-mon[27495]: osdmap e141: 8 total, 8 up, 8 in 2024-09-14T03:49:16.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:15 smithi070 ceph-mon[39288]: osdmap e140: 8 total, 8 up, 8 in 2024-09-14T03:49:16.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:15 smithi070 ceph-mon[39288]: pgmap v206: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 2.1 MiB/s wr, 10 op/s 2024-09-14T03:49:16.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:15 smithi070 ceph-mon[39288]: osdmap e141: 8 total, 8 up, 8 in 2024-09-14T03:49:16.512 INFO:tasks.rados.rados.0.smithi070.stdout:395: done (5 left) 2024-09-14T03:49:16.512 INFO:tasks.rados.rados.0.smithi070.stdout:396: done (4 left) 2024-09-14T03:49:16.512 INFO:tasks.rados.rados.0.smithi070.stdout:397: done (3 left) 2024-09-14T03:49:16.512 INFO:tasks.rados.rados.0.smithi070.stdout:398: done (2 left) 2024-09-14T03:49:16.512 INFO:tasks.rados.rados.0.smithi070.stdout:399: done (1 left) 2024-09-14T03:49:16.512 INFO:tasks.rados.rados.0.smithi070.stdout:400: done (0 left) 2024-09-14T03:49:16.512 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 13 current snap is 39 2024-09-14T03:49:16.512 INFO:tasks.rados.rados.0.smithi070.stdout:401: seq_num 138 ranges {2408448=720896,3129344=540672,3670016=557056,4227072=507904} 2024-09-14T03:49:16.528 INFO:tasks.rados.rados.0.smithi070.stdout:401: writing smithi07085154-13 from 2408448 to 3129344 tid 1 2024-09-14T03:49:16.531 INFO:tasks.rados.rados.0.smithi070.stdout:401: writing smithi07085154-13 from 3129344 to 3670016 tid 2 2024-09-14T03:49:16.534 INFO:tasks.rados.rados.0.smithi070.stdout:401: writing smithi07085154-13 from 3670016 to 4227072 tid 3 2024-09-14T03:49:16.538 INFO:tasks.rados.rados.0.smithi070.stdout:401: writing smithi07085154-13 from 4227072 to 4734976 tid 4 2024-09-14T03:49:16.538 INFO:tasks.rados.rados.0.smithi070.stdout:402: delete oid 37 current snap is 39 2024-09-14T03:49:16.538 INFO:tasks.rados.rados.0.smithi070.stdout:401: finishing write tid 1 to smithi07085154-13 2024-09-14T03:49:16.538 INFO:tasks.rados.rados.0.smithi070.stdout:401: finishing write tid 2 to smithi07085154-13 2024-09-14T03:49:16.538 INFO:tasks.rados.rados.0.smithi070.stdout:401: finishing write tid 3 to smithi07085154-13 2024-09-14T03:49:16.540 INFO:tasks.rados.rados.0.smithi070.stdout:401: finishing write tid 4 to smithi07085154-13 2024-09-14T03:49:16.540 INFO:tasks.rados.rados.0.smithi070.stdout:401: finishing write tid 5 to smithi07085154-13 2024-09-14T03:49:16.541 INFO:tasks.rados.rados.0.smithi070.stdout:401: finishing write tid 6 to smithi07085154-13 2024-09-14T03:49:16.541 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 134 (ObjNum 138 snap 39 seq_num 138) dirty exists 2024-09-14T03:49:16.542 INFO:tasks.rados.rados.0.smithi070.stdout:401: left oid 13 (ObjNum 138 snap 39 seq_num 138) 2024-09-14T03:49:16.543 INFO:tasks.rados.rados.0.smithi070.stdout:401: done (1 left) 2024-09-14T03:49:16.543 INFO:tasks.rados.rados.0.smithi070.stdout:402: done (0 left) 2024-09-14T03:49:16.543 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 25 current snap is 39 2024-09-14T03:49:16.543 INFO:tasks.rados.rados.0.smithi070.stdout:403: seq_num 140 ranges {5382144=753664,6135808=704512,6840320=524288,7364608=376832} 2024-09-14T03:49:16.575 INFO:tasks.rados.rados.0.smithi070.stdout:403: writing smithi07085154-25 from 5382144 to 6135808 tid 1 2024-09-14T03:49:16.579 INFO:tasks.rados.rados.0.smithi070.stdout:403: writing smithi07085154-25 from 6135808 to 6840320 tid 2 2024-09-14T03:49:16.582 INFO:tasks.rados.rados.0.smithi070.stdout:403: writing smithi07085154-25 from 6840320 to 7364608 tid 3 2024-09-14T03:49:16.584 INFO:tasks.rados.rados.0.smithi070.stdout:403: writing smithi07085154-25 from 7364608 to 7741440 tid 4 2024-09-14T03:49:16.584 INFO:tasks.rados.rados.0.smithi070.stdout:404: setattr oid 48 current snap is 39 2024-09-14T03:49:16.585 INFO:tasks.rados.rados.0.smithi070.stdout:403: finishing write tid 1 to smithi07085154-25 2024-09-14T03:49:16.585 INFO:tasks.rados.rados.0.smithi070.stdout:403: finishing write tid 2 to smithi07085154-25 2024-09-14T03:49:16.585 INFO:tasks.rados.rados.0.smithi070.stdout:403: finishing write tid 3 to smithi07085154-25 2024-09-14T03:49:16.586 INFO:tasks.rados.rados.0.smithi070.stdout:405: read oid 14 snap -1 2024-09-14T03:49:16.586 INFO:tasks.rados.rados.0.smithi070.stdout:405: expect (ObjNum 13 snap 0 seq_num 13) 2024-09-14T03:49:16.586 INFO:tasks.rados.rados.0.smithi070.stdout:406: snap_create 2024-09-14T03:49:16.587 INFO:tasks.rados.rados.0.smithi070.stdout:403: finishing write tid 4 to smithi07085154-25 2024-09-14T03:49:16.587 INFO:tasks.rados.rados.0.smithi070.stdout:403: finishing write tid 5 to smithi07085154-25 2024-09-14T03:49:16.588 INFO:tasks.rados.rados.0.smithi070.stdout:403: finishing write tid 6 to smithi07085154-25 2024-09-14T03:49:16.589 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 171 (ObjNum 140 snap 39 seq_num 140) dirty exists 2024-09-14T03:49:16.589 INFO:tasks.rados.rados.0.smithi070.stdout:403: left oid 25 (ObjNum 140 snap 39 seq_num 140) 2024-09-14T03:49:16.589 INFO:tasks.rados.rados.0.smithi070.stdout:405: expect (ObjNum 13 snap 0 seq_num 13) 2024-09-14T03:49:16.595 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 78 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-14T03:49:16.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:16 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:49:16] "GET /metrics HTTP/1.1" 200 240124 "" "Prometheus/2.43.0" 2024-09-14T03:49:17.518 INFO:tasks.rados.rados.0.smithi070.stdout:403: done (3 left) 2024-09-14T03:49:17.519 INFO:tasks.rados.rados.0.smithi070.stdout:404: done (2 left) 2024-09-14T03:49:17.519 INFO:tasks.rados.rados.0.smithi070.stdout:405: done (1 left) 2024-09-14T03:49:17.519 INFO:tasks.rados.rados.0.smithi070.stdout:406: done (0 left) 2024-09-14T03:49:17.519 INFO:tasks.rados.rados.0.smithi070.stdout:407: delete oid 27 current snap is 40 2024-09-14T03:49:17.521 INFO:tasks.rados.rados.0.smithi070.stdout:407: done (0 left) 2024-09-14T03:49:17.521 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 3 current snap is 40 2024-09-14T03:49:17.521 INFO:tasks.rados.rados.0.smithi070.stdout:408: seq_num 142 ranges {106496=786432,892928=475136,1368064=540672,1908736=589824} 2024-09-14T03:49:17.525 INFO:tasks.rados.rados.0.smithi070.stdout:408: writing smithi07085154-3 from 106496 to 892928 tid 1 2024-09-14T03:49:17.528 INFO:tasks.rados.rados.0.smithi070.stdout:408: writing smithi07085154-3 from 892928 to 1368064 tid 2 2024-09-14T03:49:17.531 INFO:tasks.rados.rados.0.smithi070.stdout:408: writing smithi07085154-3 from 1368064 to 1908736 tid 3 2024-09-14T03:49:17.535 INFO:tasks.rados.rados.0.smithi070.stdout:408: writing smithi07085154-3 from 1908736 to 2498560 tid 4 2024-09-14T03:49:17.535 INFO:tasks.rados.rados.0.smithi070.stdout:409: snap_create 2024-09-14T03:49:17.535 INFO:tasks.rados.rados.0.smithi070.stdout:408: finishing write tid 1 to smithi07085154-3 2024-09-14T03:49:17.535 INFO:tasks.rados.rados.0.smithi070.stdout:408: finishing write tid 2 to smithi07085154-3 2024-09-14T03:49:17.535 INFO:tasks.rados.rados.0.smithi070.stdout:408: finishing write tid 3 to smithi07085154-3 2024-09-14T03:49:17.537 INFO:tasks.rados.rados.0.smithi070.stdout:408: finishing write tid 4 to smithi07085154-3 2024-09-14T03:49:17.538 INFO:tasks.rados.rados.0.smithi070.stdout:408: finishing write tid 5 to smithi07085154-3 2024-09-14T03:49:17.538 INFO:tasks.rados.rados.0.smithi070.stdout:408: finishing write tid 6 to smithi07085154-3 2024-09-14T03:49:17.538 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 3 v 76 (ObjNum 142 snap 40 seq_num 142) dirty exists 2024-09-14T03:49:17.538 INFO:tasks.rados.rados.0.smithi070.stdout:408: left oid 3 (ObjNum 142 snap 40 seq_num 142) 2024-09-14T03:49:17.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:17 smithi094 ceph-mon[31053]: osdmap e142: 8 total, 8 up, 8 in 2024-09-14T03:49:17.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:17 smithi094 ceph-mon[31053]: pgmap v209: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:49:17.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:17 smithi070 ceph-mon[39288]: osdmap e142: 8 total, 8 up, 8 in 2024-09-14T03:49:17.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:17 smithi070 ceph-mon[39288]: pgmap v209: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:49:17.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:17 smithi070 ceph-mon[27495]: osdmap e142: 8 total, 8 up, 8 in 2024-09-14T03:49:17.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:17 smithi070 ceph-mon[27495]: pgmap v209: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:49:18.522 INFO:tasks.rados.rados.0.smithi070.stdout:408: done (1 left) 2024-09-14T03:49:18.522 INFO:tasks.rados.rados.0.smithi070.stdout:409: done (0 left) 2024-09-14T03:49:18.522 INFO:tasks.rados.rados.0.smithi070.stdout:410: rollback oid 27 current snap is 41 2024-09-14T03:49:18.522 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 27 to 40 2024-09-14T03:49:18.522 INFO:tasks.rados.rados.0.smithi070.stdout:411: setattr oid 20 current snap is 41 2024-09-14T03:49:18.523 INFO:tasks.rados.rados.0.smithi070.stdout:412: snap_create 2024-09-14T03:49:18.525 INFO:tasks.rados.rados.0.smithi070.stdout:410: finishing rollback tid 1 to smithi07085154-27 2024-09-14T03:49:18.525 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 262 (ObjNum 129 snap 0 seq_num 2930371192) dirty dne 2024-09-14T03:49:18.526 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 20 v 94 (ObjNum 129 snap 0 seq_num 2930410528) dirty exists 2024-09-14T03:49:18.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:18 smithi094 ceph-mon[31053]: osdmap e143: 8 total, 8 up, 8 in 2024-09-14T03:49:18.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:18 smithi070 ceph-mon[27495]: osdmap e143: 8 total, 8 up, 8 in 2024-09-14T03:49:18.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:18 smithi070 ceph-mon[39288]: osdmap e143: 8 total, 8 up, 8 in 2024-09-14T03:49:19.527 INFO:tasks.rados.rados.0.smithi070.stdout:410: done (2 left) 2024-09-14T03:49:19.527 INFO:tasks.rados.rados.0.smithi070.stdout:411: done (1 left) 2024-09-14T03:49:19.527 INFO:tasks.rados.rados.0.smithi070.stdout:412: done (0 left) 2024-09-14T03:49:19.527 INFO:tasks.rados.rados.0.smithi070.stdout:413: read oid 46 snap -1 2024-09-14T03:49:19.527 INFO:tasks.rados.rados.0.smithi070.stdout:413: expect deleted 2024-09-14T03:49:19.527 INFO:tasks.rados.rados.0.smithi070.stdout:414: snap_remove snap 33 2024-09-14T03:49:19.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:19 smithi094 ceph-mon[31053]: osdmap e144: 8 total, 8 up, 8 in 2024-09-14T03:49:19.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:19 smithi094 ceph-mon[31053]: pgmap v212: 121 pgs: 121 active+clean; 141 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.7 MiB/s wr, 24 op/s 2024-09-14T03:49:19.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:19 smithi070 ceph-mon[27495]: osdmap e144: 8 total, 8 up, 8 in 2024-09-14T03:49:19.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:19 smithi070 ceph-mon[27495]: pgmap v212: 121 pgs: 121 active+clean; 141 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.7 MiB/s wr, 24 op/s 2024-09-14T03:49:19.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:19 smithi070 ceph-mon[39288]: osdmap e144: 8 total, 8 up, 8 in 2024-09-14T03:49:19.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:19 smithi070 ceph-mon[39288]: pgmap v212: 121 pgs: 121 active+clean; 141 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.7 MiB/s wr, 24 op/s 2024-09-14T03:49:20.512 INFO:tasks.rados.rados.0.smithi070.stdout:413: done (1 left) 2024-09-14T03:49:20.512 INFO:tasks.rados.rados.0.smithi070.stdout:414: done (0 left) 2024-09-14T03:49:20.512 INFO:tasks.rados.rados.0.smithi070.stdout:415: read oid 45 snap -1 2024-09-14T03:49:20.512 INFO:tasks.rados.rados.0.smithi070.stdout:415: expect (ObjNum 49 snap 0 seq_num 49) 2024-09-14T03:49:20.512 INFO:tasks.rados.rados.0.smithi070.stdout:416: read oid 43 snap 41 2024-09-14T03:49:20.513 INFO:tasks.rados.rados.0.smithi070.stdout:416: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-14T03:49:20.513 INFO:tasks.rados.rados.0.smithi070.stdout:417: snap_remove snap 22 2024-09-14T03:49:20.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:20 smithi094 ceph-mon[31053]: osdmap e145: 8 total, 8 up, 8 in 2024-09-14T03:49:20.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:20 smithi094 ceph-mon[31053]: osdmap e146: 8 total, 8 up, 8 in 2024-09-14T03:49:20.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:20 smithi070 ceph-mon[27495]: osdmap e145: 8 total, 8 up, 8 in 2024-09-14T03:49:20.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:20 smithi070 ceph-mon[27495]: osdmap e146: 8 total, 8 up, 8 in 2024-09-14T03:49:20.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:20 smithi070 ceph-mon[39288]: osdmap e145: 8 total, 8 up, 8 in 2024-09-14T03:49:20.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:20 smithi070 ceph-mon[39288]: osdmap e146: 8 total, 8 up, 8 in 2024-09-14T03:49:21.514 INFO:tasks.rados.rados.0.smithi070.stdout:417: done (2 left) 2024-09-14T03:49:21.514 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 7 current snap is 42 2024-09-14T03:49:21.514 INFO:tasks.rados.rados.0.smithi070.stdout:418: seq_num 143 ranges {0=802816,802816=761856,1564672=745472,2310144=98304} 2024-09-14T03:49:21.520 INFO:tasks.rados.rados.0.smithi070.stdout:418: writing smithi07085154-7 from 0 to 802816 tid 1 2024-09-14T03:49:21.525 INFO:tasks.rados.rados.0.smithi070.stdout:418: writing smithi07085154-7 from 802816 to 1564672 tid 2 2024-09-14T03:49:21.531 INFO:tasks.rados.rados.0.smithi070.stdout:418: writing smithi07085154-7 from 1564672 to 2310144 tid 3 2024-09-14T03:49:21.531 INFO:tasks.rados.rados.0.smithi070.stdout:418: writing smithi07085154-7 from 2310144 to 2408448 tid 4 2024-09-14T03:49:21.531 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 29 current snap is 42 2024-09-14T03:49:21.531 INFO:tasks.rados.rados.0.smithi070.stdout:419: seq_num 144 ranges {761856=16384} 2024-09-14T03:49:21.535 INFO:tasks.rados.rados.0.smithi070.stdout:419: writing smithi07085154-29 from 761856 to 778240 tid 1 2024-09-14T03:49:21.535 INFO:tasks.rados.rados.0.smithi070.stdout:420: rmattr oid 16 current snap is 42 2024-09-14T03:49:21.536 INFO:tasks.rados.rados.0.smithi070.stdout:420: done (4 left) 2024-09-14T03:49:21.536 INFO:tasks.rados.rados.0.smithi070.stdout:421: delete oid 33 current snap is 42 2024-09-14T03:49:21.536 INFO:tasks.rados.rados.0.smithi070.stdout:416: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-14T03:49:21.557 INFO:tasks.rados.rados.0.smithi070.stdout:415: expect (ObjNum 49 snap 0 seq_num 49) 2024-09-14T03:49:21.579 INFO:tasks.rados.rados.0.smithi070.stdout:418: finishing write tid 1 to smithi07085154-7 2024-09-14T03:49:21.579 INFO:tasks.rados.rados.0.smithi070.stdout:418: finishing write tid 2 to smithi07085154-7 2024-09-14T03:49:21.579 INFO:tasks.rados.rados.0.smithi070.stdout:418: finishing write tid 3 to smithi07085154-7 2024-09-14T03:49:21.579 INFO:tasks.rados.rados.0.smithi070.stdout:418: finishing write tid 4 to smithi07085154-7 2024-09-14T03:49:21.579 INFO:tasks.rados.rados.0.smithi070.stdout:419: finishing write tid 1 to smithi07085154-29 2024-09-14T03:49:21.579 INFO:tasks.rados.rados.0.smithi070.stdout:418: finishing write tid 5 to smithi07085154-7 2024-09-14T03:49:21.579 INFO:tasks.rados.rados.0.smithi070.stdout:419: finishing write tid 2 to smithi07085154-29 2024-09-14T03:49:21.579 INFO:tasks.rados.rados.0.smithi070.stdout:418: finishing write tid 6 to smithi07085154-7 2024-09-14T03:49:21.579 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 107 (ObjNum 143 snap 42 seq_num 143) dirty exists 2024-09-14T03:49:21.580 INFO:tasks.rados.rados.0.smithi070.stdout:418: left oid 7 (ObjNum 143 snap 42 seq_num 143) 2024-09-14T03:49:21.580 INFO:tasks.rados.rados.0.smithi070.stdout:419: finishing write tid 3 to smithi07085154-29 2024-09-14T03:49:21.580 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 182 (ObjNum 144 snap 42 seq_num 144) dirty exists 2024-09-14T03:49:21.580 INFO:tasks.rados.rados.0.smithi070.stdout:419: left oid 29 (ObjNum 144 snap 42 seq_num 144) 2024-09-14T03:49:21.580 INFO:tasks.rados.rados.0.smithi070.stdout:415: done (4 left) 2024-09-14T03:49:21.580 INFO:tasks.rados.rados.0.smithi070.stdout:416: done (3 left) 2024-09-14T03:49:21.580 INFO:tasks.rados.rados.0.smithi070.stdout:418: done (2 left) 2024-09-14T03:49:21.580 INFO:tasks.rados.rados.0.smithi070.stdout:419: done (1 left) 2024-09-14T03:49:21.580 INFO:tasks.rados.rados.0.smithi070.stdout:421: done (0 left) 2024-09-14T03:49:21.580 INFO:tasks.rados.rados.0.smithi070.stdout:422: read oid 24 snap -1 2024-09-14T03:49:21.580 INFO:tasks.rados.rados.0.smithi070.stdout:422: expect deleted 2024-09-14T03:49:21.580 INFO:tasks.rados.rados.0.smithi070.stdout:423: snap_create 2024-09-14T03:49:21.771 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:21 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[33031]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:49:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:49:21.772 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:21 smithi094 ceph-mon[31053]: pgmap v215: 121 pgs: 121 active+clean; 141 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.7 MiB/s wr, 24 op/s 2024-09-14T03:49:21.772 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:21 smithi094 ceph-mon[31053]: osdmap e147: 8 total, 8 up, 8 in 2024-09-14T03:49:21.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:21 smithi070 ceph-mon[27495]: pgmap v215: 121 pgs: 121 active+clean; 141 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.7 MiB/s wr, 24 op/s 2024-09-14T03:49:21.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:21 smithi070 ceph-mon[27495]: osdmap e147: 8 total, 8 up, 8 in 2024-09-14T03:49:21.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:21 smithi070 ceph-mon[39288]: pgmap v215: 121 pgs: 121 active+clean; 141 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.7 MiB/s wr, 24 op/s 2024-09-14T03:49:21.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:21 smithi070 ceph-mon[39288]: osdmap e147: 8 total, 8 up, 8 in 2024-09-14T03:49:22.544 INFO:tasks.rados.rados.0.smithi070.stdout:422: done (1 left) 2024-09-14T03:49:22.544 INFO:tasks.rados.rados.0.smithi070.stdout:423: done (0 left) 2024-09-14T03:49:22.544 INFO:tasks.rados.rados.0.smithi070.stdout:424: rmattr oid 6 current snap is 43 2024-09-14T03:49:22.544 INFO:tasks.rados.rados.0.smithi070.stdout:425: rollback oid 3 current snap is 43 2024-09-14T03:49:22.544 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 3 to 20 2024-09-14T03:49:22.544 INFO:tasks.rados.rados.0.smithi070.stdout:426: rollback oid 16 current snap is 43 2024-09-14T03:49:22.544 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 16 to 28 2024-09-14T03:49:22.544 INFO:tasks.rados.rados.0.smithi070.stdout:427: snap_create 2024-09-14T03:49:22.547 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 181 (ObjNum 122 snap 31 seq_num 122) dirty exists 2024-09-14T03:49:22.549 INFO:tasks.rados.rados.0.smithi070.stdout:425: finishing rollback tid 0 to smithi07085154-3 2024-09-14T03:49:22.549 INFO:tasks.rados.rados.0.smithi070.stdout:425: finishing rollback tid 1 to smithi07085154-3 2024-09-14T03:49:22.549 INFO:tasks.rados.rados.0.smithi070.stdout:426: finishing rollback tid 0 to smithi07085154-16 2024-09-14T03:49:22.549 INFO:tasks.rados.rados.0.smithi070.stdout:425: finishing rollback tid 2 to smithi07085154-3 2024-09-14T03:49:22.549 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 3 v 88 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-09-14T03:49:22.549 INFO:tasks.rados.rados.0.smithi070.stdout:426: finishing rollback tid 1 to smithi07085154-16 2024-09-14T03:49:22.550 INFO:tasks.rados.rados.0.smithi070.stdout:426: finishing rollback tid 2 to smithi07085154-16 2024-09-14T03:49:22.550 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 285 (ObjNum 114 snap 24 seq_num 114) dirty exists 2024-09-14T03:49:22.746 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:22 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:49:23.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:22 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:49:23.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:22 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:49:23.549 INFO:tasks.rados.rados.0.smithi070.stdout:424: done (3 left) 2024-09-14T03:49:23.550 INFO:tasks.rados.rados.0.smithi070.stdout:425: done (2 left) 2024-09-14T03:49:23.550 INFO:tasks.rados.rados.0.smithi070.stdout:426: done (1 left) 2024-09-14T03:49:23.550 INFO:tasks.rados.rados.0.smithi070.stdout:427: done (0 left) 2024-09-14T03:49:23.550 INFO:tasks.rados.rados.0.smithi070.stdout:428: rollback oid 39 current snap is 44 2024-09-14T03:49:23.550 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 39 to 42 2024-09-14T03:49:23.550 INFO:tasks.rados.rados.0.smithi070.stdout:429: rollback oid 21 current snap is 44 2024-09-14T03:49:23.550 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 21 to 36 2024-09-14T03:49:23.550 INFO:tasks.rados.rados.0.smithi070.stdout:430: delete oid 24 current snap is 44 2024-09-14T03:49:23.552 INFO:tasks.rados.rados.0.smithi070.stdout:430: done (2 left) 2024-09-14T03:49:23.552 INFO:tasks.rados.rados.0.smithi070.stdout:431: delete oid 3 current snap is 44 2024-09-14T03:49:23.553 INFO:tasks.rados.rados.0.smithi070.stdout:429: finishing rollback tid 1 to smithi07085154-21 2024-09-14T03:49:23.554 INFO:tasks.rados.rados.0.smithi070.stdout:429: finishing rollback tid 2 to smithi07085154-21 2024-09-14T03:49:23.554 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 62 (ObjNum 107 snap 23 seq_num 107) dirty exists 2024-09-14T03:49:23.555 INFO:tasks.rados.rados.0.smithi070.stdout:428: finishing rollback tid 0 to smithi07085154-39 2024-09-14T03:49:23.555 INFO:tasks.rados.rados.0.smithi070.stdout:428: finishing rollback tid 1 to smithi07085154-39 2024-09-14T03:49:23.556 INFO:tasks.rados.rados.0.smithi070.stdout:428: finishing rollback tid 2 to smithi07085154-39 2024-09-14T03:49:23.556 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 39 v 24 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-14T03:49:23.556 INFO:tasks.rados.rados.0.smithi070.stdout:428: done (2 left) 2024-09-14T03:49:23.556 INFO:tasks.rados.rados.0.smithi070.stdout:429: done (1 left) 2024-09-14T03:49:23.556 INFO:tasks.rados.rados.0.smithi070.stdout:431: done (0 left) 2024-09-14T03:49:23.557 INFO:tasks.rados.rados.0.smithi070.stdout:432: read oid 33 snap 30 2024-09-14T03:49:23.557 INFO:tasks.rados.rados.0.smithi070.stdout:432: expect (ObjNum 116 snap 25 seq_num 116) 2024-09-14T03:49:23.557 INFO:tasks.rados.rados.0.smithi070.stdout:433: setattr oid 15 current snap is 44 2024-09-14T03:49:23.559 INFO:tasks.rados.rados.0.smithi070.stdout:434: setattr oid 29 current snap is 44 2024-09-14T03:49:23.562 INFO:tasks.rados.rados.0.smithi070.stdout:435: delete oid 34 current snap is 44 2024-09-14T03:49:23.565 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 15 v 229 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-14T03:49:23.566 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 184 (ObjNum 144 snap 42 seq_num 144) dirty exists 2024-09-14T03:49:23.566 INFO:tasks.rados.rados.0.smithi070.stdout:433: done (3 left) 2024-09-14T03:49:23.567 INFO:tasks.rados.rados.0.smithi070.stdout:434: done (2 left) 2024-09-14T03:49:23.567 INFO:tasks.rados.rados.0.smithi070.stdout:435: done (1 left) 2024-09-14T03:49:23.567 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 28 current snap is 44 2024-09-14T03:49:23.567 INFO:tasks.rados.rados.0.smithi070.stdout:436: seq_num 149 ranges {0=98304} 2024-09-14T03:49:23.567 INFO:tasks.rados.rados.0.smithi070.stdout:436: writing smithi07085154-28 from 0 to 98304 tid 1 2024-09-14T03:49:23.567 INFO:tasks.rados.rados.0.smithi070.stdout:437: snap_remove snap 27 2024-09-14T03:49:23.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:23 smithi094 ceph-mon[31053]: osdmap e148: 8 total, 8 up, 8 in 2024-09-14T03:49:23.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:23 smithi094 ceph-mon[31053]: pgmap v218: 121 pgs: 121 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 592 KiB/s wr, 19 op/s 2024-09-14T03:49:23.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:23 smithi094 ceph-mon[31053]: Upgrade: Updating mgr.x 2024-09-14T03:49:23.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:23 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:23.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:23 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:49:23.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:23 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:49:23.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:23 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:23.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:23 smithi094 ceph-mon[31053]: Deploying daemon mgr.x on smithi094 2024-09-14T03:49:24.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:23 smithi070 ceph-mon[27495]: osdmap e148: 8 total, 8 up, 8 in 2024-09-14T03:49:24.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:23 smithi070 ceph-mon[27495]: pgmap v218: 121 pgs: 121 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 592 KiB/s wr, 19 op/s 2024-09-14T03:49:24.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:23 smithi070 ceph-mon[27495]: Upgrade: Updating mgr.x 2024-09-14T03:49:24.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:23 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:24.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:23 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:49:24.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:23 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:49:24.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:23 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:24.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:23 smithi070 ceph-mon[27495]: Deploying daemon mgr.x on smithi094 2024-09-14T03:49:24.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:23 smithi070 ceph-mon[39288]: osdmap e148: 8 total, 8 up, 8 in 2024-09-14T03:49:24.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:23 smithi070 ceph-mon[39288]: pgmap v218: 121 pgs: 121 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 592 KiB/s wr, 19 op/s 2024-09-14T03:49:24.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:23 smithi070 ceph-mon[39288]: Upgrade: Updating mgr.x 2024-09-14T03:49:24.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:23 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:24.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:23 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:49:24.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:23 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:49:24.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:23 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:24.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:23 smithi070 ceph-mon[39288]: Deploying daemon mgr.x on smithi094 2024-09-14T03:49:24.527 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:24 smithi094 systemd[1]: Stopping Ceph mgr.x for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:49:24.553 INFO:tasks.rados.rados.0.smithi070.stdout:437: done (2 left) 2024-09-14T03:49:24.553 INFO:tasks.rados.rados.0.smithi070.stdout:438: copy_from oid 22 from oid 13 current snap is 44 2024-09-14T03:49:24.553 INFO:tasks.rados.rados.0.smithi070.stdout:432: expect (ObjNum 116 snap 25 seq_num 116) 2024-09-14T03:49:24.624 INFO:tasks.rados.rados.0.smithi070.stdout:436: finishing write tid 1 to smithi07085154-28 2024-09-14T03:49:24.625 INFO:tasks.rados.rados.0.smithi070.stdout:432: done (2 left) 2024-09-14T03:49:24.625 INFO:tasks.rados.rados.0.smithi070.stdout:439: read oid 29 snap -1 2024-09-14T03:49:24.625 INFO:tasks.rados.rados.0.smithi070.stdout:439: expect (ObjNum 144 snap 42 seq_num 144) 2024-09-14T03:49:24.625 INFO:tasks.rados.rados.0.smithi070.stdout:440: rollback oid 20 current snap is 44 2024-09-14T03:49:24.625 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 20 to 20 2024-09-14T03:49:24.625 INFO:tasks.rados.rados.0.smithi070.stdout:436: finishing write tid 2 to smithi07085154-28 2024-09-14T03:49:24.625 INFO:tasks.rados.rados.0.smithi070.stdout:436: finishing write tid 3 to smithi07085154-28 2024-09-14T03:49:24.625 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 28 v 111 (ObjNum 149 snap 44 seq_num 149) dirty exists 2024-09-14T03:49:24.625 INFO:tasks.rados.rados.0.smithi070.stdout:436: left oid 28 (ObjNum 149 snap 44 seq_num 149) 2024-09-14T03:49:24.625 INFO:tasks.rados.rados.0.smithi070.stdout:438: finishing copy_from racing read to smithi07085154-22 2024-09-14T03:49:24.625 INFO:tasks.rados.rados.0.smithi070.stdout:438: finishing copy_from to smithi07085154-22 2024-09-14T03:49:24.625 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 134 (ObjNum 138 snap 39 seq_num 138) dirty exists 2024-09-14T03:49:24.626 INFO:tasks.rados.rados.0.smithi070.stdout:436: done (3 left) 2024-09-14T03:49:24.626 INFO:tasks.rados.rados.0.smithi070.stdout:438: done (2 left) 2024-09-14T03:49:24.626 INFO:tasks.rados.rados.0.smithi070.stdout:441: snap_create 2024-09-14T03:49:24.627 INFO:tasks.rados.rados.0.smithi070.stdout:440: finishing rollback tid 0 to smithi07085154-20 2024-09-14T03:49:24.627 INFO:tasks.rados.rados.0.smithi070.stdout:440: finishing rollback tid 1 to smithi07085154-20 2024-09-14T03:49:24.627 INFO:tasks.rados.rados.0.smithi070.stdout:440: finishing rollback tid 2 to smithi07085154-20 2024-09-14T03:49:24.627 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 20 v 121 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-14T03:49:24.629 INFO:tasks.rados.rados.0.smithi070.stdout:439: expect (ObjNum 144 snap 42 seq_num 144) 2024-09-14T03:49:24.891 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:24 smithi094 ceph-mon[31053]: osdmap e149: 8 total, 8 up, 8 in 2024-09-14T03:49:24.891 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:24 smithi094 podman[67003]: 2024-09-14 03:49:24.620222474 +0000 UTC m=+0.139512338 container died 8a5c729a2096fe7c37e852dfe69eceec7e330a20db989aed67cf6cd618161755 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=quincy-eb16b97, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-14T03:49:25.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:24 smithi070 ceph-mon[27495]: osdmap e149: 8 total, 8 up, 8 in 2024-09-14T03:49:25.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:24 smithi070 ceph-mon[39288]: osdmap e149: 8 total, 8 up, 8 in 2024-09-14T03:49:25.143 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:24 smithi094 podman[67003]: 2024-09-14 03:49:24.964122053 +0000 UTC m=+0.483411924 container cleanup 8a5c729a2096fe7c37e852dfe69eceec7e330a20db989aed67cf6cd618161755 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=quincy-eb16b97, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-14T03:49:25.143 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:24 smithi094 bash[67003]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x 2024-09-14T03:49:25.143 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:25 smithi094 podman[67022]: 2024-09-14 03:49:25.140386929 +0000 UTC m=+0.513526278 container remove 8a5c729a2096fe7c37e852dfe69eceec7e330a20db989aed67cf6cd618161755 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-eb16b97) 2024-09-14T03:49:25.143 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:25 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-14T03:49:25.513 INFO:tasks.rados.rados.0.smithi070.stdout:439: done (2 left) 2024-09-14T03:49:25.514 INFO:tasks.rados.rados.0.smithi070.stdout:440: done (1 left) 2024-09-14T03:49:25.514 INFO:tasks.rados.rados.0.smithi070.stdout:441: done (0 left) 2024-09-14T03:49:25.514 INFO:tasks.rados.rados.0.smithi070.stdout:442: read oid 14 snap -1 2024-09-14T03:49:25.514 INFO:tasks.rados.rados.0.smithi070.stdout:442: expect (ObjNum 13 snap 0 seq_num 13) 2024-09-14T03:49:25.514 INFO:tasks.rados.rados.0.smithi070.stdout:443: delete oid 12 current snap is 45 2024-09-14T03:49:25.517 INFO:tasks.rados.rados.0.smithi070.stdout:443: done (1 left) 2024-09-14T03:49:25.517 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 47 current snap is 45 2024-09-14T03:49:25.518 INFO:tasks.rados.rados.0.smithi070.stdout:444: seq_num 151 ranges {2662400=131072} 2024-09-14T03:49:25.526 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:25 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-09-14T03:49:25.526 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:25 smithi094 systemd[1]: Stopped Ceph mgr.x for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:49:25.526 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:25 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@mgr.x.service: Consumed 10.209s CPU time. 2024-09-14T03:49:25.531 INFO:tasks.rados.rados.0.smithi070.stdout:444: writing smithi07085154-47 from 2662400 to 2793472 tid 1 2024-09-14T03:49:25.531 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 44 current snap is 45 2024-09-14T03:49:25.531 INFO:tasks.rados.rados.0.smithi070.stdout:445: seq_num 152 ranges {729088=147456} 2024-09-14T03:49:25.536 INFO:tasks.rados.rados.0.smithi070.stdout:445: writing smithi07085154-44 from 729088 to 876544 tid 1 2024-09-14T03:49:25.536 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 9 current snap is 45 2024-09-14T03:49:25.536 INFO:tasks.rados.rados.0.smithi070.stdout:446: seq_num 153 ranges {2686976=163840} 2024-09-14T03:49:25.551 INFO:tasks.rados.rados.0.smithi070.stdout:446: writing smithi07085154-9 from 2686976 to 2850816 tid 1 2024-09-14T03:49:25.551 INFO:tasks.rados.rados.0.smithi070.stdout:447: snap_remove snap 42 2024-09-14T03:49:25.834 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:25 smithi094 ceph-mon[31053]: osdmap e150: 8 total, 8 up, 8 in 2024-09-14T03:49:25.834 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:25 smithi094 ceph-mon[31053]: pgmap v221: 121 pgs: 121 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 592 KiB/s wr, 19 op/s 2024-09-14T03:49:25.834 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:25 smithi094 ceph-mon[31053]: osdmap e151: 8 total, 8 up, 8 in 2024-09-14T03:49:25.834 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:25 smithi094 systemd[1]: Starting Ceph mgr.x for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:49:25.834 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:25 smithi094 podman[67152]: 2024-09-14 03:49:25.701724978 +0000 UTC m=+0.016050254 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:49:25.834 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:25 smithi094 podman[67152]: 2024-09-14 03:49:25.833364555 +0000 UTC m=+0.147689832 container create 5aadf84f680bdbdaa74c49e4fa946dea914357f472c7da94476c4ceab5fa3fd3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=main-a278231, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-14T03:49:26.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:25 smithi070 ceph-mon[39288]: osdmap e150: 8 total, 8 up, 8 in 2024-09-14T03:49:26.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:25 smithi070 ceph-mon[39288]: pgmap v221: 121 pgs: 121 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 592 KiB/s wr, 19 op/s 2024-09-14T03:49:26.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:25 smithi070 ceph-mon[39288]: osdmap e151: 8 total, 8 up, 8 in 2024-09-14T03:49:26.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:25 smithi070 ceph-mon[27495]: osdmap e150: 8 total, 8 up, 8 in 2024-09-14T03:49:26.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:25 smithi070 ceph-mon[27495]: pgmap v221: 121 pgs: 121 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 592 KiB/s wr, 19 op/s 2024-09-14T03:49:26.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:25 smithi070 ceph-mon[27495]: osdmap e151: 8 total, 8 up, 8 in 2024-09-14T03:49:26.099 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:26 smithi094 podman[67152]: 2024-09-14 03:49:26.06960448 +0000 UTC m=+0.383929756 container init 5aadf84f680bdbdaa74c49e4fa946dea914357f472c7da94476c4ceab5fa3fd3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-14T03:49:26.099 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:26 smithi094 podman[67152]: 2024-09-14 03:49:26.074407165 +0000 UTC m=+0.388732444 container start 5aadf84f680bdbdaa74c49e4fa946dea914357f472c7da94476c4ceab5fa3fd3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, RELEASE=main-a278231, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-14T03:49:26.526 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:26 smithi094 ceph-mgr[67189]: -- 172.21.15.94:0/3771338125 <== mon.1 v2:172.21.15.94:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55efc5bbf0e0 con 0x55efc5b9d000 2024-09-14T03:49:26.526 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:26 smithi094 bash[67152]: 5aadf84f680bdbdaa74c49e4fa946dea914357f472c7da94476c4ceab5fa3fd3 2024-09-14T03:49:26.527 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:26 smithi094 systemd[1]: Started Ceph mgr.x for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:49:26.555 INFO:tasks.rados.rados.0.smithi070.stdout:447: done (4 left) 2024-09-14T03:49:26.555 INFO:tasks.rados.rados.0.smithi070.stdout:448: rollback oid 29 current snap is 45 2024-09-14T03:49:26.556 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 29 to 41 2024-09-14T03:49:26.556 INFO:tasks.rados.rados.0.smithi070.stdout:442: expect (ObjNum 13 snap 0 seq_num 13) 2024-09-14T03:49:26.566 INFO:tasks.rados.rados.0.smithi070.stdout:444: finishing write tid 1 to smithi07085154-47 2024-09-14T03:49:26.566 INFO:tasks.rados.rados.0.smithi070.stdout:444: finishing write tid 2 to smithi07085154-47 2024-09-14T03:49:26.566 INFO:tasks.rados.rados.0.smithi070.stdout:444: finishing write tid 3 to smithi07085154-47 2024-09-14T03:49:26.567 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 47 v 95 (ObjNum 151 snap 45 seq_num 151) dirty exists 2024-09-14T03:49:26.567 INFO:tasks.rados.rados.0.smithi070.stdout:444: left oid 47 (ObjNum 151 snap 45 seq_num 151) 2024-09-14T03:49:26.567 INFO:tasks.rados.rados.0.smithi070.stdout:445: finishing write tid 1 to smithi07085154-44 2024-09-14T03:49:26.567 INFO:tasks.rados.rados.0.smithi070.stdout:442: done (4 left) 2024-09-14T03:49:26.567 INFO:tasks.rados.rados.0.smithi070.stdout:444: done (3 left) 2024-09-14T03:49:26.567 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 21 current snap is 45 2024-09-14T03:49:26.567 INFO:tasks.rados.rados.0.smithi070.stdout:445: finishing write tid 2 to smithi07085154-44 2024-09-14T03:49:26.567 INFO:tasks.rados.rados.0.smithi070.stdout:445: finishing write tid 3 to smithi07085154-44 2024-09-14T03:49:26.567 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 105 (ObjNum 152 snap 45 seq_num 152) dirty exists 2024-09-14T03:49:26.567 INFO:tasks.rados.rados.0.smithi070.stdout:445: left oid 44 (ObjNum 152 snap 45 seq_num 152) 2024-09-14T03:49:26.567 INFO:tasks.rados.rados.0.smithi070.stdout:446: finishing write tid 1 to smithi07085154-9 2024-09-14T03:49:26.567 INFO:tasks.rados.rados.0.smithi070.stdout:449: seq_num 154 ranges {4489216=180224} 2024-09-14T03:49:26.592 INFO:tasks.rados.rados.0.smithi070.stdout:449: writing smithi07085154-21 from 4489216 to 4669440 tid 1 2024-09-14T03:49:26.592 INFO:tasks.rados.rados.0.smithi070.stdout:445: done (3 left) 2024-09-14T03:49:26.592 INFO:tasks.rados.rados.0.smithi070.stdout:450: copy_from oid 32 from oid 15 current snap is 45 2024-09-14T03:49:26.593 INFO:tasks.rados.rados.0.smithi070.stdout:446: finishing write tid 2 to smithi07085154-9 2024-09-14T03:49:26.593 INFO:tasks.rados.rados.0.smithi070.stdout:446: finishing write tid 3 to smithi07085154-9 2024-09-14T03:49:26.593 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 300 (ObjNum 153 snap 45 seq_num 153) dirty exists 2024-09-14T03:49:26.593 INFO:tasks.rados.rados.0.smithi070.stdout:446: left oid 9 (ObjNum 153 snap 45 seq_num 153) 2024-09-14T03:49:26.593 INFO:tasks.rados.rados.0.smithi070.stdout:448: finishing rollback tid 0 to smithi07085154-29 2024-09-14T03:49:26.593 INFO:tasks.rados.rados.0.smithi070.stdout:448: finishing rollback tid 1 to smithi07085154-29 2024-09-14T03:49:26.593 INFO:tasks.rados.rados.0.smithi070.stdout:448: finishing rollback tid 2 to smithi07085154-29 2024-09-14T03:49:26.593 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 202 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-14T03:49:26.593 INFO:tasks.rados.rados.0.smithi070.stdout:446: done (3 left) 2024-09-14T03:49:26.593 INFO:tasks.rados.rados.0.smithi070.stdout:448: done (2 left) 2024-09-14T03:49:26.593 INFO:tasks.rados.rados.0.smithi070.stdout:451: snap_create 2024-09-14T03:49:26.595 INFO:tasks.rados.rados.0.smithi070.stdout:449: finishing write tid 1 to smithi07085154-21 2024-09-14T03:49:26.595 INFO:tasks.rados.rados.0.smithi070.stdout:450: finishing copy_from racing read to smithi07085154-32 2024-09-14T03:49:26.595 INFO:tasks.rados.rados.0.smithi070.stdout:449: finishing write tid 2 to smithi07085154-21 2024-09-14T03:49:26.596 INFO:tasks.rados.rados.0.smithi070.stdout:449: finishing write tid 3 to smithi07085154-21 2024-09-14T03:49:26.596 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 64 (ObjNum 154 snap 45 seq_num 154) dirty exists 2024-09-14T03:49:26.597 INFO:tasks.rados.rados.0.smithi070.stdout:449: left oid 21 (ObjNum 154 snap 45 seq_num 154) 2024-09-14T03:49:26.597 INFO:tasks.rados.rados.0.smithi070.stdout:450: finishing copy_from to smithi07085154-32 2024-09-14T03:49:26.597 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 229 (ObjNum 65 snap 0 seq_num 1436215969) dirty exists 2024-09-14T03:49:26.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:26 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:49:26] "GET /metrics HTTP/1.1" 200 240185 "" "Prometheus/2.43.0" 2024-09-14T03:49:27.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:27 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:27.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:27 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:27.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:27 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:27.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:27 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:27.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:27 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:27.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:27 smithi094 ceph-mon[31053]: osdmap e152: 8 total, 8 up, 8 in 2024-09-14T03:49:27.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:27 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:27.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:27 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:27.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:27 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:27.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:27 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:27.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:27 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:27.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:27 smithi070 ceph-mon[27495]: osdmap e152: 8 total, 8 up, 8 in 2024-09-14T03:49:27.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:27 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:27.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:27 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:27.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:27 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:27.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:27 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:27.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:27 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:27.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:27 smithi070 ceph-mon[39288]: osdmap e152: 8 total, 8 up, 8 in 2024-09-14T03:49:27.558 INFO:tasks.rados.rados.0.smithi070.stdout:449: done (2 left) 2024-09-14T03:49:27.559 INFO:tasks.rados.rados.0.smithi070.stdout:450: done (1 left) 2024-09-14T03:49:27.559 INFO:tasks.rados.rados.0.smithi070.stdout:451: done (0 left) 2024-09-14T03:49:27.559 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 12 current snap is 46 2024-09-14T03:49:27.559 INFO:tasks.rados.rados.0.smithi070.stdout:452: seq_num 155 ranges {0=204800} 2024-09-14T03:49:27.560 INFO:tasks.rados.rados.0.smithi070.stdout:452: writing smithi07085154-12 from 0 to 204800 tid 1 2024-09-14T03:49:27.560 INFO:tasks.rados.rados.0.smithi070.stdout:453: read oid 9 snap -1 2024-09-14T03:49:27.560 INFO:tasks.rados.rados.0.smithi070.stdout:453: expect (ObjNum 153 snap 45 seq_num 153) 2024-09-14T03:49:27.560 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 26 current snap is 46 2024-09-14T03:49:27.560 INFO:tasks.rados.rados.0.smithi070.stdout:454: seq_num 156 ranges {1613824=221184} 2024-09-14T03:49:27.571 INFO:tasks.rados.rados.0.smithi070.stdout:454: writing smithi07085154-26 from 1613824 to 1835008 tid 1 2024-09-14T03:49:27.571 INFO:tasks.rados.rados.0.smithi070.stdout:455: snap_create 2024-09-14T03:49:27.571 INFO:tasks.rados.rados.0.smithi070.stdout:452: finishing write tid 1 to smithi07085154-12 2024-09-14T03:49:27.571 INFO:tasks.rados.rados.0.smithi070.stdout:452: finishing write tid 2 to smithi07085154-12 2024-09-14T03:49:27.571 INFO:tasks.rados.rados.0.smithi070.stdout:452: finishing write tid 3 to smithi07085154-12 2024-09-14T03:49:27.571 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 96 (ObjNum 155 snap 46 seq_num 155) dirty exists 2024-09-14T03:49:27.571 INFO:tasks.rados.rados.0.smithi070.stdout:452: left oid 12 (ObjNum 155 snap 46 seq_num 155) 2024-09-14T03:49:27.571 INFO:tasks.rados.rados.0.smithi070.stdout:453: expect (ObjNum 153 snap 45 seq_num 153) 2024-09-14T03:49:27.706 INFO:tasks.rados.rados.0.smithi070.stdout:454: finishing write tid 1 to smithi07085154-26 2024-09-14T03:49:27.706 INFO:tasks.rados.rados.0.smithi070.stdout:454: finishing write tid 2 to smithi07085154-26 2024-09-14T03:49:27.706 INFO:tasks.rados.rados.0.smithi070.stdout:454: finishing write tid 3 to smithi07085154-26 2024-09-14T03:49:27.706 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 307 (ObjNum 156 snap 46 seq_num 156) dirty exists 2024-09-14T03:49:27.706 INFO:tasks.rados.rados.0.smithi070.stdout:454: left oid 26 (ObjNum 156 snap 46 seq_num 156) 2024-09-14T03:49:28.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:28 smithi094 ceph-mon[31053]: pgmap v224: 121 pgs: 121 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:49:28.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:28 smithi094 ceph-mon[31053]: osdmap e153: 8 total, 8 up, 8 in 2024-09-14T03:49:28.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:28 smithi070 ceph-mon[39288]: pgmap v224: 121 pgs: 121 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:49:28.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:28 smithi070 ceph-mon[39288]: osdmap e153: 8 total, 8 up, 8 in 2024-09-14T03:49:28.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:28 smithi070 ceph-mon[27495]: pgmap v224: 121 pgs: 121 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:49:28.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:28 smithi070 ceph-mon[27495]: osdmap e153: 8 total, 8 up, 8 in 2024-09-14T03:49:28.561 INFO:tasks.rados.rados.0.smithi070.stdout:452: done (3 left) 2024-09-14T03:49:28.561 INFO:tasks.rados.rados.0.smithi070.stdout:453: done (2 left) 2024-09-14T03:49:28.561 INFO:tasks.rados.rados.0.smithi070.stdout:454: done (1 left) 2024-09-14T03:49:28.561 INFO:tasks.rados.rados.0.smithi070.stdout:455: done (0 left) 2024-09-14T03:49:28.561 INFO:tasks.rados.rados.0.smithi070.stdout:456: copy_from oid 15 from oid 40 current snap is 47 2024-09-14T03:49:28.561 INFO:tasks.rados.rados.0.smithi070.stdout:457: read oid 7 snap 30 2024-09-14T03:49:28.561 INFO:tasks.rados.rados.0.smithi070.stdout:457: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-14T03:49:28.561 INFO:tasks.rados.rados.0.smithi070.stdout:458: rollback oid 31 current snap is 47 2024-09-14T03:49:28.561 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 31 to 39 2024-09-14T03:49:28.561 INFO:tasks.rados.rados.0.smithi070.stdout:459: delete oid 35 current snap is 47 2024-09-14T03:49:28.562 INFO:tasks.rados.rados.0.smithi070.stdout:458: finishing rollback tid 1 to smithi07085154-31 2024-09-14T03:49:28.562 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 125 (ObjNum 129 snap 0 seq_num 2930520608) dirty dne 2024-09-14T03:49:28.565 INFO:tasks.rados.rados.0.smithi070.stdout:458: done (3 left) 2024-09-14T03:49:28.565 INFO:tasks.rados.rados.0.smithi070.stdout:459: done (2 left) 2024-09-14T03:49:28.565 INFO:tasks.rados.rados.0.smithi070.stdout:460: read oid 8 snap -1 2024-09-14T03:49:28.565 INFO:tasks.rados.rados.0.smithi070.stdout:460: expect (ObjNum 133 snap 37 seq_num 133) 2024-09-14T03:49:28.565 INFO:tasks.rados.rados.0.smithi070.stdout:461: delete oid 17 current snap is 47 2024-09-14T03:49:28.566 INFO:tasks.rados.rados.0.smithi070.stdout:457: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-14T03:49:28.570 INFO:tasks.rados.rados.0.smithi070.stdout:457: done (3 left) 2024-09-14T03:49:28.570 INFO:tasks.rados.rados.0.smithi070.stdout:461: done (2 left) 2024-09-14T03:49:28.570 INFO:tasks.rados.rados.0.smithi070.stdout:462: rmattr oid 6 current snap is 47 2024-09-14T03:49:28.570 INFO:tasks.rados.rados.0.smithi070.stdout:462: done (2 left) 2024-09-14T03:49:28.570 INFO:tasks.rados.rados.0.smithi070.stdout:463: copy_from oid 14 from oid 44 current snap is 47 2024-09-14T03:49:28.570 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 16 current snap is 47 2024-09-14T03:49:28.570 INFO:tasks.rados.rados.0.smithi070.stdout:464: seq_num 159 ranges {4784128=270336} 2024-09-14T03:49:28.599 INFO:tasks.rados.rados.0.smithi070.stdout:464: writing smithi07085154-16 from 4784128 to 5054464 tid 1 2024-09-14T03:49:28.599 INFO:tasks.rados.rados.0.smithi070.stdout:465: rollback oid 28 current snap is 47 2024-09-14T03:49:28.599 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 28 to 43 2024-09-14T03:49:28.599 INFO:tasks.rados.rados.0.smithi070.stdout:460: expect (ObjNum 133 snap 37 seq_num 133) 2024-09-14T03:49:28.661 INFO:tasks.rados.rados.0.smithi070.stdout:463: finishing copy_from racing read to smithi07085154-14 2024-09-14T03:49:28.662 INFO:tasks.rados.rados.0.smithi070.stdout:463: finishing copy_from to smithi07085154-14 2024-09-14T03:49:28.662 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 105 (ObjNum 152 snap 45 seq_num 152) dirty exists 2024-09-14T03:49:28.662 INFO:tasks.rados.rados.0.smithi070.stdout:456: finishing copy_from racing read to smithi07085154-15 2024-09-14T03:49:28.662 INFO:tasks.rados.rados.0.smithi070.stdout:456: finishing copy_from to smithi07085154-15 2024-09-14T03:49:28.662 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 15 v 62 (ObjNum 131 snap 36 seq_num 131) dirty exists 2024-09-14T03:49:28.662 INFO:tasks.rados.rados.0.smithi070.stdout:456: done (4 left) 2024-09-14T03:49:28.662 INFO:tasks.rados.rados.0.smithi070.stdout:460: done (3 left) 2024-09-14T03:49:28.662 INFO:tasks.rados.rados.0.smithi070.stdout:463: done (2 left) 2024-09-14T03:49:28.662 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 42 current snap is 47 2024-09-14T03:49:28.662 INFO:tasks.rados.rados.0.smithi070.stdout:465: finishing rollback tid 0 to smithi07085154-28 2024-09-14T03:49:28.662 INFO:tasks.rados.rados.0.smithi070.stdout:465: finishing rollback tid 1 to smithi07085154-28 2024-09-14T03:49:28.662 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 28 v 126 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-14T03:49:28.663 INFO:tasks.rados.rados.0.smithi070.stdout:464: finishing write tid 1 to smithi07085154-16 2024-09-14T03:49:28.663 INFO:tasks.rados.rados.0.smithi070.stdout:466: seq_num 160 ranges {2744320=286720} 2024-09-14T03:49:28.678 INFO:tasks.rados.rados.0.smithi070.stdout:466: writing smithi07085154-42 from 2744320 to 3031040 tid 1 2024-09-14T03:49:28.678 INFO:tasks.rados.rados.0.smithi070.stdout:465: done (2 left) 2024-09-14T03:49:28.678 INFO:tasks.rados.rados.0.smithi070.stdout:467: copy_from oid 32 from oid 45 current snap is 47 2024-09-14T03:49:28.678 INFO:tasks.rados.rados.0.smithi070.stdout:464: finishing write tid 2 to smithi07085154-16 2024-09-14T03:49:28.678 INFO:tasks.rados.rados.0.smithi070.stdout:464: finishing write tid 3 to smithi07085154-16 2024-09-14T03:49:28.678 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 310 (ObjNum 159 snap 47 seq_num 159) dirty exists 2024-09-14T03:49:28.678 INFO:tasks.rados.rados.0.smithi070.stdout:464: left oid 16 (ObjNum 159 snap 47 seq_num 159) 2024-09-14T03:49:28.678 INFO:tasks.rados.rados.0.smithi070.stdout:464: done (2 left) 2024-09-14T03:49:28.678 INFO:tasks.rados.rados.0.smithi070.stdout:468: read oid 27 snap -1 2024-09-14T03:49:28.678 INFO:tasks.rados.rados.0.smithi070.stdout:468: expect deleted 2024-09-14T03:49:28.678 INFO:tasks.rados.rados.0.smithi070.stdout:469: copy_from oid 2 from oid 10 current snap is 47 2024-09-14T03:49:28.679 INFO:tasks.rados.rados.0.smithi070.stdout:470: rmattr oid 39 current snap is 47 2024-09-14T03:49:28.679 INFO:tasks.rados.rados.0.smithi070.stdout:471: rmattr oid 44 current snap is 47 2024-09-14T03:49:28.679 INFO:tasks.rados.rados.0.smithi070.stdout:471: done (5 left) 2024-09-14T03:49:28.679 INFO:tasks.rados.rados.0.smithi070.stdout:472: setattr oid 17 current snap is 47 2024-09-14T03:49:28.679 INFO:tasks.rados.rados.0.smithi070.stdout:473: rollback oid 4 current snap is 47 2024-09-14T03:49:28.679 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 4 to 30 2024-09-14T03:49:28.679 INFO:tasks.rados.rados.0.smithi070.stdout:474: delete oid 37 current snap is 47 2024-09-14T03:49:28.680 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 39 v 30 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-14T03:49:28.680 INFO:tasks.rados.rados.0.smithi070.stdout:466: finishing write tid 1 to smithi07085154-42 2024-09-14T03:49:28.680 INFO:tasks.rados.rados.0.smithi070.stdout:466: finishing write tid 2 to smithi07085154-42 2024-09-14T03:49:28.682 INFO:tasks.rados.rados.0.smithi070.stdout:468: done (7 left) 2024-09-14T03:49:28.682 INFO:tasks.rados.rados.0.smithi070.stdout:470: done (6 left) 2024-09-14T03:49:28.682 INFO:tasks.rados.rados.0.smithi070.stdout:474: done (5 left) 2024-09-14T03:49:28.682 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 5 current snap is 47 2024-09-14T03:49:28.682 INFO:tasks.rados.rados.0.smithi070.stdout:475: seq_num 162 ranges {2121728=319488} 2024-09-14T03:49:28.695 INFO:tasks.rados.rados.0.smithi070.stdout:475: writing smithi07085154-5 from 2121728 to 2441216 tid 1 2024-09-14T03:49:28.696 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 14 current snap is 47 2024-09-14T03:49:28.696 INFO:tasks.rados.rados.0.smithi070.stdout:476: seq_num 163 ranges {876544=335872} 2024-09-14T03:49:28.703 INFO:tasks.rados.rados.0.smithi070.stdout:476: writing smithi07085154-14 from 876544 to 1212416 tid 1 2024-09-14T03:49:28.703 INFO:tasks.rados.rados.0.smithi070.stdout:477: read oid 43 snap -1 2024-09-14T03:49:28.703 INFO:tasks.rados.rados.0.smithi070.stdout:477: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-14T03:49:28.703 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 1 current snap is 47 2024-09-14T03:49:28.703 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 193 (ObjNum 593 snap 0 seq_num 2930323320) dirty exists 2024-09-14T03:49:28.703 INFO:tasks.rados.rados.0.smithi070.stdout:466: finishing write tid 3 to smithi07085154-42 2024-09-14T03:49:28.703 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 313 (ObjNum 160 snap 47 seq_num 160) dirty exists 2024-09-14T03:49:28.703 INFO:tasks.rados.rados.0.smithi070.stdout:466: left oid 42 (ObjNum 160 snap 47 seq_num 160) 2024-09-14T03:49:28.703 INFO:tasks.rados.rados.0.smithi070.stdout:473: finishing rollback tid 0 to smithi07085154-4 2024-09-14T03:49:28.703 INFO:tasks.rados.rados.0.smithi070.stdout:473: finishing rollback tid 1 to smithi07085154-4 2024-09-14T03:49:28.704 INFO:tasks.rados.rados.0.smithi070.stdout:478: seq_num 164 ranges {1400832=352256} 2024-09-14T03:49:28.713 INFO:tasks.rados.rados.0.smithi070.stdout:478: writing smithi07085154-1 from 1400832 to 1753088 tid 1 2024-09-14T03:49:28.713 INFO:tasks.rados.rados.0.smithi070.stdout:466: done (8 left) 2024-09-14T03:49:28.713 INFO:tasks.rados.rados.0.smithi070.stdout:472: done (7 left) 2024-09-14T03:49:28.713 INFO:tasks.rados.rados.0.smithi070.stdout:479: read oid 29 snap 36 2024-09-14T03:49:28.713 INFO:tasks.rados.rados.0.smithi070.stdout:479: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-14T03:49:28.713 INFO:tasks.rados.rados.0.smithi070.stdout:480: rollback oid 16 current snap is 47 2024-09-14T03:49:28.713 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 16 to 30 2024-09-14T03:49:28.713 INFO:tasks.rados.rados.0.smithi070.stdout:473: finishing rollback tid 2 to smithi07085154-4 2024-09-14T03:49:28.713 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 130 (ObjNum 3 snap 0 seq_num 3) dirty exists 2024-09-14T03:49:28.713 INFO:tasks.rados.rados.0.smithi070.stdout:467: finishing copy_from racing read to smithi07085154-32 2024-09-14T03:49:28.713 INFO:tasks.rados.rados.0.smithi070.stdout:469: finishing copy_from racing read to smithi07085154-2 2024-09-14T03:49:28.714 INFO:tasks.rados.rados.0.smithi070.stdout:469: finishing copy_from to smithi07085154-2 2024-09-14T03:49:28.714 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 2 v 128 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-14T03:49:28.714 INFO:tasks.rados.rados.0.smithi070.stdout:467: finishing copy_from to smithi07085154-32 2024-09-14T03:49:28.714 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 11 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-14T03:49:28.714 INFO:tasks.rados.rados.0.smithi070.stdout:475: finishing write tid 1 to smithi07085154-5 2024-09-14T03:49:28.714 INFO:tasks.rados.rados.0.smithi070.stdout:475: finishing write tid 2 to smithi07085154-5 2024-09-14T03:49:28.714 INFO:tasks.rados.rados.0.smithi070.stdout:467: done (8 left) 2024-09-14T03:49:28.714 INFO:tasks.rados.rados.0.smithi070.stdout:469: done (7 left) 2024-09-14T03:49:28.714 INFO:tasks.rados.rados.0.smithi070.stdout:473: done (6 left) 2024-09-14T03:49:28.714 INFO:tasks.rados.rados.0.smithi070.stdout:481: delete oid 39 current snap is 47 2024-09-14T03:49:28.714 INFO:tasks.rados.rados.0.smithi070.stdout:475: finishing write tid 3 to smithi07085154-5 2024-09-14T03:49:28.714 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 24 (ObjNum 162 snap 47 seq_num 162) dirty exists 2024-09-14T03:49:28.714 INFO:tasks.rados.rados.0.smithi070.stdout:475: left oid 5 (ObjNum 162 snap 47 seq_num 162) 2024-09-14T03:49:28.714 INFO:tasks.rados.rados.0.smithi070.stdout:476: finishing write tid 1 to smithi07085154-14 2024-09-14T03:49:28.715 INFO:tasks.rados.rados.0.smithi070.stdout:476: finishing write tid 2 to smithi07085154-14 2024-09-14T03:49:28.715 INFO:tasks.rados.rados.0.smithi070.stdout:477: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-14T03:49:28.732 INFO:tasks.rados.rados.0.smithi070.stdout:476: finishing write tid 3 to smithi07085154-14 2024-09-14T03:49:28.732 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 107 (ObjNum 163 snap 47 seq_num 163) dirty exists 2024-09-14T03:49:28.733 INFO:tasks.rados.rados.0.smithi070.stdout:476: left oid 14 (ObjNum 163 snap 47 seq_num 163) 2024-09-14T03:49:28.733 INFO:tasks.rados.rados.0.smithi070.stdout:480: finishing rollback tid 0 to smithi07085154-16 2024-09-14T03:49:28.733 INFO:tasks.rados.rados.0.smithi070.stdout:480: finishing rollback tid 1 to smithi07085154-16 2024-09-14T03:49:28.733 INFO:tasks.rados.rados.0.smithi070.stdout:480: finishing rollback tid 2 to smithi07085154-16 2024-09-14T03:49:28.733 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 318 (ObjNum 114 snap 24 seq_num 114) dirty exists 2024-09-14T03:49:28.733 INFO:tasks.rados.rados.0.smithi070.stdout:475: done (6 left) 2024-09-14T03:49:28.733 INFO:tasks.rados.rados.0.smithi070.stdout:476: done (5 left) 2024-09-14T03:49:28.733 INFO:tasks.rados.rados.0.smithi070.stdout:477: done (4 left) 2024-09-14T03:49:28.733 INFO:tasks.rados.rados.0.smithi070.stdout:480: done (3 left) 2024-09-14T03:49:28.733 INFO:tasks.rados.rados.0.smithi070.stdout:481: done (2 left) 2024-09-14T03:49:28.733 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 23 current snap is 47 2024-09-14T03:49:28.733 INFO:tasks.rados.rados.0.smithi070.stdout:478: finishing write tid 1 to smithi07085154-1 2024-09-14T03:49:28.733 INFO:tasks.rados.rados.0.smithi070.stdout:478: finishing write tid 2 to smithi07085154-1 2024-09-14T03:49:28.733 INFO:tasks.rados.rados.0.smithi070.stdout:479: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-14T03:49:28.751 INFO:tasks.rados.rados.0.smithi070.stdout:478: finishing write tid 3 to smithi07085154-1 2024-09-14T03:49:28.752 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 1 v 110 (ObjNum 164 snap 47 seq_num 164) dirty exists 2024-09-14T03:49:28.752 INFO:tasks.rados.rados.0.smithi070.stdout:478: left oid 1 (ObjNum 164 snap 47 seq_num 164) 2024-09-14T03:49:28.752 INFO:tasks.rados.rados.0.smithi070.stdout:482: seq_num 166 ranges {3244032=385024} 2024-09-14T03:49:28.771 INFO:tasks.rados.rados.0.smithi070.stdout:482: writing smithi07085154-23 from 3244032 to 3629056 tid 1 2024-09-14T03:49:28.771 INFO:tasks.rados.rados.0.smithi070.stdout:478: done (2 left) 2024-09-14T03:49:28.771 INFO:tasks.rados.rados.0.smithi070.stdout:479: done (1 left) 2024-09-14T03:49:28.771 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 17 current snap is 47 2024-09-14T03:49:28.771 INFO:tasks.rados.rados.0.smithi070.stdout:483: seq_num 167 ranges {0=401408} 2024-09-14T03:49:28.773 INFO:tasks.rados.rados.0.smithi070.stdout:483: writing smithi07085154-17 from 0 to 401408 tid 1 2024-09-14T03:49:28.773 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 12 current snap is 47 2024-09-14T03:49:28.773 INFO:tasks.rados.rados.0.smithi070.stdout:484: seq_num 168 ranges {204800=417792} 2024-09-14T03:49:28.776 INFO:tasks.rados.rados.0.smithi070.stdout:484: writing smithi07085154-12 from 204800 to 622592 tid 1 2024-09-14T03:49:28.776 INFO:tasks.rados.rados.0.smithi070.stdout:485: read oid 34 snap -1 2024-09-14T03:49:28.777 INFO:tasks.rados.rados.0.smithi070.stdout:485: expect deleted 2024-09-14T03:49:28.777 INFO:tasks.rados.rados.0.smithi070.stdout:486: snap_remove snap 28 2024-09-14T03:49:29.568 INFO:tasks.rados.rados.0.smithi070.stdout:486: done (4 left) 2024-09-14T03:49:29.568 INFO:tasks.rados.rados.0.smithi070.stdout:487: rollback oid 1 current snap is 47 2024-09-14T03:49:29.568 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 1 to 44 2024-09-14T03:49:29.568 INFO:tasks.rados.rados.0.smithi070.stdout:482: finishing write tid 1 to smithi07085154-23 2024-09-14T03:49:29.568 INFO:tasks.rados.rados.0.smithi070.stdout:482: finishing write tid 2 to smithi07085154-23 2024-09-14T03:49:29.568 INFO:tasks.rados.rados.0.smithi070.stdout:482: finishing write tid 3 to smithi07085154-23 2024-09-14T03:49:29.568 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 109 (ObjNum 166 snap 47 seq_num 166) dirty exists 2024-09-14T03:49:29.569 INFO:tasks.rados.rados.0.smithi070.stdout:482: left oid 23 (ObjNum 166 snap 47 seq_num 166) 2024-09-14T03:49:29.569 INFO:tasks.rados.rados.0.smithi070.stdout:483: finishing write tid 1 to smithi07085154-17 2024-09-14T03:49:29.569 INFO:tasks.rados.rados.0.smithi070.stdout:483: finishing write tid 2 to smithi07085154-17 2024-09-14T03:49:29.569 INFO:tasks.rados.rados.0.smithi070.stdout:483: finishing write tid 3 to smithi07085154-17 2024-09-14T03:49:29.569 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 195 (ObjNum 167 snap 47 seq_num 167) dirty exists 2024-09-14T03:49:29.569 INFO:tasks.rados.rados.0.smithi070.stdout:483: left oid 17 (ObjNum 167 snap 47 seq_num 167) 2024-09-14T03:49:29.569 INFO:tasks.rados.rados.0.smithi070.stdout:484: finishing write tid 1 to smithi07085154-12 2024-09-14T03:49:29.569 INFO:tasks.rados.rados.0.smithi070.stdout:484: finishing write tid 2 to smithi07085154-12 2024-09-14T03:49:29.569 INFO:tasks.rados.rados.0.smithi070.stdout:484: finishing write tid 3 to smithi07085154-12 2024-09-14T03:49:29.569 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 99 (ObjNum 168 snap 47 seq_num 168) dirty exists 2024-09-14T03:49:29.569 INFO:tasks.rados.rados.0.smithi070.stdout:484: left oid 12 (ObjNum 168 snap 47 seq_num 168) 2024-09-14T03:49:29.569 INFO:tasks.rados.rados.0.smithi070.stdout:482: done (4 left) 2024-09-14T03:49:29.569 INFO:tasks.rados.rados.0.smithi070.stdout:483: done (3 left) 2024-09-14T03:49:29.570 INFO:tasks.rados.rados.0.smithi070.stdout:484: done (2 left) 2024-09-14T03:49:29.570 INFO:tasks.rados.rados.0.smithi070.stdout:485: done (1 left) 2024-09-14T03:49:29.570 INFO:tasks.rados.rados.0.smithi070.stdout:488: delete oid 32 current snap is 47 2024-09-14T03:49:29.573 INFO:tasks.rados.rados.0.smithi070.stdout:488: done (1 left) 2024-09-14T03:49:29.573 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 4 current snap is 47 2024-09-14T03:49:29.573 INFO:tasks.rados.rados.0.smithi070.stdout:489: seq_num 170 ranges {57344=450560} 2024-09-14T03:49:29.577 INFO:tasks.rados.rados.0.smithi070.stdout:489: writing smithi07085154-4 from 57344 to 507904 tid 1 2024-09-14T03:49:29.577 INFO:tasks.rados.rados.0.smithi070.stdout:490: snap_remove snap 45 2024-09-14T03:49:29.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:29 smithi094 ceph-mon[31053]: osdmap e154: 8 total, 8 up, 8 in 2024-09-14T03:49:29.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:29 smithi094 ceph-mon[31053]: pgmap v227: 121 pgs: 121 active+clean; 145 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.3 MiB/s wr, 22 op/s 2024-09-14T03:49:30.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:29 smithi070 ceph-mon[27495]: osdmap e154: 8 total, 8 up, 8 in 2024-09-14T03:49:30.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:29 smithi070 ceph-mon[27495]: pgmap v227: 121 pgs: 121 active+clean; 145 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.3 MiB/s wr, 22 op/s 2024-09-14T03:49:30.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:29 smithi070 ceph-mon[39288]: osdmap e154: 8 total, 8 up, 8 in 2024-09-14T03:49:30.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:29 smithi070 ceph-mon[39288]: pgmap v227: 121 pgs: 121 active+clean; 145 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.3 MiB/s wr, 22 op/s 2024-09-14T03:49:30.515 INFO:tasks.rados.rados.0.smithi070.stdout:490: done (2 left) 2024-09-14T03:49:30.515 INFO:tasks.rados.rados.0.smithi070.stdout:491: copy_from oid 14 from oid 43 current snap is 47 2024-09-14T03:49:30.515 INFO:tasks.rados.rados.0.smithi070.stdout:492: setattr oid 48 current snap is 47 2024-09-14T03:49:30.515 INFO:tasks.rados.rados.0.smithi070.stdout:487: finishing rollback tid 0 to smithi07085154-1 2024-09-14T03:49:30.515 INFO:tasks.rados.rados.0.smithi070.stdout:487: finishing rollback tid 1 to smithi07085154-1 2024-09-14T03:49:30.515 INFO:tasks.rados.rados.0.smithi070.stdout:487: finishing rollback tid 2 to smithi07085154-1 2024-09-14T03:49:30.515 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 1 v 117 (ObjNum 83 snap 17 seq_num 83) dirty exists 2024-09-14T03:49:30.515 INFO:tasks.rados.rados.0.smithi070.stdout:489: finishing write tid 1 to smithi07085154-4 2024-09-14T03:49:30.516 INFO:tasks.rados.rados.0.smithi070.stdout:489: finishing write tid 2 to smithi07085154-4 2024-09-14T03:49:30.516 INFO:tasks.rados.rados.0.smithi070.stdout:489: finishing write tid 3 to smithi07085154-4 2024-09-14T03:49:30.516 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 136 (ObjNum 170 snap 47 seq_num 170) dirty exists 2024-09-14T03:49:30.516 INFO:tasks.rados.rados.0.smithi070.stdout:489: left oid 4 (ObjNum 170 snap 47 seq_num 170) 2024-09-14T03:49:30.516 INFO:tasks.rados.rados.0.smithi070.stdout:487: done (3 left) 2024-09-14T03:49:30.517 INFO:tasks.rados.rados.0.smithi070.stdout:489: done (2 left) 2024-09-14T03:49:30.517 INFO:tasks.rados.rados.0.smithi070.stdout:493: rollback oid 29 current snap is 47 2024-09-14T03:49:30.517 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 29 to 36 2024-09-14T03:49:30.517 INFO:tasks.rados.rados.0.smithi070.stdout:494: setattr oid 4 current snap is 47 2024-09-14T03:49:30.517 INFO:tasks.rados.rados.0.smithi070.stdout:495: snap_create 2024-09-14T03:49:30.526 INFO:tasks.rados.rados.0.smithi070.stdout:491: finishing copy_from racing read to smithi07085154-14 2024-09-14T03:49:30.526 INFO:tasks.rados.rados.0.smithi070.stdout:493: finishing rollback tid 0 to smithi07085154-29 2024-09-14T03:49:30.526 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 105 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-14T03:49:30.527 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 141 (ObjNum 170 snap 47 seq_num 170) dirty exists 2024-09-14T03:49:30.528 INFO:tasks.rados.rados.0.smithi070.stdout:493: finishing rollback tid 1 to smithi07085154-29 2024-09-14T03:49:30.528 INFO:tasks.rados.rados.0.smithi070.stdout:493: finishing rollback tid 2 to smithi07085154-29 2024-09-14T03:49:30.528 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 216 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-14T03:49:30.529 INFO:tasks.rados.rados.0.smithi070.stdout:491: finishing copy_from to smithi07085154-14 2024-09-14T03:49:30.529 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 128 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-14T03:49:30.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:30 smithi094 ceph-mon[31053]: osdmap e155: 8 total, 8 up, 8 in 2024-09-14T03:49:30.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:30 smithi094 ceph-mon[31053]: osdmap e156: 8 total, 8 up, 8 in 2024-09-14T03:49:31.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:30 smithi070 ceph-mon[27495]: osdmap e155: 8 total, 8 up, 8 in 2024-09-14T03:49:31.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:30 smithi070 ceph-mon[27495]: osdmap e156: 8 total, 8 up, 8 in 2024-09-14T03:49:31.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:30 smithi070 ceph-mon[39288]: osdmap e155: 8 total, 8 up, 8 in 2024-09-14T03:49:31.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:30 smithi070 ceph-mon[39288]: osdmap e156: 8 total, 8 up, 8 in 2024-09-14T03:49:31.516 INFO:tasks.rados.rados.0.smithi070.stdout:491: done (4 left) 2024-09-14T03:49:31.516 INFO:tasks.rados.rados.0.smithi070.stdout:492: done (3 left) 2024-09-14T03:49:31.517 INFO:tasks.rados.rados.0.smithi070.stdout:493: done (2 left) 2024-09-14T03:49:31.517 INFO:tasks.rados.rados.0.smithi070.stdout:494: done (1 left) 2024-09-14T03:49:31.517 INFO:tasks.rados.rados.0.smithi070.stdout:495: done (0 left) 2024-09-14T03:49:31.517 INFO:tasks.rados.rados.0.smithi070.stdout:496: snap_create 2024-09-14T03:49:31.740 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:31 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: [14/Sep/2024:03:49:31] ENGINE Bus STARTING 2024-09-14T03:49:31.740 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:31 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: CherryPy Checker: 2024-09-14T03:49:31.740 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:31 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: The Application mounted at '' has an empty config. 2024-09-14T03:49:31.740 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:31 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: 2024-09-14T03:49:31.740 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:31 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: [14/Sep/2024:03:49:31] ENGINE Serving on http://:::9283 2024-09-14T03:49:31.740 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:31 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: [14/Sep/2024:03:49:31] ENGINE Bus STARTED 2024-09-14T03:49:31.740 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:31 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:49:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:49:32.004 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:31 smithi094 ceph-mon[31053]: pgmap v230: 121 pgs: 121 active+clean; 145 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.3 MiB/s wr, 22 op/s 2024-09-14T03:49:32.004 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:31 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:32.004 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:31 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:32.004 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:31 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:32.004 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:31 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:32.004 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:31 smithi094 ceph-mon[31053]: Standby manager daemon x restarted 2024-09-14T03:49:32.004 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:31 smithi094 ceph-mon[31053]: Standby manager daemon x started 2024-09-14T03:49:32.004 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:31 smithi094 ceph-mon[31053]: from='mgr.? 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:49:32.004 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:31 smithi094 ceph-mon[31053]: from='mgr.? 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:49:32.004 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:31 smithi094 ceph-mon[31053]: from='mgr.? 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:49:32.004 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:31 smithi094 ceph-mon[31053]: from='mgr.? 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:49:32.005 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:31 smithi094 ceph-mon[31053]: osdmap e157: 8 total, 8 up, 8 in 2024-09-14T03:49:32.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:31 smithi070 ceph-mon[27495]: pgmap v230: 121 pgs: 121 active+clean; 145 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.3 MiB/s wr, 22 op/s 2024-09-14T03:49:32.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:31 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:32.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:31 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:32.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:31 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:32.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:31 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:32.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:31 smithi070 ceph-mon[27495]: Standby manager daemon x restarted 2024-09-14T03:49:32.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:31 smithi070 ceph-mon[27495]: Standby manager daemon x started 2024-09-14T03:49:32.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:31 smithi070 ceph-mon[27495]: from='mgr.? 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:49:32.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:31 smithi070 ceph-mon[27495]: from='mgr.? 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:49:32.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:31 smithi070 ceph-mon[27495]: from='mgr.? 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:49:32.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:31 smithi070 ceph-mon[27495]: from='mgr.? 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:49:32.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:31 smithi070 ceph-mon[27495]: osdmap e157: 8 total, 8 up, 8 in 2024-09-14T03:49:32.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:31 smithi070 ceph-mon[39288]: pgmap v230: 121 pgs: 121 active+clean; 145 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.3 MiB/s wr, 22 op/s 2024-09-14T03:49:32.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:31 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:32.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:31 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:32.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:31 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:32.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:31 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:32.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:31 smithi070 ceph-mon[39288]: Standby manager daemon x restarted 2024-09-14T03:49:32.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:31 smithi070 ceph-mon[39288]: Standby manager daemon x started 2024-09-14T03:49:32.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:31 smithi070 ceph-mon[39288]: from='mgr.? 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:49:32.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:31 smithi070 ceph-mon[39288]: from='mgr.? 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:49:32.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:31 smithi070 ceph-mon[39288]: from='mgr.? 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:49:32.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:31 smithi070 ceph-mon[39288]: from='mgr.? 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:49:32.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:31 smithi070 ceph-mon[39288]: osdmap e157: 8 total, 8 up, 8 in 2024-09-14T03:49:32.517 INFO:tasks.rados.rados.0.smithi070.stdout:496: done (0 left) 2024-09-14T03:49:32.517 INFO:tasks.rados.rados.0.smithi070.stdout:497: setattr oid 4 current snap is 49 2024-09-14T03:49:32.518 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 14 current snap is 49 2024-09-14T03:49:32.518 INFO:tasks.rados.rados.0.smithi070.stdout:498: seq_num 171 ranges {638976=466944} 2024-09-14T03:49:32.525 INFO:tasks.rados.rados.0.smithi070.stdout:498: writing smithi07085154-14 from 638976 to 1105920 tid 1 2024-09-14T03:49:32.525 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 147 (ObjNum 170 snap 47 seq_num 170) dirty exists 2024-09-14T03:49:32.525 INFO:tasks.rados.rados.0.smithi070.stdout:499: read oid 16 snap 35 2024-09-14T03:49:32.525 INFO:tasks.rados.rados.0.smithi070.stdout:499: expect (ObjNum 120 snap 31 seq_num 120) 2024-09-14T03:49:32.526 INFO:tasks.rados.rados.0.smithi070.stdout:497: done (2 left) 2024-09-14T03:49:32.526 INFO:tasks.rados.rados.0.smithi070.stdout:500: rmattr oid 36 current snap is 49 2024-09-14T03:49:32.526 INFO:tasks.rados.rados.0.smithi070.stdout:500: done (2 left) 2024-09-14T03:49:32.526 INFO:tasks.rados.rados.0.smithi070.stdout:501: copy_from oid 37 from oid 50 current snap is 49 2024-09-14T03:49:32.526 INFO:tasks.rados.rados.0.smithi070.stdout:502: delete oid 21 current snap is 49 2024-09-14T03:49:32.529 INFO:tasks.rados.rados.0.smithi070.stdout:502: done (3 left) 2024-09-14T03:49:32.529 INFO:tasks.rados.rados.0.smithi070.stdout:503: rmattr oid 27 current snap is 49 2024-09-14T03:49:32.529 INFO:tasks.rados.rados.0.smithi070.stdout:503: done (3 left) 2024-09-14T03:49:32.529 INFO:tasks.rados.rados.0.smithi070.stdout:504: rollback oid 13 current snap is 49 2024-09-14T03:49:32.529 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 13 to 48 2024-09-14T03:49:32.529 INFO:tasks.rados.rados.0.smithi070.stdout:505: copy_from oid 22 from oid 29 current snap is 49 2024-09-14T03:49:32.529 INFO:tasks.rados.rados.0.smithi070.stdout:506: read oid 6 snap -1 2024-09-14T03:49:32.529 INFO:tasks.rados.rados.0.smithi070.stdout:506: expect (ObjNum 122 snap 31 seq_num 122) 2024-09-14T03:49:32.529 INFO:tasks.rados.rados.0.smithi070.stdout:507: rollback oid 49 current snap is 49 2024-09-14T03:49:32.529 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 49 to 46 2024-09-14T03:49:32.529 INFO:tasks.rados.rados.0.smithi070.stdout:508: setattr oid 20 current snap is 49 2024-09-14T03:49:32.531 INFO:tasks.rados.rados.0.smithi070.stdout:509: snap_create 2024-09-14T03:49:32.533 INFO:tasks.rados.rados.0.smithi070.stdout:498: finishing write tid 1 to smithi07085154-14 2024-09-14T03:49:32.534 INFO:tasks.rados.rados.0.smithi070.stdout:507: finishing rollback tid 1 to smithi07085154-49 2024-09-14T03:49:32.535 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 49 v 116 (ObjNum 33 snap 0 seq_num 2930535536) dirty dne 2024-09-14T03:49:32.535 INFO:tasks.rados.rados.0.smithi070.stdout:504: finishing rollback tid 0 to smithi07085154-13 2024-09-14T03:49:32.535 INFO:tasks.rados.rados.0.smithi070.stdout:504: finishing rollback tid 1 to smithi07085154-13 2024-09-14T03:49:32.535 INFO:tasks.rados.rados.0.smithi070.stdout:498: finishing write tid 2 to smithi07085154-14 2024-09-14T03:49:32.535 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 20 v 149 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-14T03:49:32.535 INFO:tasks.rados.rados.0.smithi070.stdout:504: finishing rollback tid 2 to smithi07085154-13 2024-09-14T03:49:32.535 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 233 (ObjNum 138 snap 39 seq_num 138) dirty exists 2024-09-14T03:49:32.535 INFO:tasks.rados.rados.0.smithi070.stdout:505: finishing copy_from racing read to smithi07085154-22 2024-09-14T03:49:32.536 INFO:tasks.rados.rados.0.smithi070.stdout:498: finishing write tid 3 to smithi07085154-14 2024-09-14T03:49:32.536 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 130 (ObjNum 171 snap 49 seq_num 171) dirty exists 2024-09-14T03:49:32.536 INFO:tasks.rados.rados.0.smithi070.stdout:498: left oid 14 (ObjNum 171 snap 49 seq_num 171) 2024-09-14T03:49:32.537 INFO:tasks.rados.rados.0.smithi070.stdout:505: finishing copy_from to smithi07085154-22 2024-09-14T03:49:32.537 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 216 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-14T03:49:32.545 INFO:tasks.rados.rados.0.smithi070.stdout:501: finishing copy_from racing read to smithi07085154-37 2024-09-14T03:49:32.546 INFO:tasks.rados.rados.0.smithi070.stdout:501: finishing copy_from to smithi07085154-37 2024-09-14T03:49:32.546 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 37 v 11 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-14T03:49:32.547 INFO:tasks.rados.rados.0.smithi070.stdout:506: expect (ObjNum 122 snap 31 seq_num 122) 2024-09-14T03:49:32.627 INFO:tasks.rados.rados.0.smithi070.stdout:499: expect (ObjNum 120 snap 31 seq_num 120) 2024-09-14T03:49:33.523 INFO:tasks.rados.rados.0.smithi070.stdout:498: done (8 left) 2024-09-14T03:49:33.523 INFO:tasks.rados.rados.0.smithi070.stdout:499: done (7 left) 2024-09-14T03:49:33.523 INFO:tasks.rados.rados.0.smithi070.stdout:501: done (6 left) 2024-09-14T03:49:33.523 INFO:tasks.rados.rados.0.smithi070.stdout:504: done (5 left) 2024-09-14T03:49:33.523 INFO:tasks.rados.rados.0.smithi070.stdout:505: done (4 left) 2024-09-14T03:49:33.523 INFO:tasks.rados.rados.0.smithi070.stdout:506: done (3 left) 2024-09-14T03:49:33.523 INFO:tasks.rados.rados.0.smithi070.stdout:507: done (2 left) 2024-09-14T03:49:33.523 INFO:tasks.rados.rados.0.smithi070.stdout:508: done (1 left) 2024-09-14T03:49:33.523 INFO:tasks.rados.rados.0.smithi070.stdout:509: done (0 left) 2024-09-14T03:49:33.523 INFO:tasks.rados.rados.0.smithi070.stdout:510: rmattr oid 4 current snap is 50 2024-09-14T03:49:33.523 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 47 current snap is 50 2024-09-14T03:49:33.524 INFO:tasks.rados.rados.0.smithi070.stdout:511: seq_num 173 ranges {2793472=499712} 2024-09-14T03:49:33.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:33 smithi094 ceph-mon[31053]: mgrmap e23: y(active, since 4m), standbys: x 2024-09-14T03:49:33.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:33 smithi094 ceph-mon[31053]: osdmap e158: 8 total, 8 up, 8 in 2024-09-14T03:49:33.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:33 smithi094 ceph-mon[31053]: pgmap v233: 121 pgs: 121 active+clean; 155 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 3.1 MiB/s wr, 48 op/s 2024-09-14T03:49:33.541 INFO:tasks.rados.rados.0.smithi070.stdout:511: writing smithi07085154-47 from 2793472 to 3293184 tid 1 2024-09-14T03:49:33.541 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 40 current snap is 50 2024-09-14T03:49:33.541 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 151 (ObjNum 170 snap 47 seq_num 170) dirty exists 2024-09-14T03:49:33.541 INFO:tasks.rados.rados.0.smithi070.stdout:512: seq_num 174 ranges {4653056=516096} 2024-09-14T03:49:33.568 INFO:tasks.rados.rados.0.smithi070.stdout:512: writing smithi07085154-40 from 4653056 to 5169152 tid 1 2024-09-14T03:49:33.568 INFO:tasks.rados.rados.0.smithi070.stdout:510: done (2 left) 2024-09-14T03:49:33.569 INFO:tasks.rados.rados.0.smithi070.stdout:513: read oid 33 snap 34 2024-09-14T03:49:33.569 INFO:tasks.rados.rados.0.smithi070.stdout:513: expect (ObjNum 116 snap 25 seq_num 116) 2024-09-14T03:49:33.569 INFO:tasks.rados.rados.0.smithi070.stdout:514: delete oid 23 current snap is 50 2024-09-14T03:49:33.569 INFO:tasks.rados.rados.0.smithi070.stdout:511: finishing write tid 1 to smithi07085154-47 2024-09-14T03:49:33.569 INFO:tasks.rados.rados.0.smithi070.stdout:511: finishing write tid 2 to smithi07085154-47 2024-09-14T03:49:33.569 INFO:tasks.rados.rados.0.smithi070.stdout:511: finishing write tid 3 to smithi07085154-47 2024-09-14T03:49:33.569 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 47 v 106 (ObjNum 173 snap 50 seq_num 173) dirty exists 2024-09-14T03:49:33.569 INFO:tasks.rados.rados.0.smithi070.stdout:511: left oid 47 (ObjNum 173 snap 50 seq_num 173) 2024-09-14T03:49:33.572 INFO:tasks.rados.rados.0.smithi070.stdout:511: done (3 left) 2024-09-14T03:49:33.573 INFO:tasks.rados.rados.0.smithi070.stdout:514: done (2 left) 2024-09-14T03:49:33.573 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 6 current snap is 50 2024-09-14T03:49:33.573 INFO:tasks.rados.rados.0.smithi070.stdout:515: seq_num 176 ranges {2662400=557056} 2024-09-14T03:49:33.590 INFO:tasks.rados.rados.0.smithi070.stdout:515: writing smithi07085154-6 from 2662400 to 3219456 tid 1 2024-09-14T03:49:33.590 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 14 current snap is 50 2024-09-14T03:49:33.590 INFO:tasks.rados.rados.0.smithi070.stdout:516: seq_num 177 ranges {1105920=573440} 2024-09-14T03:49:33.599 INFO:tasks.rados.rados.0.smithi070.stdout:516: writing smithi07085154-14 from 1105920 to 1679360 tid 1 2024-09-14T03:49:33.599 INFO:tasks.rados.rados.0.smithi070.stdout:517: rollback oid 17 current snap is 50 2024-09-14T03:49:33.599 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 17 to 48 2024-09-14T03:49:33.599 INFO:tasks.rados.rados.0.smithi070.stdout:512: finishing write tid 1 to smithi07085154-40 2024-09-14T03:49:33.599 INFO:tasks.rados.rados.0.smithi070.stdout:512: finishing write tid 2 to smithi07085154-40 2024-09-14T03:49:33.600 INFO:tasks.rados.rados.0.smithi070.stdout:512: finishing write tid 3 to smithi07085154-40 2024-09-14T03:49:33.600 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 124 (ObjNum 174 snap 50 seq_num 174) dirty exists 2024-09-14T03:49:33.600 INFO:tasks.rados.rados.0.smithi070.stdout:512: left oid 40 (ObjNum 174 snap 50 seq_num 174) 2024-09-14T03:49:33.600 INFO:tasks.rados.rados.0.smithi070.stdout:513: expect (ObjNum 116 snap 25 seq_num 116) 2024-09-14T03:49:33.660 INFO:tasks.rados.rados.0.smithi070.stdout:515: finishing write tid 1 to smithi07085154-6 2024-09-14T03:49:33.660 INFO:tasks.rados.rados.0.smithi070.stdout:515: finishing write tid 2 to smithi07085154-6 2024-09-14T03:49:33.660 INFO:tasks.rados.rados.0.smithi070.stdout:515: finishing write tid 3 to smithi07085154-6 2024-09-14T03:49:33.660 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 208 (ObjNum 176 snap 50 seq_num 176) dirty exists 2024-09-14T03:49:33.660 INFO:tasks.rados.rados.0.smithi070.stdout:515: left oid 6 (ObjNum 176 snap 50 seq_num 176) 2024-09-14T03:49:33.660 INFO:tasks.rados.rados.0.smithi070.stdout:517: finishing rollback tid 0 to smithi07085154-17 2024-09-14T03:49:33.661 INFO:tasks.rados.rados.0.smithi070.stdout:517: finishing rollback tid 1 to smithi07085154-17 2024-09-14T03:49:33.661 INFO:tasks.rados.rados.0.smithi070.stdout:518: read oid 26 snap -1 2024-09-14T03:49:33.661 INFO:tasks.rados.rados.0.smithi070.stdout:518: expect (ObjNum 156 snap 46 seq_num 156) 2024-09-14T03:49:33.661 INFO:tasks.rados.rados.0.smithi070.stdout:512: done (5 left) 2024-09-14T03:49:33.661 INFO:tasks.rados.rados.0.smithi070.stdout:513: done (4 left) 2024-09-14T03:49:33.661 INFO:tasks.rados.rados.0.smithi070.stdout:515: done (3 left) 2024-09-14T03:49:33.661 INFO:tasks.rados.rados.0.smithi070.stdout:519: copy_from oid 11 from oid 7 current snap is 50 2024-09-14T03:49:33.661 INFO:tasks.rados.rados.0.smithi070.stdout:516: finishing write tid 1 to smithi07085154-14 2024-09-14T03:49:33.661 INFO:tasks.rados.rados.0.smithi070.stdout:517: finishing rollback tid 2 to smithi07085154-17 2024-09-14T03:49:33.661 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 212 (ObjNum 167 snap 47 seq_num 167) dirty exists 2024-09-14T03:49:33.661 INFO:tasks.rados.rados.0.smithi070.stdout:516: finishing write tid 2 to smithi07085154-14 2024-09-14T03:49:33.661 INFO:tasks.rados.rados.0.smithi070.stdout:517: done (3 left) 2024-09-14T03:49:33.661 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 47 current snap is 50 2024-09-14T03:49:33.662 INFO:tasks.rados.rados.0.smithi070.stdout:516: finishing write tid 3 to smithi07085154-14 2024-09-14T03:49:33.662 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 133 (ObjNum 177 snap 50 seq_num 177) dirty exists 2024-09-14T03:49:33.662 INFO:tasks.rados.rados.0.smithi070.stdout:516: left oid 14 (ObjNum 177 snap 50 seq_num 177) 2024-09-14T03:49:33.662 INFO:tasks.rados.rados.0.smithi070.stdout:520: seq_num 178 ranges {3293184=589824} 2024-09-14T03:49:33.681 INFO:tasks.rados.rados.0.smithi070.stdout:520: writing smithi07085154-47 from 3293184 to 3883008 tid 1 2024-09-14T03:49:33.681 INFO:tasks.rados.rados.0.smithi070.stdout:516: done (3 left) 2024-09-14T03:49:33.681 INFO:tasks.rados.rados.0.smithi070.stdout:521: read oid 38 snap -1 2024-09-14T03:49:33.681 INFO:tasks.rados.rados.0.smithi070.stdout:521: expect deleted 2024-09-14T03:49:33.681 INFO:tasks.rados.rados.0.smithi070.stdout:522: copy_from oid 29 from oid 48 current snap is 50 2024-09-14T03:49:33.681 INFO:tasks.rados.rados.0.smithi070.stdout:518: expect (ObjNum 156 snap 46 seq_num 156) 2024-09-14T03:49:33.756 INFO:tasks.rados.rados.0.smithi070.stdout:519: finishing copy_from racing read to smithi07085154-11 2024-09-14T03:49:33.757 INFO:tasks.rados.rados.0.smithi070.stdout:519: finishing copy_from to smithi07085154-11 2024-09-14T03:49:33.757 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 11 v 107 (ObjNum 143 snap 42 seq_num 143) dirty exists 2024-09-14T03:49:33.757 INFO:tasks.rados.rados.0.smithi070.stdout:518: done (4 left) 2024-09-14T03:49:33.757 INFO:tasks.rados.rados.0.smithi070.stdout:519: done (3 left) 2024-09-14T03:49:33.757 INFO:tasks.rados.rados.0.smithi070.stdout:523: read oid 3 snap -1 2024-09-14T03:49:33.757 INFO:tasks.rados.rados.0.smithi070.stdout:523: expect deleted 2024-09-14T03:49:33.757 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 6 current snap is 50 2024-09-14T03:49:33.757 INFO:tasks.rados.rados.0.smithi070.stdout:524: seq_num 179 ranges {3219456=606208} 2024-09-14T03:49:33.776 INFO:tasks.rados.rados.0.smithi070.stdout:524: writing smithi07085154-6 from 3219456 to 3825664 tid 1 2024-09-14T03:49:33.776 INFO:tasks.rados.rados.0.smithi070.stdout:521: done (4 left) 2024-09-14T03:49:33.776 INFO:tasks.rados.rados.0.smithi070.stdout:525: read oid 20 snap 46 2024-09-14T03:49:33.776 INFO:tasks.rados.rados.0.smithi070.stdout:525: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-14T03:49:33.777 INFO:tasks.rados.rados.0.smithi070.stdout:526: snap_remove snap 39 2024-09-14T03:49:33.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:33 smithi070 ceph-mon[27495]: mgrmap e23: y(active, since 4m), standbys: x 2024-09-14T03:49:33.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:33 smithi070 ceph-mon[27495]: osdmap e158: 8 total, 8 up, 8 in 2024-09-14T03:49:33.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:33 smithi070 ceph-mon[27495]: pgmap v233: 121 pgs: 121 active+clean; 155 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 3.1 MiB/s wr, 48 op/s 2024-09-14T03:49:33.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:33 smithi070 ceph-mon[39288]: mgrmap e23: y(active, since 4m), standbys: x 2024-09-14T03:49:33.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:33 smithi070 ceph-mon[39288]: osdmap e158: 8 total, 8 up, 8 in 2024-09-14T03:49:33.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:33 smithi070 ceph-mon[39288]: pgmap v233: 121 pgs: 121 active+clean; 155 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 3.1 MiB/s wr, 48 op/s 2024-09-14T03:49:34.514 INFO:tasks.rados.rados.0.smithi070.stdout:526: done (5 left) 2024-09-14T03:49:34.514 INFO:tasks.rados.rados.0.smithi070.stdout:527: snap_remove snap 35 2024-09-14T03:49:34.776 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:34 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: [14/Sep/2024:03:49:34] ENGINE Bus STOPPING 2024-09-14T03:49:34.777 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:34 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: [14/Sep/2024:03:49:34] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:49:34.777 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:34 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: [14/Sep/2024:03:49:34] ENGINE Bus STOPPED 2024-09-14T03:49:34.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:34 smithi094 ceph-mon[31053]: osdmap e159: 8 total, 8 up, 8 in 2024-09-14T03:49:34.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:34 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:34.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:34 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:34.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:34 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:34.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:34 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:49:34.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:34 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:34.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:34 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:34.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:34 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-14T03:49:34.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:34 smithi094 ceph-mon[31053]: Activating manager daemon x 2024-09-14T03:49:34.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:34 smithi094 ceph-mon[31053]: osdmap e160: 8 total, 8 up, 8 in 2024-09-14T03:49:34.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:34 smithi094 ceph-mon[31053]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-14T03:49:34.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:34 smithi094 ceph-mon[31053]: mgrmap e24: x(active, starting, since 0.00796339s) 2024-09-14T03:49:34.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:34 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:49:34.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:34 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:49:34.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:34 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:49:34.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:34 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-14T03:49:34.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:34 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-14T03:49:34.779 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:49:34.508+0000 7fe7ae236640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-14T03:49:34.779 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: ignoring --setuser ceph since I am not root 2024-09-14T03:49:34.779 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: ignoring --setgroup ceph since I am not root 2024-09-14T03:49:34.779 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mgr[27770]: -- 172.21.15.70:0/3406293258 <== mon.2 v2:172.21.15.70:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5605e23e0340 con 0x5605e23ecc00 2024-09-14T03:49:34.779 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:49:34.656+0000 7fc152af4ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-14T03:49:34.779 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:49:34.710+0000 7fc152af4ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-14T03:49:34.779 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:49:34.776+0000 7fc152af4ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-14T03:49:34.780 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mon[39288]: osdmap e159: 8 total, 8 up, 8 in 2024-09-14T03:49:34.780 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:34.780 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:34.780 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:34.780 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:49:34.780 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:34.780 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:34.780 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-14T03:49:34.780 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mon[39288]: Activating manager daemon x 2024-09-14T03:49:34.780 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mon[39288]: osdmap e160: 8 total, 8 up, 8 in 2024-09-14T03:49:34.780 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mon[39288]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-14T03:49:34.781 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mon[39288]: mgrmap e24: x(active, starting, since 0.00796339s) 2024-09-14T03:49:34.781 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:49:34.781 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:49:34.781 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:49:34.781 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-14T03:49:34.781 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-14T03:49:34.781 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mon[27495]: osdmap e159: 8 total, 8 up, 8 in 2024-09-14T03:49:34.781 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:34.781 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:34.781 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:34.781 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:49:34.781 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' 2024-09-14T03:49:34.782 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:34.782 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-14T03:49:34.782 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mon[27495]: Activating manager daemon x 2024-09-14T03:49:34.782 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mon[27495]: osdmap e160: 8 total, 8 up, 8 in 2024-09-14T03:49:34.782 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mon[27495]: from='mgr.14550 172.21.15.70:0/1416969541' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-14T03:49:34.782 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mon[27495]: mgrmap e24: x(active, starting, since 0.00796339s) 2024-09-14T03:49:34.782 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:49:34.782 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:49:34.782 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:49:34.782 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-14T03:49:34.782 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-14T03:49:35.035 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:34 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:49:34.898+0000 7fc152af4ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:49:35.285 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:49:35.033+0000 7fc152af4ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-14T03:49:35.517 INFO:tasks.rados.rados.0.smithi070.stdout:527: done (5 left) 2024-09-14T03:49:35.518 INFO:tasks.rados.rados.0.smithi070.stdout:528: copy_from oid 23 from oid 44 current snap is 50 2024-09-14T03:49:35.518 INFO:tasks.rados.rados.0.smithi070.stdout:520: finishing write tid 1 to smithi07085154-47 2024-09-14T03:49:35.518 INFO:tasks.rados.rados.0.smithi070.stdout:520: finishing write tid 2 to smithi07085154-47 2024-09-14T03:49:35.518 INFO:tasks.rados.rados.0.smithi070.stdout:520: finishing write tid 3 to smithi07085154-47 2024-09-14T03:49:35.518 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 47 v 108 (ObjNum 178 snap 50 seq_num 178) dirty exists 2024-09-14T03:49:35.518 INFO:tasks.rados.rados.0.smithi070.stdout:520: left oid 47 (ObjNum 178 snap 50 seq_num 178) 2024-09-14T03:49:35.518 INFO:tasks.rados.rados.0.smithi070.stdout:522: finishing copy_from racing read to smithi07085154-29 2024-09-14T03:49:35.518 INFO:tasks.rados.rados.0.smithi070.stdout:522: finishing copy_from to smithi07085154-29 2024-09-14T03:49:35.518 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 105 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-14T03:49:35.518 INFO:tasks.rados.rados.0.smithi070.stdout:525: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-14T03:49:35.527 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: [14/Sep/2024:03:49:35] ENGINE Bus STARTING 2024-09-14T03:49:35.527 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: 2024-09-14T03:49:35.330+0000 7f9274889640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:49:35.527 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: 2024-09-14T03:49:35.330+0000 7f9274889640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:49:35.527 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: 2024-09-14T03:49:35.330+0000 7f9274889640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:49:35.527 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: 2024-09-14T03:49:35.330+0000 7f9274889640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:49:35.527 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: 2024-09-14T03:49:35.330+0000 7f9274889640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:49:35.527 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: 2024-09-14T03:49:35.332+0000 7f9278090640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:49:35.527 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: 2024-09-14T03:49:35.332+0000 7f9278090640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:49:35.527 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: 2024-09-14T03:49:35.332+0000 7f9278090640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:49:35.528 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: 2024-09-14T03:49:35.332+0000 7f9278090640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:49:35.528 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: 2024-09-14T03:49:35.332+0000 7f9278090640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:49:35.528 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: [14/Sep/2024:03:49:35] ENGINE Serving on http://:::9283 2024-09-14T03:49:35.528 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: [14/Sep/2024:03:49:35] ENGINE Bus STARTED 2024-09-14T03:49:35.541 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:49:35.290+0000 7fc152af4ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-14T03:49:35.542 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:49:35.352+0000 7fc152af4ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-14T03:49:35.542 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:49:35.523+0000 7fc152af4ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-14T03:49:35.548 INFO:tasks.rados.rados.0.smithi070.stdout:524: finishing write tid 1 to smithi07085154-6 2024-09-14T03:49:35.549 INFO:tasks.rados.rados.0.smithi070.stdout:524: finishing write tid 2 to smithi07085154-6 2024-09-14T03:49:35.549 INFO:tasks.rados.rados.0.smithi070.stdout:524: finishing write tid 3 to smithi07085154-6 2024-09-14T03:49:35.549 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 214 (ObjNum 179 snap 50 seq_num 179) dirty exists 2024-09-14T03:49:35.549 INFO:tasks.rados.rados.0.smithi070.stdout:524: left oid 6 (ObjNum 179 snap 50 seq_num 179) 2024-09-14T03:49:35.549 INFO:tasks.rados.rados.0.smithi070.stdout:520: done (5 left) 2024-09-14T03:49:35.549 INFO:tasks.rados.rados.0.smithi070.stdout:522: done (4 left) 2024-09-14T03:49:35.549 INFO:tasks.rados.rados.0.smithi070.stdout:523: done (3 left) 2024-09-14T03:49:35.549 INFO:tasks.rados.rados.0.smithi070.stdout:524: done (2 left) 2024-09-14T03:49:35.549 INFO:tasks.rados.rados.0.smithi070.stdout:525: done (1 left) 2024-09-14T03:49:35.549 INFO:tasks.rados.rados.0.smithi070.stdout:529: rollback oid 8 current snap is 50 2024-09-14T03:49:35.549 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 8 to 41 2024-09-14T03:49:35.550 INFO:tasks.rados.rados.0.smithi070.stdout:528: finishing copy_from racing read to smithi07085154-23 2024-09-14T03:49:35.550 INFO:tasks.rados.rados.0.smithi070.stdout:528: finishing copy_from to smithi07085154-23 2024-09-14T03:49:35.550 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 105 (ObjNum 152 snap 45 seq_num 152) dirty exists 2024-09-14T03:49:35.550 INFO:tasks.rados.rados.0.smithi070.stdout:528: done (1 left) 2024-09-14T03:49:35.550 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 20 current snap is 50 2024-09-14T03:49:35.550 INFO:tasks.rados.rados.0.smithi070.stdout:530: seq_num 180 ranges {638976=622592} 2024-09-14T03:49:35.557 INFO:tasks.rados.rados.0.smithi070.stdout:530: writing smithi07085154-20 from 638976 to 1261568 tid 1 2024-09-14T03:49:35.557 INFO:tasks.rados.rados.0.smithi070.stdout:531: snap_create 2024-09-14T03:49:35.557 INFO:tasks.rados.rados.0.smithi070.stdout:529: finishing rollback tid 0 to smithi07085154-8 2024-09-14T03:49:35.557 INFO:tasks.rados.rados.0.smithi070.stdout:529: finishing rollback tid 1 to smithi07085154-8 2024-09-14T03:49:35.558 INFO:tasks.rados.rados.0.smithi070.stdout:529: finishing rollback tid 2 to smithi07085154-8 2024-09-14T03:49:35.558 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 8 v 275 (ObjNum 133 snap 37 seq_num 133) dirty exists 2024-09-14T03:49:35.562 INFO:tasks.rados.rados.0.smithi070.stdout:530: finishing write tid 1 to smithi07085154-20 2024-09-14T03:49:35.563 INFO:tasks.rados.rados.0.smithi070.stdout:530: finishing write tid 2 to smithi07085154-20 2024-09-14T03:49:35.564 INFO:tasks.rados.rados.0.smithi070.stdout:530: finishing write tid 3 to smithi07085154-20 2024-09-14T03:49:35.564 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 20 v 162 (ObjNum 180 snap 50 seq_num 180) dirty exists 2024-09-14T03:49:35.564 INFO:tasks.rados.rados.0.smithi070.stdout:530: left oid 20 (ObjNum 180 snap 50 seq_num 180) 2024-09-14T03:49:36.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:49:36.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:49:36.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:49:36.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:49:36.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:49:36.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:49:36.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:49:36.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:49:36.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:49:36.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:49:36.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:49:36.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:49:36.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-mon[31053]: Manager daemon x is now available 2024-09-14T03:49:36.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:36.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:36.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:36.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:36.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:36.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:36.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:36.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:49:36.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:49:36.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:49:36.029 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-14T03:49:36.029 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-14T03:49:36.029 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-mon[31053]: osdmap e161: 8 total, 8 up, 8 in 2024-09-14T03:49:36.029 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:35 smithi094 ceph-mon[31053]: osdmap e162: 8 total, 8 up, 8 in 2024-09-14T03:49:36.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:49:36.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:49:36.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:49:36.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:49:36.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:49:36.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:49:36.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:49:36.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:49:36.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:49:36.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:49:36.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:49:36.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:49:36.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[39288]: Manager daemon x is now available 2024-09-14T03:49:36.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:36.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:36.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:36.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:36.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:36.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:36.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:36.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:49:36.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:49:36.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:49:36.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-14T03:49:36.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-14T03:49:36.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[39288]: osdmap e161: 8 total, 8 up, 8 in 2024-09-14T03:49:36.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[39288]: osdmap e162: 8 total, 8 up, 8 in 2024-09-14T03:49:36.038 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:49:36.038 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:49:36.038 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:49:36.038 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:49:36.038 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:49:36.039 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:49:36.039 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:49:36.039 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:49:36.039 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:49:36.039 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:49:36.039 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:49:36.039 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:49:36.039 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[27495]: Manager daemon x is now available 2024-09-14T03:49:36.039 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:36.039 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:36.039 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:36.039 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:36.039 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:36.040 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:36.040 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:36.040 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:49:36.040 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:49:36.040 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:49:36.040 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-14T03:49:36.040 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-14T03:49:36.040 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[27495]: osdmap e161: 8 total, 8 up, 8 in 2024-09-14T03:49:36.040 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:35 smithi070 ceph-mon[27495]: osdmap e162: 8 total, 8 up, 8 in 2024-09-14T03:49:36.563 INFO:tasks.rados.rados.0.smithi070.stdout:529: done (2 left) 2024-09-14T03:49:36.563 INFO:tasks.rados.rados.0.smithi070.stdout:530: done (1 left) 2024-09-14T03:49:36.563 INFO:tasks.rados.rados.0.smithi070.stdout:531: done (0 left) 2024-09-14T03:49:36.563 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 17 current snap is 51 2024-09-14T03:49:36.563 INFO:tasks.rados.rados.0.smithi070.stdout:532: seq_num 181 ranges {401408=638976} 2024-09-14T03:49:36.570 INFO:tasks.rados.rados.0.smithi070.stdout:532: writing smithi07085154-17 from 401408 to 1040384 tid 1 2024-09-14T03:49:36.570 INFO:tasks.rados.rados.0.smithi070.stdout:533: copy_from oid 28 from oid 42 current snap is 51 2024-09-14T03:49:36.571 INFO:tasks.rados.rados.0.smithi070.stdout:534: rollback oid 15 current snap is 51 2024-09-14T03:49:36.571 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 15 to 46 2024-09-14T03:49:36.571 INFO:tasks.rados.rados.0.smithi070.stdout:535: rollback oid 33 current snap is 51 2024-09-14T03:49:36.571 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 33 to 43 2024-09-14T03:49:36.571 INFO:tasks.rados.rados.0.smithi070.stdout:536: delete oid 46 current snap is 51 2024-09-14T03:49:36.574 INFO:tasks.rados.rados.0.smithi070.stdout:536: done (4 left) 2024-09-14T03:49:36.574 INFO:tasks.rados.rados.0.smithi070.stdout:537: read oid 11 snap 48 2024-09-14T03:49:36.574 INFO:tasks.rados.rados.0.smithi070.stdout:537: expect (ObjNum 111 snap 23 seq_num 111) 2024-09-14T03:49:36.574 INFO:tasks.rados.rados.0.smithi070.stdout:538: read oid 19 snap -1 2024-09-14T03:49:36.575 INFO:tasks.rados.rados.0.smithi070.stdout:538: expect (ObjNum 129 snap 35 seq_num 129) 2024-09-14T03:49:36.575 INFO:tasks.rados.rados.0.smithi070.stdout:539: copy_from oid 14 from oid 7 current snap is 51 2024-09-14T03:49:36.575 INFO:tasks.rados.rados.0.smithi070.stdout:540: delete oid 47 current snap is 51 2024-09-14T03:49:36.575 INFO:tasks.rados.rados.0.smithi070.stdout:535: finishing rollback tid 1 to smithi07085154-33 2024-09-14T03:49:36.575 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 33 v 166 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-09-14T03:49:36.577 INFO:tasks.rados.rados.0.smithi070.stdout:534: finishing rollback tid 0 to smithi07085154-15 2024-09-14T03:49:36.577 INFO:tasks.rados.rados.0.smithi070.stdout:534: finishing rollback tid 1 to smithi07085154-15 2024-09-14T03:49:36.578 INFO:tasks.rados.rados.0.smithi070.stdout:532: finishing write tid 1 to smithi07085154-17 2024-09-14T03:49:36.579 INFO:tasks.rados.rados.0.smithi070.stdout:532: finishing write tid 2 to smithi07085154-17 2024-09-14T03:49:36.579 INFO:tasks.rados.rados.0.smithi070.stdout:534: finishing rollback tid 2 to smithi07085154-15 2024-09-14T03:49:36.579 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 15 v 279 (ObjNum 273 snap 0 seq_num 0) dirty exists 2024-09-14T03:49:36.581 INFO:tasks.rados.rados.0.smithi070.stdout:532: finishing write tid 3 to smithi07085154-17 2024-09-14T03:49:36.581 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 227 (ObjNum 181 snap 51 seq_num 181) dirty exists 2024-09-14T03:49:36.581 INFO:tasks.rados.rados.0.smithi070.stdout:532: left oid 17 (ObjNum 181 snap 51 seq_num 181) 2024-09-14T03:49:36.584 INFO:tasks.rados.rados.0.smithi070.stdout:537: expect (ObjNum 111 snap 23 seq_num 111) 2024-09-14T03:49:36.643 INFO:tasks.rados.rados.0.smithi070.stdout:533: finishing copy_from racing read to smithi07085154-28 2024-09-14T03:49:36.643 INFO:tasks.rados.rados.0.smithi070.stdout:538: expect (ObjNum 129 snap 35 seq_num 129) 2024-09-14T03:49:36.705 INFO:tasks.rados.rados.0.smithi070.stdout:533: finishing copy_from to smithi07085154-28 2024-09-14T03:49:36.706 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 28 v 313 (ObjNum 160 snap 47 seq_num 160) dirty exists 2024-09-14T03:49:36.706 INFO:tasks.rados.rados.0.smithi070.stdout:539: finishing copy_from racing read to smithi07085154-14 2024-09-14T03:49:36.706 INFO:tasks.rados.rados.0.smithi070.stdout:532: done (7 left) 2024-09-14T03:49:36.706 INFO:tasks.rados.rados.0.smithi070.stdout:533: done (6 left) 2024-09-14T03:49:36.706 INFO:tasks.rados.rados.0.smithi070.stdout:534: done (5 left) 2024-09-14T03:49:36.706 INFO:tasks.rados.rados.0.smithi070.stdout:535: done (4 left) 2024-09-14T03:49:36.706 INFO:tasks.rados.rados.0.smithi070.stdout:537: done (3 left) 2024-09-14T03:49:36.706 INFO:tasks.rados.rados.0.smithi070.stdout:538: done (2 left) 2024-09-14T03:49:36.706 INFO:tasks.rados.rados.0.smithi070.stdout:540: done (1 left) 2024-09-14T03:49:36.706 INFO:tasks.rados.rados.0.smithi070.stdout:541: read oid 47 snap -1 2024-09-14T03:49:36.706 INFO:tasks.rados.rados.0.smithi070.stdout:541: expect deleted 2024-09-14T03:49:36.706 INFO:tasks.rados.rados.0.smithi070.stdout:542: delete oid 10 current snap is 51 2024-09-14T03:49:36.707 INFO:tasks.rados.rados.0.smithi070.stdout:539: finishing copy_from to smithi07085154-14 2024-09-14T03:49:36.707 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 107 (ObjNum 143 snap 42 seq_num 143) dirty exists 2024-09-14T03:49:36.707 INFO:tasks.rados.rados.0.smithi070.stdout:539: done (2 left) 2024-09-14T03:49:36.707 INFO:tasks.rados.rados.0.smithi070.stdout:541: done (1 left) 2024-09-14T03:49:36.707 INFO:tasks.rados.rados.0.smithi070.stdout:542: done (0 left) 2024-09-14T03:49:36.707 INFO:tasks.rados.rados.0.smithi070.stdout:543: copy_from oid 8 from oid 29 current snap is 51 2024-09-14T03:49:36.707 INFO:tasks.rados.rados.0.smithi070.stdout:544: delete oid 10 current snap is 51 2024-09-14T03:49:36.708 INFO:tasks.rados.rados.0.smithi070.stdout:544: done (1 left) 2024-09-14T03:49:36.708 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 47 current snap is 51 2024-09-14T03:49:36.708 INFO:tasks.rados.rados.0.smithi070.stdout:545: seq_num 186 ranges {0=720896} 2024-09-14T03:49:36.714 INFO:tasks.rados.rados.0.smithi070.stdout:545: writing smithi07085154-47 from 0 to 720896 tid 1 2024-09-14T03:49:36.714 INFO:tasks.rados.rados.0.smithi070.stdout:546: rollback oid 21 current snap is 51 2024-09-14T03:49:36.714 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 21 to 49 2024-09-14T03:49:36.714 INFO:tasks.rados.rados.0.smithi070.stdout:543: finishing copy_from racing read to smithi07085154-8 2024-09-14T03:49:36.714 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 42 current snap is 51 2024-09-14T03:49:36.714 INFO:tasks.rados.rados.0.smithi070.stdout:547: seq_num 187 ranges {3031040=737280} 2024-09-14T03:49:36.736 INFO:tasks.rados.rados.0.smithi070.stdout:547: writing smithi07085154-42 from 3031040 to 3768320 tid 1 2024-09-14T03:49:36.736 INFO:tasks.rados.rados.0.smithi070.stdout:548: delete oid 3 current snap is 51 2024-09-14T03:49:36.736 INFO:tasks.rados.rados.0.smithi070.stdout:543: finishing copy_from to smithi07085154-8 2024-09-14T03:49:36.736 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 8 v 105 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-14T03:49:36.736 INFO:tasks.rados.rados.0.smithi070.stdout:546: finishing rollback tid 1 to smithi07085154-21 2024-09-14T03:49:36.736 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 65 (ObjNum -76911946 snap 32644 seq_num 304) dirty dne 2024-09-14T03:49:36.736 INFO:tasks.rados.rados.0.smithi070.stdout:545: finishing write tid 1 to smithi07085154-47 2024-09-14T03:49:36.737 INFO:tasks.rados.rados.0.smithi070.stdout:545: finishing write tid 2 to smithi07085154-47 2024-09-14T03:49:36.737 INFO:tasks.rados.rados.0.smithi070.stdout:545: finishing write tid 3 to smithi07085154-47 2024-09-14T03:49:36.737 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 47 v 120 (ObjNum 186 snap 51 seq_num 186) dirty exists 2024-09-14T03:49:36.737 INFO:tasks.rados.rados.0.smithi070.stdout:545: left oid 47 (ObjNum 186 snap 51 seq_num 186) 2024-09-14T03:49:36.737 INFO:tasks.rados.rados.0.smithi070.stdout:543: done (4 left) 2024-09-14T03:49:36.737 INFO:tasks.rados.rados.0.smithi070.stdout:545: done (3 left) 2024-09-14T03:49:36.737 INFO:tasks.rados.rados.0.smithi070.stdout:546: done (2 left) 2024-09-14T03:49:36.738 INFO:tasks.rados.rados.0.smithi070.stdout:548: done (1 left) 2024-09-14T03:49:36.738 INFO:tasks.rados.rados.0.smithi070.stdout:549: delete oid 10 current snap is 51 2024-09-14T03:49:36.739 INFO:tasks.rados.rados.0.smithi070.stdout:547: finishing write tid 1 to smithi07085154-42 2024-09-14T03:49:36.739 INFO:tasks.rados.rados.0.smithi070.stdout:547: finishing write tid 2 to smithi07085154-42 2024-09-14T03:49:36.739 INFO:tasks.rados.rados.0.smithi070.stdout:549: done (1 left) 2024-09-14T03:49:36.739 INFO:tasks.rados.rados.0.smithi070.stdout:550: rmattr oid 29 current snap is 51 2024-09-14T03:49:36.740 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 14 current snap is 51 2024-09-14T03:49:36.740 INFO:tasks.rados.rados.0.smithi070.stdout:551: seq_num 190 ranges {2408448=786432} 2024-09-14T03:49:36.758 INFO:tasks.rados.rados.0.smithi070.stdout:551: writing smithi07085154-14 from 2408448 to 3194880 tid 1 2024-09-14T03:49:36.759 INFO:tasks.rados.rados.0.smithi070.stdout:552: read oid 26 snap -1 2024-09-14T03:49:36.759 INFO:tasks.rados.rados.0.smithi070.stdout:552: expect (ObjNum 156 snap 46 seq_num 156) 2024-09-14T03:49:36.759 INFO:tasks.rados.rados.0.smithi070.stdout:553: read oid 12 snap -1 2024-09-14T03:49:36.759 INFO:tasks.rados.rados.0.smithi070.stdout:553: expect (ObjNum 168 snap 47 seq_num 168) 2024-09-14T03:49:36.759 INFO:tasks.rados.rados.0.smithi070.stdout:554: rollback oid 40 current snap is 51 2024-09-14T03:49:36.759 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 40 to 46 2024-09-14T03:49:36.759 INFO:tasks.rados.rados.0.smithi070.stdout:547: finishing write tid 3 to smithi07085154-42 2024-09-14T03:49:36.759 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 373 (ObjNum 187 snap 51 seq_num 187) dirty exists 2024-09-14T03:49:36.760 INFO:tasks.rados.rados.0.smithi070.stdout:547: left oid 42 (ObjNum 187 snap 51 seq_num 187) 2024-09-14T03:49:36.760 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 234 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-14T03:49:36.760 INFO:tasks.rados.rados.0.smithi070.stdout:547: done (5 left) 2024-09-14T03:49:36.760 INFO:tasks.rados.rados.0.smithi070.stdout:550: done (4 left) 2024-09-14T03:49:36.760 INFO:tasks.rados.rados.0.smithi070.stdout:555: delete oid 36 current snap is 51 2024-09-14T03:49:36.762 INFO:tasks.rados.rados.0.smithi070.stdout:551: finishing write tid 1 to smithi07085154-14 2024-09-14T03:49:36.762 INFO:tasks.rados.rados.0.smithi070.stdout:554: finishing rollback tid 0 to smithi07085154-40 2024-09-14T03:49:36.762 INFO:tasks.rados.rados.0.smithi070.stdout:554: finishing rollback tid 1 to smithi07085154-40 2024-09-14T03:49:36.762 INFO:tasks.rados.rados.0.smithi070.stdout:554: finishing rollback tid 2 to smithi07085154-40 2024-09-14T03:49:36.762 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 140 (ObjNum 131 snap 36 seq_num 131) dirty exists 2024-09-14T03:49:36.763 INFO:tasks.rados.rados.0.smithi070.stdout:551: finishing write tid 2 to smithi07085154-14 2024-09-14T03:49:36.763 INFO:tasks.rados.rados.0.smithi070.stdout:554: done (4 left) 2024-09-14T03:49:36.763 INFO:tasks.rados.rados.0.smithi070.stdout:555: done (3 left) 2024-09-14T03:49:36.764 INFO:tasks.rados.rados.0.smithi070.stdout:556: rmattr oid 8 current snap is 51 2024-09-14T03:49:36.764 INFO:tasks.rados.rados.0.smithi070.stdout:553: expect (ObjNum 168 snap 47 seq_num 168) 2024-09-14T03:49:36.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:36 smithi094 ceph-mon[31053]: Checking for cert/key for grafana.a 2024-09-14T03:49:36.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:36 smithi094 ceph-mon[31053]: Migrating grafana.a cert to cert store 2024-09-14T03:49:36.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:36 smithi094 ceph-mon[31053]: Migrating grafana.a key to cert store 2024-09-14T03:49:36.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:36 smithi094 ceph-mon[31053]: mgrmap e25: x(active, since 1.05469s) 2024-09-14T03:49:36.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:36 smithi094 ceph-mon[31053]: pgmap v5: 121 pgs: 121 active+clean; 163 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:49:36.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:36 smithi094 ceph-mon[31053]: Deploying cephadm binary to smithi094 2024-09-14T03:49:36.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:36 smithi094 ceph-mon[31053]: Deploying cephadm binary to smithi070 2024-09-14T03:49:36.777 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:36 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: 2024-09-14T03:49:36.465+0000 7f925b5da640 -1 log_channel(cephadm) log [ERR] : [14/Sep/2024:03:49:36] ENGINE Error in HTTPServer.serve 2024-09-14T03:49:36.777 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:36 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: Traceback (most recent call last): 2024-09-14T03:49:36.778 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:36 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:49:36.778 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:36 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: self._connections.run(self.expiration_interval) 2024-09-14T03:49:36.778 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:36 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:49:36.778 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:36 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: self._run(expiration_interval) 2024-09-14T03:49:36.778 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:36 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:49:36.778 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:36 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:49:36.778 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:36 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:49:36.778 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:36 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:49:36.778 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:36 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:49:36.778 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:36 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: s = self.context.wrap_socket( 2024-09-14T03:49:36.779 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:36 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:49:36.779 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:36 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: return self.sslsocket_class._create( 2024-09-14T03:49:36.779 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:36 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:49:36.779 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:36 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: self.do_handshake() 2024-09-14T03:49:36.779 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:36 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:49:36.779 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:36 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: self._sslobj.do_handshake() 2024-09-14T03:49:36.779 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:36 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:49:36.779 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:36 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: 2024-09-14T03:49:36.784 INFO:tasks.rados.rados.0.smithi070.stdout:551: finishing write tid 3 to smithi07085154-14 2024-09-14T03:49:36.784 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 135 (ObjNum 190 snap 51 seq_num 190) dirty exists 2024-09-14T03:49:36.784 INFO:tasks.rados.rados.0.smithi070.stdout:551: left oid 14 (ObjNum 190 snap 51 seq_num 190) 2024-09-14T03:49:36.784 INFO:tasks.rados.rados.0.smithi070.stdout:552: expect (ObjNum 156 snap 46 seq_num 156) 2024-09-14T03:49:36.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:36 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:49:36.393+0000 7fc152af4ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-14T03:49:36.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:36 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:49:36.456+0000 7fc152af4ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-14T03:49:36.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:36 smithi070 ceph-mon[27495]: Checking for cert/key for grafana.a 2024-09-14T03:49:36.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:36 smithi070 ceph-mon[27495]: Migrating grafana.a cert to cert store 2024-09-14T03:49:36.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:36 smithi070 ceph-mon[27495]: Migrating grafana.a key to cert store 2024-09-14T03:49:36.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:36 smithi070 ceph-mon[27495]: mgrmap e25: x(active, since 1.05469s) 2024-09-14T03:49:36.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:36 smithi070 ceph-mon[27495]: pgmap v5: 121 pgs: 121 active+clean; 163 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:49:36.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:36 smithi070 ceph-mon[27495]: Deploying cephadm binary to smithi094 2024-09-14T03:49:36.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:36 smithi070 ceph-mon[27495]: Deploying cephadm binary to smithi070 2024-09-14T03:49:36.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:36 smithi070 ceph-mon[39288]: Checking for cert/key for grafana.a 2024-09-14T03:49:36.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:36 smithi070 ceph-mon[39288]: Migrating grafana.a cert to cert store 2024-09-14T03:49:36.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:36 smithi070 ceph-mon[39288]: Migrating grafana.a key to cert store 2024-09-14T03:49:36.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:36 smithi070 ceph-mon[39288]: mgrmap e25: x(active, since 1.05469s) 2024-09-14T03:49:36.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:36 smithi070 ceph-mon[39288]: pgmap v5: 121 pgs: 121 active+clean; 163 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:49:36.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:36 smithi070 ceph-mon[39288]: Deploying cephadm binary to smithi094 2024-09-14T03:49:36.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:36 smithi070 ceph-mon[39288]: Deploying cephadm binary to smithi070 2024-09-14T03:49:36.860 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 8 v 282 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-14T03:49:36.860 INFO:tasks.rados.rados.0.smithi070.stdout:551: done (3 left) 2024-09-14T03:49:36.860 INFO:tasks.rados.rados.0.smithi070.stdout:552: done (2 left) 2024-09-14T03:49:36.860 INFO:tasks.rados.rados.0.smithi070.stdout:553: done (1 left) 2024-09-14T03:49:36.861 INFO:tasks.rados.rados.0.smithi070.stdout:556: done (0 left) 2024-09-14T03:49:36.861 INFO:tasks.rados.rados.0.smithi070.stdout:557: snap_create 2024-09-14T03:49:37.452 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:37 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:49:37.199+0000 7fc152af4ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-14T03:49:37.452 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:37 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:49:37.317+0000 7fc152af4ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-14T03:49:37.452 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:37 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:49:37.450+0000 7fc152af4ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:49:37.571 INFO:tasks.rados.rados.0.smithi070.stdout:557: done (0 left) 2024-09-14T03:49:37.571 INFO:tasks.rados.rados.0.smithi070.stdout:558: delete oid 37 current snap is 52 2024-09-14T03:49:37.573 INFO:tasks.rados.rados.0.smithi070.stdout:558: done (0 left) 2024-09-14T03:49:37.573 INFO:tasks.rados.rados.0.smithi070.stdout:559: read oid 28 snap 46 2024-09-14T03:49:37.573 INFO:tasks.rados.rados.0.smithi070.stdout:559: expect (ObjNum 149 snap 44 seq_num 149) 2024-09-14T03:49:37.573 INFO:tasks.rados.rados.0.smithi070.stdout:560: read oid 2 snap 30 2024-09-14T03:49:37.573 INFO:tasks.rados.rados.0.smithi070.stdout:560: expect (ObjNum 63 snap 3 seq_num 63) 2024-09-14T03:49:37.573 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 42 current snap is 52 2024-09-14T03:49:37.573 INFO:tasks.rados.rados.0.smithi070.stdout:561: seq_num 193 ranges {3768320=434176,4202496=401408} 2024-09-14T03:49:37.596 INFO:tasks.rados.rados.0.smithi070.stdout:561: writing smithi07085154-42 from 3768320 to 4202496 tid 1 2024-09-14T03:49:37.598 INFO:tasks.rados.rados.0.smithi070.stdout:561: writing smithi07085154-42 from 4202496 to 4603904 tid 2 2024-09-14T03:49:37.599 INFO:tasks.rados.rados.0.smithi070.stdout:562: snap_create 2024-09-14T03:49:37.599 INFO:tasks.rados.rados.0.smithi070.stdout:559: expect (ObjNum 149 snap 44 seq_num 149) 2024-09-14T03:49:37.601 INFO:tasks.rados.rados.0.smithi070.stdout:560: expect (ObjNum 63 snap 3 seq_num 63) 2024-09-14T03:49:37.662 INFO:tasks.rados.rados.0.smithi070.stdout:561: finishing write tid 1 to smithi07085154-42 2024-09-14T03:49:37.663 INFO:tasks.rados.rados.0.smithi070.stdout:561: finishing write tid 2 to smithi07085154-42 2024-09-14T03:49:37.663 INFO:tasks.rados.rados.0.smithi070.stdout:561: finishing write tid 3 to smithi07085154-42 2024-09-14T03:49:37.663 INFO:tasks.rados.rados.0.smithi070.stdout:561: finishing write tid 4 to smithi07085154-42 2024-09-14T03:49:37.663 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 378 (ObjNum 193 snap 52 seq_num 193) dirty exists 2024-09-14T03:49:37.663 INFO:tasks.rados.rados.0.smithi070.stdout:561: left oid 42 (ObjNum 193 snap 52 seq_num 193) 2024-09-14T03:49:37.726 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:37 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:49:37.516+0000 7fc152af4ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-14T03:49:37.726 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:37 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:49:37.658+0000 7fc152af4ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-14T03:49:37.726 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:37 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:49:37.723+0000 7fc152af4ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-14T03:49:37.727 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:37 smithi070 ceph-mon[39288]: [14/Sep/2024:03:49:36] ENGINE Bus STARTING 2024-09-14T03:49:37.727 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:37 smithi070 ceph-mon[39288]: [14/Sep/2024:03:49:36] ENGINE Serving on http://172.21.15.94:8765 2024-09-14T03:49:37.727 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:37 smithi070 ceph-mon[39288]: [14/Sep/2024:03:49:36] ENGINE Serving on https://172.21.15.94:7150 2024-09-14T03:49:37.727 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:37 smithi070 ceph-mon[39288]: [14/Sep/2024:03:49:36] ENGINE Bus STARTED 2024-09-14T03:49:37.727 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:37 smithi070 ceph-mon[39288]: [14/Sep/2024:03:49:36] ENGINE Error in HTTPServer.serve 2024-09-14T03:49:37.727 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-09-14T03:49:37.727 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:49:37.727 INFO:journalctl@ceph.mon.c.smithi070.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:49:37.727 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:49:37.727 INFO:journalctl@ceph.mon.c.smithi070.stdout: self._run(expiration_interval) 2024-09-14T03:49:37.727 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:49:37.727 INFO:journalctl@ceph.mon.c.smithi070.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:49:37.728 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:49:37.728 INFO:journalctl@ceph.mon.c.smithi070.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:49:37.728 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:49:37.728 INFO:journalctl@ceph.mon.c.smithi070.stdout: s = self.context.wrap_socket( 2024-09-14T03:49:37.728 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:49:37.728 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.sslsocket_class._create( 2024-09-14T03:49:37.728 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:49:37.728 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.do_handshake() 2024-09-14T03:49:37.728 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:49:37.728 INFO:journalctl@ceph.mon.c.smithi070.stdout: self._sslobj.do_handshake() 2024-09-14T03:49:37.728 INFO:journalctl@ceph.mon.c.smithi070.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:49:37.728 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:37 smithi070 ceph-mon[39288]: pgmap v6: 121 pgs: 121 active+clean; 163 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:49:37.728 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:37 smithi070 ceph-mon[39288]: osdmap e163: 8 total, 8 up, 8 in 2024-09-14T03:49:37.729 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:37 smithi070 ceph-mon[27495]: [14/Sep/2024:03:49:36] ENGINE Bus STARTING 2024-09-14T03:49:37.729 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:37 smithi070 ceph-mon[27495]: [14/Sep/2024:03:49:36] ENGINE Serving on http://172.21.15.94:8765 2024-09-14T03:49:37.729 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:37 smithi070 ceph-mon[27495]: [14/Sep/2024:03:49:36] ENGINE Serving on https://172.21.15.94:7150 2024-09-14T03:49:37.729 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:37 smithi070 ceph-mon[27495]: [14/Sep/2024:03:49:36] ENGINE Bus STARTED 2024-09-14T03:49:37.729 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:37 smithi070 ceph-mon[27495]: [14/Sep/2024:03:49:36] ENGINE Error in HTTPServer.serve 2024-09-14T03:49:37.729 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-09-14T03:49:37.729 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:49:37.729 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:49:37.730 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:49:37.730 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._run(expiration_interval) 2024-09-14T03:49:37.730 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:49:37.730 INFO:journalctl@ceph.mon.a.smithi070.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:49:37.730 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:49:37.730 INFO:journalctl@ceph.mon.a.smithi070.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:49:37.730 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:49:37.730 INFO:journalctl@ceph.mon.a.smithi070.stdout: s = self.context.wrap_socket( 2024-09-14T03:49:37.730 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:49:37.730 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.sslsocket_class._create( 2024-09-14T03:49:37.730 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:49:37.730 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.do_handshake() 2024-09-14T03:49:37.731 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:49:37.731 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._sslobj.do_handshake() 2024-09-14T03:49:37.731 INFO:journalctl@ceph.mon.a.smithi070.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:49:37.731 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:37 smithi070 ceph-mon[27495]: pgmap v6: 121 pgs: 121 active+clean; 163 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:49:37.731 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:37 smithi070 ceph-mon[27495]: osdmap e163: 8 total, 8 up, 8 in 2024-09-14T03:49:37.913 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:37 smithi094 ceph-mon[31053]: [14/Sep/2024:03:49:36] ENGINE Bus STARTING 2024-09-14T03:49:37.914 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:37 smithi094 ceph-mon[31053]: [14/Sep/2024:03:49:36] ENGINE Serving on http://172.21.15.94:8765 2024-09-14T03:49:37.914 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:37 smithi094 ceph-mon[31053]: [14/Sep/2024:03:49:36] ENGINE Serving on https://172.21.15.94:7150 2024-09-14T03:49:37.914 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:37 smithi094 ceph-mon[31053]: [14/Sep/2024:03:49:36] ENGINE Bus STARTED 2024-09-14T03:49:37.914 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:37 smithi094 ceph-mon[31053]: [14/Sep/2024:03:49:36] ENGINE Error in HTTPServer.serve 2024-09-14T03:49:37.914 INFO:journalctl@ceph.mon.b.smithi094.stdout: Traceback (most recent call last): 2024-09-14T03:49:37.914 INFO:journalctl@ceph.mon.b.smithi094.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:49:37.914 INFO:journalctl@ceph.mon.b.smithi094.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:49:37.914 INFO:journalctl@ceph.mon.b.smithi094.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:49:37.914 INFO:journalctl@ceph.mon.b.smithi094.stdout: self._run(expiration_interval) 2024-09-14T03:49:37.914 INFO:journalctl@ceph.mon.b.smithi094.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:49:37.915 INFO:journalctl@ceph.mon.b.smithi094.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:49:37.915 INFO:journalctl@ceph.mon.b.smithi094.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:49:37.915 INFO:journalctl@ceph.mon.b.smithi094.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:49:37.915 INFO:journalctl@ceph.mon.b.smithi094.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:49:37.915 INFO:journalctl@ceph.mon.b.smithi094.stdout: s = self.context.wrap_socket( 2024-09-14T03:49:37.915 INFO:journalctl@ceph.mon.b.smithi094.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:49:37.915 INFO:journalctl@ceph.mon.b.smithi094.stdout: return self.sslsocket_class._create( 2024-09-14T03:49:37.915 INFO:journalctl@ceph.mon.b.smithi094.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:49:37.915 INFO:journalctl@ceph.mon.b.smithi094.stdout: self.do_handshake() 2024-09-14T03:49:37.915 INFO:journalctl@ceph.mon.b.smithi094.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:49:37.915 INFO:journalctl@ceph.mon.b.smithi094.stdout: self._sslobj.do_handshake() 2024-09-14T03:49:37.915 INFO:journalctl@ceph.mon.b.smithi094.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:49:37.915 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:37 smithi094 ceph-mon[31053]: pgmap v6: 121 pgs: 121 active+clean; 163 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:49:37.916 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:37 smithi094 ceph-mon[31053]: osdmap e163: 8 total, 8 up, 8 in 2024-09-14T03:49:37.992 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:37 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:49:37.782+0000 7fc152af4ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-14T03:49:37.992 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:37 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:49:37.928+0000 7fc152af4ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-14T03:49:38.259 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:37 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:49:37.994+0000 7fc152af4ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-14T03:49:38.259 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:38 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:49:38.059+0000 7fc152af4ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-14T03:49:38.259 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:38 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:49:38.141+0000 7fc152af4ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-14T03:49:38.260 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:38 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:49:38.199+0000 7fc152af4ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-14T03:49:38.260 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:38 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:49:38.256+0000 7fc152af4ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-14T03:49:38.575 INFO:tasks.rados.rados.0.smithi070.stdout:559: done (3 left) 2024-09-14T03:49:38.575 INFO:tasks.rados.rados.0.smithi070.stdout:560: done (2 left) 2024-09-14T03:49:38.575 INFO:tasks.rados.rados.0.smithi070.stdout:561: done (1 left) 2024-09-14T03:49:38.575 INFO:tasks.rados.rados.0.smithi070.stdout:562: done (0 left) 2024-09-14T03:49:38.575 INFO:tasks.rados.rados.0.smithi070.stdout:563: setattr oid 5 current snap is 53 2024-09-14T03:49:38.576 INFO:tasks.rados.rados.0.smithi070.stdout:564: rollback oid 50 current snap is 53 2024-09-14T03:49:38.577 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 50 to 44 2024-09-14T03:49:38.577 INFO:tasks.rados.rados.0.smithi070.stdout:565: setattr oid 47 current snap is 53 2024-09-14T03:49:38.579 INFO:tasks.rados.rados.0.smithi070.stdout:566: rollback oid 10 current snap is 53 2024-09-14T03:49:38.579 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 10 to 34 2024-09-14T03:49:38.579 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 11 current snap is 53 2024-09-14T03:49:38.579 INFO:tasks.rados.rados.0.smithi070.stdout:567: seq_num 194 ranges {2408448=458752,2867200=401408} 2024-09-14T03:49:38.596 INFO:tasks.rados.rados.0.smithi070.stdout:567: writing smithi07085154-11 from 2408448 to 2867200 tid 1 2024-09-14T03:49:38.599 INFO:tasks.rados.rados.0.smithi070.stdout:567: writing smithi07085154-11 from 2867200 to 3268608 tid 2 2024-09-14T03:49:38.599 INFO:tasks.rados.rados.0.smithi070.stdout:564: finishing rollback tid 0 to smithi07085154-50 2024-09-14T03:49:38.599 INFO:tasks.rados.rados.0.smithi070.stdout:564: finishing rollback tid 1 to smithi07085154-50 2024-09-14T03:49:38.599 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 47 v 123 (ObjNum 186 snap 51 seq_num 186) dirty exists 2024-09-14T03:49:38.600 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 32 (ObjNum 162 snap 47 seq_num 162) dirty exists 2024-09-14T03:49:38.600 INFO:tasks.rados.rados.0.smithi070.stdout:566: finishing rollback tid 1 to smithi07085154-10 2024-09-14T03:49:38.600 INFO:tasks.rados.rados.0.smithi070.stdout:566: finishing rollback tid 2 to smithi07085154-10 2024-09-14T03:49:38.600 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 380 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-14T03:49:38.600 INFO:tasks.rados.rados.0.smithi070.stdout:564: finishing rollback tid 2 to smithi07085154-50 2024-09-14T03:49:38.600 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 238 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-14T03:49:38.600 INFO:tasks.rados.rados.0.smithi070.stdout:567: finishing write tid 1 to smithi07085154-11 2024-09-14T03:49:38.600 INFO:tasks.rados.rados.0.smithi070.stdout:563: done (4 left) 2024-09-14T03:49:38.600 INFO:tasks.rados.rados.0.smithi070.stdout:564: done (3 left) 2024-09-14T03:49:38.600 INFO:tasks.rados.rados.0.smithi070.stdout:565: done (2 left) 2024-09-14T03:49:38.600 INFO:tasks.rados.rados.0.smithi070.stdout:566: done (1 left) 2024-09-14T03:49:38.601 INFO:tasks.rados.rados.0.smithi070.stdout:568: snap_remove snap 13 2024-09-14T03:49:38.833 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:38 smithi094 ceph-mon[31053]: osdmap e164: 8 total, 8 up, 8 in 2024-09-14T03:49:38.833 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:38 smithi094 ceph-mon[31053]: mgrmap e26: x(active, since 3s) 2024-09-14T03:49:38.921 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:38 smithi070 ceph-mon[27495]: osdmap e164: 8 total, 8 up, 8 in 2024-09-14T03:49:38.921 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:38 smithi070 ceph-mon[27495]: mgrmap e26: x(active, since 3s) 2024-09-14T03:49:38.921 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:38 smithi070 ceph-mon[39288]: osdmap e164: 8 total, 8 up, 8 in 2024-09-14T03:49:38.921 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:38 smithi070 ceph-mon[39288]: mgrmap e26: x(active, since 3s) 2024-09-14T03:49:39.181 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:39 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:49:39.070+0000 7fc152af4ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-14T03:49:39.181 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:39 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:49:39] ENGINE Bus STARTING 2024-09-14T03:49:39.181 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:39 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: CherryPy Checker: 2024-09-14T03:49:39.181 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:39 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: The Application mounted at '' has an empty config. 2024-09-14T03:49:39.181 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:39 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: 2024-09-14T03:49:39.181 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:39 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:49:39] ENGINE Serving on http://:::9283 2024-09-14T03:49:39.181 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:39 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: [14/Sep/2024:03:49:39] ENGINE Bus STARTED 2024-09-14T03:49:39.583 INFO:tasks.rados.rados.0.smithi070.stdout:568: done (1 left) 2024-09-14T03:49:39.583 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 49 current snap is 53 2024-09-14T03:49:39.583 INFO:tasks.rados.rados.0.smithi070.stdout:569: seq_num 195 ranges {0=475136,475136=401408} 2024-09-14T03:49:39.585 INFO:tasks.rados.rados.0.smithi070.stdout:569: writing smithi07085154-49 from 0 to 475136 tid 1 2024-09-14T03:49:39.589 INFO:tasks.rados.rados.0.smithi070.stdout:569: writing smithi07085154-49 from 475136 to 876544 tid 2 2024-09-14T03:49:39.589 INFO:tasks.rados.rados.0.smithi070.stdout:570: rollback oid 5 current snap is 53 2024-09-14T03:49:39.589 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 5 to 47 2024-09-14T03:49:39.589 INFO:tasks.rados.rados.0.smithi070.stdout:567: finishing write tid 2 to smithi07085154-11 2024-09-14T03:49:39.589 INFO:tasks.rados.rados.0.smithi070.stdout:567: finishing write tid 3 to smithi07085154-11 2024-09-14T03:49:39.589 INFO:tasks.rados.rados.0.smithi070.stdout:567: finishing write tid 4 to smithi07085154-11 2024-09-14T03:49:39.589 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 11 v 236 (ObjNum 194 snap 53 seq_num 194) dirty exists 2024-09-14T03:49:39.589 INFO:tasks.rados.rados.0.smithi070.stdout:567: left oid 11 (ObjNum 194 snap 53 seq_num 194) 2024-09-14T03:49:39.589 INFO:tasks.rados.rados.0.smithi070.stdout:567: done (2 left) 2024-09-14T03:49:39.589 INFO:tasks.rados.rados.0.smithi070.stdout:571: rmattr oid 4 current snap is 53 2024-09-14T03:49:39.590 INFO:tasks.rados.rados.0.smithi070.stdout:572: read oid 18 snap -1 2024-09-14T03:49:39.590 INFO:tasks.rados.rados.0.smithi070.stdout:572: expect (ObjNum 78 snap 13 seq_num 78) 2024-09-14T03:49:39.590 INFO:tasks.rados.rados.0.smithi070.stdout:573: snap_create 2024-09-14T03:49:39.595 INFO:tasks.rados.rados.0.smithi070.stdout:569: finishing write tid 1 to smithi07085154-49 2024-09-14T03:49:39.595 INFO:tasks.rados.rados.0.smithi070.stdout:569: finishing write tid 2 to smithi07085154-49 2024-09-14T03:49:39.597 INFO:tasks.rados.rados.0.smithi070.stdout:570: finishing rollback tid 0 to smithi07085154-5 2024-09-14T03:49:39.597 INFO:tasks.rados.rados.0.smithi070.stdout:569: finishing write tid 3 to smithi07085154-49 2024-09-14T03:49:39.597 INFO:tasks.rados.rados.0.smithi070.stdout:570: finishing rollback tid 1 to smithi07085154-5 2024-09-14T03:49:39.597 INFO:tasks.rados.rados.0.smithi070.stdout:570: finishing rollback tid 2 to smithi07085154-5 2024-09-14T03:49:39.597 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 35 (ObjNum 162 snap 47 seq_num 162) dirty exists 2024-09-14T03:49:39.598 INFO:tasks.rados.rados.0.smithi070.stdout:569: finishing write tid 4 to smithi07085154-49 2024-09-14T03:49:39.598 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 49 v 193 (ObjNum 195 snap 53 seq_num 195) dirty exists 2024-09-14T03:49:39.598 INFO:tasks.rados.rados.0.smithi070.stdout:569: left oid 49 (ObjNum 195 snap 53 seq_num 195) 2024-09-14T03:49:39.599 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 314 (ObjNum 170 snap 47 seq_num 170) dirty exists 2024-09-14T03:49:39.602 INFO:tasks.rados.rados.0.smithi070.stdout:572: expect (ObjNum 78 snap 13 seq_num 78) 2024-09-14T03:49:39.739 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-09-14T03:49:39.846 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:39 smithi094 ceph-mon[31053]: pgmap v9: 121 pgs: 121 active+clean; 163 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:49:39.846 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:39 smithi094 ceph-mon[31053]: osdmap e165: 8 total, 8 up, 8 in 2024-09-14T03:49:39.846 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:39 smithi094 ceph-mon[31053]: Standby manager daemon y started 2024-09-14T03:49:39.847 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:39 smithi094 ceph-mon[31053]: from='mgr.? 172.21.15.70:0/4070485953' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-14T03:49:39.847 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:39 smithi094 ceph-mon[31053]: from='mgr.? 172.21.15.70:0/4070485953' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:49:39.847 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:39 smithi094 ceph-mon[31053]: from='mgr.? 172.21.15.70:0/4070485953' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-14T03:49:39.847 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:39 smithi094 ceph-mon[31053]: from='mgr.? 172.21.15.70:0/4070485953' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:49:39.866 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:39 smithi070 ceph-mon[27495]: pgmap v9: 121 pgs: 121 active+clean; 163 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:49:39.867 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:39 smithi070 ceph-mon[27495]: osdmap e165: 8 total, 8 up, 8 in 2024-09-14T03:49:39.867 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:39 smithi070 ceph-mon[27495]: Standby manager daemon y started 2024-09-14T03:49:39.867 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:39 smithi070 ceph-mon[27495]: from='mgr.? 172.21.15.70:0/4070485953' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-14T03:49:39.867 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:39 smithi070 ceph-mon[27495]: from='mgr.? 172.21.15.70:0/4070485953' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:49:39.867 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:39 smithi070 ceph-mon[27495]: from='mgr.? 172.21.15.70:0/4070485953' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-14T03:49:39.867 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:39 smithi070 ceph-mon[27495]: from='mgr.? 172.21.15.70:0/4070485953' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:49:39.868 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:39 smithi070 ceph-mon[39288]: pgmap v9: 121 pgs: 121 active+clean; 163 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:49:39.868 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:39 smithi070 ceph-mon[39288]: osdmap e165: 8 total, 8 up, 8 in 2024-09-14T03:49:39.868 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:39 smithi070 ceph-mon[39288]: Standby manager daemon y started 2024-09-14T03:49:39.868 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:39 smithi070 ceph-mon[39288]: from='mgr.? 172.21.15.70:0/4070485953' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-14T03:49:39.868 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:39 smithi070 ceph-mon[39288]: from='mgr.? 172.21.15.70:0/4070485953' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:49:39.868 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:39 smithi070 ceph-mon[39288]: from='mgr.? 172.21.15.70:0/4070485953' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-14T03:49:39.868 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:39 smithi070 ceph-mon[39288]: from='mgr.? 172.21.15.70:0/4070485953' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:49:40.298 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:49:40.299 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (3m) 0s ago 4m 27.9M - 0.25.0 c8568f914cd2 fef584d3d5c6 2024-09-14T03:49:40.299 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi094 *:3000 running (4m) 9s ago 4m 87.4M - 9.4.7 954c08fa6188 846770f48257 2024-09-14T03:49:40.299 INFO:teuthology.orchestra.run.smithi070.stdout:mds.a smithi070 running (5m) 0s ago 5m 18.4M - 17.2.7-1629-geb16b97b ab51d1bebd12 52e0dbb7be50 2024-09-14T03:49:40.299 INFO:teuthology.orchestra.run.smithi070.stdout:mds.b smithi094 running (5m) 9s ago 5m 14.4M - 17.2.7-1629-geb16b97b ab51d1bebd12 e3b5e3e00189 2024-09-14T03:49:40.299 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi094 *:8443,9283 running (14s) 9s ago 9m 66.7M - 19.3.0-4907-ga2782315 8801d2529e43 5aadf84f680b 2024-09-14T03:49:40.299 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:9283,8765 running (10m) 0s ago 10m 270M - 17.2.7-1629-geb16b97b ab51d1bebd12 dc71810017b9 2024-09-14T03:49:40.299 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (11m) 0s ago 11m 55.2M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 12d19db91f54 2024-09-14T03:49:40.299 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi094 running (9m) 9s ago 9m 40.2M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 90ea2e164909 2024-09-14T03:49:40.299 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (9m) 0s ago 9m 46.2M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 44ee80c29ee0 2024-09-14T03:49:40.299 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (3m) 0s ago 3m 16.6M - 1.5.0 0da6a335fe13 315c3f32399f 2024-09-14T03:49:40.299 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi094 *:9100 running (3m) 9s ago 3m 15.6M - 1.5.0 0da6a335fe13 3dcf1723ea35 2024-09-14T03:49:40.299 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (8m) 0s ago 8m 105M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 741c06b6adbf 2024-09-14T03:49:40.299 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (8m) 0s ago 8m 110M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 af459ea7ef68 2024-09-14T03:49:40.299 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (8m) 0s ago 8m 93.8M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 109b66e4e60f 2024-09-14T03:49:40.300 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (7m) 0s ago 7m 87.9M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 2e4199b58392 2024-09-14T03:49:40.300 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi094 running (7m) 9s ago 7m 97.0M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 52b64527e24c 2024-09-14T03:49:40.300 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi094 running (6m) 9s ago 6m 80.0M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 b543f152ba87 2024-09-14T03:49:40.300 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi094 running (6m) 9s ago 6m 56.7M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 e66b592ac940 2024-09-14T03:49:40.300 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi094 running (6m) 9s ago 6m 80.6M 2992M 17.2.7-1629-geb16b97b ab51d1bebd12 037c6208e9ce 2024-09-14T03:49:40.300 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi094 *:9095 running (3m) 9s ago 4m 59.0M - 2.43.0 a07b618ecd1d 9c88e2923e9d 2024-09-14T03:49:40.522 INFO:tasks.rados.rados.0.smithi070.stdout:569: done (4 left) 2024-09-14T03:49:40.522 INFO:tasks.rados.rados.0.smithi070.stdout:570: done (3 left) 2024-09-14T03:49:40.523 INFO:tasks.rados.rados.0.smithi070.stdout:571: done (2 left) 2024-09-14T03:49:40.523 INFO:tasks.rados.rados.0.smithi070.stdout:572: done (1 left) 2024-09-14T03:49:40.523 INFO:tasks.rados.rados.0.smithi070.stdout:573: done (0 left) 2024-09-14T03:49:40.523 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 3 current snap is 54 2024-09-14T03:49:40.523 INFO:tasks.rados.rados.0.smithi070.stdout:574: seq_num 196 ranges {0=491520,491520=401408} 2024-09-14T03:49:40.525 INFO:tasks.rados.rados.0.smithi070.stdout:574: writing smithi07085154-3 from 0 to 491520 tid 1 2024-09-14T03:49:40.527 INFO:tasks.rados.rados.0.smithi070.stdout:574: writing smithi07085154-3 from 491520 to 892928 tid 2 2024-09-14T03:49:40.527 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 49 current snap is 54 2024-09-14T03:49:40.528 INFO:tasks.rados.rados.0.smithi070.stdout:575: seq_num 197 ranges {876544=507904,1384448=401408} 2024-09-14T03:49:40.535 INFO:tasks.rados.rados.0.smithi070.stdout:575: writing smithi07085154-49 from 876544 to 1384448 tid 1 2024-09-14T03:49:40.538 INFO:tasks.rados.rados.0.smithi070.stdout:575: writing smithi07085154-49 from 1384448 to 1785856 tid 2 2024-09-14T03:49:40.539 INFO:tasks.rados.rados.0.smithi070.stdout:576: read oid 32 snap 40 2024-09-14T03:49:40.539 INFO:tasks.rados.rados.0.smithi070.stdout:576: expect deleted 2024-09-14T03:49:40.539 INFO:tasks.rados.rados.0.smithi070.stdout:577: rollback oid 50 current snap is 54 2024-09-14T03:49:40.539 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 50 to 47 2024-09-14T03:49:40.539 INFO:tasks.rados.rados.0.smithi070.stdout:574: finishing write tid 1 to smithi07085154-3 2024-09-14T03:49:40.539 INFO:tasks.rados.rados.0.smithi070.stdout:574: finishing write tid 2 to smithi07085154-3 2024-09-14T03:49:40.539 INFO:tasks.rados.rados.0.smithi070.stdout:574: finishing write tid 3 to smithi07085154-3 2024-09-14T03:49:40.539 INFO:tasks.rados.rados.0.smithi070.stdout:574: finishing write tid 4 to smithi07085154-3 2024-09-14T03:49:40.539 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 3 v 132 (ObjNum 196 snap 54 seq_num 196) dirty exists 2024-09-14T03:49:40.539 INFO:tasks.rados.rados.0.smithi070.stdout:574: left oid 3 (ObjNum 196 snap 54 seq_num 196) 2024-09-14T03:49:40.539 INFO:tasks.rados.rados.0.smithi070.stdout:574: done (3 left) 2024-09-14T03:49:40.539 INFO:tasks.rados.rados.0.smithi070.stdout:578: snap_create 2024-09-14T03:49:40.540 INFO:tasks.rados.rados.0.smithi070.stdout:577: finishing rollback tid 0 to smithi07085154-50 2024-09-14T03:49:40.542 INFO:tasks.rados.rados.0.smithi070.stdout:575: finishing write tid 1 to smithi07085154-49 2024-09-14T03:49:40.542 INFO:tasks.rados.rados.0.smithi070.stdout:577: finishing rollback tid 1 to smithi07085154-50 2024-09-14T03:49:40.542 INFO:tasks.rados.rados.0.smithi070.stdout:577: finishing rollback tid 2 to smithi07085154-50 2024-09-14T03:49:40.542 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 248 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-14T03:49:40.543 INFO:tasks.rados.rados.0.smithi070.stdout:575: finishing write tid 2 to smithi07085154-49 2024-09-14T03:49:40.543 INFO:tasks.rados.rados.0.smithi070.stdout:575: finishing write tid 3 to smithi07085154-49 2024-09-14T03:49:40.544 INFO:tasks.rados.rados.0.smithi070.stdout:575: finishing write tid 4 to smithi07085154-49 2024-09-14T03:49:40.544 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 49 v 199 (ObjNum 197 snap 54 seq_num 197) dirty exists 2024-09-14T03:49:40.544 INFO:tasks.rados.rados.0.smithi070.stdout:575: left oid 49 (ObjNum 197 snap 54 seq_num 197) 2024-09-14T03:49:40.649 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:49:40.650 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-09-14T03:49:40.650 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 3 2024-09-14T03:49:40.650 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:49:40.650 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-09-14T03:49:40.650 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 1, 2024-09-14T03:49:40.650 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 1 2024-09-14T03:49:40.650 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:49:40.650 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-09-14T03:49:40.650 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 8 2024-09-14T03:49:40.650 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:49:40.650 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": { 2024-09-14T03:49:40.651 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:49:40.651 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:49:40.651 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-09-14T03:49:40.651 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 14, 2024-09-14T03:49:40.651 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 1 2024-09-14T03:49:40.651 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-09-14T03:49:40.651 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:49:40.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:40 smithi094 ceph-mon[31053]: mgrmap e27: x(active, since 5s), standbys: y 2024-09-14T03:49:40.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:40 smithi094 ceph-mon[31053]: osdmap e166: 8 total, 8 up, 8 in 2024-09-14T03:49:40.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:40 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:49:40.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:40 smithi094 ceph-mon[31053]: from='client.24980 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:40.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:40 smithi094 ceph-mon[31053]: from='client.15081 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:40.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:40 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:40.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:40 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:40.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:40 smithi094 ceph-mon[31053]: osdmap e167: 8 total, 8 up, 8 in 2024-09-14T03:49:40.935 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:49:40.935 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:49:40.936 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-09-14T03:49:40.936 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:49:40.936 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [], 2024-09-14T03:49:40.936 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "1/20 daemons upgraded", 2024-09-14T03:49:40.936 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "", 2024-09-14T03:49:40.936 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-09-14T03:49:40.936 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:49:40.936 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:40 smithi070 ceph-mon[27495]: mgrmap e27: x(active, since 5s), standbys: y 2024-09-14T03:49:40.936 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:40 smithi070 ceph-mon[27495]: osdmap e166: 8 total, 8 up, 8 in 2024-09-14T03:49:40.937 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:40 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:49:40.937 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:40 smithi070 ceph-mon[27495]: from='client.24980 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:40.937 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:40 smithi070 ceph-mon[27495]: from='client.15081 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:40.937 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:40 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:40.937 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:40 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:40.937 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:40 smithi070 ceph-mon[27495]: osdmap e167: 8 total, 8 up, 8 in 2024-09-14T03:49:40.937 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:40 smithi070 ceph-mon[39288]: mgrmap e27: x(active, since 5s), standbys: y 2024-09-14T03:49:40.938 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:40 smithi070 ceph-mon[39288]: osdmap e166: 8 total, 8 up, 8 in 2024-09-14T03:49:40.938 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:40 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:49:40.938 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:40 smithi070 ceph-mon[39288]: from='client.24980 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:40.938 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:40 smithi070 ceph-mon[39288]: from='client.15081 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:40.938 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:40 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:40.938 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:40 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:40.938 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:40 smithi070 ceph-mon[39288]: osdmap e167: 8 total, 8 up, 8 in 2024-09-14T03:49:41.525 INFO:tasks.rados.rados.0.smithi070.stdout:575: done (3 left) 2024-09-14T03:49:41.526 INFO:tasks.rados.rados.0.smithi070.stdout:576: done (2 left) 2024-09-14T03:49:41.526 INFO:tasks.rados.rados.0.smithi070.stdout:577: done (1 left) 2024-09-14T03:49:41.526 INFO:tasks.rados.rados.0.smithi070.stdout:578: done (0 left) 2024-09-14T03:49:41.526 INFO:tasks.rados.rados.0.smithi070.stdout:579: read oid 22 snap -1 2024-09-14T03:49:41.526 INFO:tasks.rados.rados.0.smithi070.stdout:579: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-14T03:49:41.526 INFO:tasks.rados.rados.0.smithi070.stdout:580: rollback oid 47 current snap is 55 2024-09-14T03:49:41.526 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 47 to 48 2024-09-14T03:49:41.526 INFO:tasks.rados.rados.0.smithi070.stdout:581: snap_remove snap 46 2024-09-14T03:49:41.901 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:49:41] "GET /metrics HTTP/1.1" 200 33244 "" "Prometheus/2.43.0" 2024-09-14T03:49:41.901 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:41 smithi094 ceph-mon[31053]: from='client.15087 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:41.901 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:41 smithi094 ceph-mon[31053]: pgmap v13: 121 pgs: 121 active+clean; 163 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:49:41.901 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:41 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/651703616' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:41.901 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:41 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:41.901 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:41 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:41.901 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:41 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:41.901 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:41 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:41.901 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:41 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:41.902 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:41 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:41.902 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:41 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:41.902 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:41 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:41.902 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:41 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:41.902 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:41 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:41.902 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:41 smithi094 ceph-mon[31053]: Adjusting osd_memory_target on smithi070 to 2480M 2024-09-14T03:49:41.902 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:41 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:41.902 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:41 smithi094 ceph-mon[31053]: from='client.15099 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:41.902 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:41 smithi094 ceph-mon[31053]: osdmap e168: 8 total, 8 up, 8 in 2024-09-14T03:49:42.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:41 smithi070 ceph-mon[27495]: from='client.15087 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:42.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:41 smithi070 ceph-mon[27495]: pgmap v13: 121 pgs: 121 active+clean; 163 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:49:42.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:41 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/651703616' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:42.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:41 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:42.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:41 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:42.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:41 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:42.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:41 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:42.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:41 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:42.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:41 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:42.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:41 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:42.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:41 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:42.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:41 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:42.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:41 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:42.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:41 smithi070 ceph-mon[27495]: Adjusting osd_memory_target on smithi070 to 2480M 2024-09-14T03:49:42.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:41 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:42.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:41 smithi070 ceph-mon[27495]: from='client.15099 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:42.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:41 smithi070 ceph-mon[27495]: osdmap e168: 8 total, 8 up, 8 in 2024-09-14T03:49:42.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:41 smithi070 ceph-mon[39288]: from='client.15087 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:42.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:41 smithi070 ceph-mon[39288]: pgmap v13: 121 pgs: 121 active+clean; 163 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:49:42.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:41 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/651703616' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:49:42.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:41 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:42.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:41 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:42.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:41 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:42.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:41 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:42.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:41 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:42.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:41 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:42.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:41 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:42.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:41 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:42.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:41 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:42.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:41 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:42.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:41 smithi070 ceph-mon[39288]: Adjusting osd_memory_target on smithi070 to 2480M 2024-09-14T03:49:42.039 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:41 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:42.039 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:41 smithi070 ceph-mon[39288]: from='client.15099 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:42.039 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:41 smithi070 ceph-mon[39288]: osdmap e168: 8 total, 8 up, 8 in 2024-09-14T03:49:42.532 INFO:tasks.rados.rados.0.smithi070.stdout:581: done (2 left) 2024-09-14T03:49:42.533 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 6 current snap is 55 2024-09-14T03:49:42.533 INFO:tasks.rados.rados.0.smithi070.stdout:582: seq_num 198 ranges {3825664=524288,4349952=401408} 2024-09-14T03:49:42.553 INFO:tasks.rados.rados.0.smithi070.stdout:582: writing smithi07085154-6 from 3825664 to 4349952 tid 1 2024-09-14T03:49:42.556 INFO:tasks.rados.rados.0.smithi070.stdout:582: writing smithi07085154-6 from 4349952 to 4751360 tid 2 2024-09-14T03:49:42.556 INFO:tasks.rados.rados.0.smithi070.stdout:583: copy_from oid 41 from oid 46 current snap is 55 2024-09-14T03:49:42.556 INFO:tasks.rados.rados.0.smithi070.stdout:579: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-14T03:49:42.577 INFO:tasks.rados.rados.0.smithi070.stdout:580: finishing rollback tid 0 to smithi07085154-47 2024-09-14T03:49:42.577 INFO:tasks.rados.rados.0.smithi070.stdout:580: finishing rollback tid 1 to smithi07085154-47 2024-09-14T03:49:42.577 INFO:tasks.rados.rados.0.smithi070.stdout:580: finishing rollback tid 2 to smithi07085154-47 2024-09-14T03:49:42.577 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 47 v 136 (ObjNum 151 snap 45 seq_num 151) dirty exists 2024-09-14T03:49:42.577 INFO:tasks.rados.rados.0.smithi070.stdout:583: finishing copy_from to smithi07085154-41 2024-09-14T03:49:42.577 INFO:tasks.rados.rados.0.smithi070.stdout:583: got expected ENOENT (src dne) 2024-09-14T03:49:42.577 INFO:tasks.rados.rados.0.smithi070.stdout:583: finishing copy_from racing read to smithi07085154-41 2024-09-14T03:49:42.577 INFO:tasks.rados.rados.0.smithi070.stdout:582: finishing write tid 1 to smithi07085154-6 2024-09-14T03:49:42.577 INFO:tasks.rados.rados.0.smithi070.stdout:579: done (3 left) 2024-09-14T03:49:42.578 INFO:tasks.rados.rados.0.smithi070.stdout:580: done (2 left) 2024-09-14T03:49:42.578 INFO:tasks.rados.rados.0.smithi070.stdout:583: done (1 left) 2024-09-14T03:49:42.578 INFO:tasks.rados.rados.0.smithi070.stdout:584: rollback oid 21 current snap is 55 2024-09-14T03:49:42.578 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 21 to 50 2024-09-14T03:49:42.578 INFO:tasks.rados.rados.0.smithi070.stdout:582: finishing write tid 2 to smithi07085154-6 2024-09-14T03:49:42.578 INFO:tasks.rados.rados.0.smithi070.stdout:582: finishing write tid 3 to smithi07085154-6 2024-09-14T03:49:42.578 INFO:tasks.rados.rados.0.smithi070.stdout:582: finishing write tid 4 to smithi07085154-6 2024-09-14T03:49:42.578 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 243 (ObjNum 198 snap 55 seq_num 198) dirty exists 2024-09-14T03:49:42.578 INFO:tasks.rados.rados.0.smithi070.stdout:582: left oid 6 (ObjNum 198 snap 55 seq_num 198) 2024-09-14T03:49:42.578 INFO:tasks.rados.rados.0.smithi070.stdout:582: done (1 left) 2024-09-14T03:49:42.578 INFO:tasks.rados.rados.0.smithi070.stdout:585: snap_remove snap 54 2024-09-14T03:49:42.909 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:42 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:42.909 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:42 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:42.910 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:42 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:42.910 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:42 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:42.910 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:42 smithi094 ceph-mon[31053]: osdmap e169: 8 total, 8 up, 8 in 2024-09-14T03:49:43.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:42 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:43.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:42 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:43.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:42 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:43.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:42 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:43.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:42 smithi070 ceph-mon[27495]: osdmap e169: 8 total, 8 up, 8 in 2024-09-14T03:49:43.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:42 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:43.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:42 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:43.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:42 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:43.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:42 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:43.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:42 smithi070 ceph-mon[39288]: osdmap e169: 8 total, 8 up, 8 in 2024-09-14T03:49:43.532 INFO:tasks.rados.rados.0.smithi070.stdout:585: done (1 left) 2024-09-14T03:49:43.532 INFO:tasks.rados.rados.0.smithi070.stdout:586: snap_remove snap 52 2024-09-14T03:49:43.532 INFO:tasks.rados.rados.0.smithi070.stdout:584: finishing rollback tid 1 to smithi07085154-21 2024-09-14T03:49:43.532 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 65 (ObjNum 289 snap 0 seq_num 4214980279) dirty dne 2024-09-14T03:49:44.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:43 smithi094 ceph-mon[31053]: pgmap v16: 121 pgs: 121 active+clean; 170 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 3.2 MiB/s wr, 56 op/s 2024-09-14T03:49:44.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:43 smithi094 ceph-mon[31053]: osdmap e170: 8 total, 8 up, 8 in 2024-09-14T03:49:44.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:43 smithi070 ceph-mon[39288]: pgmap v16: 121 pgs: 121 active+clean; 170 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 3.2 MiB/s wr, 56 op/s 2024-09-14T03:49:44.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:43 smithi070 ceph-mon[39288]: osdmap e170: 8 total, 8 up, 8 in 2024-09-14T03:49:44.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:43 smithi070 ceph-mon[27495]: pgmap v16: 121 pgs: 121 active+clean; 170 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 3.2 MiB/s wr, 56 op/s 2024-09-14T03:49:44.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:43 smithi070 ceph-mon[27495]: osdmap e170: 8 total, 8 up, 8 in 2024-09-14T03:49:44.533 INFO:tasks.rados.rados.0.smithi070.stdout:584: done (1 left) 2024-09-14T03:49:44.534 INFO:tasks.rados.rados.0.smithi070.stdout:586: done (0 left) 2024-09-14T03:49:44.534 INFO:tasks.rados.rados.0.smithi070.stdout:587: snap_create 2024-09-14T03:49:45.524 INFO:tasks.rados.rados.0.smithi070.stdout:587: done (0 left) 2024-09-14T03:49:45.524 INFO:tasks.rados.rados.0.smithi070.stdout:588: read oid 47 snap -1 2024-09-14T03:49:45.525 INFO:tasks.rados.rados.0.smithi070.stdout:588: expect (ObjNum 151 snap 45 seq_num 151) 2024-09-14T03:49:45.525 INFO:tasks.rados.rados.0.smithi070.stdout:589: delete oid 33 current snap is 56 2024-09-14T03:49:45.528 INFO:tasks.rados.rados.0.smithi070.stdout:589: done (1 left) 2024-09-14T03:49:45.528 INFO:tasks.rados.rados.0.smithi070.stdout:590: rollback oid 8 current snap is 56 2024-09-14T03:49:45.528 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 8 to 53 2024-09-14T03:49:45.528 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 12 current snap is 56 2024-09-14T03:49:45.528 INFO:tasks.rados.rados.0.smithi070.stdout:591: seq_num 200 ranges {622592=557056,1179648=401408} 2024-09-14T03:49:45.535 INFO:tasks.rados.rados.0.smithi070.stdout:591: writing smithi07085154-12 from 622592 to 1179648 tid 1 2024-09-14T03:49:45.538 INFO:tasks.rados.rados.0.smithi070.stdout:591: writing smithi07085154-12 from 1179648 to 1581056 tid 2 2024-09-14T03:49:45.538 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 31 current snap is 56 2024-09-14T03:49:45.538 INFO:tasks.rados.rados.0.smithi070.stdout:592: seq_num 201 ranges {0=573440,573440=401408} 2024-09-14T03:49:45.542 INFO:tasks.rados.rados.0.smithi070.stdout:592: writing smithi07085154-31 from 0 to 573440 tid 1 2024-09-14T03:49:45.546 INFO:tasks.rados.rados.0.smithi070.stdout:592: writing smithi07085154-31 from 573440 to 974848 tid 2 2024-09-14T03:49:45.546 INFO:tasks.rados.rados.0.smithi070.stdout:590: finishing rollback tid 0 to smithi07085154-8 2024-09-14T03:49:45.546 INFO:tasks.rados.rados.0.smithi070.stdout:590: finishing rollback tid 1 to smithi07085154-8 2024-09-14T03:49:45.546 INFO:tasks.rados.rados.0.smithi070.stdout:590: finishing rollback tid 2 to smithi07085154-8 2024-09-14T03:49:45.546 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 8 v 300 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-14T03:49:45.546 INFO:tasks.rados.rados.0.smithi070.stdout:588: expect (ObjNum 151 snap 45 seq_num 151) 2024-09-14T03:49:45.672 INFO:tasks.rados.rados.0.smithi070.stdout:591: finishing write tid 1 to smithi07085154-12 2024-09-14T03:49:45.672 INFO:tasks.rados.rados.0.smithi070.stdout:591: finishing write tid 2 to smithi07085154-12 2024-09-14T03:49:45.672 INFO:tasks.rados.rados.0.smithi070.stdout:591: finishing write tid 3 to smithi07085154-12 2024-09-14T03:49:45.672 INFO:tasks.rados.rados.0.smithi070.stdout:591: finishing write tid 4 to smithi07085154-12 2024-09-14T03:49:45.673 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 125 (ObjNum 200 snap 56 seq_num 200) dirty exists 2024-09-14T03:49:45.673 INFO:tasks.rados.rados.0.smithi070.stdout:591: left oid 12 (ObjNum 200 snap 56 seq_num 200) 2024-09-14T03:49:45.673 INFO:tasks.rados.rados.0.smithi070.stdout:592: finishing write tid 1 to smithi07085154-31 2024-09-14T03:49:45.673 INFO:tasks.rados.rados.0.smithi070.stdout:592: finishing write tid 2 to smithi07085154-31 2024-09-14T03:49:45.673 INFO:tasks.rados.rados.0.smithi070.stdout:592: finishing write tid 3 to smithi07085154-31 2024-09-14T03:49:45.673 INFO:tasks.rados.rados.0.smithi070.stdout:588: done (3 left) 2024-09-14T03:49:45.673 INFO:tasks.rados.rados.0.smithi070.stdout:590: done (2 left) 2024-09-14T03:49:45.674 INFO:tasks.rados.rados.0.smithi070.stdout:591: done (1 left) 2024-09-14T03:49:45.674 INFO:tasks.rados.rados.0.smithi070.stdout:593: rollback oid 43 current snap is 56 2024-09-14T03:49:45.674 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 43 to 41 2024-09-14T03:49:45.674 INFO:tasks.rados.rados.0.smithi070.stdout:592: finishing write tid 4 to smithi07085154-31 2024-09-14T03:49:45.674 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 250 (ObjNum 201 snap 56 seq_num 201) dirty exists 2024-09-14T03:49:45.674 INFO:tasks.rados.rados.0.smithi070.stdout:592: left oid 31 (ObjNum 201 snap 56 seq_num 201) 2024-09-14T03:49:45.674 INFO:tasks.rados.rados.0.smithi070.stdout:592: done (1 left) 2024-09-14T03:49:45.675 INFO:tasks.rados.rados.0.smithi070.stdout:594: read oid 9 snap -1 2024-09-14T03:49:45.675 INFO:tasks.rados.rados.0.smithi070.stdout:594: expect (ObjNum 153 snap 45 seq_num 153) 2024-09-14T03:49:45.675 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 8 current snap is 56 2024-09-14T03:49:45.675 INFO:tasks.rados.rados.0.smithi070.stdout:595: seq_num 202 ranges {778240=589824,1368064=401408} 2024-09-14T03:49:45.678 INFO:tasks.rados.rados.0.smithi070.stdout:595: writing smithi07085154-8 from 778240 to 1368064 tid 1 2024-09-14T03:49:45.681 INFO:tasks.rados.rados.0.smithi070.stdout:595: writing smithi07085154-8 from 1368064 to 1769472 tid 2 2024-09-14T03:49:45.681 INFO:tasks.rados.rados.0.smithi070.stdout:596: snap_create 2024-09-14T03:49:45.681 INFO:tasks.rados.rados.0.smithi070.stdout:593: finishing rollback tid 0 to smithi07085154-43 2024-09-14T03:49:45.681 INFO:tasks.rados.rados.0.smithi070.stdout:593: finishing rollback tid 1 to smithi07085154-43 2024-09-14T03:49:45.682 INFO:tasks.rados.rados.0.smithi070.stdout:593: finishing rollback tid 2 to smithi07085154-43 2024-09-14T03:49:45.682 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 304 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-14T03:49:45.682 INFO:tasks.rados.rados.0.smithi070.stdout:594: expect (ObjNum 153 snap 45 seq_num 153) 2024-09-14T03:49:45.817 INFO:tasks.rados.rados.0.smithi070.stdout:595: finishing write tid 1 to smithi07085154-8 2024-09-14T03:49:45.817 INFO:tasks.rados.rados.0.smithi070.stdout:595: finishing write tid 2 to smithi07085154-8 2024-09-14T03:49:45.817 INFO:tasks.rados.rados.0.smithi070.stdout:595: finishing write tid 3 to smithi07085154-8 2024-09-14T03:49:45.818 INFO:tasks.rados.rados.0.smithi070.stdout:595: finishing write tid 4 to smithi07085154-8 2024-09-14T03:49:45.818 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 8 v 307 (ObjNum 202 snap 56 seq_num 202) dirty exists 2024-09-14T03:49:45.818 INFO:tasks.rados.rados.0.smithi070.stdout:595: left oid 8 (ObjNum 202 snap 56 seq_num 202) 2024-09-14T03:49:46.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:45 smithi094 ceph-mon[31053]: pgmap v18: 121 pgs: 121 active+clean; 170 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s rd, 3.2 MiB/s wr, 55 op/s 2024-09-14T03:49:46.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:45 smithi094 ceph-mon[31053]: osdmap e171: 8 total, 8 up, 8 in 2024-09-14T03:49:46.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:45 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:46.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:45 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:49:46.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:45 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:49:46.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:45 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:49:46.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:45 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:49:46.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:45 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:49:46.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:45 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:49:46.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:45 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:49:46.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:45 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:49:46.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:45 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:49:46.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:45 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:49:46.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:45 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:49:46.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:45 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:49:46.029 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:45 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:49:46.029 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:45 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:49:46.029 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:45 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:49:46.029 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:45 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:49:46.029 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:45 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:49:46.029 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:45 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:49:46.029 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:45 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:49:46.029 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:45 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:49:46.029 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:45 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:49:46.029 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:45 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:49:46.029 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:45 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:49:46.030 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:45 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:49:46.030 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:45 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:49:46.030 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:45 smithi094 ceph-mon[31053]: osdmap e172: 8 total, 8 up, 8 in 2024-09-14T03:49:46.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[39288]: pgmap v18: 121 pgs: 121 active+clean; 170 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s rd, 3.2 MiB/s wr, 55 op/s 2024-09-14T03:49:46.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[39288]: osdmap e171: 8 total, 8 up, 8 in 2024-09-14T03:49:46.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:46.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:49:46.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:49:46.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:49:46.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:49:46.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:49:46.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:49:46.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:49:46.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:49:46.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:49:46.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:49:46.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:49:46.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:49:46.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:49:46.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:49:46.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:49:46.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:49:46.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:49:46.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:49:46.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:49:46.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:49:46.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:49:46.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:49:46.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:49:46.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:49:46.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:49:46.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[39288]: osdmap e172: 8 total, 8 up, 8 in 2024-09-14T03:49:46.038 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[27495]: pgmap v18: 121 pgs: 121 active+clean; 170 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s rd, 3.2 MiB/s wr, 55 op/s 2024-09-14T03:49:46.038 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[27495]: osdmap e171: 8 total, 8 up, 8 in 2024-09-14T03:49:46.038 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:46.038 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:49:46.038 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:49:46.038 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:49:46.039 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:49:46.039 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:49:46.039 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:49:46.039 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:49:46.039 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:49:46.039 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:49:46.039 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:49:46.039 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:49:46.039 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:49:46.039 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:49:46.039 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:49:46.039 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:49:46.039 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:49:46.039 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:49:46.040 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:49:46.040 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:49:46.040 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:49:46.040 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:49:46.040 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:49:46.040 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:49:46.040 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:49:46.040 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:49:46.040 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:45 smithi070 ceph-mon[27495]: osdmap e172: 8 total, 8 up, 8 in 2024-09-14T03:49:46.546 INFO:tasks.rados.rados.0.smithi070.stdout:593: done (3 left) 2024-09-14T03:49:46.546 INFO:tasks.rados.rados.0.smithi070.stdout:594: done (2 left) 2024-09-14T03:49:46.546 INFO:tasks.rados.rados.0.smithi070.stdout:595: done (1 left) 2024-09-14T03:49:46.547 INFO:tasks.rados.rados.0.smithi070.stdout:596: done (0 left) 2024-09-14T03:49:46.547 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 23 current snap is 57 2024-09-14T03:49:46.547 INFO:tasks.rados.rados.0.smithi070.stdout:597: seq_num 203 ranges {876544=606208,1482752=401408} 2024-09-14T03:49:46.554 INFO:tasks.rados.rados.0.smithi070.stdout:597: writing smithi07085154-23 from 876544 to 1482752 tid 1 2024-09-14T03:49:46.557 INFO:tasks.rados.rados.0.smithi070.stdout:597: writing smithi07085154-23 from 1482752 to 1884160 tid 2 2024-09-14T03:49:46.557 INFO:tasks.rados.rados.0.smithi070.stdout:598: copy_from oid 26 from oid 22 current snap is 57 2024-09-14T03:49:46.557 INFO:tasks.rados.rados.0.smithi070.stdout:599: snap_create 2024-09-14T03:49:46.563 INFO:tasks.rados.rados.0.smithi070.stdout:597: finishing write tid 1 to smithi07085154-23 2024-09-14T03:49:46.568 INFO:tasks.rados.rados.0.smithi070.stdout:598: finishing copy_from racing read to smithi07085154-26 2024-09-14T03:49:46.568 INFO:tasks.rados.rados.0.smithi070.stdout:597: finishing write tid 2 to smithi07085154-23 2024-09-14T03:49:46.568 INFO:tasks.rados.rados.0.smithi070.stdout:597: finishing write tid 3 to smithi07085154-23 2024-09-14T03:49:46.572 INFO:tasks.rados.rados.0.smithi070.stdout:597: finishing write tid 4 to smithi07085154-23 2024-09-14T03:49:46.572 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 138 (ObjNum 203 snap 57 seq_num 203) dirty exists 2024-09-14T03:49:46.573 INFO:tasks.rados.rados.0.smithi070.stdout:597: left oid 23 (ObjNum 203 snap 57 seq_num 203) 2024-09-14T03:49:46.573 INFO:tasks.rados.rados.0.smithi070.stdout:598: finishing copy_from to smithi07085154-26 2024-09-14T03:49:46.573 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 216 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-14T03:49:46.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:46 smithi094 ceph-mon[31053]: mgrmap e28: x(active, since 11s), standbys: y 2024-09-14T03:49:46.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:46 smithi070 ceph-mon[27495]: mgrmap e28: x(active, since 11s), standbys: y 2024-09-14T03:49:46.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:46 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:49:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:49:46.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:46 smithi070 ceph-mon[39288]: mgrmap e28: x(active, since 11s), standbys: y 2024-09-14T03:49:47.551 INFO:tasks.rados.rados.0.smithi070.stdout:597: done (2 left) 2024-09-14T03:49:47.551 INFO:tasks.rados.rados.0.smithi070.stdout:598: done (1 left) 2024-09-14T03:49:47.551 INFO:tasks.rados.rados.0.smithi070.stdout:599: done (0 left) 2024-09-14T03:49:47.551 INFO:tasks.rados.rados.0.smithi070.stdout:600: snap_create 2024-09-14T03:49:48.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:47 smithi094 ceph-mon[31053]: pgmap v21: 121 pgs: 121 active+clean; 175 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 697 KiB/s wr, 17 op/s 2024-09-14T03:49:48.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:47 smithi094 ceph-mon[31053]: osdmap e173: 8 total, 8 up, 8 in 2024-09-14T03:49:48.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:47 smithi070 ceph-mon[27495]: pgmap v21: 121 pgs: 121 active+clean; 175 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 697 KiB/s wr, 17 op/s 2024-09-14T03:49:48.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:47 smithi070 ceph-mon[27495]: osdmap e173: 8 total, 8 up, 8 in 2024-09-14T03:49:48.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:47 smithi070 ceph-mon[39288]: pgmap v21: 121 pgs: 121 active+clean; 175 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 697 KiB/s wr, 17 op/s 2024-09-14T03:49:48.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:47 smithi070 ceph-mon[39288]: osdmap e173: 8 total, 8 up, 8 in 2024-09-14T03:49:48.556 INFO:tasks.rados.rados.0.smithi070.stdout:600: done (0 left) 2024-09-14T03:49:48.557 INFO:tasks.rados.rados.0.smithi070.stdout:601: delete oid 4 current snap is 59 2024-09-14T03:49:48.558 INFO:tasks.rados.rados.0.smithi070.stdout:601: done (0 left) 2024-09-14T03:49:48.559 INFO:tasks.rados.rados.0.smithi070.stdout:602: delete oid 15 current snap is 59 2024-09-14T03:49:48.564 INFO:tasks.rados.rados.0.smithi070.stdout:602: done (0 left) 2024-09-14T03:49:48.564 INFO:tasks.rados.rados.0.smithi070.stdout:603: snap_create 2024-09-14T03:49:49.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:48 smithi094 ceph-mon[31053]: osdmap e174: 8 total, 8 up, 8 in 2024-09-14T03:49:49.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:48 smithi070 ceph-mon[27495]: osdmap e174: 8 total, 8 up, 8 in 2024-09-14T03:49:49.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:48 smithi070 ceph-mon[39288]: osdmap e174: 8 total, 8 up, 8 in 2024-09-14T03:49:49.561 INFO:tasks.rados.rados.0.smithi070.stdout:603: done (0 left) 2024-09-14T03:49:49.561 INFO:tasks.rados.rados.0.smithi070.stdout:604: setattr oid 19 current snap is 60 2024-09-14T03:49:49.562 INFO:tasks.rados.rados.0.smithi070.stdout:605: rmattr oid 42 current snap is 60 2024-09-14T03:49:49.562 INFO:tasks.rados.rados.0.smithi070.stdout:605: done (1 left) 2024-09-14T03:49:49.562 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 39 current snap is 60 2024-09-14T03:49:49.562 INFO:tasks.rados.rados.0.smithi070.stdout:606: seq_num 206 ranges {0=655360,655360=401408} 2024-09-14T03:49:49.565 INFO:tasks.rados.rados.0.smithi070.stdout:606: writing smithi07085154-39 from 0 to 655360 tid 1 2024-09-14T03:49:49.568 INFO:tasks.rados.rados.0.smithi070.stdout:606: writing smithi07085154-39 from 655360 to 1056768 tid 2 2024-09-14T03:49:49.569 INFO:tasks.rados.rados.0.smithi070.stdout:607: read oid 47 snap -1 2024-09-14T03:49:49.569 INFO:tasks.rados.rados.0.smithi070.stdout:607: expect (ObjNum 151 snap 45 seq_num 151) 2024-09-14T03:49:49.569 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 41 current snap is 60 2024-09-14T03:49:49.569 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 19 v 258 (ObjNum 129 snap 35 seq_num 129) dirty exists 2024-09-14T03:49:49.569 INFO:tasks.rados.rados.0.smithi070.stdout:608: seq_num 207 ranges {3153920=671744,3825664=401408} 2024-09-14T03:49:49.589 INFO:tasks.rados.rados.0.smithi070.stdout:608: writing smithi07085154-41 from 3153920 to 3825664 tid 1 2024-09-14T03:49:49.592 INFO:tasks.rados.rados.0.smithi070.stdout:608: writing smithi07085154-41 from 3825664 to 4227072 tid 2 2024-09-14T03:49:49.592 INFO:tasks.rados.rados.0.smithi070.stdout:604: done (3 left) 2024-09-14T03:49:49.592 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 23 current snap is 60 2024-09-14T03:49:49.592 INFO:tasks.rados.rados.0.smithi070.stdout:609: seq_num 208 ranges {1884160=688128,2572288=401408} 2024-09-14T03:49:49.606 INFO:tasks.rados.rados.0.smithi070.stdout:609: writing smithi07085154-23 from 1884160 to 2572288 tid 1 2024-09-14T03:49:49.608 INFO:tasks.rados.rados.0.smithi070.stdout:609: writing smithi07085154-23 from 2572288 to 2973696 tid 2 2024-09-14T03:49:49.608 INFO:tasks.rados.rados.0.smithi070.stdout:610: read oid 30 snap -1 2024-09-14T03:49:49.609 INFO:tasks.rados.rados.0.smithi070.stdout:610: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-14T03:49:49.609 INFO:tasks.rados.rados.0.smithi070.stdout:611: copy_from oid 11 from oid 2 current snap is 60 2024-09-14T03:49:49.609 INFO:tasks.rados.rados.0.smithi070.stdout:606: finishing write tid 1 to smithi07085154-39 2024-09-14T03:49:49.609 INFO:tasks.rados.rados.0.smithi070.stdout:606: finishing write tid 2 to smithi07085154-39 2024-09-14T03:49:49.609 INFO:tasks.rados.rados.0.smithi070.stdout:606: finishing write tid 3 to smithi07085154-39 2024-09-14T03:49:49.609 INFO:tasks.rados.rados.0.smithi070.stdout:606: finishing write tid 4 to smithi07085154-39 2024-09-14T03:49:49.609 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 39 v 46 (ObjNum 206 snap 60 seq_num 206) dirty exists 2024-09-14T03:49:49.609 INFO:tasks.rados.rados.0.smithi070.stdout:606: left oid 39 (ObjNum 206 snap 60 seq_num 206) 2024-09-14T03:49:49.609 INFO:tasks.rados.rados.0.smithi070.stdout:607: expect (ObjNum 151 snap 45 seq_num 151) 2024-09-14T03:49:49.734 INFO:tasks.rados.rados.0.smithi070.stdout:608: finishing write tid 1 to smithi07085154-41 2024-09-14T03:49:49.734 INFO:tasks.rados.rados.0.smithi070.stdout:608: finishing write tid 2 to smithi07085154-41 2024-09-14T03:49:49.735 INFO:tasks.rados.rados.0.smithi070.stdout:608: finishing write tid 3 to smithi07085154-41 2024-09-14T03:49:49.735 INFO:tasks.rados.rados.0.smithi070.stdout:608: finishing write tid 4 to smithi07085154-41 2024-09-14T03:49:49.735 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 313 (ObjNum 207 snap 60 seq_num 207) dirty exists 2024-09-14T03:49:49.735 INFO:tasks.rados.rados.0.smithi070.stdout:608: left oid 41 (ObjNum 207 snap 60 seq_num 207) 2024-09-14T03:49:49.735 INFO:tasks.rados.rados.0.smithi070.stdout:606: done (5 left) 2024-09-14T03:49:49.735 INFO:tasks.rados.rados.0.smithi070.stdout:607: done (4 left) 2024-09-14T03:49:49.735 INFO:tasks.rados.rados.0.smithi070.stdout:608: done (3 left) 2024-09-14T03:49:49.735 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 16 current snap is 60 2024-09-14T03:49:49.735 INFO:tasks.rados.rados.0.smithi070.stdout:609: finishing write tid 1 to smithi07085154-23 2024-09-14T03:49:49.735 INFO:tasks.rados.rados.0.smithi070.stdout:609: finishing write tid 2 to smithi07085154-23 2024-09-14T03:49:49.736 INFO:tasks.rados.rados.0.smithi070.stdout:612: seq_num 209 ranges {4784128=704512,5488640=401408} 2024-09-14T03:49:49.763 INFO:tasks.rados.rados.0.smithi070.stdout:612: writing smithi07085154-16 from 4784128 to 5488640 tid 1 2024-09-14T03:49:49.765 INFO:tasks.rados.rados.0.smithi070.stdout:612: writing smithi07085154-16 from 5488640 to 5890048 tid 2 2024-09-14T03:49:49.765 INFO:tasks.rados.rados.0.smithi070.stdout:613: delete oid 37 current snap is 60 2024-09-14T03:49:49.765 INFO:tasks.rados.rados.0.smithi070.stdout:609: finishing write tid 3 to smithi07085154-23 2024-09-14T03:49:49.765 INFO:tasks.rados.rados.0.smithi070.stdout:610: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-14T03:49:49.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:49 smithi094 ceph-mon[31053]: pgmap v24: 121 pgs: 121 active+clean; 175 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 698 KiB/s wr, 17 op/s 2024-09-14T03:49:49.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:49 smithi094 ceph-mon[31053]: osdmap e175: 8 total, 8 up, 8 in 2024-09-14T03:49:49.778 INFO:tasks.rados.rados.0.smithi070.stdout:609: finishing write tid 4 to smithi07085154-23 2024-09-14T03:49:49.778 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 141 (ObjNum 208 snap 60 seq_num 208) dirty exists 2024-09-14T03:49:49.778 INFO:tasks.rados.rados.0.smithi070.stdout:609: left oid 23 (ObjNum 208 snap 60 seq_num 208) 2024-09-14T03:49:49.779 INFO:tasks.rados.rados.0.smithi070.stdout:611: finishing copy_from racing read to smithi07085154-11 2024-09-14T03:49:49.779 INFO:tasks.rados.rados.0.smithi070.stdout:611: finishing copy_from to smithi07085154-11 2024-09-14T03:49:49.779 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 11 v 128 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-14T03:49:49.779 INFO:tasks.rados.rados.0.smithi070.stdout:612: finishing write tid 1 to smithi07085154-16 2024-09-14T03:49:49.779 INFO:tasks.rados.rados.0.smithi070.stdout:612: finishing write tid 2 to smithi07085154-16 2024-09-14T03:49:49.779 INFO:tasks.rados.rados.0.smithi070.stdout:609: done (4 left) 2024-09-14T03:49:49.779 INFO:tasks.rados.rados.0.smithi070.stdout:610: done (3 left) 2024-09-14T03:49:49.779 INFO:tasks.rados.rados.0.smithi070.stdout:611: done (2 left) 2024-09-14T03:49:49.779 INFO:tasks.rados.rados.0.smithi070.stdout:613: done (1 left) 2024-09-14T03:49:49.779 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 39 current snap is 60 2024-09-14T03:49:49.779 INFO:tasks.rados.rados.0.smithi070.stdout:612: finishing write tid 3 to smithi07085154-16 2024-09-14T03:49:49.779 INFO:tasks.rados.rados.0.smithi070.stdout:612: finishing write tid 4 to smithi07085154-16 2024-09-14T03:49:49.779 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 408 (ObjNum 209 snap 60 seq_num 209) dirty exists 2024-09-14T03:49:49.779 INFO:tasks.rados.rados.0.smithi070.stdout:612: left oid 16 (ObjNum 209 snap 60 seq_num 209) 2024-09-14T03:49:49.780 INFO:tasks.rados.rados.0.smithi070.stdout:614: seq_num 211 ranges {1056768=737280,1794048=401408} 2024-09-14T03:49:49.788 INFO:tasks.rados.rados.0.smithi070.stdout:614: writing smithi07085154-39 from 1056768 to 1794048 tid 1 2024-09-14T03:49:49.790 INFO:tasks.rados.rados.0.smithi070.stdout:614: writing smithi07085154-39 from 1794048 to 2195456 tid 2 2024-09-14T03:49:49.790 INFO:tasks.rados.rados.0.smithi070.stdout:612: done (1 left) 2024-09-14T03:49:49.790 INFO:tasks.rados.rados.0.smithi070.stdout:615: setattr oid 26 current snap is 60 2024-09-14T03:49:49.792 INFO:tasks.rados.rados.0.smithi070.stdout:616: delete oid 43 current snap is 60 2024-09-14T03:49:49.793 INFO:tasks.rados.rados.0.smithi070.stdout:614: finishing write tid 1 to smithi07085154-39 2024-09-14T03:49:49.793 INFO:tasks.rados.rados.0.smithi070.stdout:614: finishing write tid 2 to smithi07085154-39 2024-09-14T03:49:49.793 INFO:tasks.rados.rados.0.smithi070.stdout:614: finishing write tid 3 to smithi07085154-39 2024-09-14T03:49:49.794 INFO:tasks.rados.rados.0.smithi070.stdout:616: done (2 left) 2024-09-14T03:49:49.794 INFO:tasks.rados.rados.0.smithi070.stdout:617: snap_remove snap 48 2024-09-14T03:49:50.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:49 smithi070 ceph-mon[27495]: pgmap v24: 121 pgs: 121 active+clean; 175 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 698 KiB/s wr, 17 op/s 2024-09-14T03:49:50.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:49 smithi070 ceph-mon[27495]: osdmap e175: 8 total, 8 up, 8 in 2024-09-14T03:49:50.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:49 smithi070 ceph-mon[39288]: pgmap v24: 121 pgs: 121 active+clean; 175 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 698 KiB/s wr, 17 op/s 2024-09-14T03:49:50.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:49 smithi070 ceph-mon[39288]: osdmap e175: 8 total, 8 up, 8 in 2024-09-14T03:49:50.525 INFO:tasks.rados.rados.0.smithi070.stdout:617: done (2 left) 2024-09-14T03:49:50.525 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 33 current snap is 60 2024-09-14T03:49:50.525 INFO:tasks.rados.rados.0.smithi070.stdout:618: seq_num 213 ranges {0=770048,770048=401408} 2024-09-14T03:49:50.530 INFO:tasks.rados.rados.0.smithi070.stdout:618: writing smithi07085154-33 from 0 to 770048 tid 1 2024-09-14T03:49:50.534 INFO:tasks.rados.rados.0.smithi070.stdout:618: writing smithi07085154-33 from 770048 to 1171456 tid 2 2024-09-14T03:49:50.534 INFO:tasks.rados.rados.0.smithi070.stdout:614: finishing write tid 4 to smithi07085154-39 2024-09-14T03:49:50.534 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 39 v 49 (ObjNum 211 snap 60 seq_num 211) dirty exists 2024-09-14T03:49:50.534 INFO:tasks.rados.rados.0.smithi070.stdout:614: left oid 39 (ObjNum 211 snap 60 seq_num 211) 2024-09-14T03:49:50.534 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 410 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-14T03:49:50.534 INFO:tasks.rados.rados.0.smithi070.stdout:614: done (2 left) 2024-09-14T03:49:50.534 INFO:tasks.rados.rados.0.smithi070.stdout:615: done (1 left) 2024-09-14T03:49:50.534 INFO:tasks.rados.rados.0.smithi070.stdout:619: snap_create 2024-09-14T03:49:50.538 INFO:tasks.rados.rados.0.smithi070.stdout:618: finishing write tid 1 to smithi07085154-33 2024-09-14T03:49:50.541 INFO:tasks.rados.rados.0.smithi070.stdout:618: finishing write tid 2 to smithi07085154-33 2024-09-14T03:49:50.541 INFO:tasks.rados.rados.0.smithi070.stdout:618: finishing write tid 3 to smithi07085154-33 2024-09-14T03:49:50.542 INFO:tasks.rados.rados.0.smithi070.stdout:618: finishing write tid 4 to smithi07085154-33 2024-09-14T03:49:50.542 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 33 v 203 (ObjNum 213 snap 60 seq_num 213) dirty exists 2024-09-14T03:49:50.542 INFO:tasks.rados.rados.0.smithi070.stdout:618: left oid 33 (ObjNum 213 snap 60 seq_num 213) 2024-09-14T03:49:51.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:50 smithi094 ceph-mon[31053]: osdmap e176: 8 total, 8 up, 8 in 2024-09-14T03:49:51.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:50 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:49:51.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:50 smithi094 ceph-mon[31053]: osdmap e177: 8 total, 8 up, 8 in 2024-09-14T03:49:51.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:50 smithi070 ceph-mon[27495]: osdmap e176: 8 total, 8 up, 8 in 2024-09-14T03:49:51.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:50 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:49:51.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:50 smithi070 ceph-mon[27495]: osdmap e177: 8 total, 8 up, 8 in 2024-09-14T03:49:51.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:50 smithi070 ceph-mon[39288]: osdmap e176: 8 total, 8 up, 8 in 2024-09-14T03:49:51.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:50 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:49:51.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:50 smithi070 ceph-mon[39288]: osdmap e177: 8 total, 8 up, 8 in 2024-09-14T03:49:51.529 INFO:tasks.rados.rados.0.smithi070.stdout:618: done (1 left) 2024-09-14T03:49:51.530 INFO:tasks.rados.rados.0.smithi070.stdout:619: done (0 left) 2024-09-14T03:49:51.530 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 27 current snap is 61 2024-09-14T03:49:51.530 INFO:tasks.rados.rados.0.smithi070.stdout:620: seq_num 214 ranges {0=794624,794624=401408} 2024-09-14T03:49:51.533 INFO:tasks.rados.rados.0.smithi070.stdout:620: writing smithi07085154-27 from 0 to 794624 tid 1 2024-09-14T03:49:51.536 INFO:tasks.rados.rados.0.smithi070.stdout:620: writing smithi07085154-27 from 794624 to 1196032 tid 2 2024-09-14T03:49:51.536 INFO:tasks.rados.rados.0.smithi070.stdout:621: rollback oid 16 current snap is 61 2024-09-14T03:49:51.537 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 16 to 56 2024-09-14T03:49:51.537 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 32 current snap is 61 2024-09-14T03:49:51.537 INFO:tasks.rados.rados.0.smithi070.stdout:622: seq_num 215 ranges {0=409600,409600=532480,942080=270336} 2024-09-14T03:49:51.538 INFO:tasks.rados.rados.0.smithi070.stdout:622: writing smithi07085154-32 from 0 to 409600 tid 1 2024-09-14T03:49:51.542 INFO:tasks.rados.rados.0.smithi070.stdout:622: writing smithi07085154-32 from 409600 to 942080 tid 2 2024-09-14T03:49:51.544 INFO:tasks.rados.rados.0.smithi070.stdout:622: writing smithi07085154-32 from 942080 to 1212416 tid 3 2024-09-14T03:49:51.544 INFO:tasks.rados.rados.0.smithi070.stdout:623: snap_create 2024-09-14T03:49:51.545 INFO:tasks.rados.rados.0.smithi070.stdout:620: finishing write tid 1 to smithi07085154-27 2024-09-14T03:49:51.545 INFO:tasks.rados.rados.0.smithi070.stdout:620: finishing write tid 2 to smithi07085154-27 2024-09-14T03:49:51.545 INFO:tasks.rados.rados.0.smithi070.stdout:620: finishing write tid 3 to smithi07085154-27 2024-09-14T03:49:51.545 INFO:tasks.rados.rados.0.smithi070.stdout:620: finishing write tid 4 to smithi07085154-27 2024-09-14T03:49:51.545 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 421 (ObjNum 214 snap 61 seq_num 214) dirty exists 2024-09-14T03:49:51.545 INFO:tasks.rados.rados.0.smithi070.stdout:620: left oid 27 (ObjNum 214 snap 61 seq_num 214) 2024-09-14T03:49:51.545 INFO:tasks.rados.rados.0.smithi070.stdout:621: finishing rollback tid 0 to smithi07085154-16 2024-09-14T03:49:51.546 INFO:tasks.rados.rados.0.smithi070.stdout:621: finishing rollback tid 1 to smithi07085154-16 2024-09-14T03:49:51.546 INFO:tasks.rados.rados.0.smithi070.stdout:621: finishing rollback tid 2 to smithi07085154-16 2024-09-14T03:49:51.546 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 425 (ObjNum 114 snap 24 seq_num 114) dirty exists 2024-09-14T03:49:51.547 INFO:tasks.rados.rados.0.smithi070.stdout:622: finishing write tid 1 to smithi07085154-32 2024-09-14T03:49:51.547 INFO:tasks.rados.rados.0.smithi070.stdout:622: finishing write tid 2 to smithi07085154-32 2024-09-14T03:49:51.547 INFO:tasks.rados.rados.0.smithi070.stdout:622: finishing write tid 3 to smithi07085154-32 2024-09-14T03:49:51.548 INFO:tasks.rados.rados.0.smithi070.stdout:622: finishing write tid 4 to smithi07085154-32 2024-09-14T03:49:51.548 INFO:tasks.rados.rados.0.smithi070.stdout:622: finishing write tid 5 to smithi07085154-32 2024-09-14T03:49:51.548 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 240 (ObjNum 215 snap 61 seq_num 215) dirty exists 2024-09-14T03:49:51.549 INFO:tasks.rados.rados.0.smithi070.stdout:622: left oid 32 (ObjNum 215 snap 61 seq_num 215) 2024-09-14T03:49:52.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:51 smithi094 ceph-mon[31053]: pgmap v28: 121 pgs: 121 active+clean; 175 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:49:52.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:51 smithi094 ceph-mon[31053]: osdmap e178: 8 total, 8 up, 8 in 2024-09-14T03:49:52.027 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:49:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:49:51] "GET /metrics HTTP/1.1" 200 36210 "" "Prometheus/2.43.0" 2024-09-14T03:49:52.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:51 smithi070 ceph-mon[27495]: pgmap v28: 121 pgs: 121 active+clean; 175 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:49:52.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:51 smithi070 ceph-mon[27495]: osdmap e178: 8 total, 8 up, 8 in 2024-09-14T03:49:52.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:51 smithi070 ceph-mon[39288]: pgmap v28: 121 pgs: 121 active+clean; 175 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:49:52.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:51 smithi070 ceph-mon[39288]: osdmap e178: 8 total, 8 up, 8 in 2024-09-14T03:49:52.532 INFO:tasks.rados.rados.0.smithi070.stdout:620: done (3 left) 2024-09-14T03:49:52.532 INFO:tasks.rados.rados.0.smithi070.stdout:621: done (2 left) 2024-09-14T03:49:52.532 INFO:tasks.rados.rados.0.smithi070.stdout:622: done (1 left) 2024-09-14T03:49:52.532 INFO:tasks.rados.rados.0.smithi070.stdout:623: done (0 left) 2024-09-14T03:49:52.532 INFO:tasks.rados.rados.0.smithi070.stdout:624: setattr oid 7 current snap is 62 2024-09-14T03:49:52.534 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 19 current snap is 62 2024-09-14T03:49:52.534 INFO:tasks.rados.rados.0.smithi070.stdout:625: seq_num 216 ranges {2277376=425984,2703360=409600,3112960=393216} 2024-09-14T03:49:52.549 INFO:tasks.rados.rados.0.smithi070.stdout:625: writing smithi07085154-19 from 2277376 to 2703360 tid 1 2024-09-14T03:49:52.551 INFO:tasks.rados.rados.0.smithi070.stdout:625: writing smithi07085154-19 from 2703360 to 3112960 tid 2 2024-09-14T03:49:52.554 INFO:tasks.rados.rados.0.smithi070.stdout:625: writing smithi07085154-19 from 3112960 to 3506176 tid 3 2024-09-14T03:49:52.554 INFO:tasks.rados.rados.0.smithi070.stdout:626: copy_from oid 13 from oid 8 current snap is 62 2024-09-14T03:49:52.554 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 316 (ObjNum 143 snap 42 seq_num 143) dirty exists 2024-09-14T03:49:52.554 INFO:tasks.rados.rados.0.smithi070.stdout:625: finishing write tid 1 to smithi07085154-19 2024-09-14T03:49:52.554 INFO:tasks.rados.rados.0.smithi070.stdout:624: done (2 left) 2024-09-14T03:49:52.554 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 41 current snap is 62 2024-09-14T03:49:52.554 INFO:tasks.rados.rados.0.smithi070.stdout:627: seq_num 217 ranges {4227072=442368,4669440=696320,5365760=106496} 2024-09-14T03:49:52.579 INFO:tasks.rados.rados.0.smithi070.stdout:627: writing smithi07085154-41 from 4227072 to 4669440 tid 1 2024-09-14T03:49:52.583 INFO:tasks.rados.rados.0.smithi070.stdout:627: writing smithi07085154-41 from 4669440 to 5365760 tid 2 2024-09-14T03:49:52.584 INFO:tasks.rados.rados.0.smithi070.stdout:627: writing smithi07085154-41 from 5365760 to 5472256 tid 3 2024-09-14T03:49:52.584 INFO:tasks.rados.rados.0.smithi070.stdout:628: read oid 20 snap 51 2024-09-14T03:49:52.584 INFO:tasks.rados.rados.0.smithi070.stdout:628: expect (ObjNum 180 snap 50 seq_num 180) 2024-09-14T03:49:52.584 INFO:tasks.rados.rados.0.smithi070.stdout:629: snap_remove snap 50 2024-09-14T03:49:53.539 INFO:tasks.rados.rados.0.smithi070.stdout:625: finishing write tid 2 to smithi07085154-19 2024-09-14T03:49:53.539 INFO:tasks.rados.rados.0.smithi070.stdout:625: finishing write tid 3 to smithi07085154-19 2024-09-14T03:49:53.539 INFO:tasks.rados.rados.0.smithi070.stdout:625: finishing write tid 4 to smithi07085154-19 2024-09-14T03:49:53.540 INFO:tasks.rados.rados.0.smithi070.stdout:625: finishing write tid 5 to smithi07085154-19 2024-09-14T03:49:53.540 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 19 v 271 (ObjNum 216 snap 62 seq_num 216) dirty exists 2024-09-14T03:49:53.540 INFO:tasks.rados.rados.0.smithi070.stdout:625: left oid 19 (ObjNum 216 snap 62 seq_num 216) 2024-09-14T03:49:53.540 INFO:tasks.rados.rados.0.smithi070.stdout:626: finishing copy_from racing read to smithi07085154-13 2024-09-14T03:49:53.540 INFO:tasks.rados.rados.0.smithi070.stdout:626: finishing copy_from to smithi07085154-13 2024-09-14T03:49:53.540 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 307 (ObjNum 202 snap 56 seq_num 202) dirty exists 2024-09-14T03:49:53.540 INFO:tasks.rados.rados.0.smithi070.stdout:627: finishing write tid 1 to smithi07085154-41 2024-09-14T03:49:53.540 INFO:tasks.rados.rados.0.smithi070.stdout:627: finishing write tid 2 to smithi07085154-41 2024-09-14T03:49:53.540 INFO:tasks.rados.rados.0.smithi070.stdout:627: finishing write tid 3 to smithi07085154-41 2024-09-14T03:49:53.540 INFO:tasks.rados.rados.0.smithi070.stdout:627: finishing write tid 4 to smithi07085154-41 2024-09-14T03:49:53.540 INFO:tasks.rados.rados.0.smithi070.stdout:627: finishing write tid 5 to smithi07085154-41 2024-09-14T03:49:53.540 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 329 (ObjNum 217 snap 62 seq_num 217) dirty exists 2024-09-14T03:49:53.540 INFO:tasks.rados.rados.0.smithi070.stdout:627: left oid 41 (ObjNum 217 snap 62 seq_num 217) 2024-09-14T03:49:53.540 INFO:tasks.rados.rados.0.smithi070.stdout:628: expect (ObjNum 180 snap 50 seq_num 180) 2024-09-14T03:49:53.589 INFO:tasks.rados.rados.0.smithi070.stdout:625: done (4 left) 2024-09-14T03:49:53.589 INFO:tasks.rados.rados.0.smithi070.stdout:626: done (3 left) 2024-09-14T03:49:53.589 INFO:tasks.rados.rados.0.smithi070.stdout:627: done (2 left) 2024-09-14T03:49:53.589 INFO:tasks.rados.rados.0.smithi070.stdout:628: done (1 left) 2024-09-14T03:49:53.590 INFO:tasks.rados.rados.0.smithi070.stdout:629: done (0 left) 2024-09-14T03:49:53.590 INFO:tasks.rados.rados.0.smithi070.stdout:630: read oid 46 snap -1 2024-09-14T03:49:53.590 INFO:tasks.rados.rados.0.smithi070.stdout:630: expect deleted 2024-09-14T03:49:53.590 INFO:tasks.rados.rados.0.smithi070.stdout:631: rollback oid 42 current snap is 62 2024-09-14T03:49:53.590 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 42 to 30 2024-09-14T03:49:53.590 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 26 current snap is 62 2024-09-14T03:49:53.590 INFO:tasks.rados.rados.0.smithi070.stdout:632: seq_num 218 ranges {761856=458752,1220608=581632,1802240=221184} 2024-09-14T03:49:53.595 INFO:tasks.rados.rados.0.smithi070.stdout:632: writing smithi07085154-26 from 761856 to 1220608 tid 1 2024-09-14T03:49:53.599 INFO:tasks.rados.rados.0.smithi070.stdout:632: writing smithi07085154-26 from 1220608 to 1802240 tid 2 2024-09-14T03:49:53.601 INFO:tasks.rados.rados.0.smithi070.stdout:632: writing smithi07085154-26 from 1802240 to 2023424 tid 3 2024-09-14T03:49:53.601 INFO:tasks.rados.rados.0.smithi070.stdout:633: copy_from oid 41 from oid 40 current snap is 62 2024-09-14T03:49:53.601 INFO:tasks.rados.rados.0.smithi070.stdout:631: finishing rollback tid 0 to smithi07085154-42 2024-09-14T03:49:53.601 INFO:tasks.rados.rados.0.smithi070.stdout:631: finishing rollback tid 1 to smithi07085154-42 2024-09-14T03:49:53.601 INFO:tasks.rados.rados.0.smithi070.stdout:631: finishing rollback tid 2 to smithi07085154-42 2024-09-14T03:49:53.601 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 437 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-14T03:49:53.601 INFO:tasks.rados.rados.0.smithi070.stdout:632: finishing write tid 1 to smithi07085154-26 2024-09-14T03:49:53.601 INFO:tasks.rados.rados.0.smithi070.stdout:630: done (3 left) 2024-09-14T03:49:53.602 INFO:tasks.rados.rados.0.smithi070.stdout:631: done (2 left) 2024-09-14T03:49:53.602 INFO:tasks.rados.rados.0.smithi070.stdout:634: rollback oid 16 current snap is 62 2024-09-14T03:49:53.602 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 16 to 60 2024-09-14T03:49:53.602 INFO:tasks.rados.rados.0.smithi070.stdout:635: copy_from oid 18 from oid 23 current snap is 62 2024-09-14T03:49:53.602 INFO:tasks.rados.rados.0.smithi070.stdout:636: rollback oid 17 current snap is 62 2024-09-14T03:49:53.602 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 17 to 59 2024-09-14T03:49:53.602 INFO:tasks.rados.rados.0.smithi070.stdout:637: delete oid 6 current snap is 62 2024-09-14T03:49:53.603 INFO:tasks.rados.rados.0.smithi070.stdout:632: finishing write tid 2 to smithi07085154-26 2024-09-14T03:49:53.604 INFO:tasks.rados.rados.0.smithi070.stdout:632: finishing write tid 3 to smithi07085154-26 2024-09-14T03:49:53.604 INFO:tasks.rados.rados.0.smithi070.stdout:632: finishing write tid 4 to smithi07085154-26 2024-09-14T03:49:53.606 INFO:tasks.rados.rados.0.smithi070.stdout:632: finishing write tid 5 to smithi07085154-26 2024-09-14T03:49:53.606 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 442 (ObjNum 218 snap 62 seq_num 218) dirty exists 2024-09-14T03:49:53.606 INFO:tasks.rados.rados.0.smithi070.stdout:632: left oid 26 (ObjNum 218 snap 62 seq_num 218) 2024-09-14T03:49:53.606 INFO:tasks.rados.rados.0.smithi070.stdout:634: finishing rollback tid 0 to smithi07085154-16 2024-09-14T03:49:53.606 INFO:tasks.rados.rados.0.smithi070.stdout:634: finishing rollback tid 1 to smithi07085154-16 2024-09-14T03:49:53.607 INFO:tasks.rados.rados.0.smithi070.stdout:634: finishing rollback tid 2 to smithi07085154-16 2024-09-14T03:49:53.607 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 446 (ObjNum 209 snap 60 seq_num 209) dirty exists 2024-09-14T03:49:53.607 INFO:tasks.rados.rados.0.smithi070.stdout:636: finishing rollback tid 0 to smithi07085154-17 2024-09-14T03:49:53.607 INFO:tasks.rados.rados.0.smithi070.stdout:636: finishing rollback tid 1 to smithi07085154-17 2024-09-14T03:49:53.607 INFO:tasks.rados.rados.0.smithi070.stdout:636: finishing rollback tid 2 to smithi07085154-17 2024-09-14T03:49:53.607 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 262 (ObjNum 181 snap 51 seq_num 181) dirty exists 2024-09-14T03:49:53.608 INFO:tasks.rados.rados.0.smithi070.stdout:632: done (5 left) 2024-09-14T03:49:53.608 INFO:tasks.rados.rados.0.smithi070.stdout:634: done (4 left) 2024-09-14T03:49:53.608 INFO:tasks.rados.rados.0.smithi070.stdout:636: done (3 left) 2024-09-14T03:49:53.608 INFO:tasks.rados.rados.0.smithi070.stdout:637: done (2 left) 2024-09-14T03:49:53.608 INFO:tasks.rados.rados.0.smithi070.stdout:638: delete oid 42 current snap is 62 2024-09-14T03:49:53.609 INFO:tasks.rados.rados.0.smithi070.stdout:638: done (2 left) 2024-09-14T03:49:53.609 INFO:tasks.rados.rados.0.smithi070.stdout:639: read oid 4 snap -1 2024-09-14T03:49:53.609 INFO:tasks.rados.rados.0.smithi070.stdout:639: expect deleted 2024-09-14T03:49:53.609 INFO:tasks.rados.rados.0.smithi070.stdout:640: rollback oid 7 current snap is 62 2024-09-14T03:49:53.609 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 7 to 47 2024-09-14T03:49:53.610 INFO:tasks.rados.rados.0.smithi070.stdout:641: delete oid 42 current snap is 62 2024-09-14T03:49:53.611 INFO:tasks.rados.rados.0.smithi070.stdout:639: done (4 left) 2024-09-14T03:49:53.611 INFO:tasks.rados.rados.0.smithi070.stdout:641: done (3 left) 2024-09-14T03:49:53.611 INFO:tasks.rados.rados.0.smithi070.stdout:642: read oid 10 snap -1 2024-09-14T03:49:53.611 INFO:tasks.rados.rados.0.smithi070.stdout:642: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-14T03:49:53.611 INFO:tasks.rados.rados.0.smithi070.stdout:643: snap_create 2024-09-14T03:49:53.613 INFO:tasks.rados.rados.0.smithi070.stdout:640: finishing rollback tid 0 to smithi07085154-7 2024-09-14T03:49:53.613 INFO:tasks.rados.rados.0.smithi070.stdout:640: finishing rollback tid 1 to smithi07085154-7 2024-09-14T03:49:53.613 INFO:tasks.rados.rados.0.smithi070.stdout:640: finishing rollback tid 2 to smithi07085154-7 2024-09-14T03:49:53.613 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 319 (ObjNum 143 snap 42 seq_num 143) dirty exists 2024-09-14T03:49:53.615 INFO:tasks.rados.rados.0.smithi070.stdout:642: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-14T03:49:53.636 INFO:tasks.rados.rados.0.smithi070.stdout:635: finishing copy_from racing read to smithi07085154-18 2024-09-14T03:49:53.636 INFO:tasks.rados.rados.0.smithi070.stdout:633: finishing copy_from racing read to smithi07085154-41 2024-09-14T03:49:53.636 INFO:tasks.rados.rados.0.smithi070.stdout:633: finishing copy_from to smithi07085154-41 2024-09-14T03:49:53.636 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 140 (ObjNum 131 snap 36 seq_num 131) dirty exists 2024-09-14T03:49:53.637 INFO:tasks.rados.rados.0.smithi070.stdout:635: finishing copy_from to smithi07085154-18 2024-09-14T03:49:53.637 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 141 (ObjNum 208 snap 60 seq_num 208) dirty exists 2024-09-14T03:49:53.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:53 smithi094 ceph-mon[31053]: osdmap e179: 8 total, 8 up, 8 in 2024-09-14T03:49:53.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:53 smithi094 ceph-mon[31053]: pgmap v31: 121 pgs: 121 active+clean; 185 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.4 MiB/s rd, 2.4 MiB/s wr, 23 op/s 2024-09-14T03:49:53.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:53 smithi070 ceph-mon[27495]: osdmap e179: 8 total, 8 up, 8 in 2024-09-14T03:49:53.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:53 smithi070 ceph-mon[27495]: pgmap v31: 121 pgs: 121 active+clean; 185 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.4 MiB/s rd, 2.4 MiB/s wr, 23 op/s 2024-09-14T03:49:53.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:53 smithi070 ceph-mon[39288]: osdmap e179: 8 total, 8 up, 8 in 2024-09-14T03:49:53.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:53 smithi070 ceph-mon[39288]: pgmap v31: 121 pgs: 121 active+clean; 185 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.4 MiB/s rd, 2.4 MiB/s wr, 23 op/s 2024-09-14T03:49:54.542 INFO:tasks.rados.rados.0.smithi070.stdout:633: done (4 left) 2024-09-14T03:49:54.542 INFO:tasks.rados.rados.0.smithi070.stdout:635: done (3 left) 2024-09-14T03:49:54.542 INFO:tasks.rados.rados.0.smithi070.stdout:640: done (2 left) 2024-09-14T03:49:54.542 INFO:tasks.rados.rados.0.smithi070.stdout:642: done (1 left) 2024-09-14T03:49:54.542 INFO:tasks.rados.rados.0.smithi070.stdout:643: done (0 left) 2024-09-14T03:49:54.542 INFO:tasks.rados.rados.0.smithi070.stdout:644: delete oid 27 current snap is 63 2024-09-14T03:49:54.545 INFO:tasks.rados.rados.0.smithi070.stdout:644: done (0 left) 2024-09-14T03:49:54.545 INFO:tasks.rados.rados.0.smithi070.stdout:645: snap_create 2024-09-14T03:49:54.998 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:54 smithi070 ceph-mon[27495]: osdmap e180: 8 total, 8 up, 8 in 2024-09-14T03:49:54.998 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:54 smithi070 ceph-mon[39288]: osdmap e180: 8 total, 8 up, 8 in 2024-09-14T03:49:55.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:54 smithi094 ceph-mon[31053]: osdmap e180: 8 total, 8 up, 8 in 2024-09-14T03:49:55.526 INFO:tasks.rados.rados.0.smithi070.stdout:645: done (0 left) 2024-09-14T03:49:55.526 INFO:tasks.rados.rados.0.smithi070.stdout:646: read oid 45 snap -1 2024-09-14T03:49:55.526 INFO:tasks.rados.rados.0.smithi070.stdout:646: expect (ObjNum 49 snap 0 seq_num 49) 2024-09-14T03:49:55.526 INFO:tasks.rados.rados.0.smithi070.stdout:647: copy_from oid 49 from oid 4 current snap is 64 2024-09-14T03:49:55.526 INFO:tasks.rados.rados.0.smithi070.stdout:648: snap_remove snap 49 2024-09-14T03:49:55.794 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:55 smithi094 ceph-mon[31053]: pgmap v33: 121 pgs: 121 active+clean; 185 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.4 MiB/s rd, 2.4 MiB/s wr, 23 op/s 2024-09-14T03:49:55.794 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:55 smithi094 ceph-mon[31053]: osdmap e181: 8 total, 8 up, 8 in 2024-09-14T03:49:55.794 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:55 smithi094 ceph-mon[31053]: osdmap e182: 8 total, 8 up, 8 in 2024-09-14T03:49:56.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:55 smithi070 ceph-mon[27495]: pgmap v33: 121 pgs: 121 active+clean; 185 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.4 MiB/s rd, 2.4 MiB/s wr, 23 op/s 2024-09-14T03:49:56.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:55 smithi070 ceph-mon[27495]: osdmap e181: 8 total, 8 up, 8 in 2024-09-14T03:49:56.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:55 smithi070 ceph-mon[27495]: osdmap e182: 8 total, 8 up, 8 in 2024-09-14T03:49:56.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:55 smithi070 ceph-mon[39288]: pgmap v33: 121 pgs: 121 active+clean; 185 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.4 MiB/s rd, 2.4 MiB/s wr, 23 op/s 2024-09-14T03:49:56.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:55 smithi070 ceph-mon[39288]: osdmap e181: 8 total, 8 up, 8 in 2024-09-14T03:49:56.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:55 smithi070 ceph-mon[39288]: osdmap e182: 8 total, 8 up, 8 in 2024-09-14T03:49:56.529 INFO:tasks.rados.rados.0.smithi070.stdout:648: done (2 left) 2024-09-14T03:49:56.529 INFO:tasks.rados.rados.0.smithi070.stdout:649: rollback oid 14 current snap is 64 2024-09-14T03:49:56.529 INFO:tasks.rados.rados.0.smithi070.stdout:646: expect (ObjNum 49 snap 0 seq_num 49) 2024-09-14T03:49:56.558 INFO:tasks.rados.rados.0.smithi070.stdout:647: finishing copy_from to smithi07085154-49 2024-09-14T03:49:56.558 INFO:tasks.rados.rados.0.smithi070.stdout:647: got expected ENOENT (src dne) 2024-09-14T03:49:56.558 INFO:tasks.rados.rados.0.smithi070.stdout:647: finishing copy_from racing read to smithi07085154-49 2024-09-14T03:49:56.558 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 14 to 61 2024-09-14T03:49:56.558 INFO:tasks.rados.rados.0.smithi070.stdout:646: done (2 left) 2024-09-14T03:49:56.559 INFO:tasks.rados.rados.0.smithi070.stdout:647: done (1 left) 2024-09-14T03:49:56.559 INFO:tasks.rados.rados.0.smithi070.stdout:650: snap_create 2024-09-14T03:49:56.560 INFO:tasks.rados.rados.0.smithi070.stdout:649: finishing rollback tid 0 to smithi07085154-14 2024-09-14T03:49:56.561 INFO:tasks.rados.rados.0.smithi070.stdout:649: finishing rollback tid 1 to smithi07085154-14 2024-09-14T03:49:56.561 INFO:tasks.rados.rados.0.smithi070.stdout:649: finishing rollback tid 2 to smithi07085154-14 2024-09-14T03:49:56.561 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 144 (ObjNum 190 snap 51 seq_num 190) dirty exists 2024-09-14T03:49:56.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:49:56 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:49:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:49:57.529 INFO:tasks.rados.rados.0.smithi070.stdout:649: done (1 left) 2024-09-14T03:49:57.529 INFO:tasks.rados.rados.0.smithi070.stdout:650: done (0 left) 2024-09-14T03:49:57.529 INFO:tasks.rados.rados.0.smithi070.stdout:651: copy_from oid 16 from oid 10 current snap is 65 2024-09-14T03:49:57.529 INFO:tasks.rados.rados.0.smithi070.stdout:652: snap_remove snap 61 2024-09-14T03:49:57.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:57 smithi070 ceph-mon[27495]: osdmap e183: 8 total, 8 up, 8 in 2024-09-14T03:49:57.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:57 smithi070 ceph-mon[27495]: pgmap v37: 121 pgs: 121 active+clean; 197 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.7 MiB/s wr, 19 op/s 2024-09-14T03:49:57.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:57 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:57.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:57 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:57.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:57 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:57.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:57 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:57.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:57 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:57.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:57 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:57.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:57 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:57.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:57 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:57.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:57 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:57.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:57 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:57.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:57 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:57.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:57 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:57.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:57 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:49:57.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:57 smithi070 ceph-mon[39288]: osdmap e183: 8 total, 8 up, 8 in 2024-09-14T03:49:57.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:57 smithi070 ceph-mon[39288]: pgmap v37: 121 pgs: 121 active+clean; 197 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.7 MiB/s wr, 19 op/s 2024-09-14T03:49:57.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:57 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:57.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:57 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:57.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:57 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:57.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:57 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:57.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:57 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:57.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:57 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:57.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:57 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:57.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:57 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:57.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:57 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:57.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:57 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:57.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:57 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:57.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:57 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:57.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:57 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:49:58.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:57 smithi094 ceph-mon[31053]: osdmap e183: 8 total, 8 up, 8 in 2024-09-14T03:49:58.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:57 smithi094 ceph-mon[31053]: pgmap v37: 121 pgs: 121 active+clean; 197 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.7 MiB/s wr, 19 op/s 2024-09-14T03:49:58.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:57 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:58.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:57 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:58.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:57 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:58.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:57 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:58.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:57 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:58.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:57 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:58.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:57 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:58.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:57 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:58.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:57 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:58.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:57 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:58.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:57 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:58.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:57 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:58.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:57 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:49:58.531 INFO:tasks.rados.rados.0.smithi070.stdout:652: done (1 left) 2024-09-14T03:49:58.531 INFO:tasks.rados.rados.0.smithi070.stdout:653: read oid 35 snap -1 2024-09-14T03:49:58.531 INFO:tasks.rados.rados.0.smithi070.stdout:653: expect deleted 2024-09-14T03:49:58.532 INFO:tasks.rados.rados.0.smithi070.stdout:654: snap_create 2024-09-14T03:49:58.532 INFO:tasks.rados.rados.0.smithi070.stdout:651: finishing copy_from racing read to smithi07085154-16 2024-09-14T03:49:58.532 INFO:tasks.rados.rados.0.smithi070.stdout:651: finishing copy_from to smithi07085154-16 2024-09-14T03:49:58.532 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 380 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-14T03:49:58.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[39288]: Adjusting osd_memory_target on smithi094 to 2480M 2024-09-14T03:49:58.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[39288]: Updating smithi070:/etc/ceph/ceph.conf 2024-09-14T03:49:58.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[39288]: Updating smithi094:/etc/ceph/ceph.conf 2024-09-14T03:49:58.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[39288]: Updating smithi094:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:49:58.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[39288]: Updating smithi070:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:49:58.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[39288]: Updating smithi094:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:49:58.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[39288]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:49:58.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[39288]: osdmap e184: 8 total, 8 up, 8 in 2024-09-14T03:49:58.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[39288]: Updating smithi094:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:49:58.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[39288]: Updating smithi070:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:49:58.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:58.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:58.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:58.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:58.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:58.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:58.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:58.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:58.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:58.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[39288]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-14T03:49:58.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[39288]: Deploying daemon alertmanager.a on smithi070 2024-09-14T03:49:58.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[27495]: Adjusting osd_memory_target on smithi094 to 2480M 2024-09-14T03:49:58.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[27495]: Updating smithi070:/etc/ceph/ceph.conf 2024-09-14T03:49:58.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[27495]: Updating smithi094:/etc/ceph/ceph.conf 2024-09-14T03:49:58.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[27495]: Updating smithi094:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:49:58.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[27495]: Updating smithi070:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:49:58.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[27495]: Updating smithi094:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:49:58.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[27495]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:49:58.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[27495]: osdmap e184: 8 total, 8 up, 8 in 2024-09-14T03:49:58.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[27495]: Updating smithi094:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:49:58.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[27495]: Updating smithi070:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:49:58.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:58.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:58.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:58.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:58.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:58.789 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:58.789 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:58.789 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:58.789 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:58.789 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[27495]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-14T03:49:58.789 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:58 smithi070 ceph-mon[27495]: Deploying daemon alertmanager.a on smithi070 2024-09-14T03:49:59.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:58 smithi094 ceph-mon[31053]: Adjusting osd_memory_target on smithi094 to 2480M 2024-09-14T03:49:59.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:58 smithi094 ceph-mon[31053]: Updating smithi070:/etc/ceph/ceph.conf 2024-09-14T03:49:59.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:58 smithi094 ceph-mon[31053]: Updating smithi094:/etc/ceph/ceph.conf 2024-09-14T03:49:59.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:58 smithi094 ceph-mon[31053]: Updating smithi094:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:49:59.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:58 smithi094 ceph-mon[31053]: Updating smithi070:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:49:59.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:58 smithi094 ceph-mon[31053]: Updating smithi094:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:49:59.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:58 smithi094 ceph-mon[31053]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:49:59.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:58 smithi094 ceph-mon[31053]: osdmap e184: 8 total, 8 up, 8 in 2024-09-14T03:49:59.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:58 smithi094 ceph-mon[31053]: Updating smithi094:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:49:59.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:58 smithi094 ceph-mon[31053]: Updating smithi070:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:49:59.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:58 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:59.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:58 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:59.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:58 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:59.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:58 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:59.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:58 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:59.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:58 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:59.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:58 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:59.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:58 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:59.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:58 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:49:59.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:58 smithi094 ceph-mon[31053]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-14T03:49:59.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:58 smithi094 ceph-mon[31053]: Deploying daemon alertmanager.a on smithi070 2024-09-14T03:49:59.532 INFO:tasks.rados.rados.0.smithi070.stdout:651: done (2 left) 2024-09-14T03:49:59.532 INFO:tasks.rados.rados.0.smithi070.stdout:653: done (1 left) 2024-09-14T03:49:59.533 INFO:tasks.rados.rados.0.smithi070.stdout:654: done (0 left) 2024-09-14T03:49:59.533 INFO:tasks.rados.rados.0.smithi070.stdout:655: rmattr oid 33 current snap is 66 2024-09-14T03:49:59.533 INFO:tasks.rados.rados.0.smithi070.stdout:655: done (0 left) 2024-09-14T03:49:59.533 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 32 current snap is 66 2024-09-14T03:49:59.533 INFO:tasks.rados.rados.0.smithi070.stdout:656: seq_num 223 ranges {1212416=540672,1753088=450560,2203648=352256} 2024-09-14T03:49:59.542 INFO:tasks.rados.rados.0.smithi070.stdout:656: writing smithi07085154-32 from 1212416 to 1753088 tid 1 2024-09-14T03:49:59.545 INFO:tasks.rados.rados.0.smithi070.stdout:656: writing smithi07085154-32 from 1753088 to 2203648 tid 2 2024-09-14T03:49:59.548 INFO:tasks.rados.rados.0.smithi070.stdout:656: writing smithi07085154-32 from 2203648 to 2555904 tid 3 2024-09-14T03:49:59.548 INFO:tasks.rados.rados.0.smithi070.stdout:657: copy_from oid 11 from oid 45 current snap is 66 2024-09-14T03:49:59.548 INFO:tasks.rados.rados.0.smithi070.stdout:656: finishing write tid 1 to smithi07085154-32 2024-09-14T03:49:59.548 INFO:tasks.rados.rados.0.smithi070.stdout:656: finishing write tid 2 to smithi07085154-32 2024-09-14T03:49:59.548 INFO:tasks.rados.rados.0.smithi070.stdout:658: read oid 12 snap -1 2024-09-14T03:49:59.548 INFO:tasks.rados.rados.0.smithi070.stdout:658: expect (ObjNum 200 snap 56 seq_num 200) 2024-09-14T03:49:59.548 INFO:tasks.rados.rados.0.smithi070.stdout:659: rollback oid 13 current snap is 66 2024-09-14T03:49:59.548 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 13 to 57 2024-09-14T03:49:59.548 INFO:tasks.rados.rados.0.smithi070.stdout:660: setattr oid 30 current snap is 66 2024-09-14T03:49:59.549 INFO:tasks.rados.rados.0.smithi070.stdout:661: rmattr oid 41 current snap is 66 2024-09-14T03:49:59.550 INFO:tasks.rados.rados.0.smithi070.stdout:661: done (5 left) 2024-09-14T03:49:59.550 INFO:tasks.rados.rados.0.smithi070.stdout:662: read oid 31 snap -1 2024-09-14T03:49:59.550 INFO:tasks.rados.rados.0.smithi070.stdout:662: expect (ObjNum 201 snap 56 seq_num 201) 2024-09-14T03:49:59.550 INFO:tasks.rados.rados.0.smithi070.stdout:663: rollback oid 20 current snap is 66 2024-09-14T03:49:59.550 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 20 to 57 2024-09-14T03:49:59.550 INFO:tasks.rados.rados.0.smithi070.stdout:664: copy_from oid 21 from oid 50 current snap is 66 2024-09-14T03:49:59.550 INFO:tasks.rados.rados.0.smithi070.stdout:665: delete oid 44 current snap is 66 2024-09-14T03:49:59.550 INFO:tasks.rados.rados.0.smithi070.stdout:656: finishing write tid 3 to smithi07085154-32 2024-09-14T03:49:59.550 INFO:tasks.rados.rados.0.smithi070.stdout:656: finishing write tid 4 to smithi07085154-32 2024-09-14T03:49:59.553 INFO:tasks.rados.rados.0.smithi070.stdout:656: finishing write tid 5 to smithi07085154-32 2024-09-14T03:49:59.553 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 311 (ObjNum 223 snap 66 seq_num 223) dirty exists 2024-09-14T03:49:59.553 INFO:tasks.rados.rados.0.smithi070.stdout:656: left oid 32 (ObjNum 223 snap 66 seq_num 223) 2024-09-14T03:49:59.556 INFO:tasks.rados.rados.0.smithi070.stdout:657: finishing copy_from racing read to smithi07085154-11 2024-09-14T03:49:59.556 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 30 v 150 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-09-14T03:49:59.557 INFO:tasks.rados.rados.0.smithi070.stdout:663: finishing rollback tid 0 to smithi07085154-20 2024-09-14T03:49:59.557 INFO:tasks.rados.rados.0.smithi070.stdout:663: finishing rollback tid 1 to smithi07085154-20 2024-09-14T03:49:59.558 INFO:tasks.rados.rados.0.smithi070.stdout:659: finishing rollback tid 0 to smithi07085154-13 2024-09-14T03:49:59.558 INFO:tasks.rados.rados.0.smithi070.stdout:663: finishing rollback tid 2 to smithi07085154-20 2024-09-14T03:49:59.558 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 20 v 322 (ObjNum 180 snap 50 seq_num 180) dirty exists 2024-09-14T03:49:59.558 INFO:tasks.rados.rados.0.smithi070.stdout:659: finishing rollback tid 1 to smithi07085154-13 2024-09-14T03:49:59.559 INFO:tasks.rados.rados.0.smithi070.stdout:662: expect (ObjNum 201 snap 56 seq_num 201) 2024-09-14T03:49:59.586 INFO:tasks.rados.rados.0.smithi070.stdout:657: finishing copy_from to smithi07085154-11 2024-09-14T03:49:59.586 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 11 v 11 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-14T03:49:59.586 INFO:tasks.rados.rados.0.smithi070.stdout:659: finishing rollback tid 2 to smithi07085154-13 2024-09-14T03:49:59.586 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 314 (ObjNum 138 snap 39 seq_num 138) dirty exists 2024-09-14T03:49:59.586 INFO:tasks.rados.rados.0.smithi070.stdout:664: finishing copy_from racing read to smithi07085154-21 2024-09-14T03:49:59.586 INFO:tasks.rados.rados.0.smithi070.stdout:664: finishing copy_from to smithi07085154-21 2024-09-14T03:49:59.586 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 248 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-14T03:49:59.586 INFO:tasks.rados.rados.0.smithi070.stdout:658: expect (ObjNum 200 snap 56 seq_num 200) 2024-09-14T03:49:59.639 INFO:tasks.rados.rados.0.smithi070.stdout:656: done (8 left) 2024-09-14T03:49:59.639 INFO:tasks.rados.rados.0.smithi070.stdout:657: done (7 left) 2024-09-14T03:49:59.639 INFO:tasks.rados.rados.0.smithi070.stdout:658: done (6 left) 2024-09-14T03:49:59.640 INFO:tasks.rados.rados.0.smithi070.stdout:659: done (5 left) 2024-09-14T03:49:59.640 INFO:tasks.rados.rados.0.smithi070.stdout:660: done (4 left) 2024-09-14T03:49:59.640 INFO:tasks.rados.rados.0.smithi070.stdout:662: done (3 left) 2024-09-14T03:49:59.640 INFO:tasks.rados.rados.0.smithi070.stdout:663: done (2 left) 2024-09-14T03:49:59.640 INFO:tasks.rados.rados.0.smithi070.stdout:664: done (1 left) 2024-09-14T03:49:59.640 INFO:tasks.rados.rados.0.smithi070.stdout:665: done (0 left) 2024-09-14T03:49:59.640 INFO:tasks.rados.rados.0.smithi070.stdout:666: rollback oid 26 current snap is 66 2024-09-14T03:49:59.640 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 26 to 56 2024-09-14T03:49:59.640 INFO:tasks.rados.rados.0.smithi070.stdout:667: snap_remove snap 63 2024-09-14T03:50:00.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:59 smithi094 ceph-mon[31053]: osdmap e185: 8 total, 8 up, 8 in 2024-09-14T03:50:00.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:59 smithi094 ceph-mon[31053]: pgmap v40: 121 pgs: 121 active+clean; 197 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.7 MiB/s wr, 19 op/s 2024-09-14T03:50:00.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:49:59 smithi094 ceph-mon[31053]: osdmap e186: 8 total, 8 up, 8 in 2024-09-14T03:50:00.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:59 smithi070 ceph-mon[27495]: osdmap e185: 8 total, 8 up, 8 in 2024-09-14T03:50:00.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:59 smithi070 ceph-mon[27495]: pgmap v40: 121 pgs: 121 active+clean; 197 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.7 MiB/s wr, 19 op/s 2024-09-14T03:50:00.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:49:59 smithi070 ceph-mon[27495]: osdmap e186: 8 total, 8 up, 8 in 2024-09-14T03:50:00.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:59 smithi070 ceph-mon[39288]: osdmap e185: 8 total, 8 up, 8 in 2024-09-14T03:50:00.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:59 smithi070 ceph-mon[39288]: pgmap v40: 121 pgs: 121 active+clean; 197 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.7 MiB/s wr, 19 op/s 2024-09-14T03:50:00.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:49:59 smithi070 ceph-mon[39288]: osdmap e186: 8 total, 8 up, 8 in 2024-09-14T03:50:00.528 INFO:tasks.rados.rados.0.smithi070.stdout:667: done (1 left) 2024-09-14T03:50:00.528 INFO:tasks.rados.rados.0.smithi070.stdout:668: read oid 32 snap -1 2024-09-14T03:50:00.528 INFO:tasks.rados.rados.0.smithi070.stdout:668: expect (ObjNum 223 snap 66 seq_num 223) 2024-09-14T03:50:00.528 INFO:tasks.rados.rados.0.smithi070.stdout:669: delete oid 27 current snap is 66 2024-09-14T03:50:00.528 INFO:tasks.rados.rados.0.smithi070.stdout:666: finishing rollback tid 0 to smithi07085154-26 2024-09-14T03:50:00.528 INFO:tasks.rados.rados.0.smithi070.stdout:666: finishing rollback tid 1 to smithi07085154-26 2024-09-14T03:50:00.528 INFO:tasks.rados.rados.0.smithi070.stdout:666: finishing rollback tid 2 to smithi07085154-26 2024-09-14T03:50:00.529 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 472 (ObjNum 156 snap 46 seq_num 156) dirty exists 2024-09-14T03:50:00.539 INFO:tasks.rados.rados.0.smithi070.stdout:666: done (2 left) 2024-09-14T03:50:00.539 INFO:tasks.rados.rados.0.smithi070.stdout:669: done (1 left) 2024-09-14T03:50:00.539 INFO:tasks.rados.rados.0.smithi070.stdout:670: rollback oid 30 current snap is 66 2024-09-14T03:50:00.539 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 30 to 34 2024-09-14T03:50:00.542 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 48 current snap is 66 2024-09-14T03:50:00.542 INFO:tasks.rados.rados.0.smithi070.stdout:671: seq_num 226 ranges {778240=589824,1368064=507904,1875968=294912} 2024-09-14T03:50:00.547 INFO:tasks.rados.rados.0.smithi070.stdout:671: writing smithi07085154-48 from 778240 to 1368064 tid 1 2024-09-14T03:50:00.550 INFO:tasks.rados.rados.0.smithi070.stdout:671: writing smithi07085154-48 from 1368064 to 1875968 tid 2 2024-09-14T03:50:00.553 INFO:tasks.rados.rados.0.smithi070.stdout:671: writing smithi07085154-48 from 1875968 to 2170880 tid 3 2024-09-14T03:50:00.553 INFO:tasks.rados.rados.0.smithi070.stdout:672: rollback oid 43 current snap is 66 2024-09-14T03:50:00.553 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 43 to 41 2024-09-14T03:50:00.553 INFO:tasks.rados.rados.0.smithi070.stdout:670: finishing rollback tid 0 to smithi07085154-30 2024-09-14T03:50:00.553 INFO:tasks.rados.rados.0.smithi070.stdout:670: finishing rollback tid 1 to smithi07085154-30 2024-09-14T03:50:00.553 INFO:tasks.rados.rados.0.smithi070.stdout:670: finishing rollback tid 2 to smithi07085154-30 2024-09-14T03:50:00.553 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 30 v 155 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-09-14T03:50:00.553 INFO:tasks.rados.rados.0.smithi070.stdout:668: expect (ObjNum 223 snap 66 seq_num 223) 2024-09-14T03:50:00.646 INFO:tasks.rados.rados.0.smithi070.stdout:671: finishing write tid 1 to smithi07085154-48 2024-09-14T03:50:00.647 INFO:tasks.rados.rados.0.smithi070.stdout:671: finishing write tid 2 to smithi07085154-48 2024-09-14T03:50:00.647 INFO:tasks.rados.rados.0.smithi070.stdout:671: finishing write tid 3 to smithi07085154-48 2024-09-14T03:50:00.647 INFO:tasks.rados.rados.0.smithi070.stdout:671: finishing write tid 4 to smithi07085154-48 2024-09-14T03:50:00.647 INFO:tasks.rados.rados.0.smithi070.stdout:672: finishing rollback tid 1 to smithi07085154-43 2024-09-14T03:50:00.647 INFO:tasks.rados.rados.0.smithi070.stdout:672: finishing rollback tid 2 to smithi07085154-43 2024-09-14T03:50:00.647 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 360 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-14T03:50:00.647 INFO:tasks.rados.rados.0.smithi070.stdout:671: finishing write tid 5 to smithi07085154-48 2024-09-14T03:50:00.647 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 136 (ObjNum 226 snap 66 seq_num 226) dirty exists 2024-09-14T03:50:00.647 INFO:tasks.rados.rados.0.smithi070.stdout:671: left oid 48 (ObjNum 226 snap 66 seq_num 226) 2024-09-14T03:50:00.647 INFO:tasks.rados.rados.0.smithi070.stdout:668: done (3 left) 2024-09-14T03:50:00.647 INFO:tasks.rados.rados.0.smithi070.stdout:670: done (2 left) 2024-09-14T03:50:00.647 INFO:tasks.rados.rados.0.smithi070.stdout:671: done (1 left) 2024-09-14T03:50:00.647 INFO:tasks.rados.rados.0.smithi070.stdout:672: done (0 left) 2024-09-14T03:50:00.647 INFO:tasks.rados.rados.0.smithi070.stdout:673: snap_remove snap 36 2024-09-14T03:50:00.958 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:00 smithi070 ceph-mon[27495]: overall HEALTH_OK 2024-09-14T03:50:00.959 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:00 smithi070 ceph-mon[27495]: osdmap e187: 8 total, 8 up, 8 in 2024-09-14T03:50:00.959 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:00 smithi070 ceph-mon[39288]: overall HEALTH_OK 2024-09-14T03:50:00.959 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:00 smithi070 ceph-mon[39288]: osdmap e187: 8 total, 8 up, 8 in 2024-09-14T03:50:01.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:00 smithi094 ceph-mon[31053]: overall HEALTH_OK 2024-09-14T03:50:01.123 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:00 smithi094 ceph-mon[31053]: osdmap e187: 8 total, 8 up, 8 in 2024-09-14T03:50:01.545 INFO:tasks.rados.rados.0.smithi070.stdout:673: done (0 left) 2024-09-14T03:50:01.545 INFO:tasks.rados.rados.0.smithi070.stdout:674: snap_create 2024-09-14T03:50:02.027 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:50:01 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:50:01] "GET /metrics HTTP/1.1" 200 36210 "" "Prometheus/2.43.0" 2024-09-14T03:50:02.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:01 smithi094 ceph-mon[31053]: pgmap v43: 121 pgs: 121 active+clean; 197 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:50:02.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:01 smithi070 ceph-mon[27495]: pgmap v43: 121 pgs: 121 active+clean; 197 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:50:02.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:01 smithi070 ceph-mon[39288]: pgmap v43: 121 pgs: 121 active+clean; 197 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:50:02.548 INFO:tasks.rados.rados.0.smithi070.stdout:674: done (0 left) 2024-09-14T03:50:02.548 INFO:tasks.rados.rados.0.smithi070.stdout:675: rmattr oid 40 current snap is 67 2024-09-14T03:50:02.548 INFO:tasks.rados.rados.0.smithi070.stdout:675: done (0 left) 2024-09-14T03:50:02.548 INFO:tasks.rados.rados.0.smithi070.stdout:676: read oid 10 snap 30 2024-09-14T03:50:02.548 INFO:tasks.rados.rados.0.smithi070.stdout:676: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-14T03:50:02.548 INFO:tasks.rados.rados.0.smithi070.stdout:677: copy_from oid 33 from oid 45 current snap is 67 2024-09-14T03:50:02.548 INFO:tasks.rados.rados.0.smithi070.stdout:678: read oid 19 snap -1 2024-09-14T03:50:02.548 INFO:tasks.rados.rados.0.smithi070.stdout:678: expect (ObjNum 216 snap 62 seq_num 216) 2024-09-14T03:50:02.548 INFO:tasks.rados.rados.0.smithi070.stdout:679: setattr oid 31 current snap is 67 2024-09-14T03:50:02.549 INFO:tasks.rados.rados.0.smithi070.stdout:680: delete oid 48 current snap is 67 2024-09-14T03:50:02.553 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 280 (ObjNum 201 snap 56 seq_num 201) dirty exists 2024-09-14T03:50:02.554 INFO:tasks.rados.rados.0.smithi070.stdout:677: finishing copy_from racing read to smithi07085154-33 2024-09-14T03:50:02.556 INFO:tasks.rados.rados.0.smithi070.stdout:677: finishing copy_from to smithi07085154-33 2024-09-14T03:50:02.556 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 33 v 11 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-14T03:50:02.560 INFO:tasks.rados.rados.0.smithi070.stdout:676: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-14T03:50:02.578 INFO:tasks.rados.rados.0.smithi070.stdout:678: expect (ObjNum 216 snap 62 seq_num 216) 2024-09-14T03:50:02.706 INFO:tasks.rados.rados.0.smithi070.stdout:676: done (4 left) 2024-09-14T03:50:02.706 INFO:tasks.rados.rados.0.smithi070.stdout:677: done (3 left) 2024-09-14T03:50:02.706 INFO:tasks.rados.rados.0.smithi070.stdout:678: done (2 left) 2024-09-14T03:50:02.706 INFO:tasks.rados.rados.0.smithi070.stdout:679: done (1 left) 2024-09-14T03:50:02.706 INFO:tasks.rados.rados.0.smithi070.stdout:680: done (0 left) 2024-09-14T03:50:02.707 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 15 current snap is 67 2024-09-14T03:50:02.707 INFO:tasks.rados.rados.0.smithi070.stdout:681: seq_num 228 ranges {0=622592,622592=671744,1294336=131072} 2024-09-14T03:50:02.709 INFO:tasks.rados.rados.0.smithi070.stdout:681: writing smithi07085154-15 from 0 to 622592 tid 1 2024-09-14T03:50:02.713 INFO:tasks.rados.rados.0.smithi070.stdout:681: writing smithi07085154-15 from 622592 to 1294336 tid 2 2024-09-14T03:50:02.714 INFO:tasks.rados.rados.0.smithi070.stdout:681: writing smithi07085154-15 from 1294336 to 1425408 tid 3 2024-09-14T03:50:02.714 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 5 current snap is 67 2024-09-14T03:50:02.714 INFO:tasks.rados.rados.0.smithi070.stdout:681: finishing write tid 1 to smithi07085154-15 2024-09-14T03:50:02.714 INFO:tasks.rados.rados.0.smithi070.stdout:682: seq_num 229 ranges {2441216=638976,3080192=606208,3686400=196608} 2024-09-14T03:50:02.731 INFO:tasks.rados.rados.0.smithi070.stdout:682: writing smithi07085154-5 from 2441216 to 3080192 tid 1 2024-09-14T03:50:02.734 INFO:tasks.rados.rados.0.smithi070.stdout:682: writing smithi07085154-5 from 3080192 to 3686400 tid 2 2024-09-14T03:50:02.736 INFO:tasks.rados.rados.0.smithi070.stdout:682: writing smithi07085154-5 from 3686400 to 3883008 tid 3 2024-09-14T03:50:02.736 INFO:tasks.rados.rados.0.smithi070.stdout:683: setattr oid 24 current snap is 67 2024-09-14T03:50:02.736 INFO:tasks.rados.rados.0.smithi070.stdout:681: finishing write tid 2 to smithi07085154-15 2024-09-14T03:50:02.736 INFO:tasks.rados.rados.0.smithi070.stdout:681: finishing write tid 3 to smithi07085154-15 2024-09-14T03:50:02.736 INFO:tasks.rados.rados.0.smithi070.stdout:681: finishing write tid 4 to smithi07085154-15 2024-09-14T03:50:02.736 INFO:tasks.rados.rados.0.smithi070.stdout:681: finishing write tid 5 to smithi07085154-15 2024-09-14T03:50:02.736 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 15 v 376 (ObjNum 228 snap 67 seq_num 228) dirty exists 2024-09-14T03:50:02.736 INFO:tasks.rados.rados.0.smithi070.stdout:681: left oid 15 (ObjNum 228 snap 67 seq_num 228) 2024-09-14T03:50:02.736 INFO:tasks.rados.rados.0.smithi070.stdout:682: finishing write tid 1 to smithi07085154-5 2024-09-14T03:50:02.737 INFO:tasks.rados.rados.0.smithi070.stdout:681: done (2 left) 2024-09-14T03:50:02.737 INFO:tasks.rados.rados.0.smithi070.stdout:684: delete oid 7 current snap is 67 2024-09-14T03:50:02.737 INFO:tasks.rados.rados.0.smithi070.stdout:682: finishing write tid 2 to smithi07085154-5 2024-09-14T03:50:02.738 INFO:tasks.rados.rados.0.smithi070.stdout:682: finishing write tid 3 to smithi07085154-5 2024-09-14T03:50:02.738 INFO:tasks.rados.rados.0.smithi070.stdout:682: finishing write tid 4 to smithi07085154-5 2024-09-14T03:50:02.739 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 145 (ObjNum 177 snap 0 seq_num 4215235444) dirty exists 2024-09-14T03:50:02.739 INFO:tasks.rados.rados.0.smithi070.stdout:683: done (2 left) 2024-09-14T03:50:02.739 INFO:tasks.rados.rados.0.smithi070.stdout:684: done (1 left) 2024-09-14T03:50:02.739 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 49 current snap is 67 2024-09-14T03:50:02.739 INFO:tasks.rados.rados.0.smithi070.stdout:685: seq_num 231 ranges {1785856=671744,2457600=778240,3235840=24576} 2024-09-14T03:50:02.753 INFO:tasks.rados.rados.0.smithi070.stdout:685: writing smithi07085154-49 from 1785856 to 2457600 tid 1 2024-09-14T03:50:02.758 INFO:tasks.rados.rados.0.smithi070.stdout:685: writing smithi07085154-49 from 2457600 to 3235840 tid 2 2024-09-14T03:50:02.758 INFO:tasks.rados.rados.0.smithi070.stdout:685: writing smithi07085154-49 from 3235840 to 3260416 tid 3 2024-09-14T03:50:02.758 INFO:tasks.rados.rados.0.smithi070.stdout:686: copy_from oid 43 from oid 46 current snap is 67 2024-09-14T03:50:02.759 INFO:tasks.rados.rados.0.smithi070.stdout:682: finishing write tid 5 to smithi07085154-5 2024-09-14T03:50:02.759 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 54 (ObjNum 229 snap 67 seq_num 229) dirty exists 2024-09-14T03:50:02.759 INFO:tasks.rados.rados.0.smithi070.stdout:682: left oid 5 (ObjNum 229 snap 67 seq_num 229) 2024-09-14T03:50:02.759 INFO:tasks.rados.rados.0.smithi070.stdout:682: done (2 left) 2024-09-14T03:50:02.759 INFO:tasks.rados.rados.0.smithi070.stdout:687: rollback oid 9 current snap is 67 2024-09-14T03:50:02.759 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 9 to 65 2024-09-14T03:50:02.760 INFO:tasks.rados.rados.0.smithi070.stdout:685: finishing write tid 1 to smithi07085154-49 2024-09-14T03:50:02.760 INFO:tasks.rados.rados.0.smithi070.stdout:688: copy_from oid 31 from oid 18 current snap is 67 2024-09-14T03:50:02.760 INFO:tasks.rados.rados.0.smithi070.stdout:689: copy_from oid 17 from oid 7 current snap is 67 2024-09-14T03:50:02.760 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 16 current snap is 67 2024-09-14T03:50:02.760 INFO:tasks.rados.rados.0.smithi070.stdout:690: seq_num 232 ranges {638976=688128,1327104=663552,1990656=139264} 2024-09-14T03:50:02.765 INFO:tasks.rados.rados.0.smithi070.stdout:690: writing smithi07085154-16 from 638976 to 1327104 tid 1 2024-09-14T03:50:02.768 INFO:tasks.rados.rados.0.smithi070.stdout:690: writing smithi07085154-16 from 1327104 to 1990656 tid 2 2024-09-14T03:50:02.769 INFO:tasks.rados.rados.0.smithi070.stdout:690: writing smithi07085154-16 from 1990656 to 2129920 tid 3 2024-09-14T03:50:02.770 INFO:tasks.rados.rados.0.smithi070.stdout:691: snap_remove snap 53 2024-09-14T03:50:02.839 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:02 smithi070 ceph-mon[27495]: osdmap e188: 8 total, 8 up, 8 in 2024-09-14T03:50:02.840 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:02 smithi070 ceph-mon[39288]: osdmap e188: 8 total, 8 up, 8 in 2024-09-14T03:50:03.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:02 smithi094 ceph-mon[31053]: osdmap e188: 8 total, 8 up, 8 in 2024-09-14T03:50:03.556 INFO:tasks.rados.rados.0.smithi070.stdout:691: done (6 left) 2024-09-14T03:50:03.556 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 26 current snap is 67 2024-09-14T03:50:03.556 INFO:tasks.rados.rados.0.smithi070.stdout:689: finishing copy_from to smithi07085154-17 2024-09-14T03:50:03.556 INFO:tasks.rados.rados.0.smithi070.stdout:689: got expected ENOENT (src dne) 2024-09-14T03:50:03.557 INFO:tasks.rados.rados.0.smithi070.stdout:692: seq_num 233 ranges {1835008=712704,2547712=548864,3096576=253952} 2024-09-14T03:50:03.571 INFO:tasks.rados.rados.0.smithi070.stdout:692: writing smithi07085154-26 from 1835008 to 2547712 tid 1 2024-09-14T03:50:03.574 INFO:tasks.rados.rados.0.smithi070.stdout:692: writing smithi07085154-26 from 2547712 to 3096576 tid 2 2024-09-14T03:50:03.577 INFO:tasks.rados.rados.0.smithi070.stdout:692: writing smithi07085154-26 from 3096576 to 3350528 tid 3 2024-09-14T03:50:03.577 INFO:tasks.rados.rados.0.smithi070.stdout:693: read oid 33 snap -1 2024-09-14T03:50:03.577 INFO:tasks.rados.rados.0.smithi070.stdout:693: expect (ObjNum 49 snap 0 seq_num 49) 2024-09-14T03:50:03.577 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 29 current snap is 67 2024-09-14T03:50:03.577 INFO:tasks.rados.rados.0.smithi070.stdout:689: finishing copy_from racing read to smithi07085154-17 2024-09-14T03:50:03.577 INFO:tasks.rados.rados.0.smithi070.stdout:685: finishing write tid 2 to smithi07085154-49 2024-09-14T03:50:03.577 INFO:tasks.rados.rados.0.smithi070.stdout:687: finishing rollback tid 0 to smithi07085154-9 2024-09-14T03:50:03.577 INFO:tasks.rados.rados.0.smithi070.stdout:685: finishing write tid 3 to smithi07085154-49 2024-09-14T03:50:03.578 INFO:tasks.rados.rados.0.smithi070.stdout:687: finishing rollback tid 1 to smithi07085154-9 2024-09-14T03:50:03.578 INFO:tasks.rados.rados.0.smithi070.stdout:686: finishing copy_from to smithi07085154-43 2024-09-14T03:50:03.578 INFO:tasks.rados.rados.0.smithi070.stdout:686: got expected ENOENT (src dne) 2024-09-14T03:50:03.578 INFO:tasks.rados.rados.0.smithi070.stdout:685: finishing write tid 4 to smithi07085154-49 2024-09-14T03:50:03.578 INFO:tasks.rados.rados.0.smithi070.stdout:686: finishing copy_from racing read to smithi07085154-43 2024-09-14T03:50:03.578 INFO:tasks.rados.rados.0.smithi070.stdout:687: finishing rollback tid 2 to smithi07085154-9 2024-09-14T03:50:03.578 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 497 (ObjNum 153 snap 45 seq_num 153) dirty exists 2024-09-14T03:50:03.578 INFO:tasks.rados.rados.0.smithi070.stdout:685: finishing write tid 5 to smithi07085154-49 2024-09-14T03:50:03.580 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 49 v 214 (ObjNum 231 snap 67 seq_num 231) dirty exists 2024-09-14T03:50:03.580 INFO:tasks.rados.rados.0.smithi070.stdout:685: left oid 49 (ObjNum 231 snap 67 seq_num 231) 2024-09-14T03:50:03.580 INFO:tasks.rados.rados.0.smithi070.stdout:690: finishing write tid 1 to smithi07085154-16 2024-09-14T03:50:03.580 INFO:tasks.rados.rados.0.smithi070.stdout:690: finishing write tid 2 to smithi07085154-16 2024-09-14T03:50:03.580 INFO:tasks.rados.rados.0.smithi070.stdout:694: seq_num 234 ranges {778240=729088,1507328=434176,1941504=368640} 2024-09-14T03:50:03.585 INFO:tasks.rados.rados.0.smithi070.stdout:694: writing smithi07085154-29 from 778240 to 1507328 tid 1 2024-09-14T03:50:03.587 INFO:tasks.rados.rados.0.smithi070.stdout:694: writing smithi07085154-29 from 1507328 to 1941504 tid 2 2024-09-14T03:50:03.590 INFO:tasks.rados.rados.0.smithi070.stdout:694: writing smithi07085154-29 from 1941504 to 2310144 tid 3 2024-09-14T03:50:03.590 INFO:tasks.rados.rados.0.smithi070.stdout:685: done (8 left) 2024-09-14T03:50:03.590 INFO:tasks.rados.rados.0.smithi070.stdout:686: done (7 left) 2024-09-14T03:50:03.591 INFO:tasks.rados.rados.0.smithi070.stdout:687: done (6 left) 2024-09-14T03:50:03.591 INFO:tasks.rados.rados.0.smithi070.stdout:689: done (5 left) 2024-09-14T03:50:03.591 INFO:tasks.rados.rados.0.smithi070.stdout:695: snap_create 2024-09-14T03:50:03.591 INFO:tasks.rados.rados.0.smithi070.stdout:688: finishing copy_from racing read to smithi07085154-31 2024-09-14T03:50:03.591 INFO:tasks.rados.rados.0.smithi070.stdout:690: finishing write tid 3 to smithi07085154-16 2024-09-14T03:50:03.591 INFO:tasks.rados.rados.0.smithi070.stdout:690: finishing write tid 4 to smithi07085154-16 2024-09-14T03:50:03.591 INFO:tasks.rados.rados.0.smithi070.stdout:688: finishing copy_from to smithi07085154-31 2024-09-14T03:50:03.591 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 141 (ObjNum 208 snap 60 seq_num 208) dirty exists 2024-09-14T03:50:03.591 INFO:tasks.rados.rados.0.smithi070.stdout:690: finishing write tid 5 to smithi07085154-16 2024-09-14T03:50:03.591 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 502 (ObjNum 232 snap 67 seq_num 232) dirty exists 2024-09-14T03:50:03.591 INFO:tasks.rados.rados.0.smithi070.stdout:690: left oid 16 (ObjNum 232 snap 67 seq_num 232) 2024-09-14T03:50:03.591 INFO:tasks.rados.rados.0.smithi070.stdout:692: finishing write tid 1 to smithi07085154-26 2024-09-14T03:50:03.591 INFO:tasks.rados.rados.0.smithi070.stdout:692: finishing write tid 2 to smithi07085154-26 2024-09-14T03:50:03.591 INFO:tasks.rados.rados.0.smithi070.stdout:693: expect (ObjNum 49 snap 0 seq_num 49) 2024-09-14T03:50:03.612 INFO:tasks.rados.rados.0.smithi070.stdout:692: finishing write tid 3 to smithi07085154-26 2024-09-14T03:50:03.612 INFO:tasks.rados.rados.0.smithi070.stdout:692: finishing write tid 4 to smithi07085154-26 2024-09-14T03:50:03.612 INFO:tasks.rados.rados.0.smithi070.stdout:692: finishing write tid 5 to smithi07085154-26 2024-09-14T03:50:03.612 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 515 (ObjNum 233 snap 67 seq_num 233) dirty exists 2024-09-14T03:50:03.612 INFO:tasks.rados.rados.0.smithi070.stdout:692: left oid 26 (ObjNum 233 snap 67 seq_num 233) 2024-09-14T03:50:03.612 INFO:tasks.rados.rados.0.smithi070.stdout:694: finishing write tid 1 to smithi07085154-29 2024-09-14T03:50:03.612 INFO:tasks.rados.rados.0.smithi070.stdout:694: finishing write tid 2 to smithi07085154-29 2024-09-14T03:50:03.612 INFO:tasks.rados.rados.0.smithi070.stdout:694: finishing write tid 3 to smithi07085154-29 2024-09-14T03:50:03.612 INFO:tasks.rados.rados.0.smithi070.stdout:694: finishing write tid 4 to smithi07085154-29 2024-09-14T03:50:03.612 INFO:tasks.rados.rados.0.smithi070.stdout:694: finishing write tid 5 to smithi07085154-29 2024-09-14T03:50:03.613 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 304 (ObjNum 234 snap 67 seq_num 234) dirty exists 2024-09-14T03:50:03.613 INFO:tasks.rados.rados.0.smithi070.stdout:694: left oid 29 (ObjNum 234 snap 67 seq_num 234) 2024-09-14T03:50:03.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:03 smithi070 ceph-mon[27495]: pgmap v45: 121 pgs: 121 active+clean; 208 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.7 MiB/s rd, 1.2 MiB/s wr, 14 op/s 2024-09-14T03:50:03.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:03 smithi070 ceph-mon[27495]: osdmap e189: 8 total, 8 up, 8 in 2024-09-14T03:50:03.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:03 smithi070 ceph-mon[39288]: pgmap v45: 121 pgs: 121 active+clean; 208 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.7 MiB/s rd, 1.2 MiB/s wr, 14 op/s 2024-09-14T03:50:03.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:03 smithi070 ceph-mon[39288]: osdmap e189: 8 total, 8 up, 8 in 2024-09-14T03:50:04.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:03 smithi094 ceph-mon[31053]: pgmap v45: 121 pgs: 121 active+clean; 208 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.7 MiB/s rd, 1.2 MiB/s wr, 14 op/s 2024-09-14T03:50:04.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:03 smithi094 ceph-mon[31053]: osdmap e189: 8 total, 8 up, 8 in 2024-09-14T03:50:04.561 INFO:tasks.rados.rados.0.smithi070.stdout:688: done (5 left) 2024-09-14T03:50:04.561 INFO:tasks.rados.rados.0.smithi070.stdout:690: done (4 left) 2024-09-14T03:50:04.561 INFO:tasks.rados.rados.0.smithi070.stdout:692: done (3 left) 2024-09-14T03:50:04.561 INFO:tasks.rados.rados.0.smithi070.stdout:693: done (2 left) 2024-09-14T03:50:04.561 INFO:tasks.rados.rados.0.smithi070.stdout:694: done (1 left) 2024-09-14T03:50:04.561 INFO:tasks.rados.rados.0.smithi070.stdout:695: done (0 left) 2024-09-14T03:50:04.561 INFO:tasks.rados.rados.0.smithi070.stdout:696: delete oid 2 current snap is 68 2024-09-14T03:50:04.566 INFO:tasks.rados.rados.0.smithi070.stdout:696: done (0 left) 2024-09-14T03:50:04.566 INFO:tasks.rados.rados.0.smithi070.stdout:697: delete oid 5 current snap is 68 2024-09-14T03:50:04.567 INFO:tasks.rados.rados.0.smithi070.stdout:697: done (0 left) 2024-09-14T03:50:04.567 INFO:tasks.rados.rados.0.smithi070.stdout:698: read oid 43 snap -1 2024-09-14T03:50:04.568 INFO:tasks.rados.rados.0.smithi070.stdout:698: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-14T03:50:04.568 INFO:tasks.rados.rados.0.smithi070.stdout:699: read oid 45 snap 56 2024-09-14T03:50:04.568 INFO:tasks.rados.rados.0.smithi070.stdout:699: expect (ObjNum 49 snap 0 seq_num 49) 2024-09-14T03:50:04.568 INFO:tasks.rados.rados.0.smithi070.stdout:700: read oid 4 snap -1 2024-09-14T03:50:04.569 INFO:tasks.rados.rados.0.smithi070.stdout:700: expect deleted 2024-09-14T03:50:04.569 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 14 current snap is 68 2024-09-14T03:50:04.569 INFO:tasks.rados.rados.0.smithi070.stdout:701: seq_num 237 ranges {3194880=778240,3973120=532480,4505600=270336} 2024-09-14T03:50:04.589 INFO:tasks.rados.rados.0.smithi070.stdout:701: writing smithi07085154-14 from 3194880 to 3973120 tid 1 2024-09-14T03:50:04.592 INFO:tasks.rados.rados.0.smithi070.stdout:701: writing smithi07085154-14 from 3973120 to 4505600 tid 2 2024-09-14T03:50:04.593 INFO:tasks.rados.rados.0.smithi070.stdout:701: writing smithi07085154-14 from 4505600 to 4775936 tid 3 2024-09-14T03:50:04.594 INFO:tasks.rados.rados.0.smithi070.stdout:702: read oid 35 snap -1 2024-09-14T03:50:04.594 INFO:tasks.rados.rados.0.smithi070.stdout:702: expect deleted 2024-09-14T03:50:04.594 INFO:tasks.rados.rados.0.smithi070.stdout:703: rollback oid 32 current snap is 68 2024-09-14T03:50:04.594 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 32 to 43 2024-09-14T03:50:04.594 INFO:tasks.rados.rados.0.smithi070.stdout:698: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-14T03:50:04.614 INFO:tasks.rados.rados.0.smithi070.stdout:699: expect (ObjNum 49 snap 0 seq_num 49) 2024-09-14T03:50:04.635 INFO:tasks.rados.rados.0.smithi070.stdout:703: finishing rollback tid 0 to smithi07085154-32 2024-09-14T03:50:04.635 INFO:tasks.rados.rados.0.smithi070.stdout:703: finishing rollback tid 1 to smithi07085154-32 2024-09-14T03:50:04.636 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 316 (ObjNum 65 snap 0 seq_num 1365142373) dirty dne 2024-09-14T03:50:04.636 INFO:tasks.rados.rados.0.smithi070.stdout:701: finishing write tid 1 to smithi07085154-14 2024-09-14T03:50:04.636 INFO:tasks.rados.rados.0.smithi070.stdout:698: done (5 left) 2024-09-14T03:50:04.636 INFO:tasks.rados.rados.0.smithi070.stdout:699: done (4 left) 2024-09-14T03:50:04.636 INFO:tasks.rados.rados.0.smithi070.stdout:700: done (3 left) 2024-09-14T03:50:04.636 INFO:tasks.rados.rados.0.smithi070.stdout:702: done (2 left) 2024-09-14T03:50:04.636 INFO:tasks.rados.rados.0.smithi070.stdout:703: done (1 left) 2024-09-14T03:50:04.636 INFO:tasks.rados.rados.0.smithi070.stdout:704: setattr oid 40 current snap is 68 2024-09-14T03:50:04.636 INFO:tasks.rados.rados.0.smithi070.stdout:701: finishing write tid 2 to smithi07085154-14 2024-09-14T03:50:04.636 INFO:tasks.rados.rados.0.smithi070.stdout:701: finishing write tid 3 to smithi07085154-14 2024-09-14T03:50:04.636 INFO:tasks.rados.rados.0.smithi070.stdout:701: finishing write tid 4 to smithi07085154-14 2024-09-14T03:50:04.636 INFO:tasks.rados.rados.0.smithi070.stdout:701: finishing write tid 5 to smithi07085154-14 2024-09-14T03:50:04.636 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 149 (ObjNum 237 snap 68 seq_num 237) dirty exists 2024-09-14T03:50:04.637 INFO:tasks.rados.rados.0.smithi070.stdout:701: left oid 14 (ObjNum 237 snap 68 seq_num 237) 2024-09-14T03:50:04.637 INFO:tasks.rados.rados.0.smithi070.stdout:701: done (1 left) 2024-09-14T03:50:04.637 INFO:tasks.rados.rados.0.smithi070.stdout:705: delete oid 27 current snap is 68 2024-09-14T03:50:04.637 INFO:tasks.rados.rados.0.smithi070.stdout:705: done (1 left) 2024-09-14T03:50:04.637 INFO:tasks.rados.rados.0.smithi070.stdout:706: rollback oid 30 current snap is 68 2024-09-14T03:50:04.637 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 30 to 51 2024-09-14T03:50:04.637 INFO:tasks.rados.rados.0.smithi070.stdout:707: setattr oid 39 current snap is 68 2024-09-14T03:50:04.638 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 182 (ObjNum 131 snap 36 seq_num 131) dirty exists 2024-09-14T03:50:04.638 INFO:tasks.rados.rados.0.smithi070.stdout:704: done (2 left) 2024-09-14T03:50:04.638 INFO:tasks.rados.rados.0.smithi070.stdout:708: setattr oid 26 current snap is 68 2024-09-14T03:50:04.639 INFO:tasks.rados.rados.0.smithi070.stdout:709: snap_remove snap 62 2024-09-14T03:50:04.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:04 smithi070 ceph-mon[27495]: osdmap e190: 8 total, 8 up, 8 in 2024-09-14T03:50:04.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:04 smithi070 ceph-mon[39288]: osdmap e190: 8 total, 8 up, 8 in 2024-09-14T03:50:05.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:04 smithi094 ceph-mon[31053]: osdmap e190: 8 total, 8 up, 8 in 2024-09-14T03:50:05.528 INFO:tasks.rados.rados.0.smithi070.stdout:709: done (3 left) 2024-09-14T03:50:05.528 INFO:tasks.rados.rados.0.smithi070.stdout:710: delete oid 35 current snap is 68 2024-09-14T03:50:05.529 INFO:tasks.rados.rados.0.smithi070.stdout:706: finishing rollback tid 0 to smithi07085154-30 2024-09-14T03:50:05.529 INFO:tasks.rados.rados.0.smithi070.stdout:706: finishing rollback tid 1 to smithi07085154-30 2024-09-14T03:50:05.529 INFO:tasks.rados.rados.0.smithi070.stdout:706: finishing rollback tid 2 to smithi07085154-30 2024-09-14T03:50:05.529 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 30 v 169 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-09-14T03:50:05.529 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 39 v 53 (ObjNum 211 snap 60 seq_num 211) dirty exists 2024-09-14T03:50:05.529 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 520 (ObjNum 233 snap 67 seq_num 233) dirty exists 2024-09-14T03:50:05.536 INFO:tasks.rados.rados.0.smithi070.stdout:706: done (3 left) 2024-09-14T03:50:05.536 INFO:tasks.rados.rados.0.smithi070.stdout:707: done (2 left) 2024-09-14T03:50:05.537 INFO:tasks.rados.rados.0.smithi070.stdout:708: done (1 left) 2024-09-14T03:50:05.537 INFO:tasks.rados.rados.0.smithi070.stdout:710: done (0 left) 2024-09-14T03:50:05.537 INFO:tasks.rados.rados.0.smithi070.stdout:711: read oid 42 snap -1 2024-09-14T03:50:05.537 INFO:tasks.rados.rados.0.smithi070.stdout:711: expect deleted 2024-09-14T03:50:05.537 INFO:tasks.rados.rados.0.smithi070.stdout:712: read oid 24 snap -1 2024-09-14T03:50:05.537 INFO:tasks.rados.rados.0.smithi070.stdout:712: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:50:05.537 INFO:tasks.rados.rados.0.smithi070.stdout:713: rmattr oid 45 current snap is 68 2024-09-14T03:50:05.537 INFO:tasks.rados.rados.0.smithi070.stdout:713: done (2 left) 2024-09-14T03:50:05.537 INFO:tasks.rados.rados.0.smithi070.stdout:714: snap_remove snap 65 2024-09-14T03:50:05.785 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:50:05 smithi070 systemd[1]: Stopping Ceph alertmanager.a for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:50:05.786 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:50:05 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a[80300]: ts=2024-09-14T03:50:05.755Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-14T03:50:05.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:05 smithi070 ceph-mon[27495]: pgmap v48: 121 pgs: 121 active+clean; 208 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s rd, 1.2 MiB/s wr, 14 op/s 2024-09-14T03:50:05.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:05 smithi070 ceph-mon[27495]: osdmap e191: 8 total, 8 up, 8 in 2024-09-14T03:50:05.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:05 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:50:05.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:05 smithi070 ceph-mon[27495]: osdmap e192: 8 total, 8 up, 8 in 2024-09-14T03:50:05.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:05 smithi070 ceph-mon[39288]: pgmap v48: 121 pgs: 121 active+clean; 208 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s rd, 1.2 MiB/s wr, 14 op/s 2024-09-14T03:50:05.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:05 smithi070 ceph-mon[39288]: osdmap e191: 8 total, 8 up, 8 in 2024-09-14T03:50:05.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:05 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:50:05.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:05 smithi070 ceph-mon[39288]: osdmap e192: 8 total, 8 up, 8 in 2024-09-14T03:50:06.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:05 smithi094 ceph-mon[31053]: pgmap v48: 121 pgs: 121 active+clean; 208 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s rd, 1.2 MiB/s wr, 14 op/s 2024-09-14T03:50:06.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:05 smithi094 ceph-mon[31053]: osdmap e191: 8 total, 8 up, 8 in 2024-09-14T03:50:06.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:05 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:50:06.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:05 smithi094 ceph-mon[31053]: osdmap e192: 8 total, 8 up, 8 in 2024-09-14T03:50:06.106 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:50:05 smithi070 podman[87433]: 2024-09-14 03:50:05.796479835 +0000 UTC m=+0.127031033 container died fef584d3d5c685dfc50a2bc94f046b8b4893870f003ee756a2381512c7e5c231 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:50:06.531 INFO:tasks.rados.rados.0.smithi070.stdout:714: done (2 left) 2024-09-14T03:50:06.531 INFO:tasks.rados.rados.0.smithi070.stdout:715: setattr oid 22 current snap is 68 2024-09-14T03:50:06.531 INFO:tasks.rados.rados.0.smithi070.stdout:712: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:50:06.533 INFO:tasks.rados.rados.0.smithi070.stdout:711: done (2 left) 2024-09-14T03:50:06.533 INFO:tasks.rados.rados.0.smithi070.stdout:712: done (1 left) 2024-09-14T03:50:06.534 INFO:tasks.rados.rados.0.smithi070.stdout:716: snap_create 2024-09-14T03:50:06.544 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 308 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-14T03:50:06.656 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:50:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:50:06.656 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:50:06 smithi070 podman[87433]: 2024-09-14 03:50:06.370423883 +0000 UTC m=+0.700975070 container cleanup fef584d3d5c685dfc50a2bc94f046b8b4893870f003ee756a2381512c7e5c231 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:50:06.656 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:50:06 smithi070 bash[87433]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a 2024-09-14T03:50:06.923 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:50:06 smithi070 podman[87450]: 2024-09-14 03:50:06.655394641 +0000 UTC m=+0.856068203 container remove fef584d3d5c685dfc50a2bc94f046b8b4893870f003ee756a2381512c7e5c231 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:50:06.923 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:50:06 smithi070 podman[87450]: 2024-09-14 03:50:06.739530233 +0000 UTC m=+0.940203810 volume remove 45362eb6e33738f2094bb877012773b280777f866153f3160f4f375d6346d9e0 2024-09-14T03:50:07.286 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:50:06 smithi070 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-14T03:50:07.286 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:50:06 smithi070 systemd[1]: Stopped Ceph alertmanager.a for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:50:07.286 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:50:06 smithi070 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@alertmanager.a.service: Consumed 1.110s CPU time. 2024-09-14T03:50:07.286 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:50:07 smithi070 systemd[1]: Starting Ceph alertmanager.a for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:50:07.536 INFO:tasks.rados.rados.0.smithi070.stdout:715: done (1 left) 2024-09-14T03:50:07.536 INFO:tasks.rados.rados.0.smithi070.stdout:716: done (0 left) 2024-09-14T03:50:07.536 INFO:tasks.rados.rados.0.smithi070.stdout:717: rollback oid 47 current snap is 69 2024-09-14T03:50:07.537 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 47 to 43 2024-09-14T03:50:07.537 INFO:tasks.rados.rados.0.smithi070.stdout:718: read oid 10 snap -1 2024-09-14T03:50:07.537 INFO:tasks.rados.rados.0.smithi070.stdout:718: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-14T03:50:07.537 INFO:tasks.rados.rados.0.smithi070.stdout:719: read oid 2 snap -1 2024-09-14T03:50:07.537 INFO:tasks.rados.rados.0.smithi070.stdout:719: expect deleted 2024-09-14T03:50:07.537 INFO:tasks.rados.rados.0.smithi070.stdout:720: rollback oid 16 current snap is 69 2024-09-14T03:50:07.537 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 16 to 47 2024-09-14T03:50:07.537 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 3 current snap is 69 2024-09-14T03:50:07.537 INFO:tasks.rados.rados.0.smithi070.stdout:721: seq_num 240 ranges {892928=425984,1318912=589824,1908736=483328,2392064=131072} 2024-09-14T03:50:07.543 INFO:tasks.rados.rados.0.smithi070.stdout:721: writing smithi07085154-3 from 892928 to 1318912 tid 1 2024-09-14T03:50:07.546 INFO:tasks.rados.rados.0.smithi070.stdout:721: writing smithi07085154-3 from 1318912 to 1908736 tid 2 2024-09-14T03:50:07.549 INFO:tasks.rados.rados.0.smithi070.stdout:721: writing smithi07085154-3 from 1908736 to 2392064 tid 3 2024-09-14T03:50:07.550 INFO:tasks.rados.rados.0.smithi070.stdout:721: writing smithi07085154-3 from 2392064 to 2523136 tid 4 2024-09-14T03:50:07.550 INFO:tasks.rados.rados.0.smithi070.stdout:722: read oid 35 snap -1 2024-09-14T03:50:07.550 INFO:tasks.rados.rados.0.smithi070.stdout:722: expect deleted 2024-09-14T03:50:07.550 INFO:tasks.rados.rados.0.smithi070.stdout:723: read oid 6 snap -1 2024-09-14T03:50:07.550 INFO:tasks.rados.rados.0.smithi070.stdout:723: expect deleted 2024-09-14T03:50:07.550 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 36 current snap is 69 2024-09-14T03:50:07.550 INFO:tasks.rados.rados.0.smithi070.stdout:724: seq_num 241 ranges {0=442368,442368=466944,909312=688128,1597440=49152} 2024-09-14T03:50:07.552 INFO:tasks.rados.rados.0.smithi070.stdout:724: writing smithi07085154-36 from 0 to 442368 tid 1 2024-09-14T03:50:07.555 INFO:tasks.rados.rados.0.smithi070.stdout:724: writing smithi07085154-36 from 442368 to 909312 tid 2 2024-09-14T03:50:07.559 INFO:tasks.rados.rados.0.smithi070.stdout:724: writing smithi07085154-36 from 909312 to 1597440 tid 3 2024-09-14T03:50:07.560 INFO:tasks.rados.rados.0.smithi070.stdout:724: writing smithi07085154-36 from 1597440 to 1646592 tid 4 2024-09-14T03:50:07.560 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 21 current snap is 69 2024-09-14T03:50:07.560 INFO:tasks.rados.rados.0.smithi070.stdout:725: seq_num 242 ranges {827392=458752,1286144=753664,2039808=450560} 2024-09-14T03:50:07.566 INFO:tasks.rados.rados.0.smithi070.stdout:725: writing smithi07085154-21 from 827392 to 1286144 tid 1 2024-09-14T03:50:07.571 INFO:tasks.rados.rados.0.smithi070.stdout:725: writing smithi07085154-21 from 1286144 to 2039808 tid 2 2024-09-14T03:50:07.574 INFO:tasks.rados.rados.0.smithi070.stdout:725: writing smithi07085154-21 from 2039808 to 2490368 tid 3 2024-09-14T03:50:07.574 INFO:tasks.rados.rados.0.smithi070.stdout:726: snap_remove snap 55 2024-09-14T03:50:07.691 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:07 smithi070 ceph-mon[27495]: osdmap e193: 8 total, 8 up, 8 in 2024-09-14T03:50:07.692 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:07 smithi070 ceph-mon[27495]: pgmap v52: 121 pgs: 121 active+clean; 221 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 3.4 MiB/s wr, 44 op/s 2024-09-14T03:50:07.692 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:07 smithi070 ceph-mon[39288]: osdmap e193: 8 total, 8 up, 8 in 2024-09-14T03:50:07.692 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:07 smithi070 ceph-mon[39288]: pgmap v52: 121 pgs: 121 active+clean; 221 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 3.4 MiB/s wr, 44 op/s 2024-09-14T03:50:07.692 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:50:07 smithi070 podman[87584]: 2024-09-14 03:50:07.256619554 +0000 UTC m=+0.015763894 image pull 11f11916f8cdfb17c1da74b20deb2075d778357b4d60474d82453303f8d461f5 quay.io/prometheus/alertmanager:v0.27.0 2024-09-14T03:50:07.692 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:50:07 smithi070 podman[87584]: 2024-09-14 03:50:07.381944619 +0000 UTC m=+0.141088995 volume create 7ff4a966259cd587f7546743d3ee38ff5d11357344d3db734f0fbcab32e5f31a 2024-09-14T03:50:07.693 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:50:07 smithi070 podman[87584]: 2024-09-14 03:50:07.473922758 +0000 UTC m=+0.233067134 container create 45e79e51074c9b37c6986ea0160e7ddc28824a0b8309ad240c45133a9639c7d6 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:50:08.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:07 smithi094 ceph-mon[31053]: osdmap e193: 8 total, 8 up, 8 in 2024-09-14T03:50:08.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:07 smithi094 ceph-mon[31053]: pgmap v52: 121 pgs: 121 active+clean; 221 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 3.4 MiB/s wr, 44 op/s 2024-09-14T03:50:08.534 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:50:08 smithi070 podman[87584]: 2024-09-14 03:50:08.110293714 +0000 UTC m=+0.869438048 container init 45e79e51074c9b37c6986ea0160e7ddc28824a0b8309ad240c45133a9639c7d6 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:50:08.535 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:50:08 smithi070 podman[87584]: 2024-09-14 03:50:08.115876699 +0000 UTC m=+0.875021036 container start 45e79e51074c9b37c6986ea0160e7ddc28824a0b8309ad240c45133a9639c7d6 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:50:08.535 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:50:08 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a[87599]: ts=2024-09-14T03:50:08.127Z caller=main.go:181 level=info msg="Starting Alertmanager" version="(version=0.27.0, branch=HEAD, revision=0aa3c2aad14cff039931923ab16b26b7481783b5)" 2024-09-14T03:50:08.535 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:50:08 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a[87599]: ts=2024-09-14T03:50:08.127Z caller=main.go:182 level=info build_context="(go=go1.21.7, platform=linux/amd64, user=root@22cd11f671e9, date=20240228-11:51:20, tags=netgo)" 2024-09-14T03:50:08.535 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:50:08 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a[87599]: ts=2024-09-14T03:50:08.128Z caller=cluster.go:186 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.70 port=9094 2024-09-14T03:50:08.535 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:50:08 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a[87599]: ts=2024-09-14T03:50:08.129Z caller=cluster.go:683 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-14T03:50:08.535 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:50:08 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a[87599]: ts=2024-09-14T03:50:08.151Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-14T03:50:08.535 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:50:08 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a[87599]: ts=2024-09-14T03:50:08.151Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-14T03:50:08.535 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:50:08 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a[87599]: ts=2024-09-14T03:50:08.153Z caller=tls_config.go:313 level=info msg="Listening on" address=[::]:9093 2024-09-14T03:50:08.536 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:50:08 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a[87599]: ts=2024-09-14T03:50:08.153Z caller=tls_config.go:316 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-14T03:50:08.536 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:50:08 smithi070 bash[87584]: 45e79e51074c9b37c6986ea0160e7ddc28824a0b8309ad240c45133a9639c7d6 2024-09-14T03:50:08.536 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:50:08 smithi070 systemd[1]: Started Ceph alertmanager.a for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:50:08.541 INFO:tasks.rados.rados.0.smithi070.stdout:726: done (9 left) 2024-09-14T03:50:08.541 INFO:tasks.rados.rados.0.smithi070.stdout:727: snap_remove snap 56 2024-09-14T03:50:08.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:08 smithi070 ceph-mon[39288]: osdmap e194: 8 total, 8 up, 8 in 2024-09-14T03:50:08.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:08 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:08.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:08 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:08.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:08 smithi070 ceph-mon[27495]: osdmap e194: 8 total, 8 up, 8 in 2024-09-14T03:50:08.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:08 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:08.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:08 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:08.787 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:08 smithi070 systemd[1]: Stopping Ceph node-exporter.a for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:50:09.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:08 smithi094 ceph-mon[31053]: osdmap e194: 8 total, 8 up, 8 in 2024-09-14T03:50:09.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:08 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:09.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:08 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:09.285 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:09 smithi070 podman[87748]: 2024-09-14 03:50:09.01829607 +0000 UTC m=+0.113305298 container died 315c3f32399febe38ae8d87f5be4faf5d0f66b9f7f27cd6b1ae5582404a2ba04 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:50:09.544 INFO:tasks.rados.rados.0.smithi070.stdout:727: done (9 left) 2024-09-14T03:50:09.544 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 4 current snap is 69 2024-09-14T03:50:09.544 INFO:tasks.rados.rados.0.smithi070.stdout:728: seq_num 243 ranges {0=475136,475136=638976,1114112=565248} 2024-09-14T03:50:09.547 INFO:tasks.rados.rados.0.smithi070.stdout:728: writing smithi07085154-4 from 0 to 475136 tid 1 2024-09-14T03:50:09.551 INFO:tasks.rados.rados.0.smithi070.stdout:728: writing smithi07085154-4 from 475136 to 1114112 tid 2 2024-09-14T03:50:09.555 INFO:tasks.rados.rados.0.smithi070.stdout:728: writing smithi07085154-4 from 1114112 to 1679360 tid 3 2024-09-14T03:50:09.556 INFO:tasks.rados.rados.0.smithi070.stdout:719: done (9 left) 2024-09-14T03:50:09.556 INFO:tasks.rados.rados.0.smithi070.stdout:729: copy_from oid 24 from oid 22 current snap is 69 2024-09-14T03:50:09.556 INFO:tasks.rados.rados.0.smithi070.stdout:717: finishing rollback tid 0 to smithi07085154-47 2024-09-14T03:50:09.556 INFO:tasks.rados.rados.0.smithi070.stdout:717: finishing rollback tid 1 to smithi07085154-47 2024-09-14T03:50:09.556 INFO:tasks.rados.rados.0.smithi070.stdout:718: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-14T03:50:09.587 INFO:tasks.rados.rados.0.smithi070.stdout:717: finishing rollback tid 2 to smithi07085154-47 2024-09-14T03:50:09.587 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 47 v 177 (ObjNum 112 snap 23 seq_num 112) dirty exists 2024-09-14T03:50:09.587 INFO:tasks.rados.rados.0.smithi070.stdout:720: finishing rollback tid 0 to smithi07085154-16 2024-09-14T03:50:09.587 INFO:tasks.rados.rados.0.smithi070.stdout:720: finishing rollback tid 1 to smithi07085154-16 2024-09-14T03:50:09.588 INFO:tasks.rados.rados.0.smithi070.stdout:720: finishing rollback tid 2 to smithi07085154-16 2024-09-14T03:50:09.588 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 542 (ObjNum 114 snap 24 seq_num 114) dirty exists 2024-09-14T03:50:09.588 INFO:tasks.rados.rados.0.smithi070.stdout:721: finishing write tid 1 to smithi07085154-3 2024-09-14T03:50:09.588 INFO:tasks.rados.rados.0.smithi070.stdout:721: finishing write tid 2 to smithi07085154-3 2024-09-14T03:50:09.588 INFO:tasks.rados.rados.0.smithi070.stdout:717: done (9 left) 2024-09-14T03:50:09.588 INFO:tasks.rados.rados.0.smithi070.stdout:718: done (8 left) 2024-09-14T03:50:09.588 INFO:tasks.rados.rados.0.smithi070.stdout:720: done (7 left) 2024-09-14T03:50:09.588 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 14 current snap is 69 2024-09-14T03:50:09.588 INFO:tasks.rados.rados.0.smithi070.stdout:730: seq_num 244 ranges {4775936=491520,5267456=573440,5840896=458752,6299648=172032} 2024-09-14T03:50:09.615 INFO:tasks.rados.rados.0.smithi070.stdout:730: writing smithi07085154-14 from 4775936 to 5267456 tid 1 2024-09-14T03:50:09.618 INFO:tasks.rados.rados.0.smithi070.stdout:730: writing smithi07085154-14 from 5267456 to 5840896 tid 2 2024-09-14T03:50:09.621 INFO:tasks.rados.rados.0.smithi070.stdout:730: writing smithi07085154-14 from 5840896 to 6299648 tid 3 2024-09-14T03:50:09.622 INFO:tasks.rados.rados.0.smithi070.stdout:730: writing smithi07085154-14 from 6299648 to 6471680 tid 4 2024-09-14T03:50:09.622 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 7 current snap is 69 2024-09-14T03:50:09.622 INFO:tasks.rados.rados.0.smithi070.stdout:731: seq_num 245 ranges {0=507904,507904=458752,966656=671744,1638400=73728} 2024-09-14T03:50:09.625 INFO:tasks.rados.rados.0.smithi070.stdout:731: writing smithi07085154-7 from 0 to 507904 tid 1 2024-09-14T03:50:09.628 INFO:tasks.rados.rados.0.smithi070.stdout:731: writing smithi07085154-7 from 507904 to 966656 tid 2 2024-09-14T03:50:09.634 INFO:tasks.rados.rados.0.smithi070.stdout:731: writing smithi07085154-7 from 966656 to 1638400 tid 3 2024-09-14T03:50:09.635 INFO:tasks.rados.rados.0.smithi070.stdout:731: writing smithi07085154-7 from 1638400 to 1712128 tid 4 2024-09-14T03:50:09.635 INFO:tasks.rados.rados.0.smithi070.stdout:732: read oid 38 snap -1 2024-09-14T03:50:09.635 INFO:tasks.rados.rados.0.smithi070.stdout:732: expect deleted 2024-09-14T03:50:09.635 INFO:tasks.rados.rados.0.smithi070.stdout:733: copy_from oid 42 from oid 28 current snap is 69 2024-09-14T03:50:09.635 INFO:tasks.rados.rados.0.smithi070.stdout:721: finishing write tid 3 to smithi07085154-3 2024-09-14T03:50:09.635 INFO:tasks.rados.rados.0.smithi070.stdout:721: finishing write tid 4 to smithi07085154-3 2024-09-14T03:50:09.636 INFO:tasks.rados.rados.0.smithi070.stdout:721: finishing write tid 5 to smithi07085154-3 2024-09-14T03:50:09.636 INFO:tasks.rados.rados.0.smithi070.stdout:721: finishing write tid 6 to smithi07085154-3 2024-09-14T03:50:09.636 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 3 v 183 (ObjNum 240 snap 69 seq_num 240) dirty exists 2024-09-14T03:50:09.636 INFO:tasks.rados.rados.0.smithi070.stdout:721: left oid 3 (ObjNum 240 snap 69 seq_num 240) 2024-09-14T03:50:09.636 INFO:tasks.rados.rados.0.smithi070.stdout:724: finishing write tid 1 to smithi07085154-36 2024-09-14T03:50:09.636 INFO:tasks.rados.rados.0.smithi070.stdout:721: done (10 left) 2024-09-14T03:50:09.636 INFO:tasks.rados.rados.0.smithi070.stdout:722: done (9 left) 2024-09-14T03:50:09.637 INFO:tasks.rados.rados.0.smithi070.stdout:723: done (8 left) 2024-09-14T03:50:09.637 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 43 current snap is 69 2024-09-14T03:50:09.637 INFO:tasks.rados.rados.0.smithi070.stdout:724: finishing write tid 2 to smithi07085154-36 2024-09-14T03:50:09.637 INFO:tasks.rados.rados.0.smithi070.stdout:734: seq_num 246 ranges {638976=524288,1163264=745472,1908736=425984,2334720=32768} 2024-09-14T03:50:09.642 INFO:tasks.rados.rados.0.smithi070.stdout:734: writing smithi07085154-43 from 638976 to 1163264 tid 1 2024-09-14T03:50:09.648 INFO:tasks.rados.rados.0.smithi070.stdout:734: writing smithi07085154-43 from 1163264 to 1908736 tid 2 2024-09-14T03:50:09.653 INFO:tasks.rados.rados.0.smithi070.stdout:734: writing smithi07085154-43 from 1908736 to 2334720 tid 3 2024-09-14T03:50:09.653 INFO:tasks.rados.rados.0.smithi070.stdout:734: writing smithi07085154-43 from 2334720 to 2367488 tid 4 2024-09-14T03:50:09.653 INFO:tasks.rados.rados.0.smithi070.stdout:735: setattr oid 35 current snap is 69 2024-09-14T03:50:09.653 INFO:tasks.rados.rados.0.smithi070.stdout:724: finishing write tid 3 to smithi07085154-36 2024-09-14T03:50:09.653 INFO:tasks.rados.rados.0.smithi070.stdout:724: finishing write tid 4 to smithi07085154-36 2024-09-14T03:50:09.653 INFO:tasks.rados.rados.0.smithi070.stdout:724: finishing write tid 5 to smithi07085154-36 2024-09-14T03:50:09.653 INFO:tasks.rados.rados.0.smithi070.stdout:724: finishing write tid 6 to smithi07085154-36 2024-09-14T03:50:09.654 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 36 v 195 (ObjNum 241 snap 69 seq_num 241) dirty exists 2024-09-14T03:50:09.654 INFO:tasks.rados.rados.0.smithi070.stdout:724: left oid 36 (ObjNum 241 snap 69 seq_num 241) 2024-09-14T03:50:09.654 INFO:tasks.rados.rados.0.smithi070.stdout:725: finishing write tid 1 to smithi07085154-21 2024-09-14T03:50:09.654 INFO:tasks.rados.rados.0.smithi070.stdout:725: finishing write tid 2 to smithi07085154-21 2024-09-14T03:50:09.654 INFO:tasks.rados.rados.0.smithi070.stdout:725: finishing write tid 3 to smithi07085154-21 2024-09-14T03:50:09.654 INFO:tasks.rados.rados.0.smithi070.stdout:725: finishing write tid 4 to smithi07085154-21 2024-09-14T03:50:09.654 INFO:tasks.rados.rados.0.smithi070.stdout:725: finishing write tid 5 to smithi07085154-21 2024-09-14T03:50:09.654 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 252 (ObjNum 242 snap 69 seq_num 242) dirty exists 2024-09-14T03:50:09.654 INFO:tasks.rados.rados.0.smithi070.stdout:725: left oid 21 (ObjNum 242 snap 69 seq_num 242) 2024-09-14T03:50:09.654 INFO:tasks.rados.rados.0.smithi070.stdout:728: finishing write tid 1 to smithi07085154-4 2024-09-14T03:50:09.655 INFO:tasks.rados.rados.0.smithi070.stdout:728: finishing write tid 2 to smithi07085154-4 2024-09-14T03:50:09.655 INFO:tasks.rados.rados.0.smithi070.stdout:728: finishing write tid 3 to smithi07085154-4 2024-09-14T03:50:09.655 INFO:tasks.rados.rados.0.smithi070.stdout:728: finishing write tid 4 to smithi07085154-4 2024-09-14T03:50:09.655 INFO:tasks.rados.rados.0.smithi070.stdout:728: finishing write tid 5 to smithi07085154-4 2024-09-14T03:50:09.655 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 327 (ObjNum 243 snap 69 seq_num 243) dirty exists 2024-09-14T03:50:09.655 INFO:tasks.rados.rados.0.smithi070.stdout:728: left oid 4 (ObjNum 243 snap 69 seq_num 243) 2024-09-14T03:50:09.655 INFO:tasks.rados.rados.0.smithi070.stdout:729: finishing copy_from racing read to smithi07085154-24 2024-09-14T03:50:09.655 INFO:tasks.rados.rados.0.smithi070.stdout:729: finishing copy_from to smithi07085154-24 2024-09-14T03:50:09.655 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 308 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-14T03:50:09.655 INFO:tasks.rados.rados.0.smithi070.stdout:730: finishing write tid 1 to smithi07085154-14 2024-09-14T03:50:09.656 INFO:tasks.rados.rados.0.smithi070.stdout:730: finishing write tid 2 to smithi07085154-14 2024-09-14T03:50:09.656 INFO:tasks.rados.rados.0.smithi070.stdout:730: finishing write tid 3 to smithi07085154-14 2024-09-14T03:50:09.656 INFO:tasks.rados.rados.0.smithi070.stdout:730: finishing write tid 4 to smithi07085154-14 2024-09-14T03:50:09.656 INFO:tasks.rados.rados.0.smithi070.stdout:730: finishing write tid 5 to smithi07085154-14 2024-09-14T03:50:09.656 INFO:tasks.rados.rados.0.smithi070.stdout:730: finishing write tid 6 to smithi07085154-14 2024-09-14T03:50:09.656 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 313 (ObjNum 244 snap 69 seq_num 244) dirty exists 2024-09-14T03:50:09.656 INFO:tasks.rados.rados.0.smithi070.stdout:730: left oid 14 (ObjNum 244 snap 69 seq_num 244) 2024-09-14T03:50:09.656 INFO:tasks.rados.rados.0.smithi070.stdout:731: finishing write tid 1 to smithi07085154-7 2024-09-14T03:50:09.656 INFO:tasks.rados.rados.0.smithi070.stdout:731: finishing write tid 2 to smithi07085154-7 2024-09-14T03:50:09.656 INFO:tasks.rados.rados.0.smithi070.stdout:731: finishing write tid 3 to smithi07085154-7 2024-09-14T03:50:09.656 INFO:tasks.rados.rados.0.smithi070.stdout:731: finishing write tid 4 to smithi07085154-7 2024-09-14T03:50:09.657 INFO:tasks.rados.rados.0.smithi070.stdout:731: finishing write tid 5 to smithi07085154-7 2024-09-14T03:50:09.657 INFO:tasks.rados.rados.0.smithi070.stdout:731: finishing write tid 6 to smithi07085154-7 2024-09-14T03:50:09.657 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 332 (ObjNum 245 snap 69 seq_num 245) dirty exists 2024-09-14T03:50:09.657 INFO:tasks.rados.rados.0.smithi070.stdout:731: left oid 7 (ObjNum 245 snap 69 seq_num 245) 2024-09-14T03:50:09.657 INFO:tasks.rados.rados.0.smithi070.stdout:734: finishing write tid 1 to smithi07085154-43 2024-09-14T03:50:09.657 INFO:tasks.rados.rados.0.smithi070.stdout:733: finishing copy_from racing read to smithi07085154-42 2024-09-14T03:50:09.657 INFO:tasks.rados.rados.0.smithi070.stdout:733: finishing copy_from to smithi07085154-42 2024-09-14T03:50:09.657 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 313 (ObjNum 160 snap 47 seq_num 160) dirty exists 2024-09-14T03:50:09.657 INFO:tasks.rados.rados.0.smithi070.stdout:734: finishing write tid 2 to smithi07085154-43 2024-09-14T03:50:09.659 INFO:tasks.rados.rados.0.smithi070.stdout:724: done (9 left) 2024-09-14T03:50:09.659 INFO:tasks.rados.rados.0.smithi070.stdout:725: done (8 left) 2024-09-14T03:50:09.659 INFO:tasks.rados.rados.0.smithi070.stdout:728: done (7 left) 2024-09-14T03:50:09.659 INFO:tasks.rados.rados.0.smithi070.stdout:729: done (6 left) 2024-09-14T03:50:09.659 INFO:tasks.rados.rados.0.smithi070.stdout:730: done (5 left) 2024-09-14T03:50:09.659 INFO:tasks.rados.rados.0.smithi070.stdout:731: done (4 left) 2024-09-14T03:50:09.659 INFO:tasks.rados.rados.0.smithi070.stdout:732: done (3 left) 2024-09-14T03:50:09.659 INFO:tasks.rados.rados.0.smithi070.stdout:733: done (2 left) 2024-09-14T03:50:09.660 INFO:tasks.rados.rados.0.smithi070.stdout:736: setattr oid 13 current snap is 69 2024-09-14T03:50:09.660 INFO:tasks.rados.rados.0.smithi070.stdout:734: finishing write tid 3 to smithi07085154-43 2024-09-14T03:50:09.660 INFO:tasks.rados.rados.0.smithi070.stdout:734: finishing write tid 4 to smithi07085154-43 2024-09-14T03:50:09.660 INFO:tasks.rados.rados.0.smithi070.stdout:734: finishing write tid 5 to smithi07085154-43 2024-09-14T03:50:09.660 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 35 v 22 (ObjNum 129 snap 0 seq_num 2930351632) dirty exists 2024-09-14T03:50:09.660 INFO:tasks.rados.rados.0.smithi070.stdout:735: done (2 left) 2024-09-14T03:50:09.660 INFO:tasks.rados.rados.0.smithi070.stdout:737: snap_create 2024-09-14T03:50:09.660 INFO:tasks.rados.rados.0.smithi070.stdout:734: finishing write tid 6 to smithi07085154-43 2024-09-14T03:50:09.660 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 410 (ObjNum 246 snap 69 seq_num 246) dirty exists 2024-09-14T03:50:09.660 INFO:tasks.rados.rados.0.smithi070.stdout:734: left oid 43 (ObjNum 246 snap 69 seq_num 246) 2024-09-14T03:50:09.660 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 317 (ObjNum 138 snap 39 seq_num 138) dirty exists 2024-09-14T03:50:09.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:09 smithi070 ceph-mon[39288]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-14T03:50:09.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:09 smithi070 ceph-mon[39288]: Deploying daemon node-exporter.a on smithi070 2024-09-14T03:50:09.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:09 smithi070 ceph-mon[39288]: pgmap v54: 121 pgs: 121 active+clean; 221 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 3.4 MiB/s wr, 44 op/s 2024-09-14T03:50:09.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:09 smithi070 ceph-mon[39288]: osdmap e195: 8 total, 8 up, 8 in 2024-09-14T03:50:09.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:09 smithi070 ceph-mon[27495]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-14T03:50:09.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:09 smithi070 ceph-mon[27495]: Deploying daemon node-exporter.a on smithi070 2024-09-14T03:50:09.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:09 smithi070 ceph-mon[27495]: pgmap v54: 121 pgs: 121 active+clean; 221 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 3.4 MiB/s wr, 44 op/s 2024-09-14T03:50:09.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:09 smithi070 ceph-mon[27495]: osdmap e195: 8 total, 8 up, 8 in 2024-09-14T03:50:09.787 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:09 smithi070 podman[87748]: 2024-09-14 03:50:09.549399798 +0000 UTC m=+0.644409026 container cleanup 315c3f32399febe38ae8d87f5be4faf5d0f66b9f7f27cd6b1ae5582404a2ba04 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:50:09.787 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:09 smithi070 bash[87748]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a 2024-09-14T03:50:10.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:09 smithi094 ceph-mon[31053]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-14T03:50:10.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:09 smithi094 ceph-mon[31053]: Deploying daemon node-exporter.a on smithi070 2024-09-14T03:50:10.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:09 smithi094 ceph-mon[31053]: pgmap v54: 121 pgs: 121 active+clean; 221 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 3.4 MiB/s wr, 44 op/s 2024-09-14T03:50:10.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:09 smithi094 ceph-mon[31053]: osdmap e195: 8 total, 8 up, 8 in 2024-09-14T03:50:10.132 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:09 smithi070 podman[87765]: 2024-09-14 03:50:09.807618449 +0000 UTC m=+0.792915634 container remove 315c3f32399febe38ae8d87f5be4faf5d0f66b9f7f27cd6b1ae5582404a2ba04 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:50:10.132 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:09 smithi070 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-14T03:50:10.132 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:09 smithi070 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-14T03:50:10.133 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:09 smithi070 systemd[1]: Stopped Ceph node-exporter.a for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:50:10.133 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:09 smithi070 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@node-exporter.a.service: Consumed 2.242s CPU time. 2024-09-14T03:50:10.530 INFO:tasks.rados.rados.0.smithi070.stdout:734: done (2 left) 2024-09-14T03:50:10.530 INFO:tasks.rados.rados.0.smithi070.stdout:736: done (1 left) 2024-09-14T03:50:10.530 INFO:tasks.rados.rados.0.smithi070.stdout:737: done (0 left) 2024-09-14T03:50:10.530 INFO:tasks.rados.rados.0.smithi070.stdout:738: rollback oid 32 current snap is 70 2024-09-14T03:50:10.530 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 32 to 69 2024-09-14T03:50:10.530 INFO:tasks.rados.rados.0.smithi070.stdout:739: snap_remove snap 43 2024-09-14T03:50:10.535 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:50:10 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a[87599]: ts=2024-09-14T03:50:10.130Z caller=cluster.go:708 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000816239s 2024-09-14T03:50:10.536 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:10 smithi070 systemd[1]: Starting Ceph node-exporter.a for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:50:10.536 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:10 smithi070 bash[87897]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-09-14T03:50:10.928 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:10 smithi094 ceph-mon[31053]: osdmap e196: 8 total, 8 up, 8 in 2024-09-14T03:50:10.928 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:10 smithi094 ceph-mon[31053]: osdmap e197: 8 total, 8 up, 8 in 2024-09-14T03:50:11.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:10 smithi070 ceph-mon[39288]: osdmap e196: 8 total, 8 up, 8 in 2024-09-14T03:50:11.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:10 smithi070 ceph-mon[39288]: osdmap e197: 8 total, 8 up, 8 in 2024-09-14T03:50:11.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:10 smithi070 ceph-mon[27495]: osdmap e196: 8 total, 8 up, 8 in 2024-09-14T03:50:11.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:10 smithi070 ceph-mon[27495]: osdmap e197: 8 total, 8 up, 8 in 2024-09-14T03:50:11.235 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-09-14T03:50:11.535 INFO:tasks.rados.rados.0.smithi070.stdout:739: done (1 left) 2024-09-14T03:50:11.535 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 24 current snap is 70 2024-09-14T03:50:11.535 INFO:tasks.rados.rados.0.smithi070.stdout:740: seq_num 247 ranges {761856=540672,1302528=622592,1925120=581632} 2024-09-14T03:50:11.535 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:11 smithi070 bash[87897]: Getting image source signatures 2024-09-14T03:50:11.535 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:11 smithi070 bash[87897]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-09-14T03:50:11.536 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:11 smithi070 bash[87897]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-09-14T03:50:11.536 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:11 smithi070 bash[87897]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-09-14T03:50:11.542 INFO:tasks.rados.rados.0.smithi070.stdout:740: writing smithi07085154-24 from 761856 to 1302528 tid 1 2024-09-14T03:50:11.547 INFO:tasks.rados.rados.0.smithi070.stdout:740: writing smithi07085154-24 from 1302528 to 1925120 tid 2 2024-09-14T03:50:11.552 INFO:tasks.rados.rados.0.smithi070.stdout:740: writing smithi07085154-24 from 1925120 to 2506752 tid 3 2024-09-14T03:50:11.552 INFO:tasks.rados.rados.0.smithi070.stdout:741: copy_from oid 3 from oid 45 current snap is 70 2024-09-14T03:50:11.552 INFO:tasks.rados.rados.0.smithi070.stdout:738: finishing rollback tid 1 to smithi07085154-32 2024-09-14T03:50:11.552 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 316 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-09-14T03:50:11.552 INFO:tasks.rados.rados.0.smithi070.stdout:738: done (2 left) 2024-09-14T03:50:11.552 INFO:tasks.rados.rados.0.smithi070.stdout:742: snap_create 2024-09-14T03:50:11.555 INFO:tasks.rados.rados.0.smithi070.stdout:740: finishing write tid 1 to smithi07085154-24 2024-09-14T03:50:11.555 INFO:tasks.rados.rados.0.smithi070.stdout:740: finishing write tid 2 to smithi07085154-24 2024-09-14T03:50:11.555 INFO:tasks.rados.rados.0.smithi070.stdout:740: finishing write tid 3 to smithi07085154-24 2024-09-14T03:50:11.556 INFO:tasks.rados.rados.0.smithi070.stdout:740: finishing write tid 4 to smithi07085154-24 2024-09-14T03:50:11.561 INFO:tasks.rados.rados.0.smithi070.stdout:740: finishing write tid 5 to smithi07085154-24 2024-09-14T03:50:11.561 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 317 (ObjNum 247 snap 70 seq_num 247) dirty exists 2024-09-14T03:50:11.561 INFO:tasks.rados.rados.0.smithi070.stdout:740: left oid 24 (ObjNum 247 snap 70 seq_num 247) 2024-09-14T03:50:11.561 INFO:tasks.rados.rados.0.smithi070.stdout:741: finishing copy_from racing read to smithi07085154-3 2024-09-14T03:50:11.561 INFO:tasks.rados.rados.0.smithi070.stdout:741: finishing copy_from to smithi07085154-3 2024-09-14T03:50:11.561 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 3 v 11 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-14T03:50:11.822 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:50:11.822 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 starting - - - - 2024-09-14T03:50:11.822 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi094 *:3000 running (4m) 30s ago 4m 87.4M - 9.4.7 954c08fa6188 846770f48257 2024-09-14T03:50:11.822 INFO:teuthology.orchestra.run.smithi070.stdout:mds.a smithi070 running (6m) 31s ago 6m 18.4M - 17.2.7-1629-geb16b97b ab51d1bebd12 52e0dbb7be50 2024-09-14T03:50:11.822 INFO:teuthology.orchestra.run.smithi070.stdout:mds.b smithi094 running (6m) 30s ago 6m 14.5M - 17.2.7-1629-geb16b97b ab51d1bebd12 e3b5e3e00189 2024-09-14T03:50:11.822 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi094 *:8443,9283 running (46s) 30s ago 9m 497M - 19.3.0-4907-ga2782315 8801d2529e43 5aadf84f680b 2024-09-14T03:50:11.822 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:9283,8765 running (11m) 31s ago 11m 270M - 17.2.7-1629-geb16b97b ab51d1bebd12 dc71810017b9 2024-09-14T03:50:11.823 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (11m) 31s ago 11m 55.2M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 12d19db91f54 2024-09-14T03:50:11.823 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi094 running (10m) 30s ago 10m 46.1M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 90ea2e164909 2024-09-14T03:50:11.823 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (10m) 31s ago 10m 46.2M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 44ee80c29ee0 2024-09-14T03:50:11.823 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (4m) 31s ago 4m 16.6M - 1.5.0 0da6a335fe13 315c3f32399f 2024-09-14T03:50:11.823 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi094 *:9100 running (4m) 30s ago 4m 15.7M - 1.5.0 0da6a335fe13 3dcf1723ea35 2024-09-14T03:50:11.823 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (9m) 31s ago 9m 105M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 741c06b6adbf 2024-09-14T03:50:11.823 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (9m) 31s ago 9m 110M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 af459ea7ef68 2024-09-14T03:50:11.823 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (8m) 31s ago 8m 93.8M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 109b66e4e60f 2024-09-14T03:50:11.823 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (8m) 31s ago 8m 87.9M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 2e4199b58392 2024-09-14T03:50:11.823 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi094 running (7m) 30s ago 7m 112M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 52b64527e24c 2024-09-14T03:50:11.823 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi094 running (7m) 30s ago 7m 90.4M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 b543f152ba87 2024-09-14T03:50:11.823 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi094 running (7m) 30s ago 7m 68.9M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 e66b592ac940 2024-09-14T03:50:11.823 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi094 running (6m) 30s ago 6m 88.0M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 037c6208e9ce 2024-09-14T03:50:11.824 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi094 *:9095 running (3m) 30s ago 5m 59.0M - 2.43.0 a07b618ecd1d 9c88e2923e9d 2024-09-14T03:50:11.824 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:11 smithi070 ceph-mon[27495]: pgmap v58: 121 pgs: 121 active+clean; 221 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:50:11.824 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:11 smithi070 ceph-mon[27495]: osdmap e198: 8 total, 8 up, 8 in 2024-09-14T03:50:11.824 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:11 smithi070 ceph-mon[39288]: pgmap v58: 121 pgs: 121 active+clean; 221 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:50:11.824 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:11 smithi070 ceph-mon[39288]: osdmap e198: 8 total, 8 up, 8 in 2024-09-14T03:50:12.026 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:50:11 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:50:11] "GET /metrics HTTP/1.1" 200 36201 "" "Prometheus/2.43.0" 2024-09-14T03:50:12.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:11 smithi094 ceph-mon[31053]: pgmap v58: 121 pgs: 121 active+clean; 221 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:50:12.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:11 smithi094 ceph-mon[31053]: osdmap e198: 8 total, 8 up, 8 in 2024-09-14T03:50:12.191 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:50:12.191 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-09-14T03:50:12.191 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 3 2024-09-14T03:50:12.191 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:50:12.191 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-09-14T03:50:12.191 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 1, 2024-09-14T03:50:12.191 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 1 2024-09-14T03:50:12.192 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:50:12.192 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-09-14T03:50:12.192 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 8 2024-09-14T03:50:12.192 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:50:12.192 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": { 2024-09-14T03:50:12.192 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:50:12.192 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:50:12.192 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-09-14T03:50:12.192 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 14, 2024-09-14T03:50:12.192 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 1 2024-09-14T03:50:12.192 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-09-14T03:50:12.192 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:50:12.488 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:50:12.488 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:50:12.488 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-09-14T03:50:12.488 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:50:12.488 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [], 2024-09-14T03:50:12.488 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "1/20 daemons upgraded", 2024-09-14T03:50:12.488 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "", 2024-09-14T03:50:12.488 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-09-14T03:50:12.489 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:50:12.557 INFO:tasks.rados.rados.0.smithi070.stdout:740: done (2 left) 2024-09-14T03:50:12.557 INFO:tasks.rados.rados.0.smithi070.stdout:741: done (1 left) 2024-09-14T03:50:12.557 INFO:tasks.rados.rados.0.smithi070.stdout:742: done (0 left) 2024-09-14T03:50:12.557 INFO:tasks.rados.rados.0.smithi070.stdout:743: read oid 39 snap -1 2024-09-14T03:50:12.557 INFO:tasks.rados.rados.0.smithi070.stdout:743: expect (ObjNum 211 snap 60 seq_num 211) 2024-09-14T03:50:12.557 INFO:tasks.rados.rados.0.smithi070.stdout:744: rollback oid 37 current snap is 71 2024-09-14T03:50:12.557 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 37 to 47 2024-09-14T03:50:12.557 INFO:tasks.rados.rados.0.smithi070.stdout:745: copy_from oid 12 from oid 22 current snap is 71 2024-09-14T03:50:12.557 INFO:tasks.rados.rados.0.smithi070.stdout:746: snap_remove snap 20 2024-09-14T03:50:12.745 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:12 smithi070 ceph-mon[27495]: from='client.15117 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:12.745 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:12 smithi070 ceph-mon[27495]: from='client.25028 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:12.745 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:12 smithi070 ceph-mon[27495]: from='client.15129 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:12.746 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:12 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/4216730788' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:12.746 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:12 smithi070 ceph-mon[39288]: from='client.15117 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:12.746 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:12 smithi070 ceph-mon[39288]: from='client.25028 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:12.746 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:12 smithi070 ceph-mon[39288]: from='client.15129 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:12.746 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:12 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/4216730788' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:13.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:12 smithi094 ceph-mon[31053]: from='client.15117 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:13.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:12 smithi094 ceph-mon[31053]: from='client.25028 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:13.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:12 smithi094 ceph-mon[31053]: from='client.15129 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:13.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:12 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/4216730788' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:13.035 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:12 smithi070 bash[87897]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-09-14T03:50:13.036 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:12 smithi070 bash[87897]: Writing manifest to image destination 2024-09-14T03:50:13.535 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 podman[87897]: 2024-09-14 03:50:13.209479787 +0000 UTC m=+2.959828591 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-09-14T03:50:13.536 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 podman[87897]: 2024-09-14 03:50:13.324826716 +0000 UTC m=+3.075175522 container create e316760a30b3120bdddb63db6db975934176b86e50fb49f37b7200085fd1dee9 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:50:13.561 INFO:tasks.rados.rados.0.smithi070.stdout:746: done (3 left) 2024-09-14T03:50:13.561 INFO:tasks.rados.rados.0.smithi070.stdout:747: delete oid 48 current snap is 71 2024-09-14T03:50:13.561 INFO:tasks.rados.rados.0.smithi070.stdout:744: finishing rollback tid 1 to smithi07085154-37 2024-09-14T03:50:13.561 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 37 v 284 (ObjNum 49 snap 0 seq_num 4214421087) dirty dne 2024-09-14T03:50:13.561 INFO:tasks.rados.rados.0.smithi070.stdout:743: expect (ObjNum 211 snap 60 seq_num 211) 2024-09-14T03:50:13.645 INFO:tasks.rados.rados.0.smithi070.stdout:745: finishing copy_from racing read to smithi07085154-12 2024-09-14T03:50:13.645 INFO:tasks.rados.rados.0.smithi070.stdout:745: finishing copy_from to smithi07085154-12 2024-09-14T03:50:13.645 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 308 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-14T03:50:13.646 INFO:tasks.rados.rados.0.smithi070.stdout:743: done (3 left) 2024-09-14T03:50:13.646 INFO:tasks.rados.rados.0.smithi070.stdout:744: done (2 left) 2024-09-14T03:50:13.646 INFO:tasks.rados.rados.0.smithi070.stdout:745: done (1 left) 2024-09-14T03:50:13.646 INFO:tasks.rados.rados.0.smithi070.stdout:747: done (0 left) 2024-09-14T03:50:13.646 INFO:tasks.rados.rados.0.smithi070.stdout:748: snap_create 2024-09-14T03:50:14.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:13 smithi094 ceph-mon[31053]: from='client.15141 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:14.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:13 smithi094 ceph-mon[31053]: pgmap v60: 121 pgs: 121 active+clean; 243 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 488 KiB/s rd, 3.7 MiB/s wr, 27 op/s 2024-09-14T03:50:14.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:13 smithi094 ceph-mon[31053]: osdmap e199: 8 total, 8 up, 8 in 2024-09-14T03:50:14.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-mon[27495]: from='client.15141 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:14.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-mon[27495]: pgmap v60: 121 pgs: 121 active+clean; 243 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 488 KiB/s rd, 3.7 MiB/s wr, 27 op/s 2024-09-14T03:50:14.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-mon[27495]: osdmap e199: 8 total, 8 up, 8 in 2024-09-14T03:50:14.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-mon[39288]: from='client.15141 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:14.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-mon[39288]: pgmap v60: 121 pgs: 121 active+clean; 243 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 488 KiB/s rd, 3.7 MiB/s wr, 27 op/s 2024-09-14T03:50:14.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-mon[39288]: osdmap e199: 8 total, 8 up, 8 in 2024-09-14T03:50:14.037 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 podman[87897]: 2024-09-14 03:50:13.585042412 +0000 UTC m=+3.335391225 container init e316760a30b3120bdddb63db6db975934176b86e50fb49f37b7200085fd1dee9 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:50:14.037 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 podman[87897]: 2024-09-14 03:50:13.590492066 +0000 UTC m=+3.340840872 container start e316760a30b3120bdddb63db6db975934176b86e50fb49f37b7200085fd1dee9 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:50:14.037 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.593Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-09-14T03:50:14.037 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.593Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-09-14T03:50:14.037 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.593Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-14T03:50:14.037 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.593Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-14T03:50:14.038 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.594Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-14T03:50:14.038 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.594Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-14T03:50:14.038 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.594Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-14T03:50:14.038 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.594Z caller=node_exporter.go:117 level=info collector=arp 2024-09-14T03:50:14.038 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.594Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-14T03:50:14.038 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.594Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-14T03:50:14.038 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.594Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-14T03:50:14.038 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.594Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-14T03:50:14.038 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.594Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-14T03:50:14.038 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.594Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-14T03:50:14.038 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.594Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-14T03:50:14.038 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.594Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-14T03:50:14.039 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.594Z caller=node_exporter.go:117 level=info collector=edac 2024-09-14T03:50:14.039 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.594Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-14T03:50:14.039 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.594Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-14T03:50:14.039 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.595Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-14T03:50:14.039 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.595Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-14T03:50:14.039 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.595Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-14T03:50:14.039 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.595Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-14T03:50:14.039 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.595Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-14T03:50:14.039 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.595Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-14T03:50:14.039 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.595Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-14T03:50:14.039 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.595Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-14T03:50:14.039 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.595Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-14T03:50:14.039 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.595Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-14T03:50:14.039 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.595Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-14T03:50:14.039 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.595Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-14T03:50:14.040 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.595Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-14T03:50:14.040 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.595Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-14T03:50:14.040 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.595Z caller=node_exporter.go:117 level=info collector=os 2024-09-14T03:50:14.040 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.595Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-14T03:50:14.040 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.595Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-14T03:50:14.040 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.595Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-14T03:50:14.040 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.595Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-14T03:50:14.040 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.595Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-14T03:50:14.040 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.595Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-14T03:50:14.040 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.595Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-14T03:50:14.040 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.595Z caller=node_exporter.go:117 level=info collector=stat 2024-09-14T03:50:14.040 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.595Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-14T03:50:14.040 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.595Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-14T03:50:14.041 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.595Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-14T03:50:14.041 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.595Z caller=node_exporter.go:117 level=info collector=time 2024-09-14T03:50:14.041 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.595Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-14T03:50:14.041 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.595Z caller=node_exporter.go:117 level=info collector=uname 2024-09-14T03:50:14.041 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.595Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-14T03:50:14.041 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.595Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-14T03:50:14.041 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.595Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-14T03:50:14.041 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.595Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-09-14T03:50:14.041 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a[88085]: ts=2024-09-14T03:50:13.595Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-14T03:50:14.041 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 bash[87897]: e316760a30b3120bdddb63db6db975934176b86e50fb49f37b7200085fd1dee9 2024-09-14T03:50:14.041 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 03:50:13 smithi070 systemd[1]: Started Ceph node-exporter.a for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:50:14.566 INFO:tasks.rados.rados.0.smithi070.stdout:748: done (0 left) 2024-09-14T03:50:14.567 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 26 current snap is 72 2024-09-14T03:50:14.567 INFO:tasks.rados.rados.0.smithi070.stdout:749: seq_num 249 ranges {3350528=573440,3923968=794624,4718592=409600} 2024-09-14T03:50:14.587 INFO:tasks.rados.rados.0.smithi070.stdout:749: writing smithi07085154-26 from 3350528 to 3923968 tid 1 2024-09-14T03:50:14.591 INFO:tasks.rados.rados.0.smithi070.stdout:749: writing smithi07085154-26 from 3923968 to 4718592 tid 2 2024-09-14T03:50:14.594 INFO:tasks.rados.rados.0.smithi070.stdout:749: writing smithi07085154-26 from 4718592 to 5128192 tid 3 2024-09-14T03:50:14.594 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 6 current snap is 72 2024-09-14T03:50:14.594 INFO:tasks.rados.rados.0.smithi070.stdout:749: finishing write tid 1 to smithi07085154-26 2024-09-14T03:50:14.594 INFO:tasks.rados.rados.0.smithi070.stdout:750: seq_num 250 ranges {0=589824,589824=679936,1269760=409600,1679360=114688} 2024-09-14T03:50:14.598 INFO:tasks.rados.rados.0.smithi070.stdout:750: writing smithi07085154-6 from 0 to 589824 tid 1 2024-09-14T03:50:14.602 INFO:tasks.rados.rados.0.smithi070.stdout:750: writing smithi07085154-6 from 589824 to 1269760 tid 2 2024-09-14T03:50:14.604 INFO:tasks.rados.rados.0.smithi070.stdout:750: writing smithi07085154-6 from 1269760 to 1679360 tid 3 2024-09-14T03:50:14.606 INFO:tasks.rados.rados.0.smithi070.stdout:750: writing smithi07085154-6 from 1679360 to 1794048 tid 4 2024-09-14T03:50:14.606 INFO:tasks.rados.rados.0.smithi070.stdout:751: copy_from oid 41 from oid 39 current snap is 72 2024-09-14T03:50:14.606 INFO:tasks.rados.rados.0.smithi070.stdout:749: finishing write tid 2 to smithi07085154-26 2024-09-14T03:50:14.606 INFO:tasks.rados.rados.0.smithi070.stdout:749: finishing write tid 3 to smithi07085154-26 2024-09-14T03:50:14.606 INFO:tasks.rados.rados.0.smithi070.stdout:749: finishing write tid 4 to smithi07085154-26 2024-09-14T03:50:14.606 INFO:tasks.rados.rados.0.smithi070.stdout:749: finishing write tid 5 to smithi07085154-26 2024-09-14T03:50:14.606 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 594 (ObjNum 249 snap 72 seq_num 249) dirty exists 2024-09-14T03:50:14.606 INFO:tasks.rados.rados.0.smithi070.stdout:749: left oid 26 (ObjNum 249 snap 72 seq_num 249) 2024-09-14T03:50:14.606 INFO:tasks.rados.rados.0.smithi070.stdout:750: finishing write tid 1 to smithi07085154-6 2024-09-14T03:50:14.606 INFO:tasks.rados.rados.0.smithi070.stdout:749: done (2 left) 2024-09-14T03:50:14.606 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 19 current snap is 72 2024-09-14T03:50:14.606 INFO:tasks.rados.rados.0.smithi070.stdout:750: finishing write tid 2 to smithi07085154-6 2024-09-14T03:50:14.607 INFO:tasks.rados.rados.0.smithi070.stdout:752: seq_num 251 ranges {3506176=606208,4112384=614400,4726784=589824} 2024-09-14T03:50:14.628 INFO:tasks.rados.rados.0.smithi070.stdout:752: writing smithi07085154-19 from 3506176 to 4112384 tid 1 2024-09-14T03:50:14.632 INFO:tasks.rados.rados.0.smithi070.stdout:752: writing smithi07085154-19 from 4112384 to 4726784 tid 2 2024-09-14T03:50:14.635 INFO:tasks.rados.rados.0.smithi070.stdout:752: writing smithi07085154-19 from 4726784 to 5316608 tid 3 2024-09-14T03:50:14.635 INFO:tasks.rados.rados.0.smithi070.stdout:753: delete oid 46 current snap is 72 2024-09-14T03:50:14.636 INFO:tasks.rados.rados.0.smithi070.stdout:750: finishing write tid 3 to smithi07085154-6 2024-09-14T03:50:14.636 INFO:tasks.rados.rados.0.smithi070.stdout:750: finishing write tid 4 to smithi07085154-6 2024-09-14T03:50:14.636 INFO:tasks.rados.rados.0.smithi070.stdout:750: finishing write tid 5 to smithi07085154-6 2024-09-14T03:50:14.636 INFO:tasks.rados.rados.0.smithi070.stdout:750: finishing write tid 6 to smithi07085154-6 2024-09-14T03:50:14.636 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 314 (ObjNum 250 snap 72 seq_num 250) dirty exists 2024-09-14T03:50:14.636 INFO:tasks.rados.rados.0.smithi070.stdout:750: left oid 6 (ObjNum 250 snap 72 seq_num 250) 2024-09-14T03:50:14.636 INFO:tasks.rados.rados.0.smithi070.stdout:751: finishing copy_from racing read to smithi07085154-41 2024-09-14T03:50:14.636 INFO:tasks.rados.rados.0.smithi070.stdout:751: finishing copy_from to smithi07085154-41 2024-09-14T03:50:14.636 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 53 (ObjNum 211 snap 60 seq_num 211) dirty exists 2024-09-14T03:50:14.636 INFO:tasks.rados.rados.0.smithi070.stdout:752: finishing write tid 1 to smithi07085154-19 2024-09-14T03:50:14.636 INFO:tasks.rados.rados.0.smithi070.stdout:752: finishing write tid 2 to smithi07085154-19 2024-09-14T03:50:14.637 INFO:tasks.rados.rados.0.smithi070.stdout:750: done (3 left) 2024-09-14T03:50:14.637 INFO:tasks.rados.rados.0.smithi070.stdout:751: done (2 left) 2024-09-14T03:50:14.637 INFO:tasks.rados.rados.0.smithi070.stdout:753: done (1 left) 2024-09-14T03:50:14.637 INFO:tasks.rados.rados.0.smithi070.stdout:754: snap_remove snap 64 2024-09-14T03:50:15.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:14 smithi094 ceph-mon[31053]: osdmap e200: 8 total, 8 up, 8 in 2024-09-14T03:50:15.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:14 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:15.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:14 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:15.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:14 smithi094 ceph-mon[31053]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:50:15.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:14 smithi094 ceph-mon[31053]: Deploying daemon prometheus.a on smithi094 2024-09-14T03:50:15.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:14 smithi070 ceph-mon[27495]: osdmap e200: 8 total, 8 up, 8 in 2024-09-14T03:50:15.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:14 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:15.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:14 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:15.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:14 smithi070 ceph-mon[27495]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:50:15.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:14 smithi070 ceph-mon[27495]: Deploying daemon prometheus.a on smithi094 2024-09-14T03:50:15.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:14 smithi070 ceph-mon[39288]: osdmap e200: 8 total, 8 up, 8 in 2024-09-14T03:50:15.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:14 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:15.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:14 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:15.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:14 smithi070 ceph-mon[39288]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:50:15.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:14 smithi070 ceph-mon[39288]: Deploying daemon prometheus.a on smithi094 2024-09-14T03:50:15.532 INFO:tasks.rados.rados.0.smithi070.stdout:752: finishing write tid 3 to smithi07085154-19 2024-09-14T03:50:15.532 INFO:tasks.rados.rados.0.smithi070.stdout:752: finishing write tid 4 to smithi07085154-19 2024-09-14T03:50:15.533 INFO:tasks.rados.rados.0.smithi070.stdout:752: finishing write tid 5 to smithi07085154-19 2024-09-14T03:50:15.533 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 19 v 345 (ObjNum 251 snap 72 seq_num 251) dirty exists 2024-09-14T03:50:15.533 INFO:tasks.rados.rados.0.smithi070.stdout:752: left oid 19 (ObjNum 251 snap 72 seq_num 251) 2024-09-14T03:50:15.533 INFO:tasks.rados.rados.0.smithi070.stdout:752: done (1 left) 2024-09-14T03:50:15.533 INFO:tasks.rados.rados.0.smithi070.stdout:754: done (0 left) 2024-09-14T03:50:15.533 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 29 current snap is 72 2024-09-14T03:50:15.533 INFO:tasks.rados.rados.0.smithi070.stdout:755: seq_num 253 ranges {2310144=647168,2957312=786432,3743744=417792} 2024-09-14T03:50:15.550 INFO:tasks.rados.rados.0.smithi070.stdout:755: writing smithi07085154-29 from 2310144 to 2957312 tid 1 2024-09-14T03:50:15.554 INFO:tasks.rados.rados.0.smithi070.stdout:755: writing smithi07085154-29 from 2957312 to 3743744 tid 2 2024-09-14T03:50:15.557 INFO:tasks.rados.rados.0.smithi070.stdout:755: writing smithi07085154-29 from 3743744 to 4161536 tid 3 2024-09-14T03:50:15.557 INFO:tasks.rados.rados.0.smithi070.stdout:756: read oid 3 snap -1 2024-09-14T03:50:15.557 INFO:tasks.rados.rados.0.smithi070.stdout:756: expect (ObjNum 49 snap 0 seq_num 49) 2024-09-14T03:50:15.557 INFO:tasks.rados.rados.0.smithi070.stdout:757: snap_create 2024-09-14T03:50:15.557 INFO:tasks.rados.rados.0.smithi070.stdout:755: finishing write tid 1 to smithi07085154-29 2024-09-14T03:50:15.560 INFO:tasks.rados.rados.0.smithi070.stdout:755: finishing write tid 2 to smithi07085154-29 2024-09-14T03:50:15.561 INFO:tasks.rados.rados.0.smithi070.stdout:755: finishing write tid 3 to smithi07085154-29 2024-09-14T03:50:15.562 INFO:tasks.rados.rados.0.smithi070.stdout:755: finishing write tid 4 to smithi07085154-29 2024-09-14T03:50:15.563 INFO:tasks.rados.rados.0.smithi070.stdout:756: expect (ObjNum 49 snap 0 seq_num 49) 2024-09-14T03:50:15.585 INFO:tasks.rados.rados.0.smithi070.stdout:755: finishing write tid 5 to smithi07085154-29 2024-09-14T03:50:15.585 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 356 (ObjNum 253 snap 72 seq_num 253) dirty exists 2024-09-14T03:50:15.585 INFO:tasks.rados.rados.0.smithi070.stdout:755: left oid 29 (ObjNum 253 snap 72 seq_num 253) 2024-09-14T03:50:16.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:15 smithi094 ceph-mon[31053]: pgmap v63: 121 pgs: 121 active+clean; 243 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 487 KiB/s rd, 3.7 MiB/s wr, 27 op/s 2024-09-14T03:50:16.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:15 smithi094 ceph-mon[31053]: osdmap e201: 8 total, 8 up, 8 in 2024-09-14T03:50:16.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:15 smithi094 ceph-mon[31053]: osdmap e202: 8 total, 8 up, 8 in 2024-09-14T03:50:16.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:15 smithi070 ceph-mon[27495]: pgmap v63: 121 pgs: 121 active+clean; 243 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 487 KiB/s rd, 3.7 MiB/s wr, 27 op/s 2024-09-14T03:50:16.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:15 smithi070 ceph-mon[27495]: osdmap e201: 8 total, 8 up, 8 in 2024-09-14T03:50:16.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:15 smithi070 ceph-mon[27495]: osdmap e202: 8 total, 8 up, 8 in 2024-09-14T03:50:16.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:15 smithi070 ceph-mon[39288]: pgmap v63: 121 pgs: 121 active+clean; 243 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 487 KiB/s rd, 3.7 MiB/s wr, 27 op/s 2024-09-14T03:50:16.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:15 smithi070 ceph-mon[39288]: osdmap e201: 8 total, 8 up, 8 in 2024-09-14T03:50:16.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:15 smithi070 ceph-mon[39288]: osdmap e202: 8 total, 8 up, 8 in 2024-09-14T03:50:16.533 INFO:tasks.rados.rados.0.smithi070.stdout:755: done (2 left) 2024-09-14T03:50:16.533 INFO:tasks.rados.rados.0.smithi070.stdout:756: done (1 left) 2024-09-14T03:50:16.533 INFO:tasks.rados.rados.0.smithi070.stdout:757: done (0 left) 2024-09-14T03:50:16.533 INFO:tasks.rados.rados.0.smithi070.stdout:758: setattr oid 33 current snap is 73 2024-09-14T03:50:16.535 INFO:tasks.rados.rados.0.smithi070.stdout:759: read oid 6 snap -1 2024-09-14T03:50:16.535 INFO:tasks.rados.rados.0.smithi070.stdout:759: expect (ObjNum 250 snap 72 seq_num 250) 2024-09-14T03:50:16.535 INFO:tasks.rados.rados.0.smithi070.stdout:760: delete oid 27 current snap is 73 2024-09-14T03:50:16.540 INFO:tasks.rados.rados.0.smithi070.stdout:760: done (2 left) 2024-09-14T03:50:16.540 INFO:tasks.rados.rados.0.smithi070.stdout:761: rollback oid 23 current snap is 73 2024-09-14T03:50:16.540 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 23 to 41 2024-09-14T03:50:16.540 INFO:tasks.rados.rados.0.smithi070.stdout:762: read oid 39 snap -1 2024-09-14T03:50:16.540 INFO:tasks.rados.rados.0.smithi070.stdout:762: expect (ObjNum 211 snap 60 seq_num 211) 2024-09-14T03:50:16.540 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 7 current snap is 73 2024-09-14T03:50:16.540 INFO:tasks.rados.rados.0.smithi070.stdout:763: seq_num 255 ranges {1712128=679936,2392064=548864,2940928=598016,3538944=57344} 2024-09-14T03:50:16.553 INFO:tasks.rados.rados.0.smithi070.stdout:763: writing smithi07085154-7 from 1712128 to 2392064 tid 1 2024-09-14T03:50:16.556 INFO:tasks.rados.rados.0.smithi070.stdout:763: writing smithi07085154-7 from 2392064 to 2940928 tid 2 2024-09-14T03:50:16.560 INFO:tasks.rados.rados.0.smithi070.stdout:763: writing smithi07085154-7 from 2940928 to 3538944 tid 3 2024-09-14T03:50:16.560 INFO:tasks.rados.rados.0.smithi070.stdout:763: writing smithi07085154-7 from 3538944 to 3596288 tid 4 2024-09-14T03:50:16.561 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 12 current snap is 73 2024-09-14T03:50:16.561 INFO:tasks.rados.rados.0.smithi070.stdout:764: seq_num 256 ranges {761856=696320,1458176=434176,1892352=770048} 2024-09-14T03:50:16.569 INFO:tasks.rados.rados.0.smithi070.stdout:764: writing smithi07085154-12 from 761856 to 1458176 tid 1 2024-09-14T03:50:16.571 INFO:tasks.rados.rados.0.smithi070.stdout:764: writing smithi07085154-12 from 1458176 to 1892352 tid 2 2024-09-14T03:50:16.576 INFO:tasks.rados.rados.0.smithi070.stdout:764: writing smithi07085154-12 from 1892352 to 2662400 tid 3 2024-09-14T03:50:16.576 INFO:tasks.rados.rados.0.smithi070.stdout:765: snap_create 2024-09-14T03:50:16.576 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 33 v 240 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-14T03:50:16.576 INFO:tasks.rados.rados.0.smithi070.stdout:759: expect (ObjNum 250 snap 72 seq_num 250) 2024-09-14T03:50:16.622 INFO:tasks.rados.rados.0.smithi070.stdout:761: finishing rollback tid 0 to smithi07085154-23 2024-09-14T03:50:16.622 INFO:tasks.rados.rados.0.smithi070.stdout:761: finishing rollback tid 1 to smithi07085154-23 2024-09-14T03:50:16.622 INFO:tasks.rados.rados.0.smithi070.stdout:761: finishing rollback tid 2 to smithi07085154-23 2024-09-14T03:50:16.622 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 320 (ObjNum 117 snap 28 seq_num 117) dirty exists 2024-09-14T03:50:16.623 INFO:tasks.rados.rados.0.smithi070.stdout:762: expect (ObjNum 211 snap 60 seq_num 211) 2024-09-14T03:50:16.695 INFO:tasks.rados.rados.0.smithi070.stdout:763: finishing write tid 1 to smithi07085154-7 2024-09-14T03:50:16.695 INFO:tasks.rados.rados.0.smithi070.stdout:763: finishing write tid 2 to smithi07085154-7 2024-09-14T03:50:16.695 INFO:tasks.rados.rados.0.smithi070.stdout:763: finishing write tid 3 to smithi07085154-7 2024-09-14T03:50:16.695 INFO:tasks.rados.rados.0.smithi070.stdout:763: finishing write tid 4 to smithi07085154-7 2024-09-14T03:50:16.695 INFO:tasks.rados.rados.0.smithi070.stdout:763: finishing write tid 5 to smithi07085154-7 2024-09-14T03:50:16.696 INFO:tasks.rados.rados.0.smithi070.stdout:763: finishing write tid 6 to smithi07085154-7 2024-09-14T03:50:16.696 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 337 (ObjNum 255 snap 73 seq_num 255) dirty exists 2024-09-14T03:50:16.696 INFO:tasks.rados.rados.0.smithi070.stdout:763: left oid 7 (ObjNum 255 snap 73 seq_num 255) 2024-09-14T03:50:16.696 INFO:tasks.rados.rados.0.smithi070.stdout:764: finishing write tid 1 to smithi07085154-12 2024-09-14T03:50:16.696 INFO:tasks.rados.rados.0.smithi070.stdout:764: finishing write tid 2 to smithi07085154-12 2024-09-14T03:50:16.696 INFO:tasks.rados.rados.0.smithi070.stdout:764: finishing write tid 3 to smithi07085154-12 2024-09-14T03:50:16.696 INFO:tasks.rados.rados.0.smithi070.stdout:764: finishing write tid 4 to smithi07085154-12 2024-09-14T03:50:16.696 INFO:tasks.rados.rados.0.smithi070.stdout:764: finishing write tid 5 to smithi07085154-12 2024-09-14T03:50:16.696 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 312 (ObjNum 256 snap 73 seq_num 256) dirty exists 2024-09-14T03:50:16.696 INFO:tasks.rados.rados.0.smithi070.stdout:764: left oid 12 (ObjNum 256 snap 73 seq_num 256) 2024-09-14T03:50:16.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:16 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[27747]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:50:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:50:17.538 INFO:tasks.rados.rados.0.smithi070.stdout:758: done (6 left) 2024-09-14T03:50:17.538 INFO:tasks.rados.rados.0.smithi070.stdout:759: done (5 left) 2024-09-14T03:50:17.539 INFO:tasks.rados.rados.0.smithi070.stdout:761: done (4 left) 2024-09-14T03:50:17.539 INFO:tasks.rados.rados.0.smithi070.stdout:762: done (3 left) 2024-09-14T03:50:17.539 INFO:tasks.rados.rados.0.smithi070.stdout:763: done (2 left) 2024-09-14T03:50:17.539 INFO:tasks.rados.rados.0.smithi070.stdout:764: done (1 left) 2024-09-14T03:50:17.539 INFO:tasks.rados.rados.0.smithi070.stdout:765: done (0 left) 2024-09-14T03:50:17.539 INFO:tasks.rados.rados.0.smithi070.stdout:766: delete oid 35 current snap is 74 2024-09-14T03:50:17.542 INFO:tasks.rados.rados.0.smithi070.stdout:766: done (0 left) 2024-09-14T03:50:17.543 INFO:tasks.rados.rados.0.smithi070.stdout:767: snap_create 2024-09-14T03:50:17.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:17 smithi070 ceph-mon[27495]: osdmap e203: 8 total, 8 up, 8 in 2024-09-14T03:50:17.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:17 smithi070 ceph-mon[27495]: pgmap v67: 121 pgs: 121 active+clean; 240 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.1 MiB/s wr, 10 op/s 2024-09-14T03:50:17.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:17 smithi070 ceph-mon[39288]: osdmap e203: 8 total, 8 up, 8 in 2024-09-14T03:50:17.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:17 smithi070 ceph-mon[39288]: pgmap v67: 121 pgs: 121 active+clean; 240 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.1 MiB/s wr, 10 op/s 2024-09-14T03:50:18.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:17 smithi094 ceph-mon[31053]: osdmap e203: 8 total, 8 up, 8 in 2024-09-14T03:50:18.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:17 smithi094 ceph-mon[31053]: pgmap v67: 121 pgs: 121 active+clean; 240 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.1 MiB/s wr, 10 op/s 2024-09-14T03:50:18.535 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 03:50:18 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a[87599]: ts=2024-09-14T03:50:18.133Z caller=cluster.go:700 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003941583s 2024-09-14T03:50:18.543 INFO:tasks.rados.rados.0.smithi070.stdout:767: done (0 left) 2024-09-14T03:50:18.543 INFO:tasks.rados.rados.0.smithi070.stdout:768: snap_create 2024-09-14T03:50:19.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:18 smithi094 ceph-mon[31053]: osdmap e204: 8 total, 8 up, 8 in 2024-09-14T03:50:19.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:18 smithi070 ceph-mon[27495]: osdmap e204: 8 total, 8 up, 8 in 2024-09-14T03:50:19.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:18 smithi070 ceph-mon[39288]: osdmap e204: 8 total, 8 up, 8 in 2024-09-14T03:50:19.544 INFO:tasks.rados.rados.0.smithi070.stdout:768: done (0 left) 2024-09-14T03:50:19.544 INFO:tasks.rados.rados.0.smithi070.stdout:769: read oid 22 snap -1 2024-09-14T03:50:19.544 INFO:tasks.rados.rados.0.smithi070.stdout:769: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-14T03:50:19.545 INFO:tasks.rados.rados.0.smithi070.stdout:770: copy_from oid 40 from oid 5 current snap is 76 2024-09-14T03:50:19.545 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 20 current snap is 76 2024-09-14T03:50:19.545 INFO:tasks.rados.rados.0.smithi070.stdout:771: seq_num 258 ranges {1261568=729088,1990656=606208,2596864=598016} 2024-09-14T03:50:19.555 INFO:tasks.rados.rados.0.smithi070.stdout:771: writing smithi07085154-20 from 1261568 to 1990656 tid 1 2024-09-14T03:50:19.559 INFO:tasks.rados.rados.0.smithi070.stdout:771: writing smithi07085154-20 from 1990656 to 2596864 tid 2 2024-09-14T03:50:19.563 INFO:tasks.rados.rados.0.smithi070.stdout:771: writing smithi07085154-20 from 2596864 to 3194880 tid 3 2024-09-14T03:50:19.563 INFO:tasks.rados.rados.0.smithi070.stdout:772: snap_remove snap 41 2024-09-14T03:50:20.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:19 smithi094 ceph-mon[31053]: pgmap v69: 121 pgs: 121 active+clean; 240 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.0 MiB/s wr, 9 op/s 2024-09-14T03:50:20.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:19 smithi094 ceph-mon[31053]: osdmap e205: 8 total, 8 up, 8 in 2024-09-14T03:50:20.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:19 smithi070 ceph-mon[27495]: pgmap v69: 121 pgs: 121 active+clean; 240 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.0 MiB/s wr, 9 op/s 2024-09-14T03:50:20.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:19 smithi070 ceph-mon[27495]: osdmap e205: 8 total, 8 up, 8 in 2024-09-14T03:50:20.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:19 smithi070 ceph-mon[39288]: pgmap v69: 121 pgs: 121 active+clean; 240 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.0 MiB/s wr, 9 op/s 2024-09-14T03:50:20.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:19 smithi070 ceph-mon[39288]: osdmap e205: 8 total, 8 up, 8 in 2024-09-14T03:50:20.533 INFO:tasks.rados.rados.0.smithi070.stdout:772: done (3 left) 2024-09-14T03:50:20.534 INFO:tasks.rados.rados.0.smithi070.stdout:773: delete oid 24 current snap is 76 2024-09-14T03:50:20.534 INFO:tasks.rados.rados.0.smithi070.stdout:769: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-14T03:50:20.566 INFO:tasks.rados.rados.0.smithi070.stdout:770: finishing copy_from to smithi07085154-40 2024-09-14T03:50:20.566 INFO:tasks.rados.rados.0.smithi070.stdout:770: got expected ENOENT (src dne) 2024-09-14T03:50:20.566 INFO:tasks.rados.rados.0.smithi070.stdout:770: finishing copy_from racing read to smithi07085154-40 2024-09-14T03:50:20.566 INFO:tasks.rados.rados.0.smithi070.stdout:771: finishing write tid 1 to smithi07085154-20 2024-09-14T03:50:20.566 INFO:tasks.rados.rados.0.smithi070.stdout:771: finishing write tid 2 to smithi07085154-20 2024-09-14T03:50:20.567 INFO:tasks.rados.rados.0.smithi070.stdout:769: done (3 left) 2024-09-14T03:50:20.567 INFO:tasks.rados.rados.0.smithi070.stdout:770: done (2 left) 2024-09-14T03:50:20.567 INFO:tasks.rados.rados.0.smithi070.stdout:773: done (1 left) 2024-09-14T03:50:20.567 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 32 current snap is 76 2024-09-14T03:50:20.567 INFO:tasks.rados.rados.0.smithi070.stdout:771: finishing write tid 3 to smithi07085154-20 2024-09-14T03:50:20.567 INFO:tasks.rados.rados.0.smithi070.stdout:771: finishing write tid 4 to smithi07085154-20 2024-09-14T03:50:20.567 INFO:tasks.rados.rados.0.smithi070.stdout:774: seq_num 260 ranges {0=761856,761856=417792,1179648=786432} 2024-09-14T03:50:20.570 INFO:tasks.rados.rados.0.smithi070.stdout:774: writing smithi07085154-32 from 0 to 761856 tid 1 2024-09-14T03:50:20.572 INFO:tasks.rados.rados.0.smithi070.stdout:774: writing smithi07085154-32 from 761856 to 1179648 tid 2 2024-09-14T03:50:20.578 INFO:tasks.rados.rados.0.smithi070.stdout:774: writing smithi07085154-32 from 1179648 to 1966080 tid 3 2024-09-14T03:50:20.578 INFO:tasks.rados.rados.0.smithi070.stdout:775: copy_from oid 43 from oid 19 current snap is 76 2024-09-14T03:50:20.578 INFO:tasks.rados.rados.0.smithi070.stdout:771: finishing write tid 5 to smithi07085154-20 2024-09-14T03:50:20.578 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 20 v 341 (ObjNum 258 snap 76 seq_num 258) dirty exists 2024-09-14T03:50:20.578 INFO:tasks.rados.rados.0.smithi070.stdout:771: left oid 20 (ObjNum 258 snap 76 seq_num 258) 2024-09-14T03:50:20.578 INFO:tasks.rados.rados.0.smithi070.stdout:771: done (2 left) 2024-09-14T03:50:20.578 INFO:tasks.rados.rados.0.smithi070.stdout:776: delete oid 35 current snap is 76 2024-09-14T03:50:20.578 INFO:tasks.rados.rados.0.smithi070.stdout:774: finishing write tid 1 to smithi07085154-32 2024-09-14T03:50:20.578 INFO:tasks.rados.rados.0.smithi070.stdout:774: finishing write tid 2 to smithi07085154-32 2024-09-14T03:50:20.579 INFO:tasks.rados.rados.0.smithi070.stdout:776: done (2 left) 2024-09-14T03:50:20.579 INFO:tasks.rados.rados.0.smithi070.stdout:777: read oid 11 snap -1 2024-09-14T03:50:20.579 INFO:tasks.rados.rados.0.smithi070.stdout:777: expect (ObjNum 49 snap 0 seq_num 49) 2024-09-14T03:50:20.579 INFO:tasks.rados.rados.0.smithi070.stdout:778: snap_remove snap 34 2024-09-14T03:50:20.930 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:20 smithi094 ceph-mon[31053]: osdmap e206: 8 total, 8 up, 8 in 2024-09-14T03:50:20.930 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:20 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:50:20.930 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:20 smithi094 ceph-mon[31053]: osdmap e207: 8 total, 8 up, 8 in 2024-09-14T03:50:21.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:20 smithi070 ceph-mon[27495]: osdmap e206: 8 total, 8 up, 8 in 2024-09-14T03:50:21.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:20 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:50:21.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:20 smithi070 ceph-mon[27495]: osdmap e207: 8 total, 8 up, 8 in 2024-09-14T03:50:21.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:20 smithi070 ceph-mon[39288]: osdmap e206: 8 total, 8 up, 8 in 2024-09-14T03:50:21.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:20 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:50:21.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:20 smithi070 ceph-mon[39288]: osdmap e207: 8 total, 8 up, 8 in 2024-09-14T03:50:21.555 INFO:tasks.rados.rados.0.smithi070.stdout:774: finishing write tid 3 to smithi07085154-32 2024-09-14T03:50:21.555 INFO:tasks.rados.rados.0.smithi070.stdout:774: finishing write tid 4 to smithi07085154-32 2024-09-14T03:50:21.555 INFO:tasks.rados.rados.0.smithi070.stdout:774: finishing write tid 5 to smithi07085154-32 2024-09-14T03:50:21.555 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 321 (ObjNum 260 snap 76 seq_num 260) dirty exists 2024-09-14T03:50:21.555 INFO:tasks.rados.rados.0.smithi070.stdout:774: left oid 32 (ObjNum 260 snap 76 seq_num 260) 2024-09-14T03:50:21.556 INFO:tasks.rados.rados.0.smithi070.stdout:777: expect (ObjNum 49 snap 0 seq_num 49) 2024-09-14T03:50:21.586 INFO:tasks.rados.rados.0.smithi070.stdout:775: finishing copy_from racing read to smithi07085154-43 2024-09-14T03:50:21.586 INFO:tasks.rados.rados.0.smithi070.stdout:775: finishing copy_from to smithi07085154-43 2024-09-14T03:50:21.586 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 345 (ObjNum 251 snap 72 seq_num 251) dirty exists 2024-09-14T03:50:21.586 INFO:tasks.rados.rados.0.smithi070.stdout:774: done (3 left) 2024-09-14T03:50:21.586 INFO:tasks.rados.rados.0.smithi070.stdout:775: done (2 left) 2024-09-14T03:50:21.587 INFO:tasks.rados.rados.0.smithi070.stdout:777: done (1 left) 2024-09-14T03:50:21.587 INFO:tasks.rados.rados.0.smithi070.stdout:778: done (0 left) 2024-09-14T03:50:21.587 INFO:tasks.rados.rados.0.smithi070.stdout:779: snap_remove snap 72 2024-09-14T03:50:21.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:21 smithi094 ceph-mon[31053]: pgmap v73: 121 pgs: 121 active+clean; 240 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:50:21.777 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:50:21 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:50:21] "GET /metrics HTTP/1.1" 200 36208 "" "Prometheus/2.43.0" 2024-09-14T03:50:22.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:21 smithi070 ceph-mon[39288]: pgmap v73: 121 pgs: 121 active+clean; 240 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:50:22.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:21 smithi070 ceph-mon[27495]: pgmap v73: 121 pgs: 121 active+clean; 240 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:50:22.560 INFO:tasks.rados.rados.0.smithi070.stdout:779: done (0 left) 2024-09-14T03:50:22.560 INFO:tasks.rados.rados.0.smithi070.stdout:780: setattr oid 48 current snap is 76 2024-09-14T03:50:22.563 INFO:tasks.rados.rados.0.smithi070.stdout:781: copy_from oid 14 from oid 10 current snap is 76 2024-09-14T03:50:22.563 INFO:tasks.rados.rados.0.smithi070.stdout:782: rollback oid 18 current snap is 76 2024-09-14T03:50:22.563 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 18 to 30 2024-09-14T03:50:22.563 INFO:tasks.rados.rados.0.smithi070.stdout:783: read oid 35 snap -1 2024-09-14T03:50:22.563 INFO:tasks.rados.rados.0.smithi070.stdout:783: expect deleted 2024-09-14T03:50:22.564 INFO:tasks.rados.rados.0.smithi070.stdout:784: snap_remove snap 38 2024-09-14T03:50:22.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:22 smithi094 ceph-mon[31053]: osdmap e208: 8 total, 8 up, 8 in 2024-09-14T03:50:23.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:22 smithi070 ceph-mon[27495]: osdmap e208: 8 total, 8 up, 8 in 2024-09-14T03:50:23.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:22 smithi070 ceph-mon[39288]: osdmap e208: 8 total, 8 up, 8 in 2024-09-14T03:50:23.564 INFO:tasks.rados.rados.0.smithi070.stdout:784: done (4 left) 2024-09-14T03:50:23.564 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 19 current snap is 76 2024-09-14T03:50:23.565 INFO:tasks.rados.rados.0.smithi070.stdout:785: seq_num 262 ranges {5316608=794624,6111232=589824,6701056=614400} 2024-09-14T03:50:23.598 INFO:tasks.rados.rados.0.smithi070.stdout:785: writing smithi07085154-19 from 5316608 to 6111232 tid 1 2024-09-14T03:50:23.601 INFO:tasks.rados.rados.0.smithi070.stdout:785: writing smithi07085154-19 from 6111232 to 6701056 tid 2 2024-09-14T03:50:23.605 INFO:tasks.rados.rados.0.smithi070.stdout:785: writing smithi07085154-19 from 6701056 to 7315456 tid 3 2024-09-14T03:50:23.605 INFO:tasks.rados.rados.0.smithi070.stdout:786: read oid 40 snap -1 2024-09-14T03:50:23.606 INFO:tasks.rados.rados.0.smithi070.stdout:786: expect (ObjNum 131 snap 36 seq_num 131) 2024-09-14T03:50:23.606 INFO:tasks.rados.rados.0.smithi070.stdout:787: snap_create 2024-09-14T03:50:23.606 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 313 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-14T03:50:23.606 INFO:tasks.rados.rados.0.smithi070.stdout:782: finishing rollback tid 0 to smithi07085154-18 2024-09-14T03:50:23.606 INFO:tasks.rados.rados.0.smithi070.stdout:782: finishing rollback tid 1 to smithi07085154-18 2024-09-14T03:50:23.606 INFO:tasks.rados.rados.0.smithi070.stdout:782: finishing rollback tid 2 to smithi07085154-18 2024-09-14T03:50:23.606 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 465 (ObjNum 119 snap 30 seq_num 119) dirty exists 2024-09-14T03:50:23.606 INFO:tasks.rados.rados.0.smithi070.stdout:781: finishing copy_from racing read to smithi07085154-14 2024-09-14T03:50:23.606 INFO:tasks.rados.rados.0.smithi070.stdout:781: finishing copy_from to smithi07085154-14 2024-09-14T03:50:23.606 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 380 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-14T03:50:23.606 INFO:tasks.rados.rados.0.smithi070.stdout:785: finishing write tid 1 to smithi07085154-19 2024-09-14T03:50:23.606 INFO:tasks.rados.rados.0.smithi070.stdout:785: finishing write tid 2 to smithi07085154-19 2024-09-14T03:50:23.609 INFO:tasks.rados.rados.0.smithi070.stdout:785: finishing write tid 3 to smithi07085154-19 2024-09-14T03:50:23.609 INFO:tasks.rados.rados.0.smithi070.stdout:785: finishing write tid 4 to smithi07085154-19 2024-09-14T03:50:23.610 INFO:tasks.rados.rados.0.smithi070.stdout:785: finishing write tid 5 to smithi07085154-19 2024-09-14T03:50:23.610 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 19 v 391 (ObjNum 262 snap 76 seq_num 262) dirty exists 2024-09-14T03:50:23.610 INFO:tasks.rados.rados.0.smithi070.stdout:785: left oid 19 (ObjNum 262 snap 76 seq_num 262) 2024-09-14T03:50:23.626 INFO:tasks.rados.rados.0.smithi070.stdout:786: expect (ObjNum 131 snap 36 seq_num 131) 2024-09-14T03:50:23.653 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:23 smithi094 ceph-mon[31053]: pgmap v75: 121 pgs: 121 active+clean; 250 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-09-14T03:50:23.653 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:23 smithi094 ceph-mon[31053]: osdmap e209: 8 total, 8 up, 8 in 2024-09-14T03:50:23.936 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:23 smithi070 ceph-mon[27495]: pgmap v75: 121 pgs: 121 active+clean; 250 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-09-14T03:50:23.936 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:23 smithi070 ceph-mon[27495]: osdmap e209: 8 total, 8 up, 8 in 2024-09-14T03:50:23.937 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:23 smithi070 ceph-mon[39288]: pgmap v75: 121 pgs: 121 active+clean; 250 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-09-14T03:50:23.937 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:23 smithi070 ceph-mon[39288]: osdmap e209: 8 total, 8 up, 8 in 2024-09-14T03:50:24.262 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:24 smithi094 systemd[1]: Stopping Ceph prometheus.a for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:50:24.262 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:24 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[64889]: ts=2024-09-14T03:50:24.250Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-14T03:50:24.263 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:24 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[64889]: ts=2024-09-14T03:50:24.250Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-14T03:50:24.263 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:24 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[64889]: ts=2024-09-14T03:50:24.250Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-14T03:50:24.263 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:24 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[64889]: ts=2024-09-14T03:50:24.251Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-14T03:50:24.263 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:24 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[64889]: ts=2024-09-14T03:50:24.251Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-14T03:50:24.263 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:24 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[64889]: ts=2024-09-14T03:50:24.251Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-14T03:50:24.263 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:24 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[64889]: ts=2024-09-14T03:50:24.251Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-14T03:50:24.263 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:24 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[64889]: ts=2024-09-14T03:50:24.251Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-14T03:50:24.263 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:24 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[64889]: ts=2024-09-14T03:50:24.251Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-14T03:50:24.263 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:24 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[64889]: ts=2024-09-14T03:50:24.251Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-14T03:50:24.263 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:24 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[64889]: ts=2024-09-14T03:50:24.251Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-14T03:50:24.263 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:24 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[64889]: ts=2024-09-14T03:50:24.251Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-14T03:50:24.263 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:24 smithi094 podman[75070]: 2024-09-14 03:50:24.261031548 +0000 UTC m=+0.109759737 container died 9c88e2923e9d8ea13213e5b0be68f011c0d04d874b57eab8902d63e16e59c815 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:50:24.569 INFO:tasks.rados.rados.0.smithi070.stdout:780: done (6 left) 2024-09-14T03:50:24.569 INFO:tasks.rados.rados.0.smithi070.stdout:781: done (5 left) 2024-09-14T03:50:24.569 INFO:tasks.rados.rados.0.smithi070.stdout:782: done (4 left) 2024-09-14T03:50:24.569 INFO:tasks.rados.rados.0.smithi070.stdout:783: done (3 left) 2024-09-14T03:50:24.569 INFO:tasks.rados.rados.0.smithi070.stdout:785: done (2 left) 2024-09-14T03:50:24.569 INFO:tasks.rados.rados.0.smithi070.stdout:786: done (1 left) 2024-09-14T03:50:24.569 INFO:tasks.rados.rados.0.smithi070.stdout:787: done (0 left) 2024-09-14T03:50:24.569 INFO:tasks.rados.rados.0.smithi070.stdout:788: snap_remove snap 60 2024-09-14T03:50:24.901 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:24 smithi094 ceph-mon[31053]: osdmap e210: 8 total, 8 up, 8 in 2024-09-14T03:50:24.901 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:24 smithi094 podman[75070]: 2024-09-14 03:50:24.657359191 +0000 UTC m=+0.506087388 container cleanup 9c88e2923e9d8ea13213e5b0be68f011c0d04d874b57eab8902d63e16e59c815 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:50:24.901 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:24 smithi094 bash[75070]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a 2024-09-14T03:50:25.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:24 smithi070 ceph-mon[27495]: osdmap e210: 8 total, 8 up, 8 in 2024-09-14T03:50:25.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:24 smithi070 ceph-mon[39288]: osdmap e210: 8 total, 8 up, 8 in 2024-09-14T03:50:25.221 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:24 smithi094 podman[75087]: 2024-09-14 03:50:24.900183753 +0000 UTC m=+0.641425066 container remove 9c88e2923e9d8ea13213e5b0be68f011c0d04d874b57eab8902d63e16e59c815 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:50:25.221 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:25 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-14T03:50:25.221 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:25 smithi094 systemd[1]: Stopped Ceph prometheus.a for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:50:25.222 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:25 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 1.747s CPU time. 2024-09-14T03:50:25.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:25 smithi094 systemd[1]: Starting Ceph prometheus.a for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:50:25.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:25 smithi094 podman[75220]: 2024-09-14 03:50:25.33019766 +0000 UTC m=+0.016758848 image pull 1d3b7f56885b6dd623f1785be963aa9c195f86bc256ea454e8d02a7980b79c53 quay.io/prometheus/prometheus:v2.51.0 2024-09-14T03:50:25.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:25 smithi094 podman[75220]: 2024-09-14 03:50:25.460116715 +0000 UTC m=+0.146677896 container create aa320cf836270bc14aad9048422af4094134f3cb0ca2abbdcfe07498e79b28a4 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:50:25.533 INFO:tasks.rados.rados.0.smithi070.stdout:788: done (0 left) 2024-09-14T03:50:25.534 INFO:tasks.rados.rados.0.smithi070.stdout:789: snap_create 2024-09-14T03:50:25.829 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:25 smithi094 ceph-mon[31053]: pgmap v78: 121 pgs: 121 active+clean; 250 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-09-14T03:50:25.829 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:25 smithi094 ceph-mon[31053]: osdmap e211: 8 total, 8 up, 8 in 2024-09-14T03:50:25.829 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:25 smithi094 ceph-mon[31053]: osdmap e212: 8 total, 8 up, 8 in 2024-09-14T03:50:25.830 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:25 smithi094 podman[75220]: 2024-09-14 03:50:25.720850236 +0000 UTC m=+0.407411416 container init aa320cf836270bc14aad9048422af4094134f3cb0ca2abbdcfe07498e79b28a4 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:50:25.830 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:25 smithi094 podman[75220]: 2024-09-14 03:50:25.725941791 +0000 UTC m=+0.412502970 container start aa320cf836270bc14aad9048422af4094134f3cb0ca2abbdcfe07498e79b28a4 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:50:25.830 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:50:25.745Z caller=main.go:617 level=info msg="Starting Prometheus Server" mode=server version="(version=2.51.0, branch=HEAD, revision=c05c15512acb675e3f6cd662a6727854e93fc024)" 2024-09-14T03:50:25.830 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:50:25.745Z caller=main.go:622 level=info build_context="(go=go1.22.1, platform=linux/amd64, user=root@b5723e458358, date=20240319-10:54:45, tags=netgo,builtinassets,stringlabels)" 2024-09-14T03:50:25.830 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:50:25.745Z caller=main.go:623 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi094 (none))" 2024-09-14T03:50:25.830 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:50:25.745Z caller=main.go:624 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-14T03:50:25.830 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:50:25.745Z caller=main.go:625 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-14T03:50:25.830 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:50:25.747Z caller=web.go:568 level=info component=web msg="Start listening for connections" address=:9095 2024-09-14T03:50:25.831 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:50:25.747Z caller=main.go:1129 level=info msg="Starting TSDB ..." 2024-09-14T03:50:25.831 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:50:25.749Z caller=tls_config.go:313 level=info component=web msg="Listening on" address=[::]:9095 2024-09-14T03:50:25.831 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:50:25.749Z caller=tls_config.go:316 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-14T03:50:25.831 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:50:25.750Z caller=head.go:616 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-14T03:50:25.831 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:50:25.750Z caller=head.go:698 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.61µs 2024-09-14T03:50:25.831 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:50:25.750Z caller=head.go:706 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-14T03:50:25.831 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:50:25.751Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-09-14T03:50:25.831 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:50:25.762Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-09-14T03:50:25.831 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:50:25.763Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-09-14T03:50:25.831 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:50:25.763Z caller=head.go:815 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=22.349µs wal_replay_duration=12.168116ms wbl_replay_duration=115ns total_replay_duration=12.207935ms 2024-09-14T03:50:25.831 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:50:25.765Z caller=main.go:1150 level=info fs_type=XFS_SUPER_MAGIC 2024-09-14T03:50:25.831 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:50:25.765Z caller=main.go:1153 level=info msg="TSDB started" 2024-09-14T03:50:25.832 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:50:25.765Z caller=main.go:1335 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-14T03:50:25.832 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:50:25.781Z caller=main.go:1372 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.73052ms db_storage=1.017µs remote_storage=971ns web_handler=425ns query_engine=513ns scrape=436.92µs scrape_sd=100.535µs notify=9.285µs notify_sd=8.634µs rules=14.801031ms tracing=24.216µs 2024-09-14T03:50:25.832 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:50:25.781Z caller=main.go:1114 level=info msg="Server is ready to receive web requests." 2024-09-14T03:50:25.832 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:50:25.781Z caller=manager.go:163 level=info component="rule manager" msg="Starting rule manager..." 2024-09-14T03:50:25.832 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:25 smithi094 bash[75220]: aa320cf836270bc14aad9048422af4094134f3cb0ca2abbdcfe07498e79b28a4 2024-09-14T03:50:26.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:25 smithi070 ceph-mon[27495]: pgmap v78: 121 pgs: 121 active+clean; 250 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-09-14T03:50:26.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:25 smithi070 ceph-mon[27495]: osdmap e211: 8 total, 8 up, 8 in 2024-09-14T03:50:26.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:25 smithi070 ceph-mon[27495]: osdmap e212: 8 total, 8 up, 8 in 2024-09-14T03:50:26.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:25 smithi070 ceph-mon[39288]: pgmap v78: 121 pgs: 121 active+clean; 250 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-09-14T03:50:26.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:25 smithi070 ceph-mon[39288]: osdmap e211: 8 total, 8 up, 8 in 2024-09-14T03:50:26.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:25 smithi070 ceph-mon[39288]: osdmap e212: 8 total, 8 up, 8 in 2024-09-14T03:50:26.276 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:25 smithi094 systemd[1]: Started Ceph prometheus.a for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:50:26.536 INFO:tasks.rados.rados.0.smithi070.stdout:789: done (0 left) 2024-09-14T03:50:26.536 INFO:tasks.rados.rados.0.smithi070.stdout:790: read oid 1 snap -1 2024-09-14T03:50:26.536 INFO:tasks.rados.rados.0.smithi070.stdout:790: expect (ObjNum 83 snap 17 seq_num 83) 2024-09-14T03:50:26.536 INFO:tasks.rados.rados.0.smithi070.stdout:791: rollback oid 50 current snap is 78 2024-09-14T03:50:26.536 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 50 to 66 2024-09-14T03:50:26.536 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 22 current snap is 78 2024-09-14T03:50:26.536 INFO:tasks.rados.rados.0.smithi070.stdout:792: seq_num 263 ranges {761856=409600,1171456=475136,1646592=557056,2203648=573440} 2024-09-14T03:50:26.542 INFO:tasks.rados.rados.0.smithi070.stdout:792: writing smithi07085154-22 from 761856 to 1171456 tid 1 2024-09-14T03:50:26.545 INFO:tasks.rados.rados.0.smithi070.stdout:792: writing smithi07085154-22 from 1171456 to 1646592 tid 2 2024-09-14T03:50:26.548 INFO:tasks.rados.rados.0.smithi070.stdout:792: writing smithi07085154-22 from 1646592 to 2203648 tid 3 2024-09-14T03:50:26.552 INFO:tasks.rados.rados.0.smithi070.stdout:792: writing smithi07085154-22 from 2203648 to 2777088 tid 4 2024-09-14T03:50:26.553 INFO:tasks.rados.rados.0.smithi070.stdout:793: read oid 9 snap -1 2024-09-14T03:50:26.553 INFO:tasks.rados.rados.0.smithi070.stdout:793: expect (ObjNum 153 snap 45 seq_num 153) 2024-09-14T03:50:26.553 INFO:tasks.rados.rados.0.smithi070.stdout:794: delete oid 5 current snap is 78 2024-09-14T03:50:26.553 INFO:tasks.rados.rados.0.smithi070.stdout:791: finishing rollback tid 0 to smithi07085154-50 2024-09-14T03:50:26.553 INFO:tasks.rados.rados.0.smithi070.stdout:791: finishing rollback tid 1 to smithi07085154-50 2024-09-14T03:50:26.554 INFO:tasks.rados.rados.0.smithi070.stdout:791: finishing rollback tid 2 to smithi07085154-50 2024-09-14T03:50:26.554 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 401 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-14T03:50:26.554 INFO:tasks.rados.rados.0.smithi070.stdout:790: expect (ObjNum 83 snap 17 seq_num 83) 2024-09-14T03:50:26.591 INFO:tasks.rados.rados.0.smithi070.stdout:792: finishing write tid 1 to smithi07085154-22 2024-09-14T03:50:26.591 INFO:tasks.rados.rados.0.smithi070.stdout:792: finishing write tid 2 to smithi07085154-22 2024-09-14T03:50:26.591 INFO:tasks.rados.rados.0.smithi070.stdout:792: finishing write tid 3 to smithi07085154-22 2024-09-14T03:50:26.591 INFO:tasks.rados.rados.0.smithi070.stdout:792: finishing write tid 4 to smithi07085154-22 2024-09-14T03:50:26.591 INFO:tasks.rados.rados.0.smithi070.stdout:792: finishing write tid 5 to smithi07085154-22 2024-09-14T03:50:26.592 INFO:tasks.rados.rados.0.smithi070.stdout:792: finishing write tid 6 to smithi07085154-22 2024-09-14T03:50:26.592 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 407 (ObjNum 263 snap 78 seq_num 263) dirty exists 2024-09-14T03:50:26.592 INFO:tasks.rados.rados.0.smithi070.stdout:792: left oid 22 (ObjNum 263 snap 78 seq_num 263) 2024-09-14T03:50:26.592 INFO:tasks.rados.rados.0.smithi070.stdout:793: expect (ObjNum 153 snap 45 seq_num 153) 2024-09-14T03:50:26.727 INFO:tasks.rados.rados.0.smithi070.stdout:790: done (4 left) 2024-09-14T03:50:26.727 INFO:tasks.rados.rados.0.smithi070.stdout:791: done (3 left) 2024-09-14T03:50:26.727 INFO:tasks.rados.rados.0.smithi070.stdout:792: done (2 left) 2024-09-14T03:50:26.727 INFO:tasks.rados.rados.0.smithi070.stdout:793: done (1 left) 2024-09-14T03:50:26.727 INFO:tasks.rados.rados.0.smithi070.stdout:794: done (0 left) 2024-09-14T03:50:26.727 INFO:tasks.rados.rados.0.smithi070.stdout:795: read oid 13 snap -1 2024-09-14T03:50:26.727 INFO:tasks.rados.rados.0.smithi070.stdout:795: expect (ObjNum 138 snap 39 seq_num 138) 2024-09-14T03:50:26.728 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 34 current snap is 78 2024-09-14T03:50:26.728 INFO:tasks.rados.rados.0.smithi070.stdout:796: seq_num 265 ranges {0=442368,442368=647168,1089536=573440,1662976=385024} 2024-09-14T03:50:26.729 INFO:tasks.rados.rados.0.smithi070.stdout:796: writing smithi07085154-34 from 0 to 442368 tid 1 2024-09-14T03:50:26.733 INFO:tasks.rados.rados.0.smithi070.stdout:796: writing smithi07085154-34 from 442368 to 1089536 tid 2 2024-09-14T03:50:26.736 INFO:tasks.rados.rados.0.smithi070.stdout:796: writing smithi07085154-34 from 1089536 to 1662976 tid 3 2024-09-14T03:50:26.739 INFO:tasks.rados.rados.0.smithi070.stdout:796: writing smithi07085154-34 from 1662976 to 2048000 tid 4 2024-09-14T03:50:26.739 INFO:tasks.rados.rados.0.smithi070.stdout:797: delete oid 3 current snap is 78 2024-09-14T03:50:26.739 INFO:tasks.rados.rados.0.smithi070.stdout:796: finishing write tid 1 to smithi07085154-34 2024-09-14T03:50:26.739 INFO:tasks.rados.rados.0.smithi070.stdout:796: finishing write tid 2 to smithi07085154-34 2024-09-14T03:50:26.739 INFO:tasks.rados.rados.0.smithi070.stdout:795: expect (ObjNum 138 snap 39 seq_num 138) 2024-09-14T03:50:26.902 INFO:tasks.rados.rados.0.smithi070.stdout:796: finishing write tid 3 to smithi07085154-34 2024-09-14T03:50:26.902 INFO:tasks.rados.rados.0.smithi070.stdout:796: finishing write tid 4 to smithi07085154-34 2024-09-14T03:50:26.902 INFO:tasks.rados.rados.0.smithi070.stdout:796: finishing write tid 5 to smithi07085154-34 2024-09-14T03:50:26.902 INFO:tasks.rados.rados.0.smithi070.stdout:796: finishing write tid 6 to smithi07085154-34 2024-09-14T03:50:26.902 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 412 (ObjNum 265 snap 78 seq_num 265) dirty exists 2024-09-14T03:50:26.902 INFO:tasks.rados.rados.0.smithi070.stdout:796: left oid 34 (ObjNum 265 snap 78 seq_num 265) 2024-09-14T03:50:26.903 INFO:tasks.rados.rados.0.smithi070.stdout:795: done (2 left) 2024-09-14T03:50:26.903 INFO:tasks.rados.rados.0.smithi070.stdout:796: done (1 left) 2024-09-14T03:50:26.903 INFO:tasks.rados.rados.0.smithi070.stdout:797: done (0 left) 2024-09-14T03:50:26.903 INFO:tasks.rados.rados.0.smithi070.stdout:798: snap_create 2024-09-14T03:50:27.092 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:26 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:27.092 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:26 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:27.092 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:26 smithi094 ceph-mon[31053]: Reconfiguring grafana.a (dependencies changed)... 2024-09-14T03:50:27.092 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:26 smithi094 ceph-mon[31053]: Reconfiguring daemon grafana.a on smithi094 2024-09-14T03:50:27.092 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:26 smithi094 ceph-mon[31053]: osdmap e213: 8 total, 8 up, 8 in 2024-09-14T03:50:27.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:26 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:27.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:26 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:27.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:26 smithi070 ceph-mon[27495]: Reconfiguring grafana.a (dependencies changed)... 2024-09-14T03:50:27.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:26 smithi070 ceph-mon[27495]: Reconfiguring daemon grafana.a on smithi094 2024-09-14T03:50:27.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:26 smithi070 ceph-mon[27495]: osdmap e213: 8 total, 8 up, 8 in 2024-09-14T03:50:27.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:26 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:27.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:26 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:27.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:26 smithi070 ceph-mon[39288]: Reconfiguring grafana.a (dependencies changed)... 2024-09-14T03:50:27.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:26 smithi070 ceph-mon[39288]: Reconfiguring daemon grafana.a on smithi094 2024-09-14T03:50:27.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:26 smithi070 ceph-mon[39288]: osdmap e213: 8 total, 8 up, 8 in 2024-09-14T03:50:27.886 INFO:tasks.rados.rados.0.smithi070.stdout:798: done (0 left) 2024-09-14T03:50:27.886 INFO:tasks.rados.rados.0.smithi070.stdout:799: rmattr oid 13 current snap is 79 2024-09-14T03:50:27.886 INFO:tasks.rados.rados.0.smithi070.stdout:800: snap_create 2024-09-14T03:50:27.888 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 322 (ObjNum 138 snap 39 seq_num 138) dirty exists 2024-09-14T03:50:28.000 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:27 smithi094 ceph-mon[31053]: pgmap v82: 121 pgs: 121 active+clean; 255 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 2.1 MiB/s wr, 10 op/s 2024-09-14T03:50:28.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:27 smithi070 ceph-mon[27495]: pgmap v82: 121 pgs: 121 active+clean; 255 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 2.1 MiB/s wr, 10 op/s 2024-09-14T03:50:28.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:27 smithi070 ceph-mon[39288]: pgmap v82: 121 pgs: 121 active+clean; 255 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 2.1 MiB/s wr, 10 op/s 2024-09-14T03:50:28.891 INFO:tasks.rados.rados.0.smithi070.stdout:799: done (1 left) 2024-09-14T03:50:28.891 INFO:tasks.rados.rados.0.smithi070.stdout:800: done (0 left) 2024-09-14T03:50:28.891 INFO:tasks.rados.rados.0.smithi070.stdout:801: copy_from oid 43 from oid 39 current snap is 80 2024-09-14T03:50:28.891 INFO:tasks.rados.rados.0.smithi070.stdout:802: delete oid 49 current snap is 80 2024-09-14T03:50:28.895 INFO:tasks.rados.rados.0.smithi070.stdout:802: done (1 left) 2024-09-14T03:50:28.895 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 23 current snap is 80 2024-09-14T03:50:28.895 INFO:tasks.rados.rados.0.smithi070.stdout:803: seq_num 268 ranges {3244032=491520,3735552=745472,4481024=745472,5226496=114688} 2024-09-14T03:50:28.916 INFO:tasks.rados.rados.0.smithi070.stdout:803: writing smithi07085154-23 from 3244032 to 3735552 tid 1 2024-09-14T03:50:28.920 INFO:tasks.rados.rados.0.smithi070.stdout:803: writing smithi07085154-23 from 3735552 to 4481024 tid 2 2024-09-14T03:50:28.924 INFO:tasks.rados.rados.0.smithi070.stdout:803: writing smithi07085154-23 from 4481024 to 5226496 tid 3 2024-09-14T03:50:28.926 INFO:tasks.rados.rados.0.smithi070.stdout:803: writing smithi07085154-23 from 5226496 to 5341184 tid 4 2024-09-14T03:50:28.926 INFO:tasks.rados.rados.0.smithi070.stdout:804: rollback oid 3 current snap is 80 2024-09-14T03:50:28.926 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 3 to 51 2024-09-14T03:50:28.926 INFO:tasks.rados.rados.0.smithi070.stdout:801: finishing copy_from racing read to smithi07085154-43 2024-09-14T03:50:28.926 INFO:tasks.rados.rados.0.smithi070.stdout:801: finishing copy_from to smithi07085154-43 2024-09-14T03:50:28.926 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 53 (ObjNum 211 snap 60 seq_num 211) dirty exists 2024-09-14T03:50:28.926 INFO:tasks.rados.rados.0.smithi070.stdout:803: finishing write tid 1 to smithi07085154-23 2024-09-14T03:50:28.926 INFO:tasks.rados.rados.0.smithi070.stdout:803: finishing write tid 2 to smithi07085154-23 2024-09-14T03:50:28.926 INFO:tasks.rados.rados.0.smithi070.stdout:801: done (2 left) 2024-09-14T03:50:28.926 INFO:tasks.rados.rados.0.smithi070.stdout:805: snap_create 2024-09-14T03:50:28.933 INFO:tasks.rados.rados.0.smithi070.stdout:804: finishing rollback tid 1 to smithi07085154-3 2024-09-14T03:50:28.933 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 3 v 243 (ObjNum 49 snap 0 seq_num 2929832000) dirty dne 2024-09-14T03:50:28.933 INFO:tasks.rados.rados.0.smithi070.stdout:803: finishing write tid 3 to smithi07085154-23 2024-09-14T03:50:28.933 INFO:tasks.rados.rados.0.smithi070.stdout:803: finishing write tid 4 to smithi07085154-23 2024-09-14T03:50:28.933 INFO:tasks.rados.rados.0.smithi070.stdout:803: finishing write tid 5 to smithi07085154-23 2024-09-14T03:50:28.933 INFO:tasks.rados.rados.0.smithi070.stdout:803: finishing write tid 6 to smithi07085154-23 2024-09-14T03:50:28.933 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 385 (ObjNum 268 snap 80 seq_num 268) dirty exists 2024-09-14T03:50:28.933 INFO:tasks.rados.rados.0.smithi070.stdout:803: left oid 23 (ObjNum 268 snap 80 seq_num 268) 2024-09-14T03:50:29.231 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:28 smithi094 ceph-mon[31053]: osdmap e214: 8 total, 8 up, 8 in 2024-09-14T03:50:29.231 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:28 smithi094 systemd[1]: Stopping Ceph grafana.a for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:50:29.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:28 smithi070 ceph-mon[39288]: osdmap e214: 8 total, 8 up, 8 in 2024-09-14T03:50:29.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:28 smithi070 ceph-mon[27495]: osdmap e214: 8 total, 8 up, 8 in 2024-09-14T03:50:29.494 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:29 smithi094 podman[75485]: 2024-09-14 03:50:29.23030937 +0000 UTC m=+0.106414024 container died 846770f482574e93cdbf220736667e7b7f4dc882cb27614e62108cfefa8a003a (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a, io.openshift.tags=base rhel8, name=ubi8, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, architecture=x86_64, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, vcs-type=git, maintainer=Paul Cuzner , release=239.1651231664) 2024-09-14T03:50:29.776 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:29 smithi094 podman[75485]: 2024-09-14 03:50:29.708463668 +0000 UTC m=+0.584568319 container cleanup 846770f482574e93cdbf220736667e7b7f4dc882cb27614e62108cfefa8a003a (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a, vcs-type=git, version=8.5, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, maintainer=Paul Cuzner , description=Ceph Grafana Container, io.buildah.version=1.32.0, io.openshift.tags=base rhel8, name=ubi8, io.openshift.expose-services=, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-09-14T03:50:29.777 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:29 smithi094 bash[75485]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a 2024-09-14T03:50:29.896 INFO:tasks.rados.rados.0.smithi070.stdout:803: done (2 left) 2024-09-14T03:50:29.896 INFO:tasks.rados.rados.0.smithi070.stdout:804: done (1 left) 2024-09-14T03:50:29.896 INFO:tasks.rados.rados.0.smithi070.stdout:805: done (0 left) 2024-09-14T03:50:29.896 INFO:tasks.rados.rados.0.smithi070.stdout:806: read oid 42 snap -1 2024-09-14T03:50:29.896 INFO:tasks.rados.rados.0.smithi070.stdout:806: expect (ObjNum 160 snap 47 seq_num 160) 2024-09-14T03:50:29.896 INFO:tasks.rados.rados.0.smithi070.stdout:807: read oid 24 snap -1 2024-09-14T03:50:29.896 INFO:tasks.rados.rados.0.smithi070.stdout:807: expect deleted 2024-09-14T03:50:29.896 INFO:tasks.rados.rados.0.smithi070.stdout:808: snap_create 2024-09-14T03:50:29.906 INFO:tasks.rados.rados.0.smithi070.stdout:806: expect (ObjNum 160 snap 47 seq_num 160) 2024-09-14T03:50:30.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:29 smithi094 ceph-mon[31053]: pgmap v84: 121 pgs: 121 active+clean; 255 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 2.1 MiB/s wr, 10 op/s 2024-09-14T03:50:30.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:29 smithi094 ceph-mon[31053]: osdmap e215: 8 total, 8 up, 8 in 2024-09-14T03:50:30.277 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 podman[75503]: 2024-09-14 03:50:30.00334529 +0000 UTC m=+0.775516188 container remove 846770f482574e93cdbf220736667e7b7f4dc882cb27614e62108cfefa8a003a (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, build-date=2022-04-29T11:48:11.055897, version=8.5, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.32.0, io.openshift.expose-services=, release=239.1651231664, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, vendor=Red Hat, Inc., name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git) 2024-09-14T03:50:30.277 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-09-14T03:50:30.277 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-09-14T03:50:30.277 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 systemd[1]: Stopped Ceph grafana.a for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:50:30.277 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@grafana.a.service: Consumed 5.355s CPU time. 2024-09-14T03:50:30.278 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 systemd[1]: Starting Ceph grafana.a for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:50:30.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:29 smithi070 ceph-mon[39288]: pgmap v84: 121 pgs: 121 active+clean; 255 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 2.1 MiB/s wr, 10 op/s 2024-09-14T03:50:30.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:29 smithi070 ceph-mon[39288]: osdmap e215: 8 total, 8 up, 8 in 2024-09-14T03:50:30.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:29 smithi070 ceph-mon[27495]: pgmap v84: 121 pgs: 121 active+clean; 255 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 2.1 MiB/s wr, 10 op/s 2024-09-14T03:50:30.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:29 smithi070 ceph-mon[27495]: osdmap e215: 8 total, 8 up, 8 in 2024-09-14T03:50:30.536 INFO:tasks.rados.rados.0.smithi070.stdout:806: done (2 left) 2024-09-14T03:50:30.536 INFO:tasks.rados.rados.0.smithi070.stdout:807: done (1 left) 2024-09-14T03:50:30.536 INFO:tasks.rados.rados.0.smithi070.stdout:808: done (0 left) 2024-09-14T03:50:30.536 INFO:tasks.rados.rados.0.smithi070.stdout:809: delete oid 10 current snap is 82 2024-09-14T03:50:30.542 INFO:tasks.rados.rados.0.smithi070.stdout:809: done (0 left) 2024-09-14T03:50:30.542 INFO:tasks.rados.rados.0.smithi070.stdout:810: snap_create 2024-09-14T03:50:30.625 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 podman[75602]: 2024-09-14 03:50:30.242918348 +0000 UTC m=+0.015339176 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-09-14T03:50:30.625 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 podman[75602]: 2024-09-14 03:50:30.379609351 +0000 UTC m=+0.152030215 container create 2240810d03680ea85e39642edc3bf217af0418f8e6889c7425ca7efcd5ccf8dc (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a, com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, description=Ceph Grafana Container, vendor=Red Hat, Inc., release=239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, distribution-scope=public, io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, name=ubi8) 2024-09-14T03:50:30.625 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 podman[75602]: 2024-09-14 03:50:30.624744202 +0000 UTC m=+0.397165018 container init 2240810d03680ea85e39642edc3bf217af0418f8e6889c7425ca7efcd5ccf8dc (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a, io.openshift.tags=base rhel8, description=Ceph Grafana Container, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, vendor=Red Hat, Inc., release=239.1651231664, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-09-14T03:50:30.888 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 podman[75602]: 2024-09-14 03:50:30.629382899 +0000 UTC m=+0.401803725 container start 2240810d03680ea85e39642edc3bf217af0418f8e6889c7425ca7efcd5ccf8dc (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a, build-date=2022-04-29T11:48:11.055897, version=8.5, com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.buildah.version=1.32.0, release=239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, architecture=x86_64, io.openshift.expose-services=, description=Ceph Grafana Container, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, maintainer=Paul Cuzner , io.openshift.tags=base rhel8) 2024-09-14T03:50:30.889 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 bash[75602]: 2240810d03680ea85e39642edc3bf217af0418f8e6889c7425ca7efcd5ccf8dc 2024-09-14T03:50:30.889 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 systemd[1]: Started Ceph grafana.a for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:50:30.889 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=settings t=2024-09-14T03:50:30.784997344Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-09-14T03:50:30.889 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=settings t=2024-09-14T03:50:30.785214484Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-09-14T03:50:30.889 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=settings t=2024-09-14T03:50:30.785239058Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-14T03:50:30.889 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=settings t=2024-09-14T03:50:30.785252443Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-14T03:50:30.889 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=settings t=2024-09-14T03:50:30.785263694Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-14T03:50:30.889 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=settings t=2024-09-14T03:50:30.78527478Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-14T03:50:30.889 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=settings t=2024-09-14T03:50:30.785285463Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-14T03:50:30.889 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=settings t=2024-09-14T03:50:30.785297164Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-14T03:50:30.889 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=settings t=2024-09-14T03:50:30.78530824Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-14T03:50:30.889 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=settings t=2024-09-14T03:50:30.785323097Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-14T03:50:30.890 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=settings t=2024-09-14T03:50:30.785334054Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-14T03:50:30.890 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=settings t=2024-09-14T03:50:30.785344359Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-14T03:50:30.890 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=settings t=2024-09-14T03:50:30.785354712Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-14T03:50:30.890 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=settings t=2024-09-14T03:50:30.785366382Z level=info msg="App mode production" 2024-09-14T03:50:30.890 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=sqlstore t=2024-09-14T03:50:30.785426702Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-14T03:50:30.890 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=sqlstore t=2024-09-14T03:50:30.785449595Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-14T03:50:30.890 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=migrator t=2024-09-14T03:50:30.799503824Z level=info msg="Starting DB migrations" 2024-09-14T03:50:30.890 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=migrator t=2024-09-14T03:50:30.803937123Z level=info msg="migrations completed" performed=0 skipped=481 duration=427.908µs 2024-09-14T03:50:30.890 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=sqlstore t=2024-09-14T03:50:30.804283085Z level=info msg="Created default organization" 2024-09-14T03:50:30.890 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=plugin.loader t=2024-09-14T03:50:30.859984401Z level=info msg="Plugin registered" pluginID=input 2024-09-14T03:50:30.890 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=plugin.loader t=2024-09-14T03:50:30.887649487Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-09-14T03:50:30.890 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=plugin.loader t=2024-09-14T03:50:30.887696936Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-09-14T03:50:30.891 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=secrets t=2024-09-14T03:50:30.887733088Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-14T03:50:31.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:30 smithi094 ceph-mon[31053]: osdmap e216: 8 total, 8 up, 8 in 2024-09-14T03:50:31.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:30 smithi094 ceph-mon[31053]: osdmap e217: 8 total, 8 up, 8 in 2024-09-14T03:50:31.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:30 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:31.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:30 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:31.277 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=query_data t=2024-09-14T03:50:30.889638511Z level=info msg="Query Service initialization" 2024-09-14T03:50:31.277 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=live.push_http t=2024-09-14T03:50:30.894131898Z level=info msg="Live Push Gateway initialization" 2024-09-14T03:50:31.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:30 smithi070 ceph-mon[27495]: osdmap e216: 8 total, 8 up, 8 in 2024-09-14T03:50:31.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:30 smithi070 ceph-mon[27495]: osdmap e217: 8 total, 8 up, 8 in 2024-09-14T03:50:31.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:30 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:31.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:30 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:31.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:30 smithi070 ceph-mon[39288]: osdmap e216: 8 total, 8 up, 8 in 2024-09-14T03:50:31.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:30 smithi070 ceph-mon[39288]: osdmap e217: 8 total, 8 up, 8 in 2024-09-14T03:50:31.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:30 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:31.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:30 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:31.542 INFO:tasks.rados.rados.0.smithi070.stdout:810: done (0 left) 2024-09-14T03:50:31.542 INFO:tasks.rados.rados.0.smithi070.stdout:811: read oid 17 snap -1 2024-09-14T03:50:31.542 INFO:tasks.rados.rados.0.smithi070.stdout:811: expect (ObjNum 181 snap 51 seq_num 181) 2024-09-14T03:50:31.542 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 23 current snap is 83 2024-09-14T03:50:31.542 INFO:tasks.rados.rados.0.smithi070.stdout:812: seq_num 270 ranges {5341184=524288,5865472=516096,6381568=761856,7143424=327680} 2024-09-14T03:50:31.572 INFO:tasks.rados.rados.0.smithi070.stdout:812: writing smithi07085154-23 from 5341184 to 5865472 tid 1 2024-09-14T03:50:31.574 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:31 smithi094 systemd[1]: Stopping Ceph node-exporter.b for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:50:31.575 INFO:tasks.rados.rados.0.smithi070.stdout:812: writing smithi07085154-23 from 5865472 to 6381568 tid 2 2024-09-14T03:50:31.580 INFO:tasks.rados.rados.0.smithi070.stdout:812: writing smithi07085154-23 from 6381568 to 7143424 tid 3 2024-09-14T03:50:31.583 INFO:tasks.rados.rados.0.smithi070.stdout:812: writing smithi07085154-23 from 7143424 to 7471104 tid 4 2024-09-14T03:50:31.583 INFO:tasks.rados.rados.0.smithi070.stdout:813: rmattr oid 9 current snap is 83 2024-09-14T03:50:31.583 INFO:tasks.rados.rados.0.smithi070.stdout:813: done (2 left) 2024-09-14T03:50:31.583 INFO:tasks.rados.rados.0.smithi070.stdout:814: read oid 22 snap -1 2024-09-14T03:50:31.583 INFO:tasks.rados.rados.0.smithi070.stdout:814: expect (ObjNum 263 snap 78 seq_num 263) 2024-09-14T03:50:31.583 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 7 current snap is 83 2024-09-14T03:50:31.583 INFO:tasks.rados.rados.0.smithi070.stdout:815: seq_num 271 ranges {3596288=540672,4136960=802816,4939776=516096,5455872=286720} 2024-09-14T03:50:31.605 INFO:tasks.rados.rados.0.smithi070.stdout:815: writing smithi07085154-7 from 3596288 to 4136960 tid 1 2024-09-14T03:50:31.610 INFO:tasks.rados.rados.0.smithi070.stdout:815: writing smithi07085154-7 from 4136960 to 4939776 tid 2 2024-09-14T03:50:31.613 INFO:tasks.rados.rados.0.smithi070.stdout:815: writing smithi07085154-7 from 4939776 to 5455872 tid 3 2024-09-14T03:50:31.615 INFO:tasks.rados.rados.0.smithi070.stdout:815: writing smithi07085154-7 from 5455872 to 5742592 tid 4 2024-09-14T03:50:31.615 INFO:tasks.rados.rados.0.smithi070.stdout:816: rollback oid 47 current snap is 83 2024-09-14T03:50:31.615 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 47 to 74 2024-09-14T03:50:31.615 INFO:tasks.rados.rados.0.smithi070.stdout:811: expect (ObjNum 181 snap 51 seq_num 181) 2024-09-14T03:50:31.652 INFO:tasks.rados.rados.0.smithi070.stdout:812: finishing write tid 1 to smithi07085154-23 2024-09-14T03:50:31.652 INFO:tasks.rados.rados.0.smithi070.stdout:812: finishing write tid 2 to smithi07085154-23 2024-09-14T03:50:31.652 INFO:tasks.rados.rados.0.smithi070.stdout:812: finishing write tid 3 to smithi07085154-23 2024-09-14T03:50:31.652 INFO:tasks.rados.rados.0.smithi070.stdout:812: finishing write tid 4 to smithi07085154-23 2024-09-14T03:50:31.653 INFO:tasks.rados.rados.0.smithi070.stdout:812: finishing write tid 5 to smithi07085154-23 2024-09-14T03:50:31.653 INFO:tasks.rados.rados.0.smithi070.stdout:811: done (4 left) 2024-09-14T03:50:31.653 INFO:tasks.rados.rados.0.smithi070.stdout:817: snap_create 2024-09-14T03:50:31.653 INFO:tasks.rados.rados.0.smithi070.stdout:812: finishing write tid 6 to smithi07085154-23 2024-09-14T03:50:31.654 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 390 (ObjNum 270 snap 83 seq_num 270) dirty exists 2024-09-14T03:50:31.654 INFO:tasks.rados.rados.0.smithi070.stdout:812: left oid 23 (ObjNum 270 snap 83 seq_num 270) 2024-09-14T03:50:31.654 INFO:tasks.rados.rados.0.smithi070.stdout:814: expect (ObjNum 263 snap 78 seq_num 263) 2024-09-14T03:50:31.736 INFO:tasks.rados.rados.0.smithi070.stdout:815: finishing write tid 1 to smithi07085154-7 2024-09-14T03:50:31.736 INFO:tasks.rados.rados.0.smithi070.stdout:815: finishing write tid 2 to smithi07085154-7 2024-09-14T03:50:31.736 INFO:tasks.rados.rados.0.smithi070.stdout:815: finishing write tid 3 to smithi07085154-7 2024-09-14T03:50:31.736 INFO:tasks.rados.rados.0.smithi070.stdout:815: finishing write tid 4 to smithi07085154-7 2024-09-14T03:50:31.736 INFO:tasks.rados.rados.0.smithi070.stdout:815: finishing write tid 5 to smithi07085154-7 2024-09-14T03:50:31.737 INFO:tasks.rados.rados.0.smithi070.stdout:816: finishing rollback tid 0 to smithi07085154-47 2024-09-14T03:50:31.737 INFO:tasks.rados.rados.0.smithi070.stdout:816: finishing rollback tid 1 to smithi07085154-47 2024-09-14T03:50:31.737 INFO:tasks.rados.rados.0.smithi070.stdout:816: finishing rollback tid 2 to smithi07085154-47 2024-09-14T03:50:31.737 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 47 v 248 (ObjNum 112 snap 23 seq_num 112) dirty exists 2024-09-14T03:50:31.737 INFO:tasks.rados.rados.0.smithi070.stdout:815: finishing write tid 6 to smithi07085154-7 2024-09-14T03:50:31.737 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 346 (ObjNum 271 snap 83 seq_num 271) dirty exists 2024-09-14T03:50:31.737 INFO:tasks.rados.rados.0.smithi070.stdout:815: left oid 7 (ObjNum 271 snap 83 seq_num 271) 2024-09-14T03:50:31.826 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:50:31 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:50:31] "GET /metrics HTTP/1.1" 200 36208 "" "Prometheus/2.51.0" 2024-09-14T03:50:31.827 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:31 smithi094 podman[75763]: 2024-09-14 03:50:31.573661277 +0000 UTC m=+0.131742682 container died 3dcf1723ea3558eeb4a4360245d68a3f318ff0adad4cdebcd061821d743f0686 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:50:31.827 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:31 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=infra.usagestats.collector t=2024-09-14T03:50:31.64618828Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-14T03:50:31.827 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:31 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=provisioning.datasources t=2024-09-14T03:50:31.734345172Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-09-14T03:50:31.827 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:31 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=provisioning.datasources t=2024-09-14T03:50:31.734390933Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-09-14T03:50:31.827 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:31 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=provisioning.datasources t=2024-09-14T03:50:31.744738376Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-14T03:50:32.088 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:31 smithi094 ceph-mon[31053]: pgmap v88: 121 pgs: 121 active+clean; 255 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:50:32.088 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:31 smithi094 ceph-mon[31053]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-14T03:50:32.089 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:31 smithi094 ceph-mon[31053]: Deploying daemon node-exporter.b on smithi094 2024-09-14T03:50:32.089 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:31 smithi094 ceph-mon[31053]: osdmap e218: 8 total, 8 up, 8 in 2024-09-14T03:50:32.089 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:31 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=provisioning.alerting t=2024-09-14T03:50:31.825665245Z level=info msg="starting to provision alerting" 2024-09-14T03:50:32.089 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:31 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=provisioning.alerting t=2024-09-14T03:50:31.825690418Z level=info msg="finished to provision alerting" 2024-09-14T03:50:32.089 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:31 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=ngalert.state.manager t=2024-09-14T03:50:31.825760957Z level=info msg="Warming state cache for startup" 2024-09-14T03:50:32.089 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:31 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=grafanaStorageLogger t=2024-09-14T03:50:31.825945308Z level=info msg="storage starting" 2024-09-14T03:50:32.089 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:31 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=http.server t=2024-09-14T03:50:31.828262394Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-14T03:50:32.090 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:31 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=ngalert.state.manager t=2024-09-14T03:50:31.836849056Z level=info msg="State cache has been initialized" states=0 duration=11.086473ms 2024-09-14T03:50:32.090 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:31 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=ticker t=2024-09-14T03:50:31.836899056Z level=info msg=starting first_tick=2024-09-14T03:50:40Z 2024-09-14T03:50:32.090 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:50:31 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[75616]: logger=ngalert.multiorg.alertmanager t=2024-09-14T03:50:31.836894417Z level=info msg="starting MultiOrg Alertmanager" 2024-09-14T03:50:32.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:31 smithi070 ceph-mon[27495]: pgmap v88: 121 pgs: 121 active+clean; 255 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:50:32.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:31 smithi070 ceph-mon[27495]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-14T03:50:32.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:31 smithi070 ceph-mon[27495]: Deploying daemon node-exporter.b on smithi094 2024-09-14T03:50:32.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:31 smithi070 ceph-mon[27495]: osdmap e218: 8 total, 8 up, 8 in 2024-09-14T03:50:32.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:31 smithi070 ceph-mon[39288]: pgmap v88: 121 pgs: 121 active+clean; 255 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:50:32.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:31 smithi070 ceph-mon[39288]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-14T03:50:32.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:31 smithi070 ceph-mon[39288]: Deploying daemon node-exporter.b on smithi094 2024-09-14T03:50:32.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:31 smithi070 ceph-mon[39288]: osdmap e218: 8 total, 8 up, 8 in 2024-09-14T03:50:32.441 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:32 smithi094 podman[75763]: 2024-09-14 03:50:32.08797774 +0000 UTC m=+0.646059177 container cleanup 3dcf1723ea3558eeb4a4360245d68a3f318ff0adad4cdebcd061821d743f0686 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:50:32.442 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:32 smithi094 bash[75763]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b 2024-09-14T03:50:32.442 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:32 smithi094 podman[75781]: 2024-09-14 03:50:32.306532358 +0000 UTC m=+0.733158613 container remove 3dcf1723ea3558eeb4a4360245d68a3f318ff0adad4cdebcd061821d743f0686 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:50:32.442 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:32 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-14T03:50:32.547 INFO:tasks.rados.rados.0.smithi070.stdout:812: done (4 left) 2024-09-14T03:50:32.548 INFO:tasks.rados.rados.0.smithi070.stdout:814: done (3 left) 2024-09-14T03:50:32.548 INFO:tasks.rados.rados.0.smithi070.stdout:815: done (2 left) 2024-09-14T03:50:32.548 INFO:tasks.rados.rados.0.smithi070.stdout:816: done (1 left) 2024-09-14T03:50:32.548 INFO:tasks.rados.rados.0.smithi070.stdout:817: done (0 left) 2024-09-14T03:50:32.548 INFO:tasks.rados.rados.0.smithi070.stdout:818: copy_from oid 48 from oid 7 current snap is 84 2024-09-14T03:50:32.548 INFO:tasks.rados.rados.0.smithi070.stdout:819: read oid 2 snap -1 2024-09-14T03:50:32.548 INFO:tasks.rados.rados.0.smithi070.stdout:819: expect deleted 2024-09-14T03:50:32.548 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 13 current snap is 84 2024-09-14T03:50:32.548 INFO:tasks.rados.rados.0.smithi070.stdout:820: seq_num 272 ranges {4734976=565248,5300224=688128,5988352=729088,6717440=188416} 2024-09-14T03:50:32.576 INFO:tasks.rados.rados.0.smithi070.stdout:820: writing smithi07085154-13 from 4734976 to 5300224 tid 1 2024-09-14T03:50:32.580 INFO:tasks.rados.rados.0.smithi070.stdout:820: writing smithi07085154-13 from 5300224 to 5988352 tid 2 2024-09-14T03:50:32.584 INFO:tasks.rados.rados.0.smithi070.stdout:820: writing smithi07085154-13 from 5988352 to 6717440 tid 3 2024-09-14T03:50:32.586 INFO:tasks.rados.rados.0.smithi070.stdout:820: writing smithi07085154-13 from 6717440 to 6905856 tid 4 2024-09-14T03:50:32.586 INFO:tasks.rados.rados.0.smithi070.stdout:821: read oid 47 snap -1 2024-09-14T03:50:32.586 INFO:tasks.rados.rados.0.smithi070.stdout:821: expect (ObjNum 112 snap 23 seq_num 112) 2024-09-14T03:50:32.586 INFO:tasks.rados.rados.0.smithi070.stdout:822: delete oid 26 current snap is 84 2024-09-14T03:50:32.586 INFO:tasks.rados.rados.0.smithi070.stdout:818: finishing copy_from racing read to smithi07085154-48 2024-09-14T03:50:32.586 INFO:tasks.rados.rados.0.smithi070.stdout:818: finishing copy_from to smithi07085154-48 2024-09-14T03:50:32.586 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 346 (ObjNum 271 snap 83 seq_num 271) dirty exists 2024-09-14T03:50:32.586 INFO:tasks.rados.rados.0.smithi070.stdout:820: finishing write tid 1 to smithi07085154-13 2024-09-14T03:50:32.586 INFO:tasks.rados.rados.0.smithi070.stdout:820: finishing write tid 2 to smithi07085154-13 2024-09-14T03:50:32.587 INFO:tasks.rados.rados.0.smithi070.stdout:820: finishing write tid 3 to smithi07085154-13 2024-09-14T03:50:32.587 INFO:tasks.rados.rados.0.smithi070.stdout:820: finishing write tid 4 to smithi07085154-13 2024-09-14T03:50:32.588 INFO:tasks.rados.rados.0.smithi070.stdout:820: finishing write tid 5 to smithi07085154-13 2024-09-14T03:50:32.589 INFO:tasks.rados.rados.0.smithi070.stdout:820: finishing write tid 6 to smithi07085154-13 2024-09-14T03:50:32.589 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 327 (ObjNum 272 snap 84 seq_num 272) dirty exists 2024-09-14T03:50:32.589 INFO:tasks.rados.rados.0.smithi070.stdout:820: left oid 13 (ObjNum 272 snap 84 seq_num 272) 2024-09-14T03:50:32.590 INFO:tasks.rados.rados.0.smithi070.stdout:818: done (4 left) 2024-09-14T03:50:32.590 INFO:tasks.rados.rados.0.smithi070.stdout:819: done (3 left) 2024-09-14T03:50:32.590 INFO:tasks.rados.rados.0.smithi070.stdout:820: done (2 left) 2024-09-14T03:50:32.590 INFO:tasks.rados.rados.0.smithi070.stdout:822: done (1 left) 2024-09-14T03:50:32.590 INFO:tasks.rados.rados.0.smithi070.stdout:823: read oid 20 snap -1 2024-09-14T03:50:32.590 INFO:tasks.rados.rados.0.smithi070.stdout:823: expect (ObjNum 258 snap 76 seq_num 258) 2024-09-14T03:50:32.590 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 40 current snap is 84 2024-09-14T03:50:32.591 INFO:tasks.rados.rados.0.smithi070.stdout:824: seq_num 274 ranges {4653056=598016,5251072=499712,5750784=745472,6496256=360448} 2024-09-14T03:50:32.619 INFO:tasks.rados.rados.0.smithi070.stdout:824: writing smithi07085154-40 from 4653056 to 5251072 tid 1 2024-09-14T03:50:32.622 INFO:tasks.rados.rados.0.smithi070.stdout:824: writing smithi07085154-40 from 5251072 to 5750784 tid 2 2024-09-14T03:50:32.626 INFO:tasks.rados.rados.0.smithi070.stdout:824: writing smithi07085154-40 from 5750784 to 6496256 tid 3 2024-09-14T03:50:32.628 INFO:tasks.rados.rados.0.smithi070.stdout:824: writing smithi07085154-40 from 6496256 to 6856704 tid 4 2024-09-14T03:50:32.628 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 23 current snap is 84 2024-09-14T03:50:32.629 INFO:tasks.rados.rados.0.smithi070.stdout:825: seq_num 275 ranges {7471104=614400,8085504=786432,8871936=499712,9371648=319488} 2024-09-14T03:50:32.671 INFO:tasks.rados.rados.0.smithi070.stdout:825: writing smithi07085154-23 from 7471104 to 8085504 tid 1 2024-09-14T03:50:32.675 INFO:tasks.rados.rados.0.smithi070.stdout:825: writing smithi07085154-23 from 8085504 to 8871936 tid 2 2024-09-14T03:50:32.678 INFO:tasks.rados.rados.0.smithi070.stdout:825: writing smithi07085154-23 from 8871936 to 9371648 tid 3 2024-09-14T03:50:32.681 INFO:tasks.rados.rados.0.smithi070.stdout:825: writing smithi07085154-23 from 9371648 to 9691136 tid 4 2024-09-14T03:50:32.681 INFO:tasks.rados.rados.0.smithi070.stdout:826: read oid 6 snap -1 2024-09-14T03:50:32.681 INFO:tasks.rados.rados.0.smithi070.stdout:826: expect (ObjNum 250 snap 72 seq_num 250) 2024-09-14T03:50:32.681 INFO:tasks.rados.rados.0.smithi070.stdout:827: copy_from oid 50 from oid 39 current snap is 84 2024-09-14T03:50:32.681 INFO:tasks.rados.rados.0.smithi070.stdout:821: expect (ObjNum 112 snap 23 seq_num 112) 2024-09-14T03:50:32.760 INFO:tasks.rados.rados.0.smithi070.stdout:823: expect (ObjNum 258 snap 76 seq_num 258) 2024-09-14T03:50:32.776 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:32 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-14T03:50:32.776 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:32 smithi094 systemd[1]: Stopped Ceph node-exporter.b for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:50:32.776 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:32 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@node-exporter.b.service: Consumed 2.194s CPU time. 2024-09-14T03:50:32.776 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:32 smithi094 systemd[1]: Starting Ceph node-exporter.b for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:50:32.872 INFO:tasks.rados.rados.0.smithi070.stdout:824: finishing write tid 1 to smithi07085154-40 2024-09-14T03:50:32.872 INFO:tasks.rados.rados.0.smithi070.stdout:824: finishing write tid 2 to smithi07085154-40 2024-09-14T03:50:32.872 INFO:tasks.rados.rados.0.smithi070.stdout:824: finishing write tid 3 to smithi07085154-40 2024-09-14T03:50:32.872 INFO:tasks.rados.rados.0.smithi070.stdout:824: finishing write tid 4 to smithi07085154-40 2024-09-14T03:50:32.872 INFO:tasks.rados.rados.0.smithi070.stdout:824: finishing write tid 5 to smithi07085154-40 2024-09-14T03:50:32.872 INFO:tasks.rados.rados.0.smithi070.stdout:824: finishing write tid 6 to smithi07085154-40 2024-09-14T03:50:32.872 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 257 (ObjNum 274 snap 84 seq_num 274) dirty exists 2024-09-14T03:50:32.872 INFO:tasks.rados.rados.0.smithi070.stdout:824: left oid 40 (ObjNum 274 snap 84 seq_num 274) 2024-09-14T03:50:32.872 INFO:tasks.rados.rados.0.smithi070.stdout:825: finishing write tid 1 to smithi07085154-23 2024-09-14T03:50:32.872 INFO:tasks.rados.rados.0.smithi070.stdout:821: done (5 left) 2024-09-14T03:50:32.873 INFO:tasks.rados.rados.0.smithi070.stdout:823: done (4 left) 2024-09-14T03:50:32.873 INFO:tasks.rados.rados.0.smithi070.stdout:824: done (3 left) 2024-09-14T03:50:32.873 INFO:tasks.rados.rados.0.smithi070.stdout:828: snap_remove snap 58 2024-09-14T03:50:33.270 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:32 smithi094 bash[75913]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-09-14T03:50:33.270 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:33 smithi094 bash[75913]: Getting image source signatures 2024-09-14T03:50:33.527 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:33 smithi094 bash[75913]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-09-14T03:50:33.527 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:33 smithi094 bash[75913]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-09-14T03:50:33.527 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:33 smithi094 bash[75913]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-09-14T03:50:33.550 INFO:tasks.rados.rados.0.smithi070.stdout:828: done (3 left) 2024-09-14T03:50:33.550 INFO:tasks.rados.rados.0.smithi070.stdout:829: delete oid 31 current snap is 84 2024-09-14T03:50:33.550 INFO:tasks.rados.rados.0.smithi070.stdout:825: finishing write tid 2 to smithi07085154-23 2024-09-14T03:50:33.550 INFO:tasks.rados.rados.0.smithi070.stdout:825: finishing write tid 3 to smithi07085154-23 2024-09-14T03:50:33.550 INFO:tasks.rados.rados.0.smithi070.stdout:825: finishing write tid 4 to smithi07085154-23 2024-09-14T03:50:33.550 INFO:tasks.rados.rados.0.smithi070.stdout:825: finishing write tid 5 to smithi07085154-23 2024-09-14T03:50:33.550 INFO:tasks.rados.rados.0.smithi070.stdout:825: finishing write tid 6 to smithi07085154-23 2024-09-14T03:50:33.550 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 395 (ObjNum 275 snap 84 seq_num 275) dirty exists 2024-09-14T03:50:33.551 INFO:tasks.rados.rados.0.smithi070.stdout:825: left oid 23 (ObjNum 275 snap 84 seq_num 275) 2024-09-14T03:50:33.551 INFO:tasks.rados.rados.0.smithi070.stdout:826: expect (ObjNum 250 snap 72 seq_num 250) 2024-09-14T03:50:33.605 INFO:tasks.rados.rados.0.smithi070.stdout:827: finishing copy_from racing read to smithi07085154-50 2024-09-14T03:50:33.605 INFO:tasks.rados.rados.0.smithi070.stdout:827: finishing copy_from to smithi07085154-50 2024-09-14T03:50:33.605 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 53 (ObjNum 211 snap 60 seq_num 211) dirty exists 2024-09-14T03:50:33.605 INFO:tasks.rados.rados.0.smithi070.stdout:825: done (3 left) 2024-09-14T03:50:33.605 INFO:tasks.rados.rados.0.smithi070.stdout:826: done (2 left) 2024-09-14T03:50:33.605 INFO:tasks.rados.rados.0.smithi070.stdout:827: done (1 left) 2024-09-14T03:50:33.605 INFO:tasks.rados.rados.0.smithi070.stdout:829: done (0 left) 2024-09-14T03:50:33.605 INFO:tasks.rados.rados.0.smithi070.stdout:830: read oid 11 snap -1 2024-09-14T03:50:33.605 INFO:tasks.rados.rados.0.smithi070.stdout:830: expect (ObjNum 49 snap 0 seq_num 49) 2024-09-14T03:50:33.606 INFO:tasks.rados.rados.0.smithi070.stdout:831: snap_remove snap 69 2024-09-14T03:50:34.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:33 smithi094 ceph-mon[31053]: pgmap v90: 121 pgs: 121 active+clean; 263 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 2.0 MiB/s wr, 12 op/s 2024-09-14T03:50:34.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:33 smithi094 ceph-mon[31053]: osdmap e219: 8 total, 8 up, 8 in 2024-09-14T03:50:34.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:33 smithi070 ceph-mon[39288]: pgmap v90: 121 pgs: 121 active+clean; 263 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 2.0 MiB/s wr, 12 op/s 2024-09-14T03:50:34.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:33 smithi070 ceph-mon[39288]: osdmap e219: 8 total, 8 up, 8 in 2024-09-14T03:50:34.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:33 smithi070 ceph-mon[27495]: pgmap v90: 121 pgs: 121 active+clean; 263 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 2.0 MiB/s wr, 12 op/s 2024-09-14T03:50:34.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:33 smithi070 ceph-mon[27495]: osdmap e219: 8 total, 8 up, 8 in 2024-09-14T03:50:34.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:34 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:50:34.146Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:50:34.561 INFO:tasks.rados.rados.0.smithi070.stdout:831: done (1 left) 2024-09-14T03:50:34.561 INFO:tasks.rados.rados.0.smithi070.stdout:832: read oid 12 snap -1 2024-09-14T03:50:34.561 INFO:tasks.rados.rados.0.smithi070.stdout:832: expect (ObjNum 256 snap 73 seq_num 256) 2024-09-14T03:50:34.561 INFO:tasks.rados.rados.0.smithi070.stdout:830: expect (ObjNum 49 snap 0 seq_num 49) 2024-09-14T03:50:34.589 INFO:tasks.rados.rados.0.smithi070.stdout:830: done (1 left) 2024-09-14T03:50:34.589 INFO:tasks.rados.rados.0.smithi070.stdout:833: rmattr oid 28 current snap is 84 2024-09-14T03:50:34.589 INFO:tasks.rados.rados.0.smithi070.stdout:833: done (1 left) 2024-09-14T03:50:34.589 INFO:tasks.rados.rados.0.smithi070.stdout:834: snap_create 2024-09-14T03:50:34.601 INFO:tasks.rados.rados.0.smithi070.stdout:832: expect (ObjNum 256 snap 73 seq_num 256) 2024-09-14T03:50:35.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:34 smithi094 ceph-mon[31053]: osdmap e220: 8 total, 8 up, 8 in 2024-09-14T03:50:35.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:34 smithi070 ceph-mon[27495]: osdmap e220: 8 total, 8 up, 8 in 2024-09-14T03:50:35.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:34 smithi070 ceph-mon[39288]: osdmap e220: 8 total, 8 up, 8 in 2024-09-14T03:50:35.566 INFO:tasks.rados.rados.0.smithi070.stdout:832: done (1 left) 2024-09-14T03:50:35.566 INFO:tasks.rados.rados.0.smithi070.stdout:834: done (0 left) 2024-09-14T03:50:35.566 INFO:tasks.rados.rados.0.smithi070.stdout:835: read oid 24 snap -1 2024-09-14T03:50:35.566 INFO:tasks.rados.rados.0.smithi070.stdout:835: expect deleted 2024-09-14T03:50:35.566 INFO:tasks.rados.rados.0.smithi070.stdout:836: rollback oid 34 current snap is 85 2024-09-14T03:50:35.566 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 34 to 66 2024-09-14T03:50:35.566 INFO:tasks.rados.rados.0.smithi070.stdout:837: copy_from oid 29 from oid 25 current snap is 85 2024-09-14T03:50:35.566 INFO:tasks.rados.rados.0.smithi070.stdout:838: read oid 17 snap -1 2024-09-14T03:50:35.567 INFO:tasks.rados.rados.0.smithi070.stdout:838: expect (ObjNum 181 snap 51 seq_num 181) 2024-09-14T03:50:35.567 INFO:tasks.rados.rados.0.smithi070.stdout:839: read oid 18 snap -1 2024-09-14T03:50:35.567 INFO:tasks.rados.rados.0.smithi070.stdout:839: expect (ObjNum 119 snap 30 seq_num 119) 2024-09-14T03:50:35.567 INFO:tasks.rados.rados.0.smithi070.stdout:840: copy_from oid 23 from oid 40 current snap is 85 2024-09-14T03:50:35.567 INFO:tasks.rados.rados.0.smithi070.stdout:841: copy_from oid 9 from oid 19 current snap is 85 2024-09-14T03:50:35.567 INFO:tasks.rados.rados.0.smithi070.stdout:842: setattr oid 14 current snap is 85 2024-09-14T03:50:35.567 INFO:tasks.rados.rados.0.smithi070.stdout:843: setattr oid 38 current snap is 85 2024-09-14T03:50:35.568 INFO:tasks.rados.rados.0.smithi070.stdout:844: snap_create 2024-09-14T03:50:35.574 INFO:tasks.rados.rados.0.smithi070.stdout:838: expect (ObjNum 181 snap 51 seq_num 181) 2024-09-14T03:50:35.615 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 396 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-14T03:50:35.615 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 38 v 259 (ObjNum 33 snap 0 seq_num 2930409984) dirty exists 2024-09-14T03:50:35.615 INFO:tasks.rados.rados.0.smithi070.stdout:836: finishing rollback tid 0 to smithi07085154-34 2024-09-14T03:50:35.615 INFO:tasks.rados.rados.0.smithi070.stdout:836: finishing rollback tid 1 to smithi07085154-34 2024-09-14T03:50:35.615 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 433 (ObjNum 129 snap 0 seq_num 2930351632) dirty dne 2024-09-14T03:50:35.616 INFO:tasks.rados.rados.0.smithi070.stdout:839: expect (ObjNum 119 snap 30 seq_num 119) 2024-09-14T03:50:35.731 INFO:tasks.rados.rados.0.smithi070.stdout:840: finishing copy_from racing read to smithi07085154-23 2024-09-14T03:50:35.731 INFO:tasks.rados.rados.0.smithi070.stdout:837: finishing copy_from racing read to smithi07085154-29 2024-09-14T03:50:35.731 INFO:tasks.rados.rados.0.smithi070.stdout:841: finishing copy_from racing read to smithi07085154-9 2024-09-14T03:50:35.731 INFO:tasks.rados.rados.0.smithi070.stdout:840: finishing copy_from to smithi07085154-23 2024-09-14T03:50:35.731 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 257 (ObjNum 274 snap 84 seq_num 274) dirty exists 2024-09-14T03:50:35.731 INFO:tasks.rados.rados.0.smithi070.stdout:837: finishing copy_from to smithi07085154-29 2024-09-14T03:50:35.731 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 171 (ObjNum 140 snap 39 seq_num 140) dirty exists 2024-09-14T03:50:35.732 INFO:tasks.rados.rados.0.smithi070.stdout:841: finishing copy_from to smithi07085154-9 2024-09-14T03:50:35.732 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 391 (ObjNum 262 snap 76 seq_num 262) dirty exists 2024-09-14T03:50:36.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:35 smithi094 ceph-mon[31053]: pgmap v93: 121 pgs: 121 active+clean; 263 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 2.0 MiB/s wr, 12 op/s 2024-09-14T03:50:36.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:35 smithi094 ceph-mon[31053]: osdmap e221: 8 total, 8 up, 8 in 2024-09-14T03:50:36.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:35 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:50:36.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:35 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:50:35.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:50:36.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:35 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:50:35.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:50:36.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:35 smithi070 ceph-mon[27495]: pgmap v93: 121 pgs: 121 active+clean; 263 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 2.0 MiB/s wr, 12 op/s 2024-09-14T03:50:36.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:35 smithi070 ceph-mon[27495]: osdmap e221: 8 total, 8 up, 8 in 2024-09-14T03:50:36.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:35 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:50:36.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:35 smithi070 ceph-mon[39288]: pgmap v93: 121 pgs: 121 active+clean; 263 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 2.0 MiB/s wr, 12 op/s 2024-09-14T03:50:36.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:35 smithi070 ceph-mon[39288]: osdmap e221: 8 total, 8 up, 8 in 2024-09-14T03:50:36.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:35 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:50:36.570 INFO:tasks.rados.rados.0.smithi070.stdout:835: done (9 left) 2024-09-14T03:50:36.570 INFO:tasks.rados.rados.0.smithi070.stdout:836: done (8 left) 2024-09-14T03:50:36.571 INFO:tasks.rados.rados.0.smithi070.stdout:837: done (7 left) 2024-09-14T03:50:36.571 INFO:tasks.rados.rados.0.smithi070.stdout:838: done (6 left) 2024-09-14T03:50:36.571 INFO:tasks.rados.rados.0.smithi070.stdout:839: done (5 left) 2024-09-14T03:50:36.571 INFO:tasks.rados.rados.0.smithi070.stdout:840: done (4 left) 2024-09-14T03:50:36.571 INFO:tasks.rados.rados.0.smithi070.stdout:841: done (3 left) 2024-09-14T03:50:36.571 INFO:tasks.rados.rados.0.smithi070.stdout:842: done (2 left) 2024-09-14T03:50:36.571 INFO:tasks.rados.rados.0.smithi070.stdout:843: done (1 left) 2024-09-14T03:50:36.571 INFO:tasks.rados.rados.0.smithi070.stdout:844: done (0 left) 2024-09-14T03:50:36.571 INFO:tasks.rados.rados.0.smithi070.stdout:845: rollback oid 27 current snap is 86 2024-09-14T03:50:36.571 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 27 to 59 2024-09-14T03:50:36.571 INFO:tasks.rados.rados.0.smithi070.stdout:846: snap_remove snap 84 2024-09-14T03:50:36.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:36 smithi094 ceph-mon[31053]: osdmap e222: 8 total, 8 up, 8 in 2024-09-14T03:50:36.777 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:36 smithi094 bash[75913]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-09-14T03:50:37.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:36 smithi070 ceph-mon[39288]: osdmap e222: 8 total, 8 up, 8 in 2024-09-14T03:50:37.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:36 smithi070 ceph-mon[27495]: osdmap e222: 8 total, 8 up, 8 in 2024-09-14T03:50:37.095 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:36 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:50:36.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:50:37.096 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:36 smithi094 bash[75913]: Writing manifest to image destination 2024-09-14T03:50:37.522 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 podman[75913]: 2024-09-14 03:50:37.072204295 +0000 UTC m=+4.305941713 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-09-14T03:50:37.522 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 podman[75913]: 2024-09-14 03:50:37.195549033 +0000 UTC m=+4.429286451 container create 767f8a0931752aa06aea4956c4685619a49e9837abc3bd42fb1c88cd9bcfe05f (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:50:37.576 INFO:tasks.rados.rados.0.smithi070.stdout:846: done (1 left) 2024-09-14T03:50:37.576 INFO:tasks.rados.rados.0.smithi070.stdout:847: copy_from oid 11 from oid 41 current snap is 86 2024-09-14T03:50:37.576 INFO:tasks.rados.rados.0.smithi070.stdout:845: finishing rollback tid 1 to smithi07085154-27 2024-09-14T03:50:37.576 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 450 (ObjNum 177 snap 0 seq_num 1) dirty dne 2024-09-14T03:50:37.576 INFO:tasks.rados.rados.0.smithi070.stdout:845: done (1 left) 2024-09-14T03:50:37.576 INFO:tasks.rados.rados.0.smithi070.stdout:848: copy_from oid 46 from oid 17 current snap is 86 2024-09-14T03:50:37.576 INFO:tasks.rados.rados.0.smithi070.stdout:849: delete oid 29 current snap is 86 2024-09-14T03:50:37.580 INFO:tasks.rados.rados.0.smithi070.stdout:849: done (2 left) 2024-09-14T03:50:37.580 INFO:tasks.rados.rados.0.smithi070.stdout:850: snap_remove snap 81 2024-09-14T03:50:38.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-mon[31053]: pgmap v96: 121 pgs: 121 active+clean; 279 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 4.5 MiB/s wr, 24 op/s 2024-09-14T03:50:38.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-mon[31053]: osdmap e223: 8 total, 8 up, 8 in 2024-09-14T03:50:38.027 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 podman[75913]: 2024-09-14 03:50:37.639683791 +0000 UTC m=+4.873421209 container init 767f8a0931752aa06aea4956c4685619a49e9837abc3bd42fb1c88cd9bcfe05f (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:50:38.027 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 podman[75913]: 2024-09-14 03:50:37.644669506 +0000 UTC m=+4.878406925 container start 767f8a0931752aa06aea4956c4685619a49e9837abc3bd42fb1c88cd9bcfe05f (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:50:38.027 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.648Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-09-14T03:50:38.027 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.648Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-09-14T03:50:38.027 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.648Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-14T03:50:38.028 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.648Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-14T03:50:38.028 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-14T03:50:38.028 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-14T03:50:38.028 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-14T03:50:38.028 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=arp 2024-09-14T03:50:38.028 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-14T03:50:38.028 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-14T03:50:38.028 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-14T03:50:38.028 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-14T03:50:38.028 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-14T03:50:38.028 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-14T03:50:38.028 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-14T03:50:38.028 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-14T03:50:38.028 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=edac 2024-09-14T03:50:38.029 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-14T03:50:38.029 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-14T03:50:38.029 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-14T03:50:38.029 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-14T03:50:38.029 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-14T03:50:38.029 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-14T03:50:38.029 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-14T03:50:38.029 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-14T03:50:38.029 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-14T03:50:38.029 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-14T03:50:38.029 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-14T03:50:38.029 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-14T03:50:38.029 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-14T03:50:38.029 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-14T03:50:38.030 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-14T03:50:38.030 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-14T03:50:38.030 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=os 2024-09-14T03:50:38.030 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-14T03:50:38.030 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-14T03:50:38.030 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-14T03:50:38.030 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-14T03:50:38.030 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-14T03:50:38.030 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-14T03:50:38.030 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-14T03:50:38.031 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=stat 2024-09-14T03:50:38.031 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-14T03:50:38.031 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-14T03:50:38.031 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-14T03:50:38.031 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=time 2024-09-14T03:50:38.031 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-14T03:50:38.031 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=uname 2024-09-14T03:50:38.031 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-14T03:50:38.031 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-14T03:50:38.031 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-14T03:50:38.031 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-09-14T03:50:38.031 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b[75987]: ts=2024-09-14T03:50:37.649Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-14T03:50:38.031 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 bash[75913]: 767f8a0931752aa06aea4956c4685619a49e9837abc3bd42fb1c88cd9bcfe05f 2024-09-14T03:50:38.031 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 03:50:37 smithi094 systemd[1]: Started Ceph node-exporter.b for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:50:38.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:37 smithi070 ceph-mon[27495]: pgmap v96: 121 pgs: 121 active+clean; 279 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 4.5 MiB/s wr, 24 op/s 2024-09-14T03:50:38.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:37 smithi070 ceph-mon[27495]: osdmap e223: 8 total, 8 up, 8 in 2024-09-14T03:50:38.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:37 smithi070 ceph-mon[39288]: pgmap v96: 121 pgs: 121 active+clean; 279 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 4.5 MiB/s wr, 24 op/s 2024-09-14T03:50:38.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:37 smithi070 ceph-mon[39288]: osdmap e223: 8 total, 8 up, 8 in 2024-09-14T03:50:38.577 INFO:tasks.rados.rados.0.smithi070.stdout:850: done (2 left) 2024-09-14T03:50:38.577 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 12 current snap is 86 2024-09-14T03:50:38.577 INFO:tasks.rados.rados.0.smithi070.stdout:851: seq_num 278 ranges {2662400=663552,3325952=442368,3768320=720896,4489216=442368} 2024-09-14T03:50:38.595 INFO:tasks.rados.rados.0.smithi070.stdout:851: writing smithi07085154-12 from 2662400 to 3325952 tid 1 2024-09-14T03:50:38.598 INFO:tasks.rados.rados.0.smithi070.stdout:851: writing smithi07085154-12 from 3325952 to 3768320 tid 2 2024-09-14T03:50:38.602 INFO:tasks.rados.rados.0.smithi070.stdout:851: writing smithi07085154-12 from 3768320 to 4489216 tid 3 2024-09-14T03:50:38.605 INFO:tasks.rados.rados.0.smithi070.stdout:851: writing smithi07085154-12 from 4489216 to 4931584 tid 4 2024-09-14T03:50:38.606 INFO:tasks.rados.rados.0.smithi070.stdout:852: delete oid 22 current snap is 86 2024-09-14T03:50:38.606 INFO:tasks.rados.rados.0.smithi070.stdout:848: finishing copy_from racing read to smithi07085154-46 2024-09-14T03:50:38.606 INFO:tasks.rados.rados.0.smithi070.stdout:847: finishing copy_from racing read to smithi07085154-11 2024-09-14T03:50:38.606 INFO:tasks.rados.rados.0.smithi070.stdout:848: finishing copy_from to smithi07085154-46 2024-09-14T03:50:38.606 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 46 v 262 (ObjNum 181 snap 51 seq_num 181) dirty exists 2024-09-14T03:50:38.606 INFO:tasks.rados.rados.0.smithi070.stdout:847: finishing copy_from to smithi07085154-11 2024-09-14T03:50:38.606 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 11 v 53 (ObjNum 211 snap 60 seq_num 211) dirty exists 2024-09-14T03:50:38.606 INFO:tasks.rados.rados.0.smithi070.stdout:851: finishing write tid 1 to smithi07085154-12 2024-09-14T03:50:38.607 INFO:tasks.rados.rados.0.smithi070.stdout:851: finishing write tid 2 to smithi07085154-12 2024-09-14T03:50:38.607 INFO:tasks.rados.rados.0.smithi070.stdout:847: done (3 left) 2024-09-14T03:50:38.608 INFO:tasks.rados.rados.0.smithi070.stdout:848: done (2 left) 2024-09-14T03:50:38.608 INFO:tasks.rados.rados.0.smithi070.stdout:852: done (1 left) 2024-09-14T03:50:38.608 INFO:tasks.rados.rados.0.smithi070.stdout:853: snap_remove snap 82 2024-09-14T03:50:38.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:38 smithi070 ceph-mon[27495]: osdmap e224: 8 total, 8 up, 8 in 2024-09-14T03:50:38.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:38 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:38.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:38 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:38.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:38 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:50:38.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:38 smithi070 ceph-mon[27495]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:50:38.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:38 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:50:38.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:38 smithi070 ceph-mon[27495]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:50:38.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:38 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:50:38.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:38 smithi070 ceph-mon[27495]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:50:38.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:38 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:38.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:38 smithi070 ceph-mon[27495]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-14T03:50:38.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:38 smithi070 ceph-mon[27495]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-14T03:50:38.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:38 smithi070 ceph-mon[39288]: osdmap e224: 8 total, 8 up, 8 in 2024-09-14T03:50:38.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:38 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:38.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:38 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:38.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:38 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:50:38.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:38 smithi070 ceph-mon[39288]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:50:38.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:38 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:50:38.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:38 smithi070 ceph-mon[39288]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:50:38.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:38 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:50:38.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:38 smithi070 ceph-mon[39288]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:50:38.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:38 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:38.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:38 smithi070 ceph-mon[39288]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-14T03:50:38.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:38 smithi070 ceph-mon[39288]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-14T03:50:39.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:38 smithi094 ceph-mon[31053]: osdmap e224: 8 total, 8 up, 8 in 2024-09-14T03:50:39.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:38 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:39.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:38 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:39.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:38 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:50:39.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:38 smithi094 ceph-mon[31053]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:50:39.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:38 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:50:39.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:38 smithi094 ceph-mon[31053]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:50:39.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:38 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:50:39.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:38 smithi094 ceph-mon[31053]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:50:39.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:38 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:39.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:38 smithi094 ceph-mon[31053]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-14T03:50:39.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:38 smithi094 ceph-mon[31053]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-14T03:50:39.586 INFO:tasks.rados.rados.0.smithi070.stdout:853: done (1 left) 2024-09-14T03:50:39.586 INFO:tasks.rados.rados.0.smithi070.stdout:854: read oid 9 snap -1 2024-09-14T03:50:39.586 INFO:tasks.rados.rados.0.smithi070.stdout:854: expect (ObjNum 262 snap 76 seq_num 262) 2024-09-14T03:50:39.586 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 42 current snap is 86 2024-09-14T03:50:39.587 INFO:tasks.rados.rados.0.smithi070.stdout:851: finishing write tid 3 to smithi07085154-12 2024-09-14T03:50:39.587 INFO:tasks.rados.rados.0.smithi070.stdout:855: seq_num 280 ranges {3031040=696320,3727360=606208,4333568=688128,5021696=311296} 2024-09-14T03:50:39.610 INFO:tasks.rados.rados.0.smithi070.stdout:855: writing smithi07085154-42 from 3031040 to 3727360 tid 1 2024-09-14T03:50:39.614 INFO:tasks.rados.rados.0.smithi070.stdout:855: writing smithi07085154-42 from 3727360 to 4333568 tid 2 2024-09-14T03:50:39.619 INFO:tasks.rados.rados.0.smithi070.stdout:855: writing smithi07085154-42 from 4333568 to 5021696 tid 3 2024-09-14T03:50:39.622 INFO:tasks.rados.rados.0.smithi070.stdout:855: writing smithi07085154-42 from 5021696 to 5332992 tid 4 2024-09-14T03:50:39.622 INFO:tasks.rados.rados.0.smithi070.stdout:856: read oid 45 snap -1 2024-09-14T03:50:39.622 INFO:tasks.rados.rados.0.smithi070.stdout:856: expect (ObjNum 49 snap 0 seq_num 49) 2024-09-14T03:50:39.622 INFO:tasks.rados.rados.0.smithi070.stdout:851: finishing write tid 4 to smithi07085154-12 2024-09-14T03:50:39.622 INFO:tasks.rados.rados.0.smithi070.stdout:851: finishing write tid 5 to smithi07085154-12 2024-09-14T03:50:39.622 INFO:tasks.rados.rados.0.smithi070.stdout:851: finishing write tid 6 to smithi07085154-12 2024-09-14T03:50:39.623 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 351 (ObjNum 278 snap 86 seq_num 278) dirty exists 2024-09-14T03:50:39.623 INFO:tasks.rados.rados.0.smithi070.stdout:851: left oid 12 (ObjNum 278 snap 86 seq_num 278) 2024-09-14T03:50:39.623 INFO:tasks.rados.rados.0.smithi070.stdout:854: expect (ObjNum 262 snap 76 seq_num 262) 2024-09-14T03:50:39.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:39 smithi070 ceph-mon[27495]: pgmap v99: 121 pgs: 121 active+clean; 279 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 4.5 MiB/s wr, 24 op/s 2024-09-14T03:50:39.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:39 smithi070 ceph-mon[27495]: osdmap e225: 8 total, 8 up, 8 in 2024-09-14T03:50:39.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:39 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:39.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:39 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:50:39.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:39 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:50:39.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:39 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:50:39.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:39 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:39.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:39 smithi070 ceph-mon[39288]: pgmap v99: 121 pgs: 121 active+clean; 279 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 4.5 MiB/s wr, 24 op/s 2024-09-14T03:50:39.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:39 smithi070 ceph-mon[39288]: osdmap e225: 8 total, 8 up, 8 in 2024-09-14T03:50:39.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:39 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:39.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:39 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:50:39.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:39 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:50:39.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:39 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:50:39.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:39 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:40.005 INFO:tasks.rados.rados.0.smithi070.stdout:855: finishing write tid 1 to smithi07085154-42 2024-09-14T03:50:40.005 INFO:tasks.rados.rados.0.smithi070.stdout:855: finishing write tid 2 to smithi07085154-42 2024-09-14T03:50:40.005 INFO:tasks.rados.rados.0.smithi070.stdout:856: expect (ObjNum 49 snap 0 seq_num 49) 2024-09-14T03:50:40.025 INFO:tasks.rados.rados.0.smithi070.stdout:855: finishing write tid 3 to smithi07085154-42 2024-09-14T03:50:40.026 INFO:tasks.rados.rados.0.smithi070.stdout:855: finishing write tid 4 to smithi07085154-42 2024-09-14T03:50:40.026 INFO:tasks.rados.rados.0.smithi070.stdout:855: finishing write tid 5 to smithi07085154-42 2024-09-14T03:50:40.026 INFO:tasks.rados.rados.0.smithi070.stdout:855: finishing write tid 6 to smithi07085154-42 2024-09-14T03:50:40.026 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 698 (ObjNum 280 snap 86 seq_num 280) dirty exists 2024-09-14T03:50:40.026 INFO:tasks.rados.rados.0.smithi070.stdout:855: left oid 42 (ObjNum 280 snap 86 seq_num 280) 2024-09-14T03:50:40.026 INFO:tasks.rados.rados.0.smithi070.stdout:851: done (3 left) 2024-09-14T03:50:40.026 INFO:tasks.rados.rados.0.smithi070.stdout:854: done (2 left) 2024-09-14T03:50:40.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:39 smithi094 ceph-mon[31053]: pgmap v99: 121 pgs: 121 active+clean; 279 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 4.5 MiB/s wr, 24 op/s 2024-09-14T03:50:40.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:39 smithi094 ceph-mon[31053]: osdmap e225: 8 total, 8 up, 8 in 2024-09-14T03:50:40.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:39 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:40.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:39 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:50:40.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:39 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:50:40.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:39 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:50:40.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:39 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:40.027 INFO:tasks.rados.rados.0.smithi070.stdout:855: done (1 left) 2024-09-14T03:50:40.028 INFO:tasks.rados.rados.0.smithi070.stdout:856: done (0 left) 2024-09-14T03:50:40.028 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 9 current snap is 86 2024-09-14T03:50:40.028 INFO:tasks.rados.rados.0.smithi070.stdout:857: seq_num 281 ranges {7315456=712704,8028160=491520,8519680=491520,9011200=622592} 2024-09-14T03:50:40.069 INFO:tasks.rados.rados.0.smithi070.stdout:857: writing smithi07085154-9 from 7315456 to 8028160 tid 1 2024-09-14T03:50:40.072 INFO:tasks.rados.rados.0.smithi070.stdout:857: writing smithi07085154-9 from 8028160 to 8519680 tid 2 2024-09-14T03:50:40.075 INFO:tasks.rados.rados.0.smithi070.stdout:857: writing smithi07085154-9 from 8519680 to 9011200 tid 3 2024-09-14T03:50:40.079 INFO:tasks.rados.rados.0.smithi070.stdout:857: writing smithi07085154-9 from 9011200 to 9633792 tid 4 2024-09-14T03:50:40.079 INFO:tasks.rados.rados.0.smithi070.stdout:858: setattr oid 48 current snap is 86 2024-09-14T03:50:40.079 INFO:tasks.rados.rados.0.smithi070.stdout:857: finishing write tid 1 to smithi07085154-9 2024-09-14T03:50:40.079 INFO:tasks.rados.rados.0.smithi070.stdout:857: finishing write tid 2 to smithi07085154-9 2024-09-14T03:50:40.079 INFO:tasks.rados.rados.0.smithi070.stdout:857: finishing write tid 3 to smithi07085154-9 2024-09-14T03:50:40.080 INFO:tasks.rados.rados.0.smithi070.stdout:859: read oid 43 snap -1 2024-09-14T03:50:40.080 INFO:tasks.rados.rados.0.smithi070.stdout:859: expect (ObjNum 211 snap 60 seq_num 211) 2024-09-14T03:50:40.080 INFO:tasks.rados.rados.0.smithi070.stdout:860: snap_remove snap 30 2024-09-14T03:50:40.537 INFO:tasks.rados.rados.0.smithi070.stdout:860: done (3 left) 2024-09-14T03:50:40.538 INFO:tasks.rados.rados.0.smithi070.stdout:861: snap_remove snap 75 2024-09-14T03:50:40.928 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:40 smithi094 ceph-mon[31053]: Upgrade: Updating mgr.y 2024-09-14T03:50:40.928 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:40 smithi094 ceph-mon[31053]: Deploying daemon mgr.y on smithi070 2024-09-14T03:50:40.928 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:40 smithi094 ceph-mon[31053]: osdmap e226: 8 total, 8 up, 8 in 2024-09-14T03:50:40.928 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:40 smithi094 ceph-mon[31053]: osdmap e227: 8 total, 8 up, 8 in 2024-09-14T03:50:40.956 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:40 smithi070 ceph-mon[27495]: Upgrade: Updating mgr.y 2024-09-14T03:50:40.956 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:40 smithi070 ceph-mon[27495]: Deploying daemon mgr.y on smithi070 2024-09-14T03:50:40.956 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:40 smithi070 ceph-mon[27495]: osdmap e226: 8 total, 8 up, 8 in 2024-09-14T03:50:40.956 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:40 smithi070 ceph-mon[27495]: osdmap e227: 8 total, 8 up, 8 in 2024-09-14T03:50:40.957 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:40 smithi070 ceph-mon[39288]: Upgrade: Updating mgr.y 2024-09-14T03:50:40.957 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:40 smithi070 ceph-mon[39288]: Deploying daemon mgr.y on smithi070 2024-09-14T03:50:40.957 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:40 smithi070 ceph-mon[39288]: osdmap e226: 8 total, 8 up, 8 in 2024-09-14T03:50:40.957 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:40 smithi070 ceph-mon[39288]: osdmap e227: 8 total, 8 up, 8 in 2024-09-14T03:50:41.493 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:41 smithi070 systemd[1]: Stopping Ceph mgr.y for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:50:41.542 INFO:tasks.rados.rados.0.smithi070.stdout:861: done (3 left) 2024-09-14T03:50:41.542 INFO:tasks.rados.rados.0.smithi070.stdout:862: snap_remove snap 70 2024-09-14T03:50:41.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:41 smithi070 podman[88421]: 2024-09-14 03:50:41.491443483 +0000 UTC m=+0.126852409 container died dc71810017b9104ef94e066b0c4569085c459823e4fdae447c9d42888fc4a016 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, maintainer=Guillaume Abrioux , RELEASE=quincy-eb16b97, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-09-14T03:50:41.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:41 smithi070 ceph-mon[39288]: pgmap v103: 121 pgs: 121 active+clean; 279 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:50:41.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:41 smithi070 ceph-mon[39288]: osdmap e228: 8 total, 8 up, 8 in 2024-09-14T03:50:41.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:41 smithi070 ceph-mon[27495]: pgmap v103: 121 pgs: 121 active+clean; 279 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:50:41.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:41 smithi070 ceph-mon[27495]: osdmap e228: 8 total, 8 up, 8 in 2024-09-14T03:50:41.943 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:41 smithi094 ceph-mon[31053]: pgmap v103: 121 pgs: 121 active+clean; 279 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:50:41.943 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:41 smithi094 ceph-mon[31053]: osdmap e228: 8 total, 8 up, 8 in 2024-09-14T03:50:41.944 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:50:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:50:41] "GET /metrics HTTP/1.1" 200 36203 "" "Prometheus/2.51.0" 2024-09-14T03:50:42.137 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:41 smithi070 podman[88421]: 2024-09-14 03:50:41.884844628 +0000 UTC m=+0.520253549 container cleanup dc71810017b9104ef94e066b0c4569085c459823e4fdae447c9d42888fc4a016 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=quincy-eb16b97, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-14T03:50:42.137 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:41 smithi070 bash[88421]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y 2024-09-14T03:50:42.501 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:42 smithi070 podman[88440]: 2024-09-14 03:50:42.136438058 +0000 UTC m=+0.633891150 container remove dc71810017b9104ef94e066b0c4569085c459823e4fdae447c9d42888fc4a016 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=quincy-eb16b97, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-14T03:50:42.502 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:42 smithi070 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-09-14T03:50:42.502 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:42 smithi070 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-09-14T03:50:42.502 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:42 smithi070 systemd[1]: Stopped Ceph mgr.y for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:50:42.502 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:42 smithi070 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@mgr.y.service: Consumed 40.688s CPU time. 2024-09-14T03:50:42.544 INFO:tasks.rados.rados.0.smithi070.stdout:862: done (3 left) 2024-09-14T03:50:42.544 INFO:tasks.rados.rados.0.smithi070.stdout:863: snap_create 2024-09-14T03:50:42.544 INFO:tasks.rados.rados.0.smithi070.stdout:857: finishing write tid 4 to smithi07085154-9 2024-09-14T03:50:42.544 INFO:tasks.rados.rados.0.smithi070.stdout:857: finishing write tid 5 to smithi07085154-9 2024-09-14T03:50:42.545 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 352 (ObjNum 271 snap 83 seq_num 271) dirty exists 2024-09-14T03:50:42.545 INFO:tasks.rados.rados.0.smithi070.stdout:857: finishing write tid 6 to smithi07085154-9 2024-09-14T03:50:42.545 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 704 (ObjNum 281 snap 86 seq_num 281) dirty exists 2024-09-14T03:50:42.545 INFO:tasks.rados.rados.0.smithi070.stdout:857: left oid 9 (ObjNum 281 snap 86 seq_num 281) 2024-09-14T03:50:42.545 INFO:tasks.rados.rados.0.smithi070.stdout:859: expect (ObjNum 211 snap 60 seq_num 211) 2024-09-14T03:50:42.763 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:42 smithi070 systemd[1]: Starting Ceph mgr.y for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:50:42.763 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:42 smithi070 podman[88596]: 2024-09-14 03:50:42.638666215 +0000 UTC m=+0.023444574 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:50:42.763 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:42 smithi070 podman[88596]: 2024-09-14 03:50:42.762416588 +0000 UTC m=+0.147194937 container create 0ef4d8e0cdbeb034eb69f6ae58fd51178439b15cb034db2198d5ce8b3e5c39b4 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y, GIT_CLEAN=True, RELEASE=main-a278231, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-14T03:50:42.816 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-09-14T03:50:43.030 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:42 smithi070 podman[88596]: 2024-09-14 03:50:42.998518768 +0000 UTC m=+0.383297118 container init 0ef4d8e0cdbeb034eb69f6ae58fd51178439b15cb034db2198d5ce8b3e5c39b4 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-14T03:50:43.030 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:43 smithi070 podman[88596]: 2024-09-14 03:50:43.003159546 +0000 UTC m=+0.387937897 container start 0ef4d8e0cdbeb034eb69f6ae58fd51178439b15cb034db2198d5ce8b3e5c39b4 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=main-a278231, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731) 2024-09-14T03:50:43.286 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:43 smithi070 ceph-mgr[88658]: -- 172.21.15.70:0/678737250 <== mon.0 v2:172.21.15.70:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x557de38af0e0 con 0x557de388d400 2024-09-14T03:50:43.286 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:43 smithi070 bash[88596]: 0ef4d8e0cdbeb034eb69f6ae58fd51178439b15cb034db2198d5ce8b3e5c39b4 2024-09-14T03:50:43.286 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:43 smithi070 systemd[1]: Started Ceph mgr.y for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:50:43.403 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:50:43.403 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 starting - - - - 2024-09-14T03:50:43.404 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi094 *:3000 starting - - - - 2024-09-14T03:50:43.404 INFO:teuthology.orchestra.run.smithi070.stdout:mds.a smithi070 running (6m) 63s ago 6m 18.4M - 17.2.7-1629-geb16b97b ab51d1bebd12 52e0dbb7be50 2024-09-14T03:50:43.404 INFO:teuthology.orchestra.run.smithi070.stdout:mds.b smithi094 running (6m) 61s ago 6m 14.5M - 17.2.7-1629-geb16b97b ab51d1bebd12 e3b5e3e00189 2024-09-14T03:50:43.404 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi094 *:8443,9283 running (77s) 61s ago 10m 497M - 19.3.0-4907-ga2782315 8801d2529e43 5aadf84f680b 2024-09-14T03:50:43.404 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 starting - - - - 2024-09-14T03:50:43.404 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (12m) 63s ago 12m 55.2M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 12d19db91f54 2024-09-14T03:50:43.404 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi094 running (10m) 61s ago 10m 46.1M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 90ea2e164909 2024-09-14T03:50:43.404 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (10m) 63s ago 10m 46.2M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 44ee80c29ee0 2024-09-14T03:50:43.404 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 starting - - - - 2024-09-14T03:50:43.404 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi094 *:9100 starting - - - - 2024-09-14T03:50:43.404 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (10m) 63s ago 9m 105M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 741c06b6adbf 2024-09-14T03:50:43.404 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (9m) 63s ago 9m 110M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 af459ea7ef68 2024-09-14T03:50:43.405 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (9m) 63s ago 9m 93.8M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 109b66e4e60f 2024-09-14T03:50:43.405 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (8m) 63s ago 8m 87.9M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 2e4199b58392 2024-09-14T03:50:43.405 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi094 running (8m) 61s ago 8m 112M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 52b64527e24c 2024-09-14T03:50:43.405 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi094 running (8m) 61s ago 8m 90.4M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 b543f152ba87 2024-09-14T03:50:43.405 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi094 running (7m) 61s ago 7m 68.9M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 e66b592ac940 2024-09-14T03:50:43.405 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi094 running (7m) 61s ago 7m 88.0M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 037c6208e9ce 2024-09-14T03:50:43.405 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi094 *:9095 starting - - - - 2024-09-14T03:50:43.550 INFO:tasks.rados.rados.0.smithi070.stdout:857: done (3 left) 2024-09-14T03:50:43.551 INFO:tasks.rados.rados.0.smithi070.stdout:858: done (2 left) 2024-09-14T03:50:43.551 INFO:tasks.rados.rados.0.smithi070.stdout:859: done (1 left) 2024-09-14T03:50:43.551 INFO:tasks.rados.rados.0.smithi070.stdout:863: done (0 left) 2024-09-14T03:50:43.551 INFO:tasks.rados.rados.0.smithi070.stdout:864: rollback oid 21 current snap is 87 2024-09-14T03:50:43.551 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 21 to 78 2024-09-14T03:50:43.551 INFO:tasks.rados.rados.0.smithi070.stdout:865: copy_from oid 28 from oid 37 current snap is 87 2024-09-14T03:50:43.551 INFO:tasks.rados.rados.0.smithi070.stdout:866: delete oid 1 current snap is 87 2024-09-14T03:50:43.555 INFO:tasks.rados.rados.0.smithi070.stdout:866: done (2 left) 2024-09-14T03:50:43.555 INFO:tasks.rados.rados.0.smithi070.stdout:867: delete oid 20 current snap is 87 2024-09-14T03:50:43.557 INFO:tasks.rados.rados.0.smithi070.stdout:864: finishing rollback tid 0 to smithi07085154-21 2024-09-14T03:50:43.557 INFO:tasks.rados.rados.0.smithi070.stdout:864: finishing rollback tid 1 to smithi07085154-21 2024-09-14T03:50:43.557 INFO:tasks.rados.rados.0.smithi070.stdout:864: finishing rollback tid 2 to smithi07085154-21 2024-09-14T03:50:43.557 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 255 (ObjNum 242 snap 69 seq_num 242) dirty exists 2024-09-14T03:50:43.561 INFO:tasks.rados.rados.0.smithi070.stdout:865: finishing copy_from to smithi07085154-28 2024-09-14T03:50:43.561 INFO:tasks.rados.rados.0.smithi070.stdout:865: got expected ENOENT (src dne) 2024-09-14T03:50:43.561 INFO:tasks.rados.rados.0.smithi070.stdout:864: done (2 left) 2024-09-14T03:50:43.561 INFO:tasks.rados.rados.0.smithi070.stdout:867: done (1 left) 2024-09-14T03:50:43.562 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 24 current snap is 87 2024-09-14T03:50:43.562 INFO:tasks.rados.rados.0.smithi070.stdout:868: seq_num 284 ranges {0=761856,761856=598016,1359872=671744,2031616=335872} 2024-09-14T03:50:43.566 INFO:tasks.rados.rados.0.smithi070.stdout:868: writing smithi07085154-24 from 0 to 761856 tid 1 2024-09-14T03:50:43.569 INFO:tasks.rados.rados.0.smithi070.stdout:868: writing smithi07085154-24 from 761856 to 1359872 tid 2 2024-09-14T03:50:43.573 INFO:tasks.rados.rados.0.smithi070.stdout:868: writing smithi07085154-24 from 1359872 to 2031616 tid 3 2024-09-14T03:50:43.575 INFO:tasks.rados.rados.0.smithi070.stdout:868: writing smithi07085154-24 from 2031616 to 2367488 tid 4 2024-09-14T03:50:43.575 INFO:tasks.rados.rados.0.smithi070.stdout:869: rmattr oid 2 current snap is 87 2024-09-14T03:50:43.575 INFO:tasks.rados.rados.0.smithi070.stdout:869: done (2 left) 2024-09-14T03:50:43.575 INFO:tasks.rados.rados.0.smithi070.stdout:870: snap_create 2024-09-14T03:50:43.576 INFO:tasks.rados.rados.0.smithi070.stdout:865: finishing copy_from racing read to smithi07085154-28 2024-09-14T03:50:43.576 INFO:tasks.rados.rados.0.smithi070.stdout:868: finishing write tid 1 to smithi07085154-24 2024-09-14T03:50:43.576 INFO:tasks.rados.rados.0.smithi070.stdout:868: finishing write tid 2 to smithi07085154-24 2024-09-14T03:50:43.577 INFO:tasks.rados.rados.0.smithi070.stdout:868: finishing write tid 3 to smithi07085154-24 2024-09-14T03:50:43.577 INFO:tasks.rados.rados.0.smithi070.stdout:868: finishing write tid 4 to smithi07085154-24 2024-09-14T03:50:43.577 INFO:tasks.rados.rados.0.smithi070.stdout:868: finishing write tid 5 to smithi07085154-24 2024-09-14T03:50:43.578 INFO:tasks.rados.rados.0.smithi070.stdout:868: finishing write tid 6 to smithi07085154-24 2024-09-14T03:50:43.578 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 401 (ObjNum 284 snap 87 seq_num 284) dirty exists 2024-09-14T03:50:43.579 INFO:tasks.rados.rados.0.smithi070.stdout:868: left oid 24 (ObjNum 284 snap 87 seq_num 284) 2024-09-14T03:50:43.770 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:50:43.770 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-09-14T03:50:43.770 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 3 2024-09-14T03:50:43.770 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:50:43.770 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-09-14T03:50:43.770 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 1, 2024-09-14T03:50:43.770 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 1 2024-09-14T03:50:43.770 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:50:43.770 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-09-14T03:50:43.771 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 8 2024-09-14T03:50:43.771 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:50:43.771 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": { 2024-09-14T03:50:43.771 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:50:43.771 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:50:43.771 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-09-14T03:50:43.771 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 14, 2024-09-14T03:50:43.771 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 1 2024-09-14T03:50:43.771 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-09-14T03:50:43.771 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:50:43.772 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:43 smithi070 ceph-mon[27495]: osdmap e229: 8 total, 8 up, 8 in 2024-09-14T03:50:43.772 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:43 smithi070 ceph-mon[27495]: pgmap v106: 121 pgs: 121 active+clean; 273 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.5 MiB/s rd, 7.7 MiB/s wr, 20 op/s 2024-09-14T03:50:43.772 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:43 smithi070 ceph-mon[27495]: from='client.15147 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:43.772 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:43 smithi070 ceph-mon[27495]: from='client.25058 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:43.772 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:43 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:43.772 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:43 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:43.772 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:43 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:43.772 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:43 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:43.772 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:43 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:43.773 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:43 smithi070 ceph-mon[39288]: osdmap e229: 8 total, 8 up, 8 in 2024-09-14T03:50:43.773 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:43 smithi070 ceph-mon[39288]: pgmap v106: 121 pgs: 121 active+clean; 273 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.5 MiB/s rd, 7.7 MiB/s wr, 20 op/s 2024-09-14T03:50:43.773 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:43 smithi070 ceph-mon[39288]: from='client.15147 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:43.773 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:43 smithi070 ceph-mon[39288]: from='client.25058 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:43.773 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:43 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:43.773 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:43 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:43.773 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:43 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:43.773 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:43 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:43.773 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:43 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:44.013 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:43 smithi094 ceph-mon[31053]: osdmap e229: 8 total, 8 up, 8 in 2024-09-14T03:50:44.013 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:43 smithi094 ceph-mon[31053]: pgmap v106: 121 pgs: 121 active+clean; 273 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.5 MiB/s rd, 7.7 MiB/s wr, 20 op/s 2024-09-14T03:50:44.013 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:43 smithi094 ceph-mon[31053]: from='client.15147 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:44.013 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:43 smithi094 ceph-mon[31053]: from='client.25058 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:44.013 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:43 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:44.013 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:43 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:44.013 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:43 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:44.013 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:43 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:44.013 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:43 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:44.088 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:50:44.088 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:50:44.088 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-09-14T03:50:44.088 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:50:44.088 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [], 2024-09-14T03:50:44.088 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "", 2024-09-14T03:50:44.089 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "Currently upgrading mgr daemons", 2024-09-14T03:50:44.089 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-09-14T03:50:44.089 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:50:44.277 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:44 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:50:44.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:50:44.554 INFO:tasks.rados.rados.0.smithi070.stdout:865: done (2 left) 2024-09-14T03:50:44.554 INFO:tasks.rados.rados.0.smithi070.stdout:868: done (1 left) 2024-09-14T03:50:44.554 INFO:tasks.rados.rados.0.smithi070.stdout:870: done (0 left) 2024-09-14T03:50:44.554 INFO:tasks.rados.rados.0.smithi070.stdout:871: rmattr oid 35 current snap is 88 2024-09-14T03:50:44.554 INFO:tasks.rados.rados.0.smithi070.stdout:871: done (0 left) 2024-09-14T03:50:44.554 INFO:tasks.rados.rados.0.smithi070.stdout:872: read oid 27 snap -1 2024-09-14T03:50:44.554 INFO:tasks.rados.rados.0.smithi070.stdout:872: expect deleted 2024-09-14T03:50:44.554 INFO:tasks.rados.rados.0.smithi070.stdout:873: setattr oid 18 current snap is 88 2024-09-14T03:50:44.555 INFO:tasks.rados.rados.0.smithi070.stdout:874: rollback oid 25 current snap is 88 2024-09-14T03:50:44.555 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 25 to 51 2024-09-14T03:50:44.555 INFO:tasks.rados.rados.0.smithi070.stdout:875: snap_remove snap 40 2024-09-14T03:50:44.869 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:44 smithi094 ceph-mon[31053]: from='client.25070 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:44.869 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:44 smithi094 ceph-mon[31053]: osdmap e230: 8 total, 8 up, 8 in 2024-09-14T03:50:44.870 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:44 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/1436967665' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:44.870 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:44 smithi094 ceph-mon[31053]: from='client.15177 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:44.897 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:44 smithi070 ceph-mon[39288]: from='client.25070 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:44.897 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:44 smithi070 ceph-mon[39288]: osdmap e230: 8 total, 8 up, 8 in 2024-09-14T03:50:44.897 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:44 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/1436967665' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:44.897 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:44 smithi070 ceph-mon[39288]: from='client.15177 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:44.897 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:44 smithi070 ceph-mon[27495]: from='client.25070 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:44.897 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:44 smithi070 ceph-mon[27495]: osdmap e230: 8 total, 8 up, 8 in 2024-09-14T03:50:44.898 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:44 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/1436967665' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:50:44.898 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:44 smithi070 ceph-mon[27495]: from='client.15177 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:45.542 INFO:tasks.rados.rados.0.smithi070.stdout:875: done (3 left) 2024-09-14T03:50:45.542 INFO:tasks.rados.rados.0.smithi070.stdout:876: snap_create 2024-09-14T03:50:45.542 INFO:tasks.rados.rados.0.smithi070.stdout:874: finishing rollback tid 0 to smithi07085154-25 2024-09-14T03:50:45.542 INFO:tasks.rados.rados.0.smithi070.stdout:874: finishing rollback tid 1 to smithi07085154-25 2024-09-14T03:50:45.542 INFO:tasks.rados.rados.0.smithi070.stdout:874: finishing rollback tid 2 to smithi07085154-25 2024-09-14T03:50:45.543 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 368 (ObjNum 140 snap 39 seq_num 140) dirty exists 2024-09-14T03:50:45.543 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 521 (ObjNum 119 snap 30 seq_num 119) dirty exists 2024-09-14T03:50:45.891 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:45 smithi094 ceph-mon[31053]: pgmap v108: 121 pgs: 121 active+clean; 273 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.4 MiB/s rd, 7.6 MiB/s wr, 20 op/s 2024-09-14T03:50:45.891 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:45 smithi094 ceph-mon[31053]: osdmap e231: 8 total, 8 up, 8 in 2024-09-14T03:50:45.891 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:45 smithi094 ceph-mon[31053]: osdmap e232: 8 total, 8 up, 8 in 2024-09-14T03:50:45.892 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:45 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:50:45.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:50:45.892 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:45 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:50:45.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:50:45.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:45 smithi070 ceph-mon[27495]: pgmap v108: 121 pgs: 121 active+clean; 273 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.4 MiB/s rd, 7.6 MiB/s wr, 20 op/s 2024-09-14T03:50:45.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:45 smithi070 ceph-mon[27495]: osdmap e231: 8 total, 8 up, 8 in 2024-09-14T03:50:45.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:45 smithi070 ceph-mon[27495]: osdmap e232: 8 total, 8 up, 8 in 2024-09-14T03:50:45.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:45 smithi070 ceph-mon[39288]: pgmap v108: 121 pgs: 121 active+clean; 273 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.4 MiB/s rd, 7.6 MiB/s wr, 20 op/s 2024-09-14T03:50:45.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:45 smithi070 ceph-mon[39288]: osdmap e231: 8 total, 8 up, 8 in 2024-09-14T03:50:45.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:45 smithi070 ceph-mon[39288]: osdmap e232: 8 total, 8 up, 8 in 2024-09-14T03:50:46.540 INFO:tasks.rados.rados.0.smithi070.stdout:872: done (3 left) 2024-09-14T03:50:46.540 INFO:tasks.rados.rados.0.smithi070.stdout:873: done (2 left) 2024-09-14T03:50:46.540 INFO:tasks.rados.rados.0.smithi070.stdout:874: done (1 left) 2024-09-14T03:50:46.540 INFO:tasks.rados.rados.0.smithi070.stdout:876: done (0 left) 2024-09-14T03:50:46.540 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 1 current snap is 89 2024-09-14T03:50:46.540 INFO:tasks.rados.rados.0.smithi070.stdout:877: seq_num 285 ranges {0=778240,778240=475136,1253376=475136,1728512=655360} 2024-09-14T03:50:46.544 INFO:tasks.rados.rados.0.smithi070.stdout:877: writing smithi07085154-1 from 0 to 778240 tid 1 2024-09-14T03:50:46.546 INFO:tasks.rados.rados.0.smithi070.stdout:877: writing smithi07085154-1 from 778240 to 1253376 tid 2 2024-09-14T03:50:46.549 INFO:tasks.rados.rados.0.smithi070.stdout:877: writing smithi07085154-1 from 1253376 to 1728512 tid 3 2024-09-14T03:50:46.554 INFO:tasks.rados.rados.0.smithi070.stdout:877: writing smithi07085154-1 from 1728512 to 2383872 tid 4 2024-09-14T03:50:46.554 INFO:tasks.rados.rados.0.smithi070.stdout:878: copy_from oid 48 from oid 18 current snap is 89 2024-09-14T03:50:46.554 INFO:tasks.rados.rados.0.smithi070.stdout:877: finishing write tid 1 to smithi07085154-1 2024-09-14T03:50:46.554 INFO:tasks.rados.rados.0.smithi070.stdout:877: finishing write tid 2 to smithi07085154-1 2024-09-14T03:50:46.554 INFO:tasks.rados.rados.0.smithi070.stdout:877: finishing write tid 3 to smithi07085154-1 2024-09-14T03:50:46.554 INFO:tasks.rados.rados.0.smithi070.stdout:879: read oid 33 snap -1 2024-09-14T03:50:46.554 INFO:tasks.rados.rados.0.smithi070.stdout:879: expect (ObjNum 49 snap 0 seq_num 49) 2024-09-14T03:50:46.554 INFO:tasks.rados.rados.0.smithi070.stdout:880: copy_from oid 32 from oid 37 current snap is 89 2024-09-14T03:50:46.554 INFO:tasks.rados.rados.0.smithi070.stdout:881: delete oid 20 current snap is 89 2024-09-14T03:50:46.558 INFO:tasks.rados.rados.0.smithi070.stdout:880: finishing copy_from to smithi07085154-32 2024-09-14T03:50:46.558 INFO:tasks.rados.rados.0.smithi070.stdout:880: got expected ENOENT (src dne) 2024-09-14T03:50:46.558 INFO:tasks.rados.rados.0.smithi070.stdout:879: expect (ObjNum 49 snap 0 seq_num 49) 2024-09-14T03:50:46.585 INFO:tasks.rados.rados.0.smithi070.stdout:880: finishing copy_from racing read to smithi07085154-32 2024-09-14T03:50:46.585 INFO:tasks.rados.rados.0.smithi070.stdout:877: finishing write tid 4 to smithi07085154-1 2024-09-14T03:50:46.585 INFO:tasks.rados.rados.0.smithi070.stdout:877: finishing write tid 5 to smithi07085154-1 2024-09-14T03:50:46.585 INFO:tasks.rados.rados.0.smithi070.stdout:877: finishing write tid 6 to smithi07085154-1 2024-09-14T03:50:46.586 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 1 v 294 (ObjNum 285 snap 89 seq_num 285) dirty exists 2024-09-14T03:50:46.586 INFO:tasks.rados.rados.0.smithi070.stdout:877: left oid 1 (ObjNum 285 snap 89 seq_num 285) 2024-09-14T03:50:46.586 INFO:tasks.rados.rados.0.smithi070.stdout:878: finishing copy_from racing read to smithi07085154-48 2024-09-14T03:50:46.586 INFO:tasks.rados.rados.0.smithi070.stdout:878: finishing copy_from to smithi07085154-48 2024-09-14T03:50:46.586 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 521 (ObjNum 119 snap 30 seq_num 119) dirty exists 2024-09-14T03:50:46.586 INFO:tasks.rados.rados.0.smithi070.stdout:877: done (4 left) 2024-09-14T03:50:46.586 INFO:tasks.rados.rados.0.smithi070.stdout:878: done (3 left) 2024-09-14T03:50:46.586 INFO:tasks.rados.rados.0.smithi070.stdout:879: done (2 left) 2024-09-14T03:50:46.586 INFO:tasks.rados.rados.0.smithi070.stdout:880: done (1 left) 2024-09-14T03:50:46.586 INFO:tasks.rados.rados.0.smithi070.stdout:881: done (0 left) 2024-09-14T03:50:46.586 INFO:tasks.rados.rados.0.smithi070.stdout:882: snap_create 2024-09-14T03:50:47.276 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:46 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:50:46.951Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:50:47.546 INFO:tasks.rados.rados.0.smithi070.stdout:882: done (0 left) 2024-09-14T03:50:47.546 INFO:tasks.rados.rados.0.smithi070.stdout:883: read oid 28 snap -1 2024-09-14T03:50:47.547 INFO:tasks.rados.rados.0.smithi070.stdout:883: expect (ObjNum 160 snap 47 seq_num 160) 2024-09-14T03:50:47.547 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 2 current snap is 90 2024-09-14T03:50:47.547 INFO:tasks.rados.rados.0.smithi070.stdout:884: seq_num 287 ranges {0=8192} 2024-09-14T03:50:47.547 INFO:tasks.rados.rados.0.smithi070.stdout:884: writing smithi07085154-2 from 0 to 8192 tid 1 2024-09-14T03:50:47.547 INFO:tasks.rados.rados.0.smithi070.stdout:885: delete oid 38 current snap is 90 2024-09-14T03:50:47.553 INFO:tasks.rados.rados.0.smithi070.stdout:885: done (2 left) 2024-09-14T03:50:47.553 INFO:tasks.rados.rados.0.smithi070.stdout:886: snap_create 2024-09-14T03:50:47.554 INFO:tasks.rados.rados.0.smithi070.stdout:884: finishing write tid 1 to smithi07085154-2 2024-09-14T03:50:47.554 INFO:tasks.rados.rados.0.smithi070.stdout:884: finishing write tid 2 to smithi07085154-2 2024-09-14T03:50:47.556 INFO:tasks.rados.rados.0.smithi070.stdout:884: finishing write tid 3 to smithi07085154-2 2024-09-14T03:50:47.556 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 2 v 750 (ObjNum 287 snap 90 seq_num 287) dirty exists 2024-09-14T03:50:47.556 INFO:tasks.rados.rados.0.smithi070.stdout:884: left oid 2 (ObjNum 287 snap 90 seq_num 287) 2024-09-14T03:50:47.557 INFO:tasks.rados.rados.0.smithi070.stdout:883: expect (ObjNum 160 snap 47 seq_num 160) 2024-09-14T03:50:47.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:47 smithi094 ceph-mon[31053]: osdmap e233: 8 total, 8 up, 8 in 2024-09-14T03:50:47.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:47 smithi094 ceph-mon[31053]: pgmap v112: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 578 KiB/s wr, 8 op/s 2024-09-14T03:50:47.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:47 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:47.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:47 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:48.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:47 smithi070 ceph-mon[27495]: osdmap e233: 8 total, 8 up, 8 in 2024-09-14T03:50:48.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:47 smithi070 ceph-mon[27495]: pgmap v112: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 578 KiB/s wr, 8 op/s 2024-09-14T03:50:48.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:47 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:48.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:47 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:48.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:47 smithi070 ceph-mon[39288]: osdmap e233: 8 total, 8 up, 8 in 2024-09-14T03:50:48.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:47 smithi070 ceph-mon[39288]: pgmap v112: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 578 KiB/s wr, 8 op/s 2024-09-14T03:50:48.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:47 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:48.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:47 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:48.392 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: [14/Sep/2024:03:50:48] ENGINE Bus STARTING 2024-09-14T03:50:48.392 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: CherryPy Checker: 2024-09-14T03:50:48.393 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: The Application mounted at '' has an empty config. 2024-09-14T03:50:48.393 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:50:48.550 INFO:tasks.rados.rados.0.smithi070.stdout:883: done (2 left) 2024-09-14T03:50:48.551 INFO:tasks.rados.rados.0.smithi070.stdout:884: done (1 left) 2024-09-14T03:50:48.551 INFO:tasks.rados.rados.0.smithi070.stdout:886: done (0 left) 2024-09-14T03:50:48.551 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 27 current snap is 91 2024-09-14T03:50:48.551 INFO:tasks.rados.rados.0.smithi070.stdout:887: seq_num 289 ranges {0=40960} 2024-09-14T03:50:48.551 INFO:tasks.rados.rados.0.smithi070.stdout:887: writing smithi07085154-27 from 0 to 40960 tid 1 2024-09-14T03:50:48.551 INFO:tasks.rados.rados.0.smithi070.stdout:888: read oid 26 snap -1 2024-09-14T03:50:48.551 INFO:tasks.rados.rados.0.smithi070.stdout:888: expect deleted 2024-09-14T03:50:48.551 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 48 current snap is 91 2024-09-14T03:50:48.551 INFO:tasks.rados.rados.0.smithi070.stdout:889: seq_num 290 ranges {3530752=57344} 2024-09-14T03:50:48.570 INFO:tasks.rados.rados.0.smithi070.stdout:889: writing smithi07085154-48 from 3530752 to 3588096 tid 1 2024-09-14T03:50:48.571 INFO:tasks.rados.rados.0.smithi070.stdout:890: read oid 10 snap -1 2024-09-14T03:50:48.571 INFO:tasks.rados.rados.0.smithi070.stdout:890: expect deleted 2024-09-14T03:50:48.571 INFO:tasks.rados.rados.0.smithi070.stdout:891: rollback oid 30 current snap is 91 2024-09-14T03:50:48.571 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 30 to 83 2024-09-14T03:50:48.571 INFO:tasks.rados.rados.0.smithi070.stdout:887: finishing write tid 1 to smithi07085154-27 2024-09-14T03:50:48.571 INFO:tasks.rados.rados.0.smithi070.stdout:887: finishing write tid 2 to smithi07085154-27 2024-09-14T03:50:48.571 INFO:tasks.rados.rados.0.smithi070.stdout:887: finishing write tid 3 to smithi07085154-27 2024-09-14T03:50:48.571 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 752 (ObjNum 289 snap 91 seq_num 289) dirty exists 2024-09-14T03:50:48.571 INFO:tasks.rados.rados.0.smithi070.stdout:887: left oid 27 (ObjNum 289 snap 91 seq_num 289) 2024-09-14T03:50:48.571 INFO:tasks.rados.rados.0.smithi070.stdout:887: done (4 left) 2024-09-14T03:50:48.571 INFO:tasks.rados.rados.0.smithi070.stdout:888: done (3 left) 2024-09-14T03:50:48.571 INFO:tasks.rados.rados.0.smithi070.stdout:892: read oid 45 snap -1 2024-09-14T03:50:48.572 INFO:tasks.rados.rados.0.smithi070.stdout:892: expect (ObjNum 49 snap 0 seq_num 49) 2024-09-14T03:50:48.572 INFO:tasks.rados.rados.0.smithi070.stdout:893: read oid 26 snap -1 2024-09-14T03:50:48.572 INFO:tasks.rados.rados.0.smithi070.stdout:893: expect deleted 2024-09-14T03:50:48.572 INFO:tasks.rados.rados.0.smithi070.stdout:894: snap_remove snap 87 2024-09-14T03:50:48.779 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: [14/Sep/2024:03:50:48] ENGINE Serving on http://:::9283 2024-09-14T03:50:48.779 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:48 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: [14/Sep/2024:03:50:48] ENGINE Bus STARTED 2024-09-14T03:50:48.780 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:48 smithi070 ceph-mon[27495]: osdmap e234: 8 total, 8 up, 8 in 2024-09-14T03:50:48.780 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:48 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:48.780 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:48 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:48.780 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:48 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:48.780 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:48 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:48.780 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:48 smithi070 ceph-mon[27495]: Standby manager daemon y restarted 2024-09-14T03:50:48.780 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:48 smithi070 ceph-mon[27495]: Standby manager daemon y started 2024-09-14T03:50:48.780 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:48 smithi070 ceph-mon[27495]: from='mgr.? 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-14T03:50:48.780 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:48 smithi070 ceph-mon[27495]: from='mgr.? 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:50:48.780 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:48 smithi070 ceph-mon[27495]: from='mgr.? 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-14T03:50:48.780 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:48 smithi070 ceph-mon[27495]: from='mgr.? 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:50:48.781 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:48 smithi070 ceph-mon[39288]: osdmap e234: 8 total, 8 up, 8 in 2024-09-14T03:50:48.781 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:48 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:48.781 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:48 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:48.781 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:48 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:48.781 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:48 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:48.781 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:48 smithi070 ceph-mon[39288]: Standby manager daemon y restarted 2024-09-14T03:50:48.781 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:48 smithi070 ceph-mon[39288]: Standby manager daemon y started 2024-09-14T03:50:48.781 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:48 smithi070 ceph-mon[39288]: from='mgr.? 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-14T03:50:48.781 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:48 smithi070 ceph-mon[39288]: from='mgr.? 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:50:48.781 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:48 smithi070 ceph-mon[39288]: from='mgr.? 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-14T03:50:48.781 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:48 smithi070 ceph-mon[39288]: from='mgr.? 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:50:49.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:48 smithi094 ceph-mon[31053]: osdmap e234: 8 total, 8 up, 8 in 2024-09-14T03:50:49.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:48 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:49.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:48 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:49.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:48 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:49.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:48 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:49.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:48 smithi094 ceph-mon[31053]: Standby manager daemon y restarted 2024-09-14T03:50:49.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:48 smithi094 ceph-mon[31053]: Standby manager daemon y started 2024-09-14T03:50:49.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:48 smithi094 ceph-mon[31053]: from='mgr.? 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-14T03:50:49.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:48 smithi094 ceph-mon[31053]: from='mgr.? 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:50:49.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:48 smithi094 ceph-mon[31053]: from='mgr.? 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-14T03:50:49.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:48 smithi094 ceph-mon[31053]: from='mgr.? 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:50:49.555 INFO:tasks.rados.rados.0.smithi070.stdout:894: done (5 left) 2024-09-14T03:50:49.555 INFO:tasks.rados.rados.0.smithi070.stdout:895: snap_remove snap 74 2024-09-14T03:50:49.687 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:49 smithi070 ceph-mon[27495]: pgmap v114: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 578 KiB/s wr, 8 op/s 2024-09-14T03:50:49.687 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:49 smithi070 ceph-mon[27495]: osdmap e235: 8 total, 8 up, 8 in 2024-09-14T03:50:49.687 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:49 smithi070 ceph-mon[27495]: mgrmap e29: x(active, since 74s), standbys: y 2024-09-14T03:50:49.687 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:49 smithi070 ceph-mon[39288]: pgmap v114: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 578 KiB/s wr, 8 op/s 2024-09-14T03:50:49.687 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:49 smithi070 ceph-mon[39288]: osdmap e235: 8 total, 8 up, 8 in 2024-09-14T03:50:49.687 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:49 smithi070 ceph-mon[39288]: mgrmap e29: x(active, since 74s), standbys: y 2024-09-14T03:50:50.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:49 smithi094 ceph-mon[31053]: pgmap v114: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 578 KiB/s wr, 8 op/s 2024-09-14T03:50:50.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:49 smithi094 ceph-mon[31053]: osdmap e235: 8 total, 8 up, 8 in 2024-09-14T03:50:50.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:49 smithi094 ceph-mon[31053]: mgrmap e29: x(active, since 74s), standbys: y 2024-09-14T03:50:50.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:50 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:50:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:50:50.541 INFO:tasks.rados.rados.0.smithi070.stdout:895: done (5 left) 2024-09-14T03:50:50.541 INFO:tasks.rados.rados.0.smithi070.stdout:896: rmattr oid 25 current snap is 91 2024-09-14T03:50:50.541 INFO:tasks.rados.rados.0.smithi070.stdout:896: done (5 left) 2024-09-14T03:50:50.542 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 38 current snap is 91 2024-09-14T03:50:50.542 INFO:tasks.rados.rados.0.smithi070.stdout:897: seq_num 291 ranges {0=81920} 2024-09-14T03:50:50.542 INFO:tasks.rados.rados.0.smithi070.stdout:897: writing smithi07085154-38 from 0 to 81920 tid 1 2024-09-14T03:50:50.542 INFO:tasks.rados.rados.0.smithi070.stdout:898: read oid 23 snap 86 2024-09-14T03:50:50.542 INFO:tasks.rados.rados.0.smithi070.stdout:898: expect (ObjNum 274 snap 84 seq_num 274) 2024-09-14T03:50:50.542 INFO:tasks.rados.rados.0.smithi070.stdout:899: read oid 12 snap 59 2024-09-14T03:50:50.542 INFO:tasks.rados.rados.0.smithi070.stdout:899: expect (ObjNum 200 snap 56 seq_num 200) 2024-09-14T03:50:50.543 INFO:tasks.rados.rados.0.smithi070.stdout:900: rmattr oid 50 current snap is 91 2024-09-14T03:50:50.543 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 20 current snap is 91 2024-09-14T03:50:50.543 INFO:tasks.rados.rados.0.smithi070.stdout:901: seq_num 292 ranges {0=98304} 2024-09-14T03:50:50.543 INFO:tasks.rados.rados.0.smithi070.stdout:901: writing smithi07085154-20 from 0 to 98304 tid 1 2024-09-14T03:50:50.543 INFO:tasks.rados.rados.0.smithi070.stdout:902: rmattr oid 46 current snap is 91 2024-09-14T03:50:50.543 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 11 current snap is 91 2024-09-14T03:50:50.543 INFO:tasks.rados.rados.0.smithi070.stdout:903: seq_num 293 ranges {2195456=114688} 2024-09-14T03:50:50.557 INFO:tasks.rados.rados.0.smithi070.stdout:903: writing smithi07085154-11 from 2195456 to 2310144 tid 1 2024-09-14T03:50:50.557 INFO:tasks.rados.rados.0.smithi070.stdout:904: read oid 29 snap 71 2024-09-14T03:50:50.557 INFO:tasks.rados.rados.0.smithi070.stdout:904: expect (ObjNum 234 snap 67 seq_num 234) 2024-09-14T03:50:50.557 INFO:tasks.rados.rados.0.smithi070.stdout:905: rollback oid 41 current snap is 91 2024-09-14T03:50:50.557 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 41 to 76 2024-09-14T03:50:50.557 INFO:tasks.rados.rados.0.smithi070.stdout:891: finishing rollback tid 0 to smithi07085154-30 2024-09-14T03:50:50.558 INFO:tasks.rados.rados.0.smithi070.stdout:889: finishing write tid 1 to smithi07085154-48 2024-09-14T03:50:50.558 INFO:tasks.rados.rados.0.smithi070.stdout:889: finishing write tid 2 to smithi07085154-48 2024-09-14T03:50:50.558 INFO:tasks.rados.rados.0.smithi070.stdout:891: finishing rollback tid 1 to smithi07085154-30 2024-09-14T03:50:50.558 INFO:tasks.rados.rados.0.smithi070.stdout:891: finishing rollback tid 2 to smithi07085154-30 2024-09-14T03:50:50.558 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 30 v 289 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-09-14T03:50:50.558 INFO:tasks.rados.rados.0.smithi070.stdout:892: expect (ObjNum 49 snap 0 seq_num 49) 2024-09-14T03:50:50.593 INFO:tasks.rados.rados.0.smithi070.stdout:889: finishing write tid 3 to smithi07085154-48 2024-09-14T03:50:50.593 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 523 (ObjNum 290 snap 91 seq_num 290) dirty exists 2024-09-14T03:50:50.594 INFO:tasks.rados.rados.0.smithi070.stdout:889: left oid 48 (ObjNum 290 snap 91 seq_num 290) 2024-09-14T03:50:50.594 INFO:tasks.rados.rados.0.smithi070.stdout:901: finishing write tid 1 to smithi07085154-20 2024-09-14T03:50:50.594 INFO:tasks.rados.rados.0.smithi070.stdout:901: finishing write tid 2 to smithi07085154-20 2024-09-14T03:50:50.594 INFO:tasks.rados.rados.0.smithi070.stdout:897: finishing write tid 1 to smithi07085154-38 2024-09-14T03:50:50.594 INFO:tasks.rados.rados.0.smithi070.stdout:901: finishing write tid 3 to smithi07085154-20 2024-09-14T03:50:50.594 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 20 v 349 (ObjNum 292 snap 91 seq_num 292) dirty exists 2024-09-14T03:50:50.594 INFO:tasks.rados.rados.0.smithi070.stdout:901: left oid 20 (ObjNum 292 snap 91 seq_num 292) 2024-09-14T03:50:50.594 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 46 v 754 (ObjNum 181 snap 51 seq_num 181) dirty exists 2024-09-14T03:50:50.594 INFO:tasks.rados.rados.0.smithi070.stdout:897: finishing write tid 2 to smithi07085154-38 2024-09-14T03:50:50.594 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 495 (ObjNum 211 snap 60 seq_num 211) dirty exists 2024-09-14T03:50:50.594 INFO:tasks.rados.rados.0.smithi070.stdout:889: done (13 left) 2024-09-14T03:50:50.595 INFO:tasks.rados.rados.0.smithi070.stdout:890: done (12 left) 2024-09-14T03:50:50.595 INFO:tasks.rados.rados.0.smithi070.stdout:891: done (11 left) 2024-09-14T03:50:50.595 INFO:tasks.rados.rados.0.smithi070.stdout:892: done (10 left) 2024-09-14T03:50:50.595 INFO:tasks.rados.rados.0.smithi070.stdout:893: done (9 left) 2024-09-14T03:50:50.595 INFO:tasks.rados.rados.0.smithi070.stdout:900: done (8 left) 2024-09-14T03:50:50.595 INFO:tasks.rados.rados.0.smithi070.stdout:901: done (7 left) 2024-09-14T03:50:50.595 INFO:tasks.rados.rados.0.smithi070.stdout:902: done (6 left) 2024-09-14T03:50:50.595 INFO:tasks.rados.rados.0.smithi070.stdout:906: delete oid 14 current snap is 91 2024-09-14T03:50:50.595 INFO:tasks.rados.rados.0.smithi070.stdout:897: finishing write tid 3 to smithi07085154-38 2024-09-14T03:50:50.595 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 38 v 295 (ObjNum 291 snap 91 seq_num 291) dirty exists 2024-09-14T03:50:50.595 INFO:tasks.rados.rados.0.smithi070.stdout:897: left oid 38 (ObjNum 291 snap 91 seq_num 291) 2024-09-14T03:50:50.595 INFO:tasks.rados.rados.0.smithi070.stdout:905: finishing rollback tid 0 to smithi07085154-41 2024-09-14T03:50:50.595 INFO:tasks.rados.rados.0.smithi070.stdout:905: finishing rollback tid 1 to smithi07085154-41 2024-09-14T03:50:50.595 INFO:tasks.rados.rados.0.smithi070.stdout:899: expect (ObjNum 200 snap 56 seq_num 200) 2024-09-14T03:50:50.646 INFO:tasks.rados.rados.0.smithi070.stdout:903: finishing write tid 1 to smithi07085154-11 2024-09-14T03:50:50.647 INFO:tasks.rados.rados.0.smithi070.stdout:897: done (6 left) 2024-09-14T03:50:50.647 INFO:tasks.rados.rados.0.smithi070.stdout:899: done (5 left) 2024-09-14T03:50:50.647 INFO:tasks.rados.rados.0.smithi070.stdout:906: done (4 left) 2024-09-14T03:50:50.647 INFO:tasks.rados.rados.0.smithi070.stdout:907: snap_create 2024-09-14T03:50:50.647 INFO:tasks.rados.rados.0.smithi070.stdout:905: finishing rollback tid 2 to smithi07085154-41 2024-09-14T03:50:50.647 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 541 (ObjNum 211 snap 60 seq_num 211) dirty exists 2024-09-14T03:50:50.647 INFO:tasks.rados.rados.0.smithi070.stdout:903: finishing write tid 2 to smithi07085154-11 2024-09-14T03:50:50.647 INFO:tasks.rados.rados.0.smithi070.stdout:903: finishing write tid 3 to smithi07085154-11 2024-09-14T03:50:50.647 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 11 v 346 (ObjNum 293 snap 91 seq_num 293) dirty exists 2024-09-14T03:50:50.647 INFO:tasks.rados.rados.0.smithi070.stdout:903: left oid 11 (ObjNum 293 snap 91 seq_num 293) 2024-09-14T03:50:50.647 INFO:tasks.rados.rados.0.smithi070.stdout:904: expect (ObjNum 234 snap 67 seq_num 234) 2024-09-14T03:50:50.720 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:50 smithi070 ceph-mon[27495]: osdmap e236: 8 total, 8 up, 8 in 2024-09-14T03:50:50.720 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:50 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:50:50.720 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:50 smithi070 ceph-mon[27495]: osdmap e237: 8 total, 8 up, 8 in 2024-09-14T03:50:50.720 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:50 smithi070 ceph-mon[39288]: osdmap e236: 8 total, 8 up, 8 in 2024-09-14T03:50:50.720 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:50 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:50:50.720 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:50 smithi070 ceph-mon[39288]: osdmap e237: 8 total, 8 up, 8 in 2024-09-14T03:50:50.721 INFO:tasks.rados.rados.0.smithi070.stdout:898: expect (ObjNum 274 snap 84 seq_num 274) 2024-09-14T03:50:51.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:50 smithi094 ceph-mon[31053]: osdmap e236: 8 total, 8 up, 8 in 2024-09-14T03:50:51.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:50 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:50:51.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:50 smithi094 ceph-mon[31053]: osdmap e237: 8 total, 8 up, 8 in 2024-09-14T03:50:51.209 INFO:tasks.rados.rados.0.smithi070.stdout:898: done (4 left) 2024-09-14T03:50:51.209 INFO:tasks.rados.rados.0.smithi070.stdout:903: done (3 left) 2024-09-14T03:50:51.209 INFO:tasks.rados.rados.0.smithi070.stdout:904: done (2 left) 2024-09-14T03:50:51.209 INFO:tasks.rados.rados.0.smithi070.stdout:905: done (1 left) 2024-09-14T03:50:51.209 INFO:tasks.rados.rados.0.smithi070.stdout:907: done (0 left) 2024-09-14T03:50:51.209 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 45 current snap is 92 2024-09-14T03:50:51.209 INFO:tasks.rados.rados.0.smithi070.stdout:908: seq_num 295 ranges {827392=147456} 2024-09-14T03:50:51.214 INFO:tasks.rados.rados.0.smithi070.stdout:908: writing smithi07085154-45 from 827392 to 974848 tid 1 2024-09-14T03:50:51.214 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 10 current snap is 92 2024-09-14T03:50:51.215 INFO:tasks.rados.rados.0.smithi070.stdout:909: seq_num 296 ranges {0=163840} 2024-09-14T03:50:51.216 INFO:tasks.rados.rados.0.smithi070.stdout:909: writing smithi07085154-10 from 0 to 163840 tid 1 2024-09-14T03:50:51.216 INFO:tasks.rados.rados.0.smithi070.stdout:910: delete oid 7 current snap is 92 2024-09-14T03:50:51.220 INFO:tasks.rados.rados.0.smithi070.stdout:909: finishing write tid 1 to smithi07085154-10 2024-09-14T03:50:51.220 INFO:tasks.rados.rados.0.smithi070.stdout:910: done (2 left) 2024-09-14T03:50:51.220 INFO:tasks.rados.rados.0.smithi070.stdout:911: rmattr oid 6 current snap is 92 2024-09-14T03:50:51.220 INFO:tasks.rados.rados.0.smithi070.stdout:911: done (2 left) 2024-09-14T03:50:51.220 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 47 current snap is 92 2024-09-14T03:50:51.220 INFO:tasks.rados.rados.0.smithi070.stdout:912: seq_num 298 ranges {2662400=196608} 2024-09-14T03:50:51.236 INFO:tasks.rados.rados.0.smithi070.stdout:912: writing smithi07085154-47 from 2662400 to 2859008 tid 1 2024-09-14T03:50:51.236 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 48 current snap is 92 2024-09-14T03:50:51.236 INFO:tasks.rados.rados.0.smithi070.stdout:908: finishing write tid 1 to smithi07085154-45 2024-09-14T03:50:51.236 INFO:tasks.rados.rados.0.smithi070.stdout:908: finishing write tid 2 to smithi07085154-45 2024-09-14T03:50:51.236 INFO:tasks.rados.rados.0.smithi070.stdout:909: finishing write tid 2 to smithi07085154-10 2024-09-14T03:50:51.236 INFO:tasks.rados.rados.0.smithi070.stdout:908: finishing write tid 3 to smithi07085154-45 2024-09-14T03:50:51.236 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 45 v 281 (ObjNum 295 snap 92 seq_num 295) dirty exists 2024-09-14T03:50:51.237 INFO:tasks.rados.rados.0.smithi070.stdout:908: left oid 45 (ObjNum 295 snap 92 seq_num 295) 2024-09-14T03:50:51.237 INFO:tasks.rados.rados.0.smithi070.stdout:909: finishing write tid 3 to smithi07085154-10 2024-09-14T03:50:51.237 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 764 (ObjNum 296 snap 92 seq_num 296) dirty exists 2024-09-14T03:50:51.237 INFO:tasks.rados.rados.0.smithi070.stdout:909: left oid 10 (ObjNum 296 snap 92 seq_num 296) 2024-09-14T03:50:51.237 INFO:tasks.rados.rados.0.smithi070.stdout:913: seq_num 299 ranges {3588096=212992} 2024-09-14T03:50:51.257 INFO:tasks.rados.rados.0.smithi070.stdout:913: writing smithi07085154-48 from 3588096 to 3801088 tid 1 2024-09-14T03:50:51.257 INFO:tasks.rados.rados.0.smithi070.stdout:908: done (3 left) 2024-09-14T03:50:51.257 INFO:tasks.rados.rados.0.smithi070.stdout:909: done (2 left) 2024-09-14T03:50:51.257 INFO:tasks.rados.rados.0.smithi070.stdout:914: delete oid 20 current snap is 92 2024-09-14T03:50:51.257 INFO:tasks.rados.rados.0.smithi070.stdout:912: finishing write tid 1 to smithi07085154-47 2024-09-14T03:50:51.257 INFO:tasks.rados.rados.0.smithi070.stdout:912: finishing write tid 2 to smithi07085154-47 2024-09-14T03:50:51.257 INFO:tasks.rados.rados.0.smithi070.stdout:912: finishing write tid 3 to smithi07085154-47 2024-09-14T03:50:51.258 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 47 v 304 (ObjNum 298 snap 92 seq_num 298) dirty exists 2024-09-14T03:50:51.258 INFO:tasks.rados.rados.0.smithi070.stdout:912: left oid 47 (ObjNum 298 snap 92 seq_num 298) 2024-09-14T03:50:51.258 INFO:tasks.rados.rados.0.smithi070.stdout:912: done (2 left) 2024-09-14T03:50:51.258 INFO:tasks.rados.rados.0.smithi070.stdout:914: done (1 left) 2024-09-14T03:50:51.258 INFO:tasks.rados.rados.0.smithi070.stdout:915: snap_create 2024-09-14T03:50:51.259 INFO:tasks.rados.rados.0.smithi070.stdout:913: finishing write tid 1 to smithi07085154-48 2024-09-14T03:50:51.260 INFO:tasks.rados.rados.0.smithi070.stdout:913: finishing write tid 2 to smithi07085154-48 2024-09-14T03:50:51.260 INFO:tasks.rados.rados.0.smithi070.stdout:913: finishing write tid 3 to smithi07085154-48 2024-09-14T03:50:51.261 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 525 (ObjNum 299 snap 92 seq_num 299) dirty exists 2024-09-14T03:50:51.261 INFO:tasks.rados.rados.0.smithi070.stdout:913: left oid 48 (ObjNum 299 snap 92 seq_num 299) 2024-09-14T03:50:51.429 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:51 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: [14/Sep/2024:03:50:51] ENGINE Bus STOPPING 2024-09-14T03:50:51.527 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:50:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: 2024-09-14T03:50:51.207+0000 7f930a613640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-14T03:50:51.527 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:50:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: ignoring --setuser ceph since I am not root 2024-09-14T03:50:51.527 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:50:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: ignoring --setgroup ceph since I am not root 2024-09-14T03:50:51.527 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:50:51 smithi094 ceph-mgr[67189]: -- 172.21.15.94:0/3347034248 <== mon.1 v2:172.21.15.94:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x564f8bc050e0 con 0x564f8bbe3400 2024-09-14T03:50:51.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:51 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: [14/Sep/2024:03:50:51] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:50:51.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:51 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: [14/Sep/2024:03:50:51] ENGINE Bus STOPPED 2024-09-14T03:50:52.104 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:51 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:50:51.815+0000 7f8dec7b4640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:50:52.104 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:51 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:50:51.815+0000 7f8dec7b4640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:50:52.104 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:51 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:50:51.815+0000 7f8dec7b4640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:50:52.104 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:51 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:50:51.815+0000 7f8dec7b4640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:50:52.104 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:51 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:50:51.815+0000 7f8dec7b4640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:50:52.104 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:51 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:50:51.815+0000 7f8def7ba640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:50:52.104 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:51 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:50:51.815+0000 7f8def7ba640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:50:52.104 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:51 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:50:51.815+0000 7f8def7ba640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:50:52.104 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:51 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:50:51.815+0000 7f8def7ba640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:50:52.104 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:51 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:50:51.815+0000 7f8def7ba640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:50:52.217 INFO:tasks.rados.rados.0.smithi070.stdout:913: done (1 left) 2024-09-14T03:50:52.217 INFO:tasks.rados.rados.0.smithi070.stdout:915: done (0 left) 2024-09-14T03:50:52.217 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 13 current snap is 93 2024-09-14T03:50:52.218 INFO:tasks.rados.rados.0.smithi070.stdout:916: seq_num 301 ranges {6905856=245760} 2024-09-14T03:50:52.256 INFO:tasks.rados.rados.0.smithi070.stdout:916: writing smithi07085154-13 from 6905856 to 7151616 tid 1 2024-09-14T03:50:52.256 INFO:tasks.rados.rados.0.smithi070.stdout:917: delete oid 50 current snap is 93 2024-09-14T03:50:52.262 INFO:tasks.rados.rados.0.smithi070.stdout:917: done (1 left) 2024-09-14T03:50:52.262 INFO:tasks.rados.rados.0.smithi070.stdout:918: rollback oid 21 current snap is 93 2024-09-14T03:50:52.262 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 21 to 71 2024-09-14T03:50:52.262 INFO:tasks.rados.rados.0.smithi070.stdout:919: snap_remove snap 86 2024-09-14T03:50:52.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: pgmap v118: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:50:52.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:52.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:52.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:52.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:50:52.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:52.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:52.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-14T03:50:52.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-14T03:50:52.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: Activating manager daemon y 2024-09-14T03:50:52.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: osdmap e238: 8 total, 8 up, 8 in 2024-09-14T03:50:52.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: from='mgr.24709 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-14T03:50:52.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: mgrmap e30: y(active, starting, since 0.00579443s) 2024-09-14T03:50:52.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:50:52.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:50:52.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:50:52.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-14T03:50:52.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-14T03:50:52.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:50:52.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:50:52.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:50:52.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:50:52.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:50:52.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:50:52.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:50:52.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:50:52.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:50:52.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:50:52.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:50:52.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:50:52.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: Manager daemon y is now available 2024-09-14T03:50:52.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:52.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:50:52.280 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:50:52.280 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:50:52.280 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:50:52.280 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:50:52.280 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:52 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:50:52.536 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: [14/Sep/2024:03:50:52] ENGINE Bus STARTING 2024-09-14T03:50:52.536 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: [14/Sep/2024:03:50:52] ENGINE Serving on http://:::9283 2024-09-14T03:50:52.536 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: [14/Sep/2024:03:50:52] ENGINE Bus STARTED 2024-09-14T03:50:52.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: pgmap v118: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:50:52.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:52.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:52.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:52.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:50:52.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:52.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:52.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-14T03:50:52.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-14T03:50:52.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: Activating manager daemon y 2024-09-14T03:50:52.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: osdmap e238: 8 total, 8 up, 8 in 2024-09-14T03:50:52.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: from='mgr.24709 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-14T03:50:52.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: mgrmap e30: y(active, starting, since 0.00579443s) 2024-09-14T03:50:52.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:50:52.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:50:52.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:50:52.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-14T03:50:52.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-14T03:50:52.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:50:52.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:50:52.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:50:52.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:50:52.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:50:52.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:50:52.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:50:52.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:50:52.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:50:52.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:50:52.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:50:52.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: pgmap v118: 121 pgs: 121 active+clean; 266 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:50:52.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:52.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:52.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:52.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:50:52.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' 2024-09-14T03:50:52.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:52.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: from='mgr.24709 172.21.15.94:0/4251531500' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-14T03:50:52.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-14T03:50:52.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: Activating manager daemon y 2024-09-14T03:50:52.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: osdmap e238: 8 total, 8 up, 8 in 2024-09-14T03:50:52.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: from='mgr.24709 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-14T03:50:52.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: mgrmap e30: y(active, starting, since 0.00579443s) 2024-09-14T03:50:52.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:50:52.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:50:52.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:50:52.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-14T03:50:52.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-14T03:50:52.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:50:52.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:50:52.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:50:52.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:50:52.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:50:52.541 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:50:52.541 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:50:52.541 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:50:52.541 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:50:52.541 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:50:52.541 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:50:52.541 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:50:52.541 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: Manager daemon y is now available 2024-09-14T03:50:52.541 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:52.541 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:50:52.541 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:50:52.542 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:50:52.542 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:50:52.542 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:50:52.542 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:50:52.542 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:50:52.542 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: Manager daemon y is now available 2024-09-14T03:50:52.542 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:52.542 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:50:52.542 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:50:52.542 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:50:52.542 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:50:52.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:50:52.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:52 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:50:53.209 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:50:53.036+0000 7f8dcd4f9640 -1 log_channel(cephadm) log [ERR] : [14/Sep/2024:03:50:53] ENGINE Error in HTTPServer.serve 2024-09-14T03:50:53.210 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: Traceback (most recent call last): 2024-09-14T03:50:53.210 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:50:53.210 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: self._connections.run(self.expiration_interval) 2024-09-14T03:50:53.210 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:50:53.210 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: self._run(expiration_interval) 2024-09-14T03:50:53.210 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:50:53.210 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:50:53.210 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:50:53.210 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:50:53.210 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:50:53.210 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: s = self.context.wrap_socket( 2024-09-14T03:50:53.210 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:50:53.211 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: return self.sslsocket_class._create( 2024-09-14T03:50:53.211 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:50:53.211 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: self.do_handshake() 2024-09-14T03:50:53.211 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:50:53.211 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: self._sslobj.do_handshake() 2024-09-14T03:50:53.211 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:50:53.211 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:50:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:50:53.218 INFO:tasks.rados.rados.0.smithi070.stdout:919: done (2 left) 2024-09-14T03:50:53.218 INFO:tasks.rados.rados.0.smithi070.stdout:920: snap_create 2024-09-14T03:50:53.218 INFO:tasks.rados.rados.0.smithi070.stdout:916: finishing write tid 1 to smithi07085154-13 2024-09-14T03:50:53.218 INFO:tasks.rados.rados.0.smithi070.stdout:916: finishing write tid 2 to smithi07085154-13 2024-09-14T03:50:53.218 INFO:tasks.rados.rados.0.smithi070.stdout:916: finishing write tid 3 to smithi07085154-13 2024-09-14T03:50:53.219 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 349 (ObjNum 301 snap 93 seq_num 301) dirty exists 2024-09-14T03:50:53.219 INFO:tasks.rados.rados.0.smithi070.stdout:916: left oid 13 (ObjNum 301 snap 93 seq_num 301) 2024-09-14T03:50:53.219 INFO:tasks.rados.rados.0.smithi070.stdout:918: finishing rollback tid 0 to smithi07085154-21 2024-09-14T03:50:53.219 INFO:tasks.rados.rados.0.smithi070.stdout:918: finishing rollback tid 1 to smithi07085154-21 2024-09-14T03:50:53.219 INFO:tasks.rados.rados.0.smithi070.stdout:918: finishing rollback tid 2 to smithi07085154-21 2024-09-14T03:50:53.219 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 258 (ObjNum 242 snap 69 seq_num 242) dirty exists 2024-09-14T03:50:53.251 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:53 smithi094 ceph-mon[31053]: mgrmap e31: y(active, since 1.00907s) 2024-09-14T03:50:53.252 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:53 smithi094 ceph-mon[31053]: osdmap e239: 8 total, 8 up, 8 in 2024-09-14T03:50:53.252 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:53 smithi094 ceph-mon[31053]: [14/Sep/2024:03:50:52] ENGINE Bus STARTING 2024-09-14T03:50:53.252 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:53 smithi094 ceph-mon[31053]: [14/Sep/2024:03:50:52] ENGINE Serving on http://172.21.15.70:8765 2024-09-14T03:50:53.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:53 smithi070 ceph-mon[39288]: mgrmap e31: y(active, since 1.00907s) 2024-09-14T03:50:53.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:53 smithi070 ceph-mon[39288]: osdmap e239: 8 total, 8 up, 8 in 2024-09-14T03:50:53.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:53 smithi070 ceph-mon[39288]: [14/Sep/2024:03:50:52] ENGINE Bus STARTING 2024-09-14T03:50:53.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:53 smithi070 ceph-mon[39288]: [14/Sep/2024:03:50:52] ENGINE Serving on http://172.21.15.70:8765 2024-09-14T03:50:53.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:53 smithi070 ceph-mon[27495]: mgrmap e31: y(active, since 1.00907s) 2024-09-14T03:50:53.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:53 smithi070 ceph-mon[27495]: osdmap e239: 8 total, 8 up, 8 in 2024-09-14T03:50:53.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:53 smithi070 ceph-mon[27495]: [14/Sep/2024:03:50:52] ENGINE Bus STARTING 2024-09-14T03:50:53.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:53 smithi070 ceph-mon[27495]: [14/Sep/2024:03:50:52] ENGINE Serving on http://172.21.15.70:8765 2024-09-14T03:50:54.219 INFO:tasks.rados.rados.0.smithi070.stdout:916: done (2 left) 2024-09-14T03:50:54.219 INFO:tasks.rados.rados.0.smithi070.stdout:918: done (1 left) 2024-09-14T03:50:54.220 INFO:tasks.rados.rados.0.smithi070.stdout:920: done (0 left) 2024-09-14T03:50:54.220 INFO:tasks.rados.rados.0.smithi070.stdout:921: read oid 46 snap -1 2024-09-14T03:50:54.220 INFO:tasks.rados.rados.0.smithi070.stdout:921: expect (ObjNum 181 snap 51 seq_num 181) 2024-09-14T03:50:54.220 INFO:tasks.rados.rados.0.smithi070.stdout:922: read oid 37 snap -1 2024-09-14T03:50:54.220 INFO:tasks.rados.rados.0.smithi070.stdout:922: expect deleted 2024-09-14T03:50:54.220 INFO:tasks.rados.rados.0.smithi070.stdout:923: snap_create 2024-09-14T03:50:54.226 INFO:tasks.rados.rados.0.smithi070.stdout:921: expect (ObjNum 181 snap 51 seq_num 181) 2024-09-14T03:50:54.486 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:54 smithi070 ceph-mon[27495]: [14/Sep/2024:03:50:53] ENGINE Serving on https://172.21.15.70:7150 2024-09-14T03:50:54.486 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:54 smithi070 ceph-mon[27495]: [14/Sep/2024:03:50:53] ENGINE Bus STARTED 2024-09-14T03:50:54.486 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:54 smithi070 ceph-mon[27495]: [14/Sep/2024:03:50:53] ENGINE Error in HTTPServer.serve 2024-09-14T03:50:54.486 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-09-14T03:50:54.487 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:50:54.487 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:50:54.487 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:50:54.487 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._run(expiration_interval) 2024-09-14T03:50:54.487 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:50:54.487 INFO:journalctl@ceph.mon.a.smithi070.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:50:54.487 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:50:54.487 INFO:journalctl@ceph.mon.a.smithi070.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:50:54.487 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:50:54.487 INFO:journalctl@ceph.mon.a.smithi070.stdout: s = self.context.wrap_socket( 2024-09-14T03:50:54.487 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:50:54.487 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.sslsocket_class._create( 2024-09-14T03:50:54.487 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:50:54.488 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.do_handshake() 2024-09-14T03:50:54.488 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:50:54.488 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._sslobj.do_handshake() 2024-09-14T03:50:54.488 INFO:journalctl@ceph.mon.a.smithi070.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:50:54.488 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:54 smithi070 ceph-mon[27495]: pgmap v5: 121 pgs: 121 active+clean; 273 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 555 MiB/s rd, 323 MiB/s wr, 3.74k op/s; 582 KiB/s, 2 objects/s recovering 2024-09-14T03:50:54.488 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:54 smithi070 ceph-mon[27495]: osdmap e240: 8 total, 8 up, 8 in 2024-09-14T03:50:54.488 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:54 smithi070 ceph-mon[27495]: mgrmap e32: y(active, since 2s) 2024-09-14T03:50:54.488 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:54 smithi070 ceph-mon[39288]: [14/Sep/2024:03:50:53] ENGINE Serving on https://172.21.15.70:7150 2024-09-14T03:50:54.488 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:54 smithi070 ceph-mon[39288]: [14/Sep/2024:03:50:53] ENGINE Bus STARTED 2024-09-14T03:50:54.488 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:54 smithi070 ceph-mon[39288]: [14/Sep/2024:03:50:53] ENGINE Error in HTTPServer.serve 2024-09-14T03:50:54.488 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-09-14T03:50:54.489 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:50:54.489 INFO:journalctl@ceph.mon.c.smithi070.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:50:54.489 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:50:54.489 INFO:journalctl@ceph.mon.c.smithi070.stdout: self._run(expiration_interval) 2024-09-14T03:50:54.489 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:50:54.489 INFO:journalctl@ceph.mon.c.smithi070.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:50:54.489 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:50:54.489 INFO:journalctl@ceph.mon.c.smithi070.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:50:54.489 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:50:54.489 INFO:journalctl@ceph.mon.c.smithi070.stdout: s = self.context.wrap_socket( 2024-09-14T03:50:54.489 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:50:54.489 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.sslsocket_class._create( 2024-09-14T03:50:54.489 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:50:54.489 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.do_handshake() 2024-09-14T03:50:54.490 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:50:54.490 INFO:journalctl@ceph.mon.c.smithi070.stdout: self._sslobj.do_handshake() 2024-09-14T03:50:54.490 INFO:journalctl@ceph.mon.c.smithi070.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:50:54.490 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:54 smithi070 ceph-mon[39288]: pgmap v5: 121 pgs: 121 active+clean; 273 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 555 MiB/s rd, 323 MiB/s wr, 3.74k op/s; 582 KiB/s, 2 objects/s recovering 2024-09-14T03:50:54.490 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:54 smithi070 ceph-mon[39288]: osdmap e240: 8 total, 8 up, 8 in 2024-09-14T03:50:54.490 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:54 smithi070 ceph-mon[39288]: mgrmap e32: y(active, since 2s) 2024-09-14T03:50:54.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:54 smithi094 ceph-mon[31053]: [14/Sep/2024:03:50:53] ENGINE Serving on https://172.21.15.70:7150 2024-09-14T03:50:54.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:54 smithi094 ceph-mon[31053]: [14/Sep/2024:03:50:53] ENGINE Bus STARTED 2024-09-14T03:50:54.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:54 smithi094 ceph-mon[31053]: [14/Sep/2024:03:50:53] ENGINE Error in HTTPServer.serve 2024-09-14T03:50:54.527 INFO:journalctl@ceph.mon.b.smithi094.stdout: Traceback (most recent call last): 2024-09-14T03:50:54.527 INFO:journalctl@ceph.mon.b.smithi094.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:50:54.527 INFO:journalctl@ceph.mon.b.smithi094.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:50:54.527 INFO:journalctl@ceph.mon.b.smithi094.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:50:54.527 INFO:journalctl@ceph.mon.b.smithi094.stdout: self._run(expiration_interval) 2024-09-14T03:50:54.527 INFO:journalctl@ceph.mon.b.smithi094.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:50:54.527 INFO:journalctl@ceph.mon.b.smithi094.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:50:54.527 INFO:journalctl@ceph.mon.b.smithi094.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:50:54.528 INFO:journalctl@ceph.mon.b.smithi094.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:50:54.528 INFO:journalctl@ceph.mon.b.smithi094.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:50:54.528 INFO:journalctl@ceph.mon.b.smithi094.stdout: s = self.context.wrap_socket( 2024-09-14T03:50:54.528 INFO:journalctl@ceph.mon.b.smithi094.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:50:54.528 INFO:journalctl@ceph.mon.b.smithi094.stdout: return self.sslsocket_class._create( 2024-09-14T03:50:54.528 INFO:journalctl@ceph.mon.b.smithi094.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:50:54.528 INFO:journalctl@ceph.mon.b.smithi094.stdout: self.do_handshake() 2024-09-14T03:50:54.528 INFO:journalctl@ceph.mon.b.smithi094.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:50:54.528 INFO:journalctl@ceph.mon.b.smithi094.stdout: self._sslobj.do_handshake() 2024-09-14T03:50:54.528 INFO:journalctl@ceph.mon.b.smithi094.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:50:54.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:54 smithi094 ceph-mon[31053]: pgmap v5: 121 pgs: 121 active+clean; 273 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 555 MiB/s rd, 323 MiB/s wr, 3.74k op/s; 582 KiB/s, 2 objects/s recovering 2024-09-14T03:50:54.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:54 smithi094 ceph-mon[31053]: osdmap e240: 8 total, 8 up, 8 in 2024-09-14T03:50:54.529 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:54 smithi094 ceph-mon[31053]: mgrmap e32: y(active, since 2s) 2024-09-14T03:50:55.221 INFO:tasks.rados.rados.0.smithi070.stdout:921: done (2 left) 2024-09-14T03:50:55.221 INFO:tasks.rados.rados.0.smithi070.stdout:922: done (1 left) 2024-09-14T03:50:55.221 INFO:tasks.rados.rados.0.smithi070.stdout:923: done (0 left) 2024-09-14T03:50:55.221 INFO:tasks.rados.rados.0.smithi070.stdout:924: snap_create 2024-09-14T03:50:55.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:55 smithi094 ceph-mon[31053]: osdmap e241: 8 total, 8 up, 8 in 2024-09-14T03:50:55.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:55 smithi070 ceph-mon[27495]: osdmap e241: 8 total, 8 up, 8 in 2024-09-14T03:50:55.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:55 smithi070 ceph-mon[39288]: osdmap e241: 8 total, 8 up, 8 in 2024-09-14T03:50:55.541 INFO:tasks.rados.rados.0.smithi070.stdout:924: done (0 left) 2024-09-14T03:50:55.541 INFO:tasks.rados.rados.0.smithi070.stdout:925: delete oid 17 current snap is 96 2024-09-14T03:50:55.544 INFO:tasks.rados.rados.0.smithi070.stdout:925: done (0 left) 2024-09-14T03:50:55.544 INFO:tasks.rados.rados.0.smithi070.stdout:926: snap_create 2024-09-14T03:50:56.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:56 smithi094 ceph-mon[31053]: pgmap v8: 121 pgs: 121 active+clean; 273 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 184 MiB/s rd, 107 MiB/s wr, 1.24k op/s; 193 KiB/s, 0 objects/s recovering 2024-09-14T03:50:56.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:56 smithi094 ceph-mon[31053]: osdmap e242: 8 total, 8 up, 8 in 2024-09-14T03:50:56.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:56 smithi094 ceph-mon[31053]: mgrmap e33: y(active, since 4s) 2024-09-14T03:50:56.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:56 smithi094 ceph-mon[31053]: osdmap e243: 8 total, 8 up, 8 in 2024-09-14T03:50:56.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:56 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:50:56.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:56 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:50:56.527 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:50:56 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: [14/Sep/2024:03:50:56] ENGINE Bus STARTING 2024-09-14T03:50:56.527 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:50:56 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: CherryPy Checker: 2024-09-14T03:50:56.527 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:50:56 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: The Application mounted at '' has an empty config. 2024-09-14T03:50:56.527 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:50:56 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: 2024-09-14T03:50:56.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:56 smithi070 ceph-mon[27495]: pgmap v8: 121 pgs: 121 active+clean; 273 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 184 MiB/s rd, 107 MiB/s wr, 1.24k op/s; 193 KiB/s, 0 objects/s recovering 2024-09-14T03:50:56.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:56 smithi070 ceph-mon[27495]: osdmap e242: 8 total, 8 up, 8 in 2024-09-14T03:50:56.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:56 smithi070 ceph-mon[27495]: mgrmap e33: y(active, since 4s) 2024-09-14T03:50:56.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:56 smithi070 ceph-mon[27495]: osdmap e243: 8 total, 8 up, 8 in 2024-09-14T03:50:56.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:56 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:50:56.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:56 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:50:56.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:56 smithi070 ceph-mon[39288]: pgmap v8: 121 pgs: 121 active+clean; 273 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 184 MiB/s rd, 107 MiB/s wr, 1.24k op/s; 193 KiB/s, 0 objects/s recovering 2024-09-14T03:50:56.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:56 smithi070 ceph-mon[39288]: osdmap e242: 8 total, 8 up, 8 in 2024-09-14T03:50:56.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:56 smithi070 ceph-mon[39288]: mgrmap e33: y(active, since 4s) 2024-09-14T03:50:56.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:56 smithi070 ceph-mon[39288]: osdmap e243: 8 total, 8 up, 8 in 2024-09-14T03:50:56.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:56 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:50:56.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:56 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:50:56.543 INFO:tasks.rados.rados.0.smithi070.stdout:926: done (0 left) 2024-09-14T03:50:56.543 INFO:tasks.rados.rados.0.smithi070.stdout:927: setattr oid 42 current snap is 97 2024-09-14T03:50:56.546 INFO:tasks.rados.rados.0.smithi070.stdout:928: delete oid 12 current snap is 97 2024-09-14T03:50:56.552 INFO:tasks.rados.rados.0.smithi070.stdout:928: done (1 left) 2024-09-14T03:50:56.552 INFO:tasks.rados.rados.0.smithi070.stdout:929: read oid 16 snap 76 2024-09-14T03:50:56.552 INFO:tasks.rados.rados.0.smithi070.stdout:929: expect (ObjNum 114 snap 24 seq_num 114) 2024-09-14T03:50:56.552 INFO:tasks.rados.rados.0.smithi070.stdout:930: rollback oid 44 current snap is 97 2024-09-14T03:50:56.552 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 44 to 44 2024-09-14T03:50:56.552 INFO:tasks.rados.rados.0.smithi070.stdout:931: rollback oid 1 current snap is 97 2024-09-14T03:50:56.552 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 1 to 91 2024-09-14T03:50:56.553 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 12 current snap is 97 2024-09-14T03:50:56.553 INFO:tasks.rados.rados.0.smithi070.stdout:932: seq_num 305 ranges {0=311296} 2024-09-14T03:50:56.554 INFO:tasks.rados.rados.0.smithi070.stdout:932: writing smithi07085154-12 from 0 to 311296 tid 1 2024-09-14T03:50:56.554 INFO:tasks.rados.rados.0.smithi070.stdout:933: rollback oid 35 current snap is 97 2024-09-14T03:50:56.554 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 35 to 44 2024-09-14T03:50:56.554 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 768 (ObjNum 280 snap 86 seq_num 280) dirty exists 2024-09-14T03:50:56.554 INFO:tasks.rados.rados.0.smithi070.stdout:927: done (5 left) 2024-09-14T03:50:56.554 INFO:tasks.rados.rados.0.smithi070.stdout:934: read oid 22 snap -1 2024-09-14T03:50:56.554 INFO:tasks.rados.rados.0.smithi070.stdout:934: expect deleted 2024-09-14T03:50:56.555 INFO:tasks.rados.rados.0.smithi070.stdout:935: snap_create 2024-09-14T03:50:56.556 INFO:tasks.rados.rados.0.smithi070.stdout:933: finishing rollback tid 1 to smithi07085154-35 2024-09-14T03:50:56.556 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 35 v 24 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-14T03:50:56.557 INFO:tasks.rados.rados.0.smithi070.stdout:930: finishing rollback tid 1 to smithi07085154-44 2024-09-14T03:50:56.557 INFO:tasks.rados.rados.0.smithi070.stdout:930: finishing rollback tid 2 to smithi07085154-44 2024-09-14T03:50:56.557 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 302 (ObjNum 43 snap 0 seq_num 43) dirty exists 2024-09-14T03:50:56.557 INFO:tasks.rados.rados.0.smithi070.stdout:931: finishing rollback tid 0 to smithi07085154-1 2024-09-14T03:50:56.558 INFO:tasks.rados.rados.0.smithi070.stdout:932: finishing write tid 1 to smithi07085154-12 2024-09-14T03:50:56.558 INFO:tasks.rados.rados.0.smithi070.stdout:932: finishing write tid 2 to smithi07085154-12 2024-09-14T03:50:56.558 INFO:tasks.rados.rados.0.smithi070.stdout:931: finishing rollback tid 1 to smithi07085154-1 2024-09-14T03:50:56.558 INFO:tasks.rados.rados.0.smithi070.stdout:931: finishing rollback tid 2 to smithi07085154-1 2024-09-14T03:50:56.558 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 1 v 306 (ObjNum 285 snap 89 seq_num 285) dirty exists 2024-09-14T03:50:56.558 INFO:tasks.rados.rados.0.smithi070.stdout:932: finishing write tid 3 to smithi07085154-12 2024-09-14T03:50:56.558 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 528 (ObjNum 305 snap 97 seq_num 305) dirty exists 2024-09-14T03:50:56.559 INFO:tasks.rados.rados.0.smithi070.stdout:932: left oid 12 (ObjNum 305 snap 97 seq_num 305) 2024-09-14T03:50:56.569 INFO:tasks.rados.rados.0.smithi070.stdout:929: expect (ObjNum 114 snap 24 seq_num 114) 2024-09-14T03:50:56.950 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:50:56 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: [14/Sep/2024:03:50:56] ENGINE Serving on http://:::9283 2024-09-14T03:50:56.950 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:50:56 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: [14/Sep/2024:03:50:56] ENGINE Bus STARTED 2024-09-14T03:50:57.276 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:50:56 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:50:56.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:50:57.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:57 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:50:57.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:57 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:50:57.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:57 smithi070 ceph-mon[27495]: Standby manager daemon x started 2024-09-14T03:50:57.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:57 smithi070 ceph-mon[27495]: from='mgr.? 172.21.15.94:0/3821181381' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:50:57.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:57 smithi070 ceph-mon[27495]: from='mgr.? 172.21.15.94:0/3821181381' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:50:57.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:57 smithi070 ceph-mon[27495]: from='mgr.? 172.21.15.94:0/3821181381' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:50:57.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:57 smithi070 ceph-mon[27495]: from='mgr.? 172.21.15.94:0/3821181381' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:50:57.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:57 smithi070 ceph-mon[27495]: osdmap e244: 8 total, 8 up, 8 in 2024-09-14T03:50:57.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:57 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:50:57.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:57 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:50:57.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:57 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:50:57.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:57 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:50:57.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:57 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:50:57.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:57 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:50:57.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:57 smithi070 ceph-mon[39288]: Standby manager daemon x started 2024-09-14T03:50:57.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:57 smithi070 ceph-mon[39288]: from='mgr.? 172.21.15.94:0/3821181381' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:50:57.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:57 smithi070 ceph-mon[39288]: from='mgr.? 172.21.15.94:0/3821181381' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:50:57.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:57 smithi070 ceph-mon[39288]: from='mgr.? 172.21.15.94:0/3821181381' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:50:57.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:57 smithi070 ceph-mon[39288]: from='mgr.? 172.21.15.94:0/3821181381' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:50:57.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:57 smithi070 ceph-mon[39288]: osdmap e244: 8 total, 8 up, 8 in 2024-09-14T03:50:57.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:57 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:50:57.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:57 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:50:57.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:57 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:50:57.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:57 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:50:57.544 INFO:tasks.rados.rados.0.smithi070.stdout:929: done (6 left) 2024-09-14T03:50:57.544 INFO:tasks.rados.rados.0.smithi070.stdout:930: done (5 left) 2024-09-14T03:50:57.544 INFO:tasks.rados.rados.0.smithi070.stdout:931: done (4 left) 2024-09-14T03:50:57.544 INFO:tasks.rados.rados.0.smithi070.stdout:932: done (3 left) 2024-09-14T03:50:57.544 INFO:tasks.rados.rados.0.smithi070.stdout:933: done (2 left) 2024-09-14T03:50:57.544 INFO:tasks.rados.rados.0.smithi070.stdout:934: done (1 left) 2024-09-14T03:50:57.545 INFO:tasks.rados.rados.0.smithi070.stdout:935: done (0 left) 2024-09-14T03:50:57.545 INFO:tasks.rados.rados.0.smithi070.stdout:936: snap_remove snap 85 2024-09-14T03:50:57.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:57 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:50:57.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:57 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:50:57.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:57 smithi094 ceph-mon[31053]: Standby manager daemon x started 2024-09-14T03:50:57.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:57 smithi094 ceph-mon[31053]: from='mgr.? 172.21.15.94:0/3821181381' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:50:57.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:57 smithi094 ceph-mon[31053]: from='mgr.? 172.21.15.94:0/3821181381' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:50:57.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:57 smithi094 ceph-mon[31053]: from='mgr.? 172.21.15.94:0/3821181381' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:50:57.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:57 smithi094 ceph-mon[31053]: from='mgr.? 172.21.15.94:0/3821181381' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:50:57.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:57 smithi094 ceph-mon[31053]: osdmap e244: 8 total, 8 up, 8 in 2024-09-14T03:50:57.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:57 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:50:57.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:57 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:50:57.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:57 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:50:57.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:57 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:50:58.546 INFO:tasks.rados.rados.0.smithi070.stdout:936: done (0 left) 2024-09-14T03:50:58.547 INFO:tasks.rados.rados.0.smithi070.stdout:937: rmattr oid 41 current snap is 98 2024-09-14T03:50:58.547 INFO:tasks.rados.rados.0.smithi070.stdout:938: snap_remove snap 91 2024-09-14T03:50:58.712 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:58 smithi070 ceph-mon[27495]: pgmap v12: 121 pgs: 121 active+clean; 273 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 256 B/s rd, 60 KiB/s wr, 0 op/s 2024-09-14T03:50:58.712 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:58 smithi070 ceph-mon[27495]: mgrmap e34: y(active, since 6s), standbys: x 2024-09-14T03:50:58.712 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:58 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:50:58.712 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:58 smithi070 ceph-mon[27495]: osdmap e245: 8 total, 8 up, 8 in 2024-09-14T03:50:58.713 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:58 smithi070 ceph-mon[39288]: pgmap v12: 121 pgs: 121 active+clean; 273 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 256 B/s rd, 60 KiB/s wr, 0 op/s 2024-09-14T03:50:58.713 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:58 smithi070 ceph-mon[39288]: mgrmap e34: y(active, since 6s), standbys: x 2024-09-14T03:50:58.713 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:58 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:50:58.713 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:58 smithi070 ceph-mon[39288]: osdmap e245: 8 total, 8 up, 8 in 2024-09-14T03:50:58.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:58 smithi094 ceph-mon[31053]: pgmap v12: 121 pgs: 121 active+clean; 273 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 256 B/s rd, 60 KiB/s wr, 0 op/s 2024-09-14T03:50:58.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:58 smithi094 ceph-mon[31053]: mgrmap e34: y(active, since 6s), standbys: x 2024-09-14T03:50:58.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:58 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:50:58.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:58 smithi094 ceph-mon[31053]: osdmap e245: 8 total, 8 up, 8 in 2024-09-14T03:50:59.551 INFO:tasks.rados.rados.0.smithi070.stdout:938: done (1 left) 2024-09-14T03:50:59.551 INFO:tasks.rados.rados.0.smithi070.stdout:939: setattr oid 12 current snap is 98 2024-09-14T03:50:59.551 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 551 (ObjNum 211 snap 60 seq_num 211) dirty exists 2024-09-14T03:50:59.552 INFO:tasks.rados.rados.0.smithi070.stdout:937: done (1 left) 2024-09-14T03:50:59.552 INFO:tasks.rados.rados.0.smithi070.stdout:940: rmattr oid 31 current snap is 98 2024-09-14T03:50:59.552 INFO:tasks.rados.rados.0.smithi070.stdout:940: done (1 left) 2024-09-14T03:50:59.552 INFO:tasks.rados.rados.0.smithi070.stdout:941: rollback oid 42 current snap is 98 2024-09-14T03:50:59.552 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 42 to 83 2024-09-14T03:50:59.552 INFO:tasks.rados.rados.0.smithi070.stdout:942: rollback oid 38 current snap is 98 2024-09-14T03:50:59.553 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 38 to 77 2024-09-14T03:50:59.553 INFO:tasks.rados.rados.0.smithi070.stdout:943: snap_create 2024-09-14T03:50:59.558 INFO:tasks.rados.rados.0.smithi070.stdout:942: finishing rollback tid 0 to smithi07085154-38 2024-09-14T03:50:59.558 INFO:tasks.rados.rados.0.smithi070.stdout:942: finishing rollback tid 1 to smithi07085154-38 2024-09-14T03:50:59.558 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 38 v 321 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-14T03:50:59.558 INFO:tasks.rados.rados.0.smithi070.stdout:941: finishing rollback tid 0 to smithi07085154-42 2024-09-14T03:50:59.560 INFO:tasks.rados.rados.0.smithi070.stdout:941: finishing rollback tid 1 to smithi07085154-42 2024-09-14T03:50:59.560 INFO:tasks.rados.rados.0.smithi070.stdout:941: finishing rollback tid 2 to smithi07085154-42 2024-09-14T03:50:59.560 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 776 (ObjNum 160 snap 47 seq_num 160) dirty exists 2024-09-14T03:50:59.561 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 529 (ObjNum 305 snap 97 seq_num 305) dirty exists 2024-09-14T03:50:59.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:50:59 smithi070 ceph-mon[39288]: osdmap e246: 8 total, 8 up, 8 in 2024-09-14T03:50:59.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:50:59 smithi070 ceph-mon[27495]: osdmap e246: 8 total, 8 up, 8 in 2024-09-14T03:51:00.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:50:59 smithi094 ceph-mon[31053]: osdmap e246: 8 total, 8 up, 8 in 2024-09-14T03:51:00.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:00 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:51:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:51:00.543 INFO:tasks.rados.rados.0.smithi070.stdout:939: done (3 left) 2024-09-14T03:51:00.543 INFO:tasks.rados.rados.0.smithi070.stdout:941: done (2 left) 2024-09-14T03:51:00.543 INFO:tasks.rados.rados.0.smithi070.stdout:942: done (1 left) 2024-09-14T03:51:00.543 INFO:tasks.rados.rados.0.smithi070.stdout:943: done (0 left) 2024-09-14T03:51:00.544 INFO:tasks.rados.rados.0.smithi070.stdout:944: rollback oid 43 current snap is 99 2024-09-14T03:51:00.544 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 43 to 71 2024-09-14T03:51:00.544 INFO:tasks.rados.rados.0.smithi070.stdout:945: snap_create 2024-09-14T03:51:00.548 INFO:tasks.rados.rados.0.smithi070.stdout:944: finishing rollback tid 0 to smithi07085154-43 2024-09-14T03:51:00.548 INFO:tasks.rados.rados.0.smithi070.stdout:944: finishing rollback tid 1 to smithi07085154-43 2024-09-14T03:51:00.548 INFO:tasks.rados.rados.0.smithi070.stdout:944: finishing rollback tid 2 to smithi07085154-43 2024-09-14T03:51:00.548 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 557 (ObjNum 246 snap 69 seq_num 246) dirty exists 2024-09-14T03:51:01.005 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:00 smithi094 ceph-mon[31053]: pgmap v15: 121 pgs: 121 active+clean; 274 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.2 MiB/s rd, 136 KiB/s wr, 29 op/s 2024-09-14T03:51:01.005 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:00 smithi094 ceph-mon[31053]: osdmap e247: 8 total, 8 up, 8 in 2024-09-14T03:51:01.005 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:00 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:01.005 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:00 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:01.005 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:00 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:01.005 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:00 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:51:01.005 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:00 smithi094 ceph-mon[31053]: Updating smithi070:/etc/ceph/ceph.conf 2024-09-14T03:51:01.005 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:00 smithi094 ceph-mon[31053]: Updating smithi094:/etc/ceph/ceph.conf 2024-09-14T03:51:01.005 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:00 smithi094 ceph-mon[31053]: Updating smithi070:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:51:01.005 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:00 smithi094 ceph-mon[31053]: Updating smithi094:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:51:01.005 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:00 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:01.006 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:00 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:01.006 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:00 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:01.006 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:00 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:01.006 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:00 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:01.006 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:00 smithi094 ceph-mon[31053]: osdmap e248: 8 total, 8 up, 8 in 2024-09-14T03:51:01.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:00 smithi070 ceph-mon[27495]: pgmap v15: 121 pgs: 121 active+clean; 274 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.2 MiB/s rd, 136 KiB/s wr, 29 op/s 2024-09-14T03:51:01.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:00 smithi070 ceph-mon[27495]: osdmap e247: 8 total, 8 up, 8 in 2024-09-14T03:51:01.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:00 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:01.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:00 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:01.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:00 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:01.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:00 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:51:01.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:00 smithi070 ceph-mon[27495]: Updating smithi070:/etc/ceph/ceph.conf 2024-09-14T03:51:01.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:00 smithi070 ceph-mon[27495]: Updating smithi094:/etc/ceph/ceph.conf 2024-09-14T03:51:01.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:00 smithi070 ceph-mon[27495]: Updating smithi070:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:51:01.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:00 smithi070 ceph-mon[27495]: Updating smithi094:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:51:01.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:00 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:01.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:00 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:01.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:00 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:01.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:00 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:01.038 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:00 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:01.038 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:00 smithi070 ceph-mon[27495]: osdmap e248: 8 total, 8 up, 8 in 2024-09-14T03:51:01.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:00 smithi070 ceph-mon[39288]: pgmap v15: 121 pgs: 121 active+clean; 274 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.2 MiB/s rd, 136 KiB/s wr, 29 op/s 2024-09-14T03:51:01.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:00 smithi070 ceph-mon[39288]: osdmap e247: 8 total, 8 up, 8 in 2024-09-14T03:51:01.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:00 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:01.039 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:00 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:01.039 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:00 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:01.039 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:00 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:51:01.039 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:00 smithi070 ceph-mon[39288]: Updating smithi070:/etc/ceph/ceph.conf 2024-09-14T03:51:01.039 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:00 smithi070 ceph-mon[39288]: Updating smithi094:/etc/ceph/ceph.conf 2024-09-14T03:51:01.040 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:00 smithi070 ceph-mon[39288]: Updating smithi070:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:51:01.040 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:00 smithi070 ceph-mon[39288]: Updating smithi094:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:51:01.040 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:00 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:01.040 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:00 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:01.040 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:00 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:01.040 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:00 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:01.040 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:00 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:01.040 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:00 smithi070 ceph-mon[39288]: osdmap e248: 8 total, 8 up, 8 in 2024-09-14T03:51:01.545 INFO:tasks.rados.rados.0.smithi070.stdout:944: done (1 left) 2024-09-14T03:51:01.545 INFO:tasks.rados.rados.0.smithi070.stdout:945: done (0 left) 2024-09-14T03:51:01.546 INFO:tasks.rados.rados.0.smithi070.stdout:946: rmattr oid 29 current snap is 100 2024-09-14T03:51:01.546 INFO:tasks.rados.rados.0.smithi070.stdout:946: done (0 left) 2024-09-14T03:51:01.546 INFO:tasks.rados.rados.0.smithi070.stdout:947: snap_create 2024-09-14T03:51:01.661 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:01 smithi094 ceph-mon[31053]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:51:01.661 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:01 smithi094 ceph-mon[31053]: Updating smithi094:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:51:01.661 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:01 smithi094 ceph-mon[31053]: Updating smithi070:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:51:01.661 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:01 smithi094 ceph-mon[31053]: Updating smithi094:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:51:01.662 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:01 smithi094 ceph-mon[31053]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:51:01.662 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:01 smithi094 ceph-mon[31053]: Reconfiguring daemon prometheus.a on smithi094 2024-09-14T03:51:01.662 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:01 smithi094 ceph-mon[31053]: osdmap e249: 8 total, 8 up, 8 in 2024-09-14T03:51:02.026 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:51:01 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[67166]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:51:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.51.0" 2024-09-14T03:51:02.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:01 smithi070 ceph-mon[39288]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:51:02.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:01 smithi070 ceph-mon[39288]: Updating smithi094:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:51:02.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:01 smithi070 ceph-mon[39288]: Updating smithi070:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:51:02.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:01 smithi070 ceph-mon[39288]: Updating smithi094:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:51:02.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:01 smithi070 ceph-mon[39288]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:51:02.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:01 smithi070 ceph-mon[39288]: Reconfiguring daemon prometheus.a on smithi094 2024-09-14T03:51:02.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:01 smithi070 ceph-mon[39288]: osdmap e249: 8 total, 8 up, 8 in 2024-09-14T03:51:02.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:01 smithi070 ceph-mon[27495]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:51:02.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:01 smithi070 ceph-mon[27495]: Updating smithi094:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:51:02.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:01 smithi070 ceph-mon[27495]: Updating smithi070:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:51:02.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:01 smithi070 ceph-mon[27495]: Updating smithi094:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:51:02.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:01 smithi070 ceph-mon[27495]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:51:02.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:01 smithi070 ceph-mon[27495]: Reconfiguring daemon prometheus.a on smithi094 2024-09-14T03:51:02.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:01 smithi070 ceph-mon[27495]: osdmap e249: 8 total, 8 up, 8 in 2024-09-14T03:51:02.547 INFO:tasks.rados.rados.0.smithi070.stdout:947: done (0 left) 2024-09-14T03:51:02.547 INFO:tasks.rados.rados.0.smithi070.stdout:948: delete oid 33 current snap is 101 2024-09-14T03:51:02.551 INFO:tasks.rados.rados.0.smithi070.stdout:948: done (0 left) 2024-09-14T03:51:02.552 INFO:tasks.rados.rados.0.smithi070.stdout:949: setattr oid 48 current snap is 101 2024-09-14T03:51:02.553 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 2 current snap is 101 2024-09-14T03:51:02.553 INFO:tasks.rados.rados.0.smithi070.stdout:950: seq_num 307 ranges {8192=344064} 2024-09-14T03:51:02.555 INFO:tasks.rados.rados.0.smithi070.stdout:950: writing smithi07085154-2 from 8192 to 352256 tid 1 2024-09-14T03:51:02.556 INFO:tasks.rados.rados.0.smithi070.stdout:951: read oid 37 snap -1 2024-09-14T03:51:02.556 INFO:tasks.rados.rados.0.smithi070.stdout:951: expect deleted 2024-09-14T03:51:02.556 INFO:tasks.rados.rados.0.smithi070.stdout:952: copy_from oid 40 from oid 33 current snap is 101 2024-09-14T03:51:02.556 INFO:tasks.rados.rados.0.smithi070.stdout:953: snap_remove snap 88 2024-09-14T03:51:02.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:02 smithi094 ceph-mon[31053]: pgmap v18: 121 pgs: 121 active+clean; 274 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.2 MiB/s rd, 76 KiB/s wr, 28 op/s 2024-09-14T03:51:02.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:02 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:02.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:02 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:51:02.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:02 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:51:02.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:02 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:51:02.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:02 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:51:02.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:02 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:51:02.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:02 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:51:02.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:02 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:51:02.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:02 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:51:02.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:02 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:51:02.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:02 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:51:02.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:02 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:51:02.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:02 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:51:02.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:02 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:51:02.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:02 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:51:02.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:02 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:51:02.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:02 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:51:02.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:02 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:51:02.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:02 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:51:02.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:02 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:51:02.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:02 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:51:02.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:02 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:51:02.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:02 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:51:02.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:02 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:51:02.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:02 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:51:02.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:02 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:51:02.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:02 smithi094 ceph-mon[31053]: osdmap e250: 8 total, 8 up, 8 in 2024-09-14T03:51:03.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[27495]: pgmap v18: 121 pgs: 121 active+clean; 274 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.2 MiB/s rd, 76 KiB/s wr, 28 op/s 2024-09-14T03:51:03.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:03.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:51:03.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:51:03.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:51:03.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:51:03.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:51:03.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:51:03.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:51:03.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:51:03.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:51:03.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:51:03.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:51:03.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:51:03.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:51:03.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:51:03.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:51:03.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:51:03.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:51:03.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:51:03.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:51:03.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:51:03.038 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:51:03.038 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[27495]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:51:03.038 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:51:03.038 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:51:03.038 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:51:03.038 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[27495]: osdmap e250: 8 total, 8 up, 8 in 2024-09-14T03:51:03.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[39288]: pgmap v18: 121 pgs: 121 active+clean; 274 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.2 MiB/s rd, 76 KiB/s wr, 28 op/s 2024-09-14T03:51:03.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:03.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-14T03:51:03.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:51:03.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:51:03.039 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:51:03.039 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:51:03.039 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:51:03.039 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:51:03.039 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:51:03.039 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:51:03.039 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:51:03.039 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:51:03.039 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:51:03.039 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:51:03.039 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:51:03.039 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:51:03.039 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:51:03.040 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:51:03.040 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:51:03.040 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:51:03.040 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:51:03.040 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:51:03.040 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:51:03.040 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:51:03.040 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:51:03.040 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:51:03.040 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:02 smithi070 ceph-mon[39288]: osdmap e250: 8 total, 8 up, 8 in 2024-09-14T03:51:03.549 INFO:tasks.rados.rados.0.smithi070.stdout:952: finishing copy_from to smithi07085154-40 2024-09-14T03:51:03.550 INFO:tasks.rados.rados.0.smithi070.stdout:952: got expected ENOENT (src dne) 2024-09-14T03:51:03.550 INFO:tasks.rados.rados.0.smithi070.stdout:952: finishing copy_from racing read to smithi07085154-40 2024-09-14T03:51:03.550 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 530 (ObjNum 299 snap 92 seq_num 299) dirty exists 2024-09-14T03:51:03.550 INFO:tasks.rados.rados.0.smithi070.stdout:950: finishing write tid 1 to smithi07085154-2 2024-09-14T03:51:03.550 INFO:tasks.rados.rados.0.smithi070.stdout:950: finishing write tid 2 to smithi07085154-2 2024-09-14T03:51:03.550 INFO:tasks.rados.rados.0.smithi070.stdout:950: finishing write tid 3 to smithi07085154-2 2024-09-14T03:51:03.550 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 2 v 781 (ObjNum 307 snap 101 seq_num 307) dirty exists 2024-09-14T03:51:03.550 INFO:tasks.rados.rados.0.smithi070.stdout:950: left oid 2 (ObjNum 307 snap 101 seq_num 307) 2024-09-14T03:51:03.550 INFO:tasks.rados.rados.0.smithi070.stdout:949: done (4 left) 2024-09-14T03:51:03.550 INFO:tasks.rados.rados.0.smithi070.stdout:950: done (3 left) 2024-09-14T03:51:03.550 INFO:tasks.rados.rados.0.smithi070.stdout:951: done (2 left) 2024-09-14T03:51:03.551 INFO:tasks.rados.rados.0.smithi070.stdout:952: done (1 left) 2024-09-14T03:51:03.551 INFO:tasks.rados.rados.0.smithi070.stdout:953: done (0 left) 2024-09-14T03:51:03.551 INFO:tasks.rados.rados.0.smithi070.stdout:954: setattr oid 31 current snap is 101 2024-09-14T03:51:03.553 INFO:tasks.rados.rados.0.smithi070.stdout:955: snap_create 2024-09-14T03:51:03.559 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 393 (ObjNum 753 snap 0 seq_num 4194279360) dirty exists 2024-09-14T03:51:03.969 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:03 smithi094 ceph-mon[31053]: mgrmap e35: y(active, since 11s), standbys: x 2024-09-14T03:51:03.969 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:03 smithi094 ceph-mon[31053]: osdmap e251: 8 total, 8 up, 8 in 2024-09-14T03:51:03.969 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:03 smithi094 systemd[1]: Stopping Ceph prometheus.a for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:51:04.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:03 smithi070 ceph-mon[27495]: mgrmap e35: y(active, since 11s), standbys: x 2024-09-14T03:51:04.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:03 smithi070 ceph-mon[27495]: osdmap e251: 8 total, 8 up, 8 in 2024-09-14T03:51:04.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:03 smithi070 ceph-mon[39288]: mgrmap e35: y(active, since 11s), standbys: x 2024-09-14T03:51:04.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:03 smithi070 ceph-mon[39288]: osdmap e251: 8 total, 8 up, 8 in 2024-09-14T03:51:04.276 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:03 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:51:03.968Z caller=main.go:964 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-14T03:51:04.277 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:03 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:51:03.968Z caller=main.go:988 level=info msg="Stopping scrape discovery manager..." 2024-09-14T03:51:04.277 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:03 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:51:03.968Z caller=main.go:1002 level=info msg="Stopping notify discovery manager..." 2024-09-14T03:51:04.277 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:03 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:51:03.968Z caller=manager.go:177 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-14T03:51:04.277 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:03 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:51:03.968Z caller=main.go:984 level=info msg="Scrape discovery manager stopped" 2024-09-14T03:51:04.277 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:03 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:51:03.968Z caller=main.go:998 level=info msg="Notify discovery manager stopped" 2024-09-14T03:51:04.277 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:03 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:51:03.968Z caller=manager.go:187 level=info component="rule manager" msg="Rule manager stopped" 2024-09-14T03:51:04.277 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:03 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:51:03.968Z caller=main.go:1039 level=info msg="Stopping scrape manager..." 2024-09-14T03:51:04.277 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:03 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:51:03.969Z caller=main.go:1031 level=info msg="Scrape manager stopped" 2024-09-14T03:51:04.277 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:03 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:51:03.970Z caller=notifier.go:618 level=info component=notifier msg="Stopping notification manager..." 2024-09-14T03:51:04.277 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:03 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:51:03.970Z caller=main.go:1261 level=info msg="Notifier manager stopped" 2024-09-14T03:51:04.277 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:03 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[75235]: ts=2024-09-14T03:51:03.970Z caller=main.go:1273 level=info msg="See you next time!" 2024-09-14T03:51:04.277 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:03 smithi094 podman[78146]: 2024-09-14 03:51:03.979271255 +0000 UTC m=+0.116315256 container died aa320cf836270bc14aad9048422af4094134f3cb0ca2abbdcfe07498e79b28a4 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:51:04.550 INFO:tasks.rados.rados.0.smithi070.stdout:954: done (1 left) 2024-09-14T03:51:04.550 INFO:tasks.rados.rados.0.smithi070.stdout:955: done (0 left) 2024-09-14T03:51:04.550 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 37 current snap is 102 2024-09-14T03:51:04.550 INFO:tasks.rados.rados.0.smithi070.stdout:956: seq_num 308 ranges {0=360448} 2024-09-14T03:51:04.552 INFO:tasks.rados.rados.0.smithi070.stdout:956: writing smithi07085154-37 from 0 to 360448 tid 1 2024-09-14T03:51:04.552 INFO:tasks.rados.rados.0.smithi070.stdout:957: snap_create 2024-09-14T03:51:04.557 INFO:tasks.rados.rados.0.smithi070.stdout:956: finishing write tid 1 to smithi07085154-37 2024-09-14T03:51:04.558 INFO:tasks.rados.rados.0.smithi070.stdout:956: finishing write tid 2 to smithi07085154-37 2024-09-14T03:51:04.559 INFO:tasks.rados.rados.0.smithi070.stdout:956: finishing write tid 3 to smithi07085154-37 2024-09-14T03:51:04.559 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 37 v 563 (ObjNum 308 snap 102 seq_num 308) dirty exists 2024-09-14T03:51:04.559 INFO:tasks.rados.rados.0.smithi070.stdout:956: left oid 37 (ObjNum 308 snap 102 seq_num 308) 2024-09-14T03:51:04.573 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:04 smithi094 podman[78146]: 2024-09-14 03:51:04.391550079 +0000 UTC m=+0.528594092 container cleanup aa320cf836270bc14aad9048422af4094134f3cb0ca2abbdcfe07498e79b28a4 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:51:04.573 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:04 smithi094 bash[78146]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a 2024-09-14T03:51:04.846 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:04 smithi094 podman[78165]: 2024-09-14 03:51:04.60928778 +0000 UTC m=+0.631762584 container remove aa320cf836270bc14aad9048422af4094134f3cb0ca2abbdcfe07498e79b28a4 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:51:04.846 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:04 smithi094 ceph-mon[31053]: pgmap v21: 121 pgs: 121 active+clean; 269 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 39 KiB/s rd, 0 B/s wr, 18 op/s 2024-09-14T03:51:04.846 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:04 smithi094 ceph-mon[31053]: osdmap e252: 8 total, 8 up, 8 in 2024-09-14T03:51:04.956 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:04 smithi070 ceph-mon[27495]: pgmap v21: 121 pgs: 121 active+clean; 269 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 39 KiB/s rd, 0 B/s wr, 18 op/s 2024-09-14T03:51:04.956 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:04 smithi070 ceph-mon[27495]: osdmap e252: 8 total, 8 up, 8 in 2024-09-14T03:51:04.957 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:04 smithi070 ceph-mon[39288]: pgmap v21: 121 pgs: 121 active+clean; 269 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 39 KiB/s rd, 0 B/s wr, 18 op/s 2024-09-14T03:51:04.957 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:04 smithi070 ceph-mon[39288]: osdmap e252: 8 total, 8 up, 8 in 2024-09-14T03:51:05.211 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:04 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-14T03:51:05.211 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:04 smithi094 systemd[1]: Stopped Ceph prometheus.a for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:51:05.211 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:04 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 1.121s CPU time. 2024-09-14T03:51:05.211 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:04 smithi094 systemd[1]: Starting Ceph prometheus.a for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:51:05.211 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:05 smithi094 podman[78260]: 2024-09-14 03:51:04.975163119 +0000 UTC m=+0.016499461 image pull 1d3b7f56885b6dd623f1785be963aa9c195f86bc256ea454e8d02a7980b79c53 quay.io/prometheus/prometheus:v2.51.0 2024-09-14T03:51:05.211 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:05 smithi094 podman[78260]: 2024-09-14 03:51:05.085083637 +0000 UTC m=+0.126419980 container create ce1b353828dc3ceb0980e284aec473d26e5ca7875b6a4b810fc0df86e4912491 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:51:05.545 INFO:tasks.rados.rados.0.smithi070.stdout:956: done (1 left) 2024-09-14T03:51:05.545 INFO:tasks.rados.rados.0.smithi070.stdout:957: done (0 left) 2024-09-14T03:51:05.545 INFO:tasks.rados.rados.0.smithi070.stdout:958: read oid 11 snap 67 2024-09-14T03:51:05.545 INFO:tasks.rados.rados.0.smithi070.stdout:958: expect (ObjNum 49 snap 0 seq_num 49) 2024-09-14T03:51:05.545 INFO:tasks.rados.rados.0.smithi070.stdout:959: snap_create 2024-09-14T03:51:05.548 INFO:tasks.rados.rados.0.smithi070.stdout:958: expect (ObjNum 49 snap 0 seq_num 49) 2024-09-14T03:51:05.777 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:05 smithi094 podman[78260]: 2024-09-14 03:51:05.320699303 +0000 UTC m=+0.362035645 container init ce1b353828dc3ceb0980e284aec473d26e5ca7875b6a4b810fc0df86e4912491 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:51:05.778 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:05 smithi094 podman[78260]: 2024-09-14 03:51:05.325651609 +0000 UTC m=+0.366987958 container start ce1b353828dc3ceb0980e284aec473d26e5ca7875b6a4b810fc0df86e4912491 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:51:05.778 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:05.344Z caller=main.go:617 level=info msg="Starting Prometheus Server" mode=server version="(version=2.51.0, branch=HEAD, revision=c05c15512acb675e3f6cd662a6727854e93fc024)" 2024-09-14T03:51:05.778 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:05.344Z caller=main.go:622 level=info build_context="(go=go1.22.1, platform=linux/amd64, user=root@b5723e458358, date=20240319-10:54:45, tags=netgo,builtinassets,stringlabels)" 2024-09-14T03:51:05.778 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:05.344Z caller=main.go:623 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi094 (none))" 2024-09-14T03:51:05.778 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:05.344Z caller=main.go:624 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-14T03:51:05.778 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:05.344Z caller=main.go:625 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-14T03:51:05.778 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:05.346Z caller=web.go:568 level=info component=web msg="Start listening for connections" address=:9095 2024-09-14T03:51:05.778 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:05.346Z caller=main.go:1129 level=info msg="Starting TSDB ..." 2024-09-14T03:51:05.778 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:05.348Z caller=tls_config.go:313 level=info component=web msg="Listening on" address=[::]:9095 2024-09-14T03:51:05.778 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:05.348Z caller=tls_config.go:316 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-14T03:51:05.778 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:05.350Z caller=head.go:616 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-14T03:51:05.779 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:05.350Z caller=head.go:698 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.574µs 2024-09-14T03:51:05.779 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:05.350Z caller=head.go:706 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-14T03:51:05.779 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:05.350Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-09-14T03:51:05.779 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:05.362Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-09-14T03:51:05.779 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:05.367Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-09-14T03:51:05.779 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:05.367Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-09-14T03:51:05.779 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:05.367Z caller=head.go:815 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=23.281µs wal_replay_duration=17.159938ms wbl_replay_duration=141ns total_replay_duration=17.201253ms 2024-09-14T03:51:05.779 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:05.372Z caller=main.go:1150 level=info fs_type=XFS_SUPER_MAGIC 2024-09-14T03:51:05.779 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:05.372Z caller=main.go:1153 level=info msg="TSDB started" 2024-09-14T03:51:05.779 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:05.372Z caller=main.go:1335 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-14T03:51:05.779 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:05.385Z caller=main.go:1372 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=13.695135ms db_storage=800ns remote_storage=1.595µs web_handler=376ns query_engine=606ns scrape=399.642µs scrape_sd=117.211µs notify=15.164µs notify_sd=14.812µs rules=12.80872ms tracing=5.788µs 2024-09-14T03:51:05.779 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:05.385Z caller=main.go:1114 level=info msg="Server is ready to receive web requests." 2024-09-14T03:51:05.780 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:05.385Z caller=manager.go:163 level=info component="rule manager" msg="Starting rule manager..." 2024-09-14T03:51:05.780 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:05 smithi094 bash[78260]: ce1b353828dc3ceb0980e284aec473d26e5ca7875b6a4b810fc0df86e4912491 2024-09-14T03:51:05.780 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:05 smithi094 systemd[1]: Started Ceph prometheus.a for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:51:06.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:06 smithi094 ceph-mon[31053]: pgmap v24: 121 pgs: 121 active+clean; 269 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 39 KiB/s rd, 0 B/s wr, 18 op/s 2024-09-14T03:51:06.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:06 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:06.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:06 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:06.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:06 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:51:06.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:06 smithi094 ceph-mon[31053]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:51:06.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:06 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:06.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:06 smithi094 ceph-mon[31053]: osdmap e253: 8 total, 8 up, 8 in 2024-09-14T03:51:06.546 INFO:tasks.rados.rados.0.smithi070.stdout:958: done (1 left) 2024-09-14T03:51:06.546 INFO:tasks.rados.rados.0.smithi070.stdout:959: done (0 left) 2024-09-14T03:51:06.546 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 10 current snap is 104 2024-09-14T03:51:06.546 INFO:tasks.rados.rados.0.smithi070.stdout:960: seq_num 309 ranges {163840=376832} 2024-09-14T03:51:06.549 INFO:tasks.rados.rados.0.smithi070.stdout:960: writing smithi07085154-10 from 163840 to 540672 tid 1 2024-09-14T03:51:06.550 INFO:tasks.rados.rados.0.smithi070.stdout:961: copy_from oid 5 from oid 16 current snap is 104 2024-09-14T03:51:06.550 INFO:tasks.rados.rados.0.smithi070.stdout:962: copy_from oid 36 from oid 3 current snap is 104 2024-09-14T03:51:06.550 INFO:tasks.rados.rados.0.smithi070.stdout:963: read oid 6 snap -1 2024-09-14T03:51:06.550 INFO:tasks.rados.rados.0.smithi070.stdout:963: expect (ObjNum 250 snap 72 seq_num 250) 2024-09-14T03:51:06.550 INFO:tasks.rados.rados.0.smithi070.stdout:964: read oid 47 snap -1 2024-09-14T03:51:06.550 INFO:tasks.rados.rados.0.smithi070.stdout:964: expect (ObjNum 298 snap 92 seq_num 298) 2024-09-14T03:51:06.550 INFO:tasks.rados.rados.0.smithi070.stdout:965: delete oid 44 current snap is 104 2024-09-14T03:51:06.554 INFO:tasks.rados.rados.0.smithi070.stdout:965: done (5 left) 2024-09-14T03:51:06.554 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 1 current snap is 104 2024-09-14T03:51:06.554 INFO:tasks.rados.rados.0.smithi070.stdout:966: seq_num 311 ranges {2383872=417792} 2024-09-14T03:51:06.570 INFO:tasks.rados.rados.0.smithi070.stdout:966: writing smithi07085154-1 from 2383872 to 2801664 tid 1 2024-09-14T03:51:06.570 INFO:tasks.rados.rados.0.smithi070.stdout:967: rollback oid 17 current snap is 104 2024-09-14T03:51:06.570 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 17 to 51 2024-09-14T03:51:06.570 INFO:tasks.rados.rados.0.smithi070.stdout:962: finishing copy_from to smithi07085154-36 2024-09-14T03:51:06.570 INFO:tasks.rados.rados.0.smithi070.stdout:962: got expected ENOENT (src dne) 2024-09-14T03:51:06.570 INFO:tasks.rados.rados.0.smithi070.stdout:962: finishing copy_from racing read to smithi07085154-36 2024-09-14T03:51:06.570 INFO:tasks.rados.rados.0.smithi070.stdout:962: done (6 left) 2024-09-14T03:51:06.570 INFO:tasks.rados.rados.0.smithi070.stdout:968: setattr oid 2 current snap is 104 2024-09-14T03:51:06.570 INFO:tasks.rados.rados.0.smithi070.stdout:960: finishing write tid 1 to smithi07085154-10 2024-09-14T03:51:06.570 INFO:tasks.rados.rados.0.smithi070.stdout:960: finishing write tid 2 to smithi07085154-10 2024-09-14T03:51:06.571 INFO:tasks.rados.rados.0.smithi070.stdout:963: expect (ObjNum 250 snap 72 seq_num 250) 2024-09-14T03:51:06.619 INFO:tasks.rados.rados.0.smithi070.stdout:960: finishing write tid 3 to smithi07085154-10 2024-09-14T03:51:06.619 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 788 (ObjNum 309 snap 104 seq_num 309) dirty exists 2024-09-14T03:51:06.619 INFO:tasks.rados.rados.0.smithi070.stdout:960: left oid 10 (ObjNum 309 snap 104 seq_num 309) 2024-09-14T03:51:06.619 INFO:tasks.rados.rados.0.smithi070.stdout:964: expect (ObjNum 298 snap 92 seq_num 298) 2024-09-14T03:51:06.749 INFO:tasks.rados.rados.0.smithi070.stdout:961: finishing copy_from racing read to smithi07085154-5 2024-09-14T03:51:06.749 INFO:tasks.rados.rados.0.smithi070.stdout:967: finishing rollback tid 1 to smithi07085154-17 2024-09-14T03:51:06.750 INFO:tasks.rados.rados.0.smithi070.stdout:967: finishing rollback tid 2 to smithi07085154-17 2024-09-14T03:51:06.750 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 395 (ObjNum 181 snap 51 seq_num 181) dirty exists 2024-09-14T03:51:06.750 INFO:tasks.rados.rados.0.smithi070.stdout:961: finishing copy_from to smithi07085154-5 2024-09-14T03:51:06.750 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 542 (ObjNum 114 snap 24 seq_num 114) dirty exists 2024-09-14T03:51:06.750 INFO:tasks.rados.rados.0.smithi070.stdout:966: finishing write tid 1 to smithi07085154-1 2024-09-14T03:51:06.750 INFO:tasks.rados.rados.0.smithi070.stdout:966: finishing write tid 2 to smithi07085154-1 2024-09-14T03:51:06.750 INFO:tasks.rados.rados.0.smithi070.stdout:966: finishing write tid 3 to smithi07085154-1 2024-09-14T03:51:06.750 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 1 v 315 (ObjNum 311 snap 104 seq_num 311) dirty exists 2024-09-14T03:51:06.750 INFO:tasks.rados.rados.0.smithi070.stdout:966: left oid 1 (ObjNum 311 snap 104 seq_num 311) 2024-09-14T03:51:06.750 INFO:tasks.rados.rados.0.smithi070.stdout:960: done (6 left) 2024-09-14T03:51:06.750 INFO:tasks.rados.rados.0.smithi070.stdout:961: done (5 left) 2024-09-14T03:51:06.750 INFO:tasks.rados.rados.0.smithi070.stdout:963: done (4 left) 2024-09-14T03:51:06.750 INFO:tasks.rados.rados.0.smithi070.stdout:964: done (3 left) 2024-09-14T03:51:06.751 INFO:tasks.rados.rados.0.smithi070.stdout:966: done (2 left) 2024-09-14T03:51:06.751 INFO:tasks.rados.rados.0.smithi070.stdout:967: done (1 left) 2024-09-14T03:51:06.751 INFO:tasks.rados.rados.0.smithi070.stdout:969: snap_remove snap 100 2024-09-14T03:51:06.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:06 smithi070 ceph-mon[39288]: pgmap v24: 121 pgs: 121 active+clean; 269 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 39 KiB/s rd, 0 B/s wr, 18 op/s 2024-09-14T03:51:06.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:06 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:06.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:06 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:06.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:06 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:51:06.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:06 smithi070 ceph-mon[39288]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:51:06.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:06 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:06.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:06 smithi070 ceph-mon[39288]: osdmap e253: 8 total, 8 up, 8 in 2024-09-14T03:51:06.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:06 smithi070 ceph-mon[27495]: pgmap v24: 121 pgs: 121 active+clean; 269 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 39 KiB/s rd, 0 B/s wr, 18 op/s 2024-09-14T03:51:06.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:06 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:06.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:06 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:06.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:06 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:51:06.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:06 smithi070 ceph-mon[27495]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:51:06.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:06 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:06.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:06 smithi070 ceph-mon[27495]: osdmap e253: 8 total, 8 up, 8 in 2024-09-14T03:51:07.245 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:06 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:06.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:51:07.551 INFO:tasks.rados.rados.0.smithi070.stdout:969: done (1 left) 2024-09-14T03:51:07.551 INFO:tasks.rados.rados.0.smithi070.stdout:970: snap_create 2024-09-14T03:51:07.552 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 2 v 790 (ObjNum 307 snap 101 seq_num 307) dirty exists 2024-09-14T03:51:07.909 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:07 smithi094 ceph-mon[31053]: osdmap e254: 8 total, 8 up, 8 in 2024-09-14T03:51:07.909 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:07 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:51:07.910 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:07 smithi094 ceph-mon[31053]: Upgrade: Updating mgr.x 2024-09-14T03:51:07.910 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:07 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:51:07.910 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:07 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:51:07.910 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:07 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:51:07.910 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:07 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:07.910 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:07 smithi094 ceph-mon[31053]: Deploying daemon mgr.x on smithi094 2024-09-14T03:51:08.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:07 smithi070 ceph-mon[27495]: osdmap e254: 8 total, 8 up, 8 in 2024-09-14T03:51:08.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:07 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:51:08.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:07 smithi070 ceph-mon[27495]: Upgrade: Updating mgr.x 2024-09-14T03:51:08.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:07 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:51:08.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:07 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:51:08.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:07 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:51:08.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:07 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:08.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:07 smithi070 ceph-mon[27495]: Deploying daemon mgr.x on smithi094 2024-09-14T03:51:08.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:07 smithi070 ceph-mon[39288]: osdmap e254: 8 total, 8 up, 8 in 2024-09-14T03:51:08.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:07 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:51:08.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:07 smithi070 ceph-mon[39288]: Upgrade: Updating mgr.x 2024-09-14T03:51:08.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:07 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:51:08.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:07 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:51:08.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:07 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:51:08.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:07 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:08.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:07 smithi070 ceph-mon[39288]: Deploying daemon mgr.x on smithi094 2024-09-14T03:51:08.557 INFO:tasks.rados.rados.0.smithi070.stdout:968: done (1 left) 2024-09-14T03:51:08.557 INFO:tasks.rados.rados.0.smithi070.stdout:970: done (0 left) 2024-09-14T03:51:08.557 INFO:tasks.rados.rados.0.smithi070.stdout:971: rmattr oid 17 current snap is 105 2024-09-14T03:51:08.557 INFO:tasks.rados.rados.0.smithi070.stdout:972: read oid 13 snap -1 2024-09-14T03:51:08.557 INFO:tasks.rados.rados.0.smithi070.stdout:972: expect (ObjNum 301 snap 93 seq_num 301) 2024-09-14T03:51:08.557 INFO:tasks.rados.rados.0.smithi070.stdout:973: rmattr oid 9 current snap is 105 2024-09-14T03:51:08.557 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 37 current snap is 105 2024-09-14T03:51:08.557 INFO:tasks.rados.rados.0.smithi070.stdout:974: seq_num 312 ranges {360448=434176} 2024-09-14T03:51:08.561 INFO:tasks.rados.rados.0.smithi070.stdout:974: writing smithi07085154-37 from 360448 to 794624 tid 1 2024-09-14T03:51:08.561 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 397 (ObjNum 181 snap 51 seq_num 181) dirty exists 2024-09-14T03:51:08.561 INFO:tasks.rados.rados.0.smithi070.stdout:971: done (3 left) 2024-09-14T03:51:08.561 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 14 current snap is 105 2024-09-14T03:51:08.562 INFO:tasks.rados.rados.0.smithi070.stdout:975: seq_num 313 ranges {0=450560} 2024-09-14T03:51:08.564 INFO:tasks.rados.rados.0.smithi070.stdout:975: writing smithi07085154-14 from 0 to 450560 tid 1 2024-09-14T03:51:08.565 INFO:tasks.rados.rados.0.smithi070.stdout:976: rmattr oid 38 current snap is 105 2024-09-14T03:51:08.565 INFO:tasks.rados.rados.0.smithi070.stdout:976: done (4 left) 2024-09-14T03:51:08.565 INFO:tasks.rados.rados.0.smithi070.stdout:977: snap_create 2024-09-14T03:51:08.565 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 796 (ObjNum 281 snap 86 seq_num 281) dirty exists 2024-09-14T03:51:08.569 INFO:tasks.rados.rados.0.smithi070.stdout:975: finishing write tid 1 to smithi07085154-14 2024-09-14T03:51:08.569 INFO:tasks.rados.rados.0.smithi070.stdout:975: finishing write tid 2 to smithi07085154-14 2024-09-14T03:51:08.569 INFO:tasks.rados.rados.0.smithi070.stdout:974: finishing write tid 1 to smithi07085154-37 2024-09-14T03:51:08.569 INFO:tasks.rados.rados.0.smithi070.stdout:974: finishing write tid 2 to smithi07085154-37 2024-09-14T03:51:08.570 INFO:tasks.rados.rados.0.smithi070.stdout:975: finishing write tid 3 to smithi07085154-14 2024-09-14T03:51:08.570 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 404 (ObjNum 313 snap 105 seq_num 313) dirty exists 2024-09-14T03:51:08.570 INFO:tasks.rados.rados.0.smithi070.stdout:975: left oid 14 (ObjNum 313 snap 105 seq_num 313) 2024-09-14T03:51:08.570 INFO:tasks.rados.rados.0.smithi070.stdout:974: finishing write tid 3 to smithi07085154-37 2024-09-14T03:51:08.570 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 37 v 566 (ObjNum 312 snap 105 seq_num 312) dirty exists 2024-09-14T03:51:08.570 INFO:tasks.rados.rados.0.smithi070.stdout:974: left oid 37 (ObjNum 312 snap 105 seq_num 312) 2024-09-14T03:51:08.582 INFO:tasks.rados.rados.0.smithi070.stdout:972: expect (ObjNum 301 snap 93 seq_num 301) 2024-09-14T03:51:08.776 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:51:08 smithi094 systemd[1]: Stopping Ceph mgr.x for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:51:08.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:08 smithi094 ceph-mon[31053]: pgmap v27: 121 pgs: 121 active+clean; 269 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 606 KiB/s rd, 0 op/s 2024-09-14T03:51:08.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:08 smithi094 ceph-mon[31053]: osdmap e255: 8 total, 8 up, 8 in 2024-09-14T03:51:09.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:08 smithi070 ceph-mon[27495]: pgmap v27: 121 pgs: 121 active+clean; 269 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 606 KiB/s rd, 0 op/s 2024-09-14T03:51:09.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:08 smithi070 ceph-mon[27495]: osdmap e255: 8 total, 8 up, 8 in 2024-09-14T03:51:09.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:08 smithi070 ceph-mon[39288]: pgmap v27: 121 pgs: 121 active+clean; 269 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 606 KiB/s rd, 0 op/s 2024-09-14T03:51:09.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:08 smithi070 ceph-mon[39288]: osdmap e255: 8 total, 8 up, 8 in 2024-09-14T03:51:09.072 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:51:08 smithi094 podman[78605]: 2024-09-14 03:51:08.795870763 +0000 UTC m=+0.128767064 container died 5aadf84f680bdbdaa74c49e4fa946dea914357f472c7da94476c4ceab5fa3fd3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231) 2024-09-14T03:51:09.439 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:51:09 smithi094 podman[78605]: 2024-09-14 03:51:09.179030797 +0000 UTC m=+0.511927086 container cleanup 5aadf84f680bdbdaa74c49e4fa946dea914357f472c7da94476c4ceab5fa3fd3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-14T03:51:09.439 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:51:09 smithi094 bash[78605]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x 2024-09-14T03:51:09.561 INFO:tasks.rados.rados.0.smithi070.stdout:972: done (4 left) 2024-09-14T03:51:09.561 INFO:tasks.rados.rados.0.smithi070.stdout:973: done (3 left) 2024-09-14T03:51:09.561 INFO:tasks.rados.rados.0.smithi070.stdout:974: done (2 left) 2024-09-14T03:51:09.561 INFO:tasks.rados.rados.0.smithi070.stdout:975: done (1 left) 2024-09-14T03:51:09.561 INFO:tasks.rados.rados.0.smithi070.stdout:977: done (0 left) 2024-09-14T03:51:09.561 INFO:tasks.rados.rados.0.smithi070.stdout:978: snap_create 2024-09-14T03:51:09.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:09 smithi094 ceph-mon[31053]: osdmap e256: 8 total, 8 up, 8 in 2024-09-14T03:51:09.777 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:51:09 smithi094 podman[78622]: 2024-09-14 03:51:09.438484494 +0000 UTC m=+0.638976802 container remove 5aadf84f680bdbdaa74c49e4fa946dea914357f472c7da94476c4ceab5fa3fd3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-14T03:51:09.777 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:51:09 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-14T03:51:09.777 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:51:09 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-09-14T03:51:09.777 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:51:09 smithi094 systemd[1]: Stopped Ceph mgr.x for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:51:09.777 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:51:09 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@mgr.x.service: Consumed 17.971s CPU time. 2024-09-14T03:51:10.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:09 smithi070 ceph-mon[27495]: osdmap e256: 8 total, 8 up, 8 in 2024-09-14T03:51:10.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:09 smithi070 ceph-mon[39288]: osdmap e256: 8 total, 8 up, 8 in 2024-09-14T03:51:10.184 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:51:09 smithi094 systemd[1]: Starting Ceph mgr.x for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:51:10.184 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:51:10 smithi094 podman[78758]: 2024-09-14 03:51:09.927686492 +0000 UTC m=+0.015583545 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:51:10.184 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:51:10 smithi094 podman[78758]: 2024-09-14 03:51:10.056455312 +0000 UTC m=+0.144352357 container create 569d9c7e0f060ad8d6b06f075081d11c326448eac0ff973a390fd0df55b4c0da (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=main-a278231, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T03:51:10.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:10 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:51:10.527 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:51:10 smithi094 podman[78758]: 2024-09-14 03:51:10.29246219 +0000 UTC m=+0.380359240 container init 569d9c7e0f060ad8d6b06f075081d11c326448eac0ff973a390fd0df55b4c0da (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=main-a278231, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T03:51:10.527 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:51:10 smithi094 podman[78758]: 2024-09-14 03:51:10.297170053 +0000 UTC m=+0.385067100 container start 569d9c7e0f060ad8d6b06f075081d11c326448eac0ff973a390fd0df55b4c0da (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True) 2024-09-14T03:51:10.527 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:51:10 smithi094 ceph-mgr[78796]: -- 172.21.15.94:0/1699732874 <== mon.1 v2:172.21.15.94:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5604d3d6b0e0 con 0x5604d3d49400 2024-09-14T03:51:10.527 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:51:10 smithi094 bash[78758]: 569d9c7e0f060ad8d6b06f075081d11c326448eac0ff973a390fd0df55b4c0da 2024-09-14T03:51:10.527 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:51:10 smithi094 systemd[1]: Started Ceph mgr.x for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:51:10.547 INFO:tasks.rados.rados.0.smithi070.stdout:978: done (0 left) 2024-09-14T03:51:10.547 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 17 current snap is 107 2024-09-14T03:51:10.547 INFO:tasks.rados.rados.0.smithi070.stdout:979: seq_num 314 ranges {1040384=466944} 2024-09-14T03:51:10.555 INFO:tasks.rados.rados.0.smithi070.stdout:979: writing smithi07085154-17 from 1040384 to 1507328 tid 1 2024-09-14T03:51:10.555 INFO:tasks.rados.rados.0.smithi070.stdout:980: delete oid 15 current snap is 107 2024-09-14T03:51:10.559 INFO:tasks.rados.rados.0.smithi070.stdout:980: done (1 left) 2024-09-14T03:51:10.560 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 27 current snap is 107 2024-09-14T03:51:10.560 INFO:tasks.rados.rados.0.smithi070.stdout:981: seq_num 316 ranges {40960=499712} 2024-09-14T03:51:10.563 INFO:tasks.rados.rados.0.smithi070.stdout:981: writing smithi07085154-27 from 40960 to 540672 tid 1 2024-09-14T03:51:10.563 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 10 current snap is 107 2024-09-14T03:51:10.564 INFO:tasks.rados.rados.0.smithi070.stdout:979: finishing write tid 1 to smithi07085154-17 2024-09-14T03:51:10.564 INFO:tasks.rados.rados.0.smithi070.stdout:979: finishing write tid 2 to smithi07085154-17 2024-09-14T03:51:10.564 INFO:tasks.rados.rados.0.smithi070.stdout:982: seq_num 317 ranges {540672=516096} 2024-09-14T03:51:10.570 INFO:tasks.rados.rados.0.smithi070.stdout:982: writing smithi07085154-10 from 540672 to 1056768 tid 1 2024-09-14T03:51:10.570 INFO:tasks.rados.rados.0.smithi070.stdout:983: delete oid 32 current snap is 107 2024-09-14T03:51:10.570 INFO:tasks.rados.rados.0.smithi070.stdout:979: finishing write tid 3 to smithi07085154-17 2024-09-14T03:51:10.570 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 400 (ObjNum 314 snap 107 seq_num 314) dirty exists 2024-09-14T03:51:10.570 INFO:tasks.rados.rados.0.smithi070.stdout:979: left oid 17 (ObjNum 314 snap 107 seq_num 314) 2024-09-14T03:51:10.572 INFO:tasks.rados.rados.0.smithi070.stdout:979: done (3 left) 2024-09-14T03:51:10.572 INFO:tasks.rados.rados.0.smithi070.stdout:983: done (2 left) 2024-09-14T03:51:10.572 INFO:tasks.rados.rados.0.smithi070.stdout:984: read oid 30 snap -1 2024-09-14T03:51:10.572 INFO:tasks.rados.rados.0.smithi070.stdout:984: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-14T03:51:10.572 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 33 current snap is 107 2024-09-14T03:51:10.572 INFO:tasks.rados.rados.0.smithi070.stdout:985: seq_num 319 ranges {0=548864} 2024-09-14T03:51:10.575 INFO:tasks.rados.rados.0.smithi070.stdout:985: writing smithi07085154-33 from 0 to 548864 tid 1 2024-09-14T03:51:10.576 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 46 current snap is 107 2024-09-14T03:51:10.576 INFO:tasks.rados.rados.0.smithi070.stdout:981: finishing write tid 1 to smithi07085154-27 2024-09-14T03:51:10.576 INFO:tasks.rados.rados.0.smithi070.stdout:986: seq_num 320 ranges {1040384=565248} 2024-09-14T03:51:10.585 INFO:tasks.rados.rados.0.smithi070.stdout:986: writing smithi07085154-46 from 1040384 to 1605632 tid 1 2024-09-14T03:51:10.585 INFO:tasks.rados.rados.0.smithi070.stdout:987: delete oid 43 current snap is 107 2024-09-14T03:51:10.585 INFO:tasks.rados.rados.0.smithi070.stdout:981: finishing write tid 2 to smithi07085154-27 2024-09-14T03:51:10.585 INFO:tasks.rados.rados.0.smithi070.stdout:982: finishing write tid 1 to smithi07085154-10 2024-09-14T03:51:10.585 INFO:tasks.rados.rados.0.smithi070.stdout:982: finishing write tid 2 to smithi07085154-10 2024-09-14T03:51:10.585 INFO:tasks.rados.rados.0.smithi070.stdout:981: finishing write tid 3 to smithi07085154-27 2024-09-14T03:51:10.585 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 799 (ObjNum 316 snap 107 seq_num 316) dirty exists 2024-09-14T03:51:10.585 INFO:tasks.rados.rados.0.smithi070.stdout:981: left oid 27 (ObjNum 316 snap 107 seq_num 316) 2024-09-14T03:51:10.585 INFO:tasks.rados.rados.0.smithi070.stdout:982: finishing write tid 3 to smithi07085154-10 2024-09-14T03:51:10.586 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 802 (ObjNum 317 snap 107 seq_num 317) dirty exists 2024-09-14T03:51:10.586 INFO:tasks.rados.rados.0.smithi070.stdout:982: left oid 10 (ObjNum 317 snap 107 seq_num 317) 2024-09-14T03:51:10.586 INFO:tasks.rados.rados.0.smithi070.stdout:984: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-14T03:51:10.599 INFO:tasks.rados.rados.0.smithi070.stdout:985: finishing write tid 1 to smithi07085154-33 2024-09-14T03:51:10.599 INFO:tasks.rados.rados.0.smithi070.stdout:985: finishing write tid 2 to smithi07085154-33 2024-09-14T03:51:10.599 INFO:tasks.rados.rados.0.smithi070.stdout:985: finishing write tid 3 to smithi07085154-33 2024-09-14T03:51:10.599 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 33 v 297 (ObjNum 319 snap 107 seq_num 319) dirty exists 2024-09-14T03:51:10.600 INFO:tasks.rados.rados.0.smithi070.stdout:985: left oid 33 (ObjNum 319 snap 107 seq_num 319) 2024-09-14T03:51:10.600 INFO:tasks.rados.rados.0.smithi070.stdout:981: done (5 left) 2024-09-14T03:51:10.600 INFO:tasks.rados.rados.0.smithi070.stdout:982: done (4 left) 2024-09-14T03:51:10.600 INFO:tasks.rados.rados.0.smithi070.stdout:984: done (3 left) 2024-09-14T03:51:10.600 INFO:tasks.rados.rados.0.smithi070.stdout:985: done (2 left) 2024-09-14T03:51:10.600 INFO:tasks.rados.rados.0.smithi070.stdout:987: done (1 left) 2024-09-14T03:51:10.600 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 38 current snap is 107 2024-09-14T03:51:10.600 INFO:tasks.rados.rados.0.smithi070.stdout:986: finishing write tid 1 to smithi07085154-46 2024-09-14T03:51:10.600 INFO:tasks.rados.rados.0.smithi070.stdout:986: finishing write tid 2 to smithi07085154-46 2024-09-14T03:51:10.600 INFO:tasks.rados.rados.0.smithi070.stdout:988: seq_num 322 ranges {0=598016} 2024-09-14T03:51:10.602 INFO:tasks.rados.rados.0.smithi070.stdout:988: writing smithi07085154-38 from 0 to 598016 tid 1 2024-09-14T03:51:10.602 INFO:tasks.rados.rados.0.smithi070.stdout:989: copy_from oid 34 from oid 40 current snap is 107 2024-09-14T03:51:10.602 INFO:tasks.rados.rados.0.smithi070.stdout:986: finishing write tid 3 to smithi07085154-46 2024-09-14T03:51:10.602 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 46 v 805 (ObjNum 320 snap 107 seq_num 320) dirty exists 2024-09-14T03:51:10.602 INFO:tasks.rados.rados.0.smithi070.stdout:986: left oid 46 (ObjNum 320 snap 107 seq_num 320) 2024-09-14T03:51:10.602 INFO:tasks.rados.rados.0.smithi070.stdout:986: done (2 left) 2024-09-14T03:51:10.603 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 50 current snap is 107 2024-09-14T03:51:10.603 INFO:tasks.rados.rados.0.smithi070.stdout:990: seq_num 323 ranges {0=614400} 2024-09-14T03:51:10.606 INFO:tasks.rados.rados.0.smithi070.stdout:990: writing smithi07085154-50 from 0 to 614400 tid 1 2024-09-14T03:51:10.606 INFO:tasks.rados.rados.0.smithi070.stdout:991: delete oid 18 current snap is 107 2024-09-14T03:51:10.606 INFO:tasks.rados.rados.0.smithi070.stdout:988: finishing write tid 1 to smithi07085154-38 2024-09-14T03:51:10.606 INFO:tasks.rados.rados.0.smithi070.stdout:988: finishing write tid 2 to smithi07085154-38 2024-09-14T03:51:10.606 INFO:tasks.rados.rados.0.smithi070.stdout:988: finishing write tid 3 to smithi07085154-38 2024-09-14T03:51:10.606 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 38 v 329 (ObjNum 322 snap 107 seq_num 322) dirty exists 2024-09-14T03:51:10.606 INFO:tasks.rados.rados.0.smithi070.stdout:988: left oid 38 (ObjNum 322 snap 107 seq_num 322) 2024-09-14T03:51:10.610 INFO:tasks.rados.rados.0.smithi070.stdout:988: done (3 left) 2024-09-14T03:51:10.610 INFO:tasks.rados.rados.0.smithi070.stdout:991: done (2 left) 2024-09-14T03:51:10.610 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 17 current snap is 107 2024-09-14T03:51:10.610 INFO:tasks.rados.rados.0.smithi070.stdout:992: seq_num 325 ranges {1507328=647168} 2024-09-14T03:51:10.621 INFO:tasks.rados.rados.0.smithi070.stdout:992: writing smithi07085154-17 from 1507328 to 2154496 tid 1 2024-09-14T03:51:10.622 INFO:tasks.rados.rados.0.smithi070.stdout:993: snap_create 2024-09-14T03:51:10.622 INFO:tasks.rados.rados.0.smithi070.stdout:990: finishing write tid 1 to smithi07085154-50 2024-09-14T03:51:10.622 INFO:tasks.rados.rados.0.smithi070.stdout:990: finishing write tid 2 to smithi07085154-50 2024-09-14T03:51:10.622 INFO:tasks.rados.rados.0.smithi070.stdout:990: finishing write tid 3 to smithi07085154-50 2024-09-14T03:51:10.622 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 519 (ObjNum 323 snap 107 seq_num 323) dirty exists 2024-09-14T03:51:10.622 INFO:tasks.rados.rados.0.smithi070.stdout:990: left oid 50 (ObjNum 323 snap 107 seq_num 323) 2024-09-14T03:51:10.629 INFO:tasks.rados.rados.0.smithi070.stdout:992: finishing write tid 1 to smithi07085154-17 2024-09-14T03:51:10.629 INFO:tasks.rados.rados.0.smithi070.stdout:992: finishing write tid 2 to smithi07085154-17 2024-09-14T03:51:10.629 INFO:tasks.rados.rados.0.smithi070.stdout:992: finishing write tid 3 to smithi07085154-17 2024-09-14T03:51:10.629 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 402 (ObjNum 325 snap 107 seq_num 325) dirty exists 2024-09-14T03:51:10.629 INFO:tasks.rados.rados.0.smithi070.stdout:992: left oid 17 (ObjNum 325 snap 107 seq_num 325) 2024-09-14T03:51:10.629 INFO:tasks.rados.rados.0.smithi070.stdout:989: finishing copy_from racing read to smithi07085154-34 2024-09-14T03:51:10.631 INFO:tasks.rados.rados.0.smithi070.stdout:989: finishing copy_from to smithi07085154-34 2024-09-14T03:51:10.631 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 257 (ObjNum 274 snap 84 seq_num 274) dirty exists 2024-09-14T03:51:11.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:10 smithi094 ceph-mon[31053]: pgmap v30: 121 pgs: 121 active+clean; 271 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.5 MiB/s wr, 22 op/s 2024-09-14T03:51:11.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:10 smithi094 ceph-mon[31053]: osdmap e257: 8 total, 8 up, 8 in 2024-09-14T03:51:11.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:10 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:11.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:10 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:11.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:10 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:11.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:10 smithi094 ceph-mon[31053]: osdmap e258: 8 total, 8 up, 8 in 2024-09-14T03:51:11.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:10 smithi070 ceph-mon[27495]: pgmap v30: 121 pgs: 121 active+clean; 271 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.5 MiB/s wr, 22 op/s 2024-09-14T03:51:11.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:10 smithi070 ceph-mon[27495]: osdmap e257: 8 total, 8 up, 8 in 2024-09-14T03:51:11.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:10 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:11.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:10 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:11.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:10 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:11.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:10 smithi070 ceph-mon[27495]: osdmap e258: 8 total, 8 up, 8 in 2024-09-14T03:51:11.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:10 smithi070 ceph-mon[39288]: pgmap v30: 121 pgs: 121 active+clean; 271 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.5 MiB/s wr, 22 op/s 2024-09-14T03:51:11.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:10 smithi070 ceph-mon[39288]: osdmap e257: 8 total, 8 up, 8 in 2024-09-14T03:51:11.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:10 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:11.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:10 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:11.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:10 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:11.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:10 smithi070 ceph-mon[39288]: osdmap e258: 8 total, 8 up, 8 in 2024-09-14T03:51:11.567 INFO:tasks.rados.rados.0.smithi070.stdout:989: done (3 left) 2024-09-14T03:51:11.567 INFO:tasks.rados.rados.0.smithi070.stdout:990: done (2 left) 2024-09-14T03:51:11.568 INFO:tasks.rados.rados.0.smithi070.stdout:992: done (1 left) 2024-09-14T03:51:11.568 INFO:tasks.rados.rados.0.smithi070.stdout:993: done (0 left) 2024-09-14T03:51:11.568 INFO:tasks.rados.rados.0.smithi070.stdout:994: snap_remove snap 44 2024-09-14T03:51:12.571 INFO:tasks.rados.rados.0.smithi070.stdout:994: done (0 left) 2024-09-14T03:51:12.571 INFO:tasks.rados.rados.0.smithi070.stdout:995: rmattr oid 46 current snap is 108 2024-09-14T03:51:12.572 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 48 current snap is 108 2024-09-14T03:51:12.572 INFO:tasks.rados.rados.0.smithi070.stdout:996: seq_num 326 ranges {3801088=663552} 2024-09-14T03:51:12.597 INFO:tasks.rados.rados.0.smithi070.stdout:996: writing smithi07085154-48 from 3801088 to 4464640 tid 1 2024-09-14T03:51:12.597 INFO:tasks.rados.rados.0.smithi070.stdout:997: delete oid 18 current snap is 108 2024-09-14T03:51:12.597 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 46 v 807 (ObjNum 320 snap 107 seq_num 320) dirty exists 2024-09-14T03:51:12.600 INFO:tasks.rados.rados.0.smithi070.stdout:995: done (2 left) 2024-09-14T03:51:12.600 INFO:tasks.rados.rados.0.smithi070.stdout:997: done (1 left) 2024-09-14T03:51:12.600 INFO:tasks.rados.rados.0.smithi070.stdout:998: read oid 49 snap -1 2024-09-14T03:51:12.600 INFO:tasks.rados.rados.0.smithi070.stdout:998: expect deleted 2024-09-14T03:51:12.600 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 27 current snap is 108 2024-09-14T03:51:12.600 INFO:tasks.rados.rados.0.smithi070.stdout:999: seq_num 328 ranges {540672=696320} 2024-09-14T03:51:12.607 INFO:tasks.rados.rados.0.smithi070.stdout:999: writing smithi07085154-27 from 540672 to 1236992 tid 1 2024-09-14T03:51:12.607 INFO:tasks.rados.rados.0.smithi070.stdout:1000: read oid 38 snap -1 2024-09-14T03:51:12.607 INFO:tasks.rados.rados.0.smithi070.stdout:1000: expect (ObjNum 322 snap 107 seq_num 322) 2024-09-14T03:51:12.607 INFO:tasks.rados.rados.0.smithi070.stdout:1001: rmattr oid 12 current snap is 108 2024-09-14T03:51:12.607 INFO:tasks.rados.rados.0.smithi070.stdout:996: finishing write tid 1 to smithi07085154-48 2024-09-14T03:51:12.607 INFO:tasks.rados.rados.0.smithi070.stdout:996: finishing write tid 2 to smithi07085154-48 2024-09-14T03:51:12.607 INFO:tasks.rados.rados.0.smithi070.stdout:996: finishing write tid 3 to smithi07085154-48 2024-09-14T03:51:12.608 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 532 (ObjNum 326 snap 108 seq_num 326) dirty exists 2024-09-14T03:51:12.608 INFO:tasks.rados.rados.0.smithi070.stdout:996: left oid 48 (ObjNum 326 snap 108 seq_num 326) 2024-09-14T03:51:12.608 INFO:tasks.rados.rados.0.smithi070.stdout:996: done (4 left) 2024-09-14T03:51:12.608 INFO:tasks.rados.rados.0.smithi070.stdout:998: done (3 left) 2024-09-14T03:51:12.608 INFO:tasks.rados.rados.0.smithi070.stdout:1002: delete oid 22 current snap is 108 2024-09-14T03:51:12.609 INFO:tasks.rados.rados.0.smithi070.stdout:1002: done (3 left) 2024-09-14T03:51:12.609 INFO:tasks.rados.rados.0.smithi070.stdout:1003: copy_from oid 43 from oid 2 current snap is 108 2024-09-14T03:51:12.609 INFO:tasks.rados.rados.0.smithi070.stdout:1004: snap_remove snap 106 2024-09-14T03:51:12.961 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:12 smithi094 ceph-mon[31053]: pgmap v33: 121 pgs: 121 active+clean; 271 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.5 MiB/s wr, 21 op/s 2024-09-14T03:51:12.962 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:12 smithi094 ceph-mon[31053]: osdmap e259: 8 total, 8 up, 8 in 2024-09-14T03:51:13.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:12 smithi070 ceph-mon[27495]: pgmap v33: 121 pgs: 121 active+clean; 271 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.5 MiB/s wr, 21 op/s 2024-09-14T03:51:13.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:12 smithi070 ceph-mon[27495]: osdmap e259: 8 total, 8 up, 8 in 2024-09-14T03:51:13.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:12 smithi070 ceph-mon[39288]: pgmap v33: 121 pgs: 121 active+clean; 271 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.5 MiB/s wr, 21 op/s 2024-09-14T03:51:13.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:12 smithi070 ceph-mon[39288]: osdmap e259: 8 total, 8 up, 8 in 2024-09-14T03:51:13.581 INFO:tasks.rados.rados.0.smithi070.stdout:1004: done (4 left) 2024-09-14T03:51:13.581 INFO:tasks.rados.rados.0.smithi070.stdout:1005: snap_create 2024-09-14T03:51:13.581 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 533 (ObjNum 305 snap 97 seq_num 305) dirty exists 2024-09-14T03:51:13.581 INFO:tasks.rados.rados.0.smithi070.stdout:999: finishing write tid 1 to smithi07085154-27 2024-09-14T03:51:13.581 INFO:tasks.rados.rados.0.smithi070.stdout:999: finishing write tid 2 to smithi07085154-27 2024-09-14T03:51:13.581 INFO:tasks.rados.rados.0.smithi070.stdout:1000: expect (ObjNum 322 snap 107 seq_num 322) 2024-09-14T03:51:13.600 INFO:tasks.rados.rados.0.smithi070.stdout:999: finishing write tid 3 to smithi07085154-27 2024-09-14T03:51:13.600 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 822 (ObjNum 328 snap 108 seq_num 328) dirty exists 2024-09-14T03:51:13.600 INFO:tasks.rados.rados.0.smithi070.stdout:999: left oid 27 (ObjNum 328 snap 108 seq_num 328) 2024-09-14T03:51:13.600 INFO:tasks.rados.rados.0.smithi070.stdout:1003: finishing copy_from racing read to smithi07085154-43 2024-09-14T03:51:13.600 INFO:tasks.rados.rados.0.smithi070.stdout:1003: finishing copy_from to smithi07085154-43 2024-09-14T03:51:13.600 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 790 (ObjNum 307 snap 101 seq_num 307) dirty exists 2024-09-14T03:51:13.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:13 smithi070 ceph-mon[27495]: osdmap e260: 8 total, 8 up, 8 in 2024-09-14T03:51:13.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:51:13] "GET /metrics HTTP/1.1" 200 36193 "" "Prometheus/2.51.0" 2024-09-14T03:51:13.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:13 smithi070 ceph-mon[39288]: osdmap e260: 8 total, 8 up, 8 in 2024-09-14T03:51:13.880 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:13 smithi094 ceph-mon[31053]: osdmap e260: 8 total, 8 up, 8 in 2024-09-14T03:51:14.395 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-09-14T03:51:14.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:14 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:14.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:51:14.580 INFO:tasks.rados.rados.0.smithi070.stdout:999: done (4 left) 2024-09-14T03:51:14.580 INFO:tasks.rados.rados.0.smithi070.stdout:1000: done (3 left) 2024-09-14T03:51:14.581 INFO:tasks.rados.rados.0.smithi070.stdout:1001: done (2 left) 2024-09-14T03:51:14.581 INFO:tasks.rados.rados.0.smithi070.stdout:1003: done (1 left) 2024-09-14T03:51:14.581 INFO:tasks.rados.rados.0.smithi070.stdout:1005: done (0 left) 2024-09-14T03:51:14.581 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 21 current snap is 109 2024-09-14T03:51:14.581 INFO:tasks.rados.rados.0.smithi070.stdout:1006: seq_num 330 ranges {2490368=737280} 2024-09-14T03:51:14.598 INFO:tasks.rados.rados.0.smithi070.stdout:1006: writing smithi07085154-21 from 2490368 to 3227648 tid 1 2024-09-14T03:51:14.598 INFO:tasks.rados.rados.0.smithi070.stdout:1007: rollback oid 42 current snap is 109 2024-09-14T03:51:14.598 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 42 to 94 2024-09-14T03:51:14.598 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 47 current snap is 109 2024-09-14T03:51:14.598 INFO:tasks.rados.rados.0.smithi070.stdout:1008: seq_num 331 ranges {2859008=753664} 2024-09-14T03:51:14.618 INFO:tasks.rados.rados.0.smithi070.stdout:1008: writing smithi07085154-47 from 2859008 to 3612672 tid 1 2024-09-14T03:51:14.618 INFO:tasks.rados.rados.0.smithi070.stdout:1009: copy_from oid 13 from oid 7 current snap is 109 2024-09-14T03:51:14.618 INFO:tasks.rados.rados.0.smithi070.stdout:1006: finishing write tid 1 to smithi07085154-21 2024-09-14T03:51:14.618 INFO:tasks.rados.rados.0.smithi070.stdout:1007: finishing rollback tid 0 to smithi07085154-42 2024-09-14T03:51:14.619 INFO:tasks.rados.rados.0.smithi070.stdout:1006: finishing write tid 2 to smithi07085154-21 2024-09-14T03:51:14.619 INFO:tasks.rados.rados.0.smithi070.stdout:1007: finishing rollback tid 1 to smithi07085154-42 2024-09-14T03:51:14.619 INFO:tasks.rados.rados.0.smithi070.stdout:1007: finishing rollback tid 2 to smithi07085154-42 2024-09-14T03:51:14.619 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 832 (ObjNum 280 snap 86 seq_num 280) dirty exists 2024-09-14T03:51:14.619 INFO:tasks.rados.rados.0.smithi070.stdout:1006: finishing write tid 3 to smithi07085154-21 2024-09-14T03:51:14.619 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 260 (ObjNum 330 snap 109 seq_num 330) dirty exists 2024-09-14T03:51:14.619 INFO:tasks.rados.rados.0.smithi070.stdout:1006: left oid 21 (ObjNum 330 snap 109 seq_num 330) 2024-09-14T03:51:14.619 INFO:tasks.rados.rados.0.smithi070.stdout:1006: done (3 left) 2024-09-14T03:51:14.619 INFO:tasks.rados.rados.0.smithi070.stdout:1007: done (2 left) 2024-09-14T03:51:14.619 INFO:tasks.rados.rados.0.smithi070.stdout:1010: snap_create 2024-09-14T03:51:14.620 INFO:tasks.rados.rados.0.smithi070.stdout:1009: finishing copy_from to smithi07085154-13 2024-09-14T03:51:14.620 INFO:tasks.rados.rados.0.smithi070.stdout:1009: got expected ENOENT (src dne) 2024-09-14T03:51:14.620 INFO:tasks.rados.rados.0.smithi070.stdout:1009: finishing copy_from racing read to smithi07085154-13 2024-09-14T03:51:14.622 INFO:tasks.rados.rados.0.smithi070.stdout:1008: finishing write tid 1 to smithi07085154-47 2024-09-14T03:51:14.622 INFO:tasks.rados.rados.0.smithi070.stdout:1008: finishing write tid 2 to smithi07085154-47 2024-09-14T03:51:14.623 INFO:tasks.rados.rados.0.smithi070.stdout:1008: finishing write tid 3 to smithi07085154-47 2024-09-14T03:51:14.623 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 47 v 336 (ObjNum 331 snap 109 seq_num 331) dirty exists 2024-09-14T03:51:14.623 INFO:tasks.rados.rados.0.smithi070.stdout:1008: left oid 47 (ObjNum 331 snap 109 seq_num 331) 2024-09-14T03:51:14.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:14 smithi070 ceph-mon[27495]: pgmap v36: 121 pgs: 121 active+clean; 282 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.5 MiB/s rd, 2.9 MiB/s wr, 15 op/s 2024-09-14T03:51:14.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:14 smithi070 ceph-mon[27495]: osdmap e261: 8 total, 8 up, 8 in 2024-09-14T03:51:14.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:14 smithi070 ceph-mon[39288]: pgmap v36: 121 pgs: 121 active+clean; 282 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.5 MiB/s rd, 2.9 MiB/s wr, 15 op/s 2024-09-14T03:51:14.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:14 smithi070 ceph-mon[39288]: osdmap e261: 8 total, 8 up, 8 in 2024-09-14T03:51:14.947 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:51:14.948 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (67s) 18s ago 5m 19.7M - 0.27.0 11f11916f8cd 45e79e51074c 2024-09-14T03:51:14.948 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi094 *:3000 running (44s) 18s ago 5m 80.2M - 9.4.7 954c08fa6188 2240810d0368 2024-09-14T03:51:14.948 INFO:teuthology.orchestra.run.smithi070.stdout:mds.a smithi070 running (7m) 18s ago 7m 19.2M - 17.2.7-1629-geb16b97b ab51d1bebd12 52e0dbb7be50 2024-09-14T03:51:14.948 INFO:teuthology.orchestra.run.smithi070.stdout:mds.b smithi094 running (7m) 18s ago 7m 15.0M - 17.2.7-1629-geb16b97b ab51d1bebd12 e3b5e3e00189 2024-09-14T03:51:14.948 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi094 *:8443,9283,8765 starting - - - - 2024-09-14T03:51:14.948 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (32s) 18s ago 12m 491M - 19.3.0-4907-ga2782315 8801d2529e43 0ef4d8e0cdbe 2024-09-14T03:51:14.948 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (12m) 18s ago 12m 58.3M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 12d19db91f54 2024-09-14T03:51:14.948 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi094 running (11m) 18s ago 11m 49.9M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 90ea2e164909 2024-09-14T03:51:14.948 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (11m) 18s ago 11m 49.5M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 44ee80c29ee0 2024-09-14T03:51:14.948 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (61s) 18s ago 5m 10.1M - 1.7.0 72c9c2088986 e316760a30b3 2024-09-14T03:51:14.948 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi094 *:9100 running (37s) 18s ago 5m 6937k - 1.7.0 72c9c2088986 767f8a093175 2024-09-14T03:51:14.949 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (10m) 18s ago 10m 146M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 741c06b6adbf 2024-09-14T03:51:14.949 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (10m) 18s ago 10m 157M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 af459ea7ef68 2024-09-14T03:51:14.949 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (9m) 18s ago 9m 127M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 109b66e4e60f 2024-09-14T03:51:14.949 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (9m) 18s ago 9m 108M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 2e4199b58392 2024-09-14T03:51:14.949 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi094 running (8m) 18s ago 8m 148M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 52b64527e24c 2024-09-14T03:51:14.949 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi094 running (8m) 18s ago 8m 135M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 b543f152ba87 2024-09-14T03:51:14.949 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi094 running (8m) 18s ago 8m 102M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 e66b592ac940 2024-09-14T03:51:14.949 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi094 running (7m) 18s ago 7m 128M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 037c6208e9ce 2024-09-14T03:51:14.949 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi094 *:9095 starting - - - - 2024-09-14T03:51:15.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:14 smithi094 ceph-mon[31053]: pgmap v36: 121 pgs: 121 active+clean; 282 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.5 MiB/s rd, 2.9 MiB/s wr, 15 op/s 2024-09-14T03:51:15.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:14 smithi094 ceph-mon[31053]: osdmap e261: 8 total, 8 up, 8 in 2024-09-14T03:51:15.282 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:51:15.282 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-09-14T03:51:15.282 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 3 2024-09-14T03:51:15.282 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:51:15.282 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-09-14T03:51:15.282 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:51:15.282 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:51:15.282 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-09-14T03:51:15.282 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 8 2024-09-14T03:51:15.282 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:51:15.283 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": { 2024-09-14T03:51:15.283 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:51:15.283 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:51:15.283 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-09-14T03:51:15.283 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 13, 2024-09-14T03:51:15.283 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:51:15.283 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-09-14T03:51:15.283 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:51:15.547 INFO:tasks.rados.rados.0.smithi070.stdout:1008: done (2 left) 2024-09-14T03:51:15.547 INFO:tasks.rados.rados.0.smithi070.stdout:1009: done (1 left) 2024-09-14T03:51:15.547 INFO:tasks.rados.rados.0.smithi070.stdout:1010: done (0 left) 2024-09-14T03:51:15.547 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 15 current snap is 110 2024-09-14T03:51:15.548 INFO:tasks.rados.rados.0.smithi070.stdout:1011: seq_num 332 ranges {0=770048} 2024-09-14T03:51:15.551 INFO:tasks.rados.rados.0.smithi070.stdout:1011: writing smithi07085154-15 from 0 to 770048 tid 1 2024-09-14T03:51:15.551 INFO:tasks.rados.rados.0.smithi070.stdout:1012: delete oid 31 current snap is 110 2024-09-14T03:51:15.555 INFO:tasks.rados.rados.0.smithi070.stdout:1012: done (1 left) 2024-09-14T03:51:15.555 INFO:tasks.rados.rados.0.smithi070.stdout:1013: snap_create 2024-09-14T03:51:15.556 INFO:tasks.rados.rados.0.smithi070.stdout:1011: finishing write tid 1 to smithi07085154-15 2024-09-14T03:51:15.556 INFO:tasks.rados.rados.0.smithi070.stdout:1011: finishing write tid 2 to smithi07085154-15 2024-09-14T03:51:15.557 INFO:tasks.rados.rados.0.smithi070.stdout:1011: finishing write tid 3 to smithi07085154-15 2024-09-14T03:51:15.557 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 15 v 792 (ObjNum 332 snap 110 seq_num 332) dirty exists 2024-09-14T03:51:15.557 INFO:tasks.rados.rados.0.smithi070.stdout:1011: left oid 15 (ObjNum 332 snap 110 seq_num 332) 2024-09-14T03:51:15.566 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:51:15.567 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:51:15.567 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-09-14T03:51:15.567 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:51:15.567 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [ 2024-09-14T03:51:15.567 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr" 2024-09-14T03:51:15.567 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2024-09-14T03:51:15.568 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "2/20 daemons upgraded", 2024-09-14T03:51:15.568 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "Currently upgrading mgr daemons", 2024-09-14T03:51:15.568 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-09-14T03:51:15.569 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:51:15.928 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:51:15 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[78773]: [14/Sep/2024:03:51:15] ENGINE Bus STARTING 2024-09-14T03:51:15.928 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:51:15 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[78773]: CherryPy Checker: 2024-09-14T03:51:15.928 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:51:15 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[78773]: The Application mounted at '' has an empty config. 2024-09-14T03:51:15.928 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:51:15 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[78773]: 2024-09-14T03:51:15.928 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:51:15 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[78773]: [14/Sep/2024:03:51:15] ENGINE Serving on http://:::9283 2024-09-14T03:51:15.929 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:51:15 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[78773]: [14/Sep/2024:03:51:15] ENGINE Bus STARTED 2024-09-14T03:51:15.929 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:15 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:15.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:51:15.929 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:15 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:15.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:51:15.929 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:15 smithi094 ceph-mon[31053]: from='client.15234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:15.929 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:15 smithi094 ceph-mon[31053]: osdmap e262: 8 total, 8 up, 8 in 2024-09-14T03:51:15.929 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:15 smithi094 ceph-mon[31053]: from='client.15240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:15.929 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:15 smithi094 ceph-mon[31053]: from='client.25151 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:15.930 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:15 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:15.930 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:15 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:15.930 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:15 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/1907509631' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:15.930 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:15 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:15.930 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:15 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:15.930 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:15 smithi094 ceph-mon[31053]: osdmap e263: 8 total, 8 up, 8 in 2024-09-14T03:51:16.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:15 smithi070 ceph-mon[27495]: from='client.15234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:16.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:15 smithi070 ceph-mon[27495]: osdmap e262: 8 total, 8 up, 8 in 2024-09-14T03:51:16.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:15 smithi070 ceph-mon[27495]: from='client.15240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:16.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:15 smithi070 ceph-mon[27495]: from='client.25151 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:16.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:15 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:16.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:15 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:16.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:15 smithi070 ceph-mon[27495]: from='client.? 172.21.15.70:0/1907509631' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:16.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:15 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:16.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:15 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:16.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:15 smithi070 ceph-mon[27495]: osdmap e263: 8 total, 8 up, 8 in 2024-09-14T03:51:16.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:15 smithi070 ceph-mon[39288]: from='client.15234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:16.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:15 smithi070 ceph-mon[39288]: osdmap e262: 8 total, 8 up, 8 in 2024-09-14T03:51:16.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:15 smithi070 ceph-mon[39288]: from='client.15240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:16.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:15 smithi070 ceph-mon[39288]: from='client.25151 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:16.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:15 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:16.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:15 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:16.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:15 smithi070 ceph-mon[39288]: from='client.? 172.21.15.70:0/1907509631' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:16.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:15 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:16.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:15 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:16.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:15 smithi070 ceph-mon[39288]: osdmap e263: 8 total, 8 up, 8 in 2024-09-14T03:51:16.549 INFO:tasks.rados.rados.0.smithi070.stdout:1011: done (1 left) 2024-09-14T03:51:16.549 INFO:tasks.rados.rados.0.smithi070.stdout:1013: done (0 left) 2024-09-14T03:51:16.549 INFO:tasks.rados.rados.0.smithi070.stdout:1014: copy_from oid 33 from oid 44 current snap is 111 2024-09-14T03:51:16.549 INFO:tasks.rados.rados.0.smithi070.stdout:1015: delete oid 3 current snap is 111 2024-09-14T03:51:16.552 INFO:tasks.rados.rados.0.smithi070.stdout:1015: done (1 left) 2024-09-14T03:51:16.552 INFO:tasks.rados.rados.0.smithi070.stdout:1016: snap_remove snap 59 2024-09-14T03:51:16.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:16 smithi070 ceph-mon[27495]: pgmap v39: 121 pgs: 121 active+clean; 282 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.5 MiB/s rd, 2.9 MiB/s wr, 15 op/s 2024-09-14T03:51:16.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:16 smithi070 ceph-mon[27495]: from='client.25163 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:16.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:16 smithi070 ceph-mon[27495]: Standby manager daemon x restarted 2024-09-14T03:51:16.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:16 smithi070 ceph-mon[27495]: Standby manager daemon x started 2024-09-14T03:51:16.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:16 smithi070 ceph-mon[27495]: from='mgr.? 172.21.15.94:0/271083757' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:51:16.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:16 smithi070 ceph-mon[27495]: from='mgr.? 172.21.15.94:0/271083757' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:51:16.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:16 smithi070 ceph-mon[27495]: from='mgr.? 172.21.15.94:0/271083757' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:51:16.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:16 smithi070 ceph-mon[27495]: from='mgr.? 172.21.15.94:0/271083757' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:51:16.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:16 smithi070 ceph-mon[27495]: osdmap e264: 8 total, 8 up, 8 in 2024-09-14T03:51:16.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:16 smithi070 ceph-mon[39288]: pgmap v39: 121 pgs: 121 active+clean; 282 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.5 MiB/s rd, 2.9 MiB/s wr, 15 op/s 2024-09-14T03:51:16.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:16 smithi070 ceph-mon[39288]: from='client.25163 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:16.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:16 smithi070 ceph-mon[39288]: Standby manager daemon x restarted 2024-09-14T03:51:16.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:16 smithi070 ceph-mon[39288]: Standby manager daemon x started 2024-09-14T03:51:16.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:16 smithi070 ceph-mon[39288]: from='mgr.? 172.21.15.94:0/271083757' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:51:16.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:16 smithi070 ceph-mon[39288]: from='mgr.? 172.21.15.94:0/271083757' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:51:16.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:16 smithi070 ceph-mon[39288]: from='mgr.? 172.21.15.94:0/271083757' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:51:16.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:16 smithi070 ceph-mon[39288]: from='mgr.? 172.21.15.94:0/271083757' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:51:16.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:16 smithi070 ceph-mon[39288]: osdmap e264: 8 total, 8 up, 8 in 2024-09-14T03:51:16.860 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:16 smithi094 ceph-mon[31053]: pgmap v39: 121 pgs: 121 active+clean; 282 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.5 MiB/s rd, 2.9 MiB/s wr, 15 op/s 2024-09-14T03:51:16.860 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:16 smithi094 ceph-mon[31053]: from='client.25163 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:16.860 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:16 smithi094 ceph-mon[31053]: Standby manager daemon x restarted 2024-09-14T03:51:16.860 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:16 smithi094 ceph-mon[31053]: Standby manager daemon x started 2024-09-14T03:51:16.860 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:16 smithi094 ceph-mon[31053]: from='mgr.? 172.21.15.94:0/271083757' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:51:16.860 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:16 smithi094 ceph-mon[31053]: from='mgr.? 172.21.15.94:0/271083757' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:51:16.860 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:16 smithi094 ceph-mon[31053]: from='mgr.? 172.21.15.94:0/271083757' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:51:16.861 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:16 smithi094 ceph-mon[31053]: from='mgr.? 172.21.15.94:0/271083757' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:51:16.861 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:16 smithi094 ceph-mon[31053]: osdmap e264: 8 total, 8 up, 8 in 2024-09-14T03:51:17.164 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:16 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:16.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:51:17.551 INFO:tasks.rados.rados.0.smithi070.stdout:1016: done (1 left) 2024-09-14T03:51:17.551 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 21 current snap is 111 2024-09-14T03:51:17.551 INFO:tasks.rados.rados.0.smithi070.stdout:1017: seq_num 335 ranges {3227648=417792,3645440=401408} 2024-09-14T03:51:17.572 INFO:tasks.rados.rados.0.smithi070.stdout:1017: writing smithi07085154-21 from 3227648 to 3645440 tid 1 2024-09-14T03:51:17.575 INFO:tasks.rados.rados.0.smithi070.stdout:1017: writing smithi07085154-21 from 3645440 to 4046848 tid 2 2024-09-14T03:51:17.575 INFO:tasks.rados.rados.0.smithi070.stdout:1018: read oid 3 snap -1 2024-09-14T03:51:17.575 INFO:tasks.rados.rados.0.smithi070.stdout:1018: expect deleted 2024-09-14T03:51:17.575 INFO:tasks.rados.rados.0.smithi070.stdout:1019: read oid 23 snap -1 2024-09-14T03:51:17.575 INFO:tasks.rados.rados.0.smithi070.stdout:1019: expect (ObjNum 274 snap 84 seq_num 274) 2024-09-14T03:51:17.576 INFO:tasks.rados.rados.0.smithi070.stdout:1020: delete oid 19 current snap is 111 2024-09-14T03:51:17.576 INFO:tasks.rados.rados.0.smithi070.stdout:1014: finishing copy_from to smithi07085154-33 2024-09-14T03:51:17.576 INFO:tasks.rados.rados.0.smithi070.stdout:1014: got expected ENOENT (src dne) 2024-09-14T03:51:17.576 INFO:tasks.rados.rados.0.smithi070.stdout:1014: finishing copy_from racing read to smithi07085154-33 2024-09-14T03:51:17.576 INFO:tasks.rados.rados.0.smithi070.stdout:1017: finishing write tid 1 to smithi07085154-21 2024-09-14T03:51:17.577 INFO:tasks.rados.rados.0.smithi070.stdout:1017: finishing write tid 2 to smithi07085154-21 2024-09-14T03:51:17.577 INFO:tasks.rados.rados.0.smithi070.stdout:1017: finishing write tid 3 to smithi07085154-21 2024-09-14T03:51:17.578 INFO:tasks.rados.rados.0.smithi070.stdout:1014: done (4 left) 2024-09-14T03:51:17.578 INFO:tasks.rados.rados.0.smithi070.stdout:1018: done (3 left) 2024-09-14T03:51:17.578 INFO:tasks.rados.rados.0.smithi070.stdout:1020: done (2 left) 2024-09-14T03:51:17.578 INFO:tasks.rados.rados.0.smithi070.stdout:1021: rollback oid 41 current snap is 111 2024-09-14T03:51:17.578 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 41 to 107 2024-09-14T03:51:17.578 INFO:tasks.rados.rados.0.smithi070.stdout:1022: rollback oid 22 current snap is 111 2024-09-14T03:51:17.578 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 22 to 51 2024-09-14T03:51:17.578 INFO:tasks.rados.rados.0.smithi070.stdout:1023: rmattr oid 32 current snap is 111 2024-09-14T03:51:17.578 INFO:tasks.rados.rados.0.smithi070.stdout:1023: done (4 left) 2024-09-14T03:51:17.578 INFO:tasks.rados.rados.0.smithi070.stdout:1024: read oid 20 snap -1 2024-09-14T03:51:17.579 INFO:tasks.rados.rados.0.smithi070.stdout:1024: expect deleted 2024-09-14T03:51:17.579 INFO:tasks.rados.rados.0.smithi070.stdout:1025: snap_create 2024-09-14T03:51:17.579 INFO:tasks.rados.rados.0.smithi070.stdout:1017: finishing write tid 4 to smithi07085154-21 2024-09-14T03:51:17.579 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 263 (ObjNum 335 snap 111 seq_num 335) dirty exists 2024-09-14T03:51:17.579 INFO:tasks.rados.rados.0.smithi070.stdout:1017: left oid 21 (ObjNum 335 snap 111 seq_num 335) 2024-09-14T03:51:17.580 INFO:tasks.rados.rados.0.smithi070.stdout:1022: finishing rollback tid 1 to smithi07085154-22 2024-09-14T03:51:17.580 INFO:tasks.rados.rados.0.smithi070.stdout:1022: finishing rollback tid 2 to smithi07085154-22 2024-09-14T03:51:17.580 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 541 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-14T03:51:17.581 INFO:tasks.rados.rados.0.smithi070.stdout:1021: finishing rollback tid 0 to smithi07085154-41 2024-09-14T03:51:17.581 INFO:tasks.rados.rados.0.smithi070.stdout:1021: finishing rollback tid 1 to smithi07085154-41 2024-09-14T03:51:17.581 INFO:tasks.rados.rados.0.smithi070.stdout:1021: finishing rollback tid 2 to smithi07085154-41 2024-09-14T03:51:17.581 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 795 (ObjNum 211 snap 60 seq_num 211) dirty exists 2024-09-14T03:51:17.604 INFO:tasks.rados.rados.0.smithi070.stdout:1019: expect (ObjNum 274 snap 84 seq_num 274) 2024-09-14T03:51:17.865 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:17 smithi094 ceph-mon[31053]: mgrmap e36: y(active, since 25s), standbys: x 2024-09-14T03:51:17.865 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:17 smithi094 ceph-mon[31053]: osdmap e265: 8 total, 8 up, 8 in 2024-09-14T03:51:18.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:17 smithi070 ceph-mon[27495]: mgrmap e36: y(active, since 25s), standbys: x 2024-09-14T03:51:18.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:17 smithi070 ceph-mon[27495]: osdmap e265: 8 total, 8 up, 8 in 2024-09-14T03:51:18.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:17 smithi070 ceph-mon[39288]: mgrmap e36: y(active, since 25s), standbys: x 2024-09-14T03:51:18.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:17 smithi070 ceph-mon[39288]: osdmap e265: 8 total, 8 up, 8 in 2024-09-14T03:51:18.552 INFO:tasks.rados.rados.0.smithi070.stdout:1017: done (5 left) 2024-09-14T03:51:18.552 INFO:tasks.rados.rados.0.smithi070.stdout:1019: done (4 left) 2024-09-14T03:51:18.552 INFO:tasks.rados.rados.0.smithi070.stdout:1021: done (3 left) 2024-09-14T03:51:18.552 INFO:tasks.rados.rados.0.smithi070.stdout:1022: done (2 left) 2024-09-14T03:51:18.553 INFO:tasks.rados.rados.0.smithi070.stdout:1024: done (1 left) 2024-09-14T03:51:18.553 INFO:tasks.rados.rados.0.smithi070.stdout:1025: done (0 left) 2024-09-14T03:51:18.553 INFO:tasks.rados.rados.0.smithi070.stdout:1026: snap_remove snap 47 2024-09-14T03:51:19.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:18 smithi094 ceph-mon[31053]: pgmap v42: 121 pgs: 121 active+clean; 282 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2024-09-14T03:51:19.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:18 smithi094 ceph-mon[31053]: osdmap e266: 8 total, 8 up, 8 in 2024-09-14T03:51:19.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:18 smithi070 ceph-mon[27495]: pgmap v42: 121 pgs: 121 active+clean; 282 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2024-09-14T03:51:19.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:18 smithi070 ceph-mon[27495]: osdmap e266: 8 total, 8 up, 8 in 2024-09-14T03:51:19.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:18 smithi070 ceph-mon[39288]: pgmap v42: 121 pgs: 121 active+clean; 282 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2024-09-14T03:51:19.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:18 smithi070 ceph-mon[39288]: osdmap e266: 8 total, 8 up, 8 in 2024-09-14T03:51:19.554 INFO:tasks.rados.rados.0.smithi070.stdout:1026: done (0 left) 2024-09-14T03:51:19.554 INFO:tasks.rados.rados.0.smithi070.stdout:1027: read oid 47 snap -1 2024-09-14T03:51:19.554 INFO:tasks.rados.rados.0.smithi070.stdout:1027: expect (ObjNum 331 snap 109 seq_num 331) 2024-09-14T03:51:19.554 INFO:tasks.rados.rados.0.smithi070.stdout:1028: read oid 35 snap -1 2024-09-14T03:51:19.554 INFO:tasks.rados.rados.0.smithi070.stdout:1028: expect deleted 2024-09-14T03:51:19.554 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 4 current snap is 112 2024-09-14T03:51:19.554 INFO:tasks.rados.rados.0.smithi070.stdout:1029: seq_num 337 ranges {1679360=450560,2129920=401408} 2024-09-14T03:51:19.567 INFO:tasks.rados.rados.0.smithi070.stdout:1029: writing smithi07085154-4 from 1679360 to 2129920 tid 1 2024-09-14T03:51:19.570 INFO:tasks.rados.rados.0.smithi070.stdout:1029: writing smithi07085154-4 from 2129920 to 2531328 tid 2 2024-09-14T03:51:19.570 INFO:tasks.rados.rados.0.smithi070.stdout:1028: done (2 left) 2024-09-14T03:51:19.570 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 16 current snap is 112 2024-09-14T03:51:19.570 INFO:tasks.rados.rados.0.smithi070.stdout:1030: seq_num 338 ranges {4784128=466944,5251072=401408} 2024-09-14T03:51:19.598 INFO:tasks.rados.rados.0.smithi070.stdout:1030: writing smithi07085154-16 from 4784128 to 5251072 tid 1 2024-09-14T03:51:19.601 INFO:tasks.rados.rados.0.smithi070.stdout:1030: writing smithi07085154-16 from 5251072 to 5652480 tid 2 2024-09-14T03:51:19.601 INFO:tasks.rados.rados.0.smithi070.stdout:1031: copy_from oid 34 from oid 17 current snap is 112 2024-09-14T03:51:19.601 INFO:tasks.rados.rados.0.smithi070.stdout:1029: finishing write tid 1 to smithi07085154-4 2024-09-14T03:51:19.601 INFO:tasks.rados.rados.0.smithi070.stdout:1029: finishing write tid 2 to smithi07085154-4 2024-09-14T03:51:19.601 INFO:tasks.rados.rados.0.smithi070.stdout:1029: finishing write tid 3 to smithi07085154-4 2024-09-14T03:51:19.601 INFO:tasks.rados.rados.0.smithi070.stdout:1029: finishing write tid 4 to smithi07085154-4 2024-09-14T03:51:19.601 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 387 (ObjNum 337 snap 112 seq_num 337) dirty exists 2024-09-14T03:51:19.602 INFO:tasks.rados.rados.0.smithi070.stdout:1029: left oid 4 (ObjNum 337 snap 112 seq_num 337) 2024-09-14T03:51:19.602 INFO:tasks.rados.rados.0.smithi070.stdout:1027: expect (ObjNum 331 snap 109 seq_num 331) 2024-09-14T03:51:19.798 INFO:tasks.rados.rados.0.smithi070.stdout:1030: finishing write tid 1 to smithi07085154-16 2024-09-14T03:51:19.798 INFO:tasks.rados.rados.0.smithi070.stdout:1030: finishing write tid 2 to smithi07085154-16 2024-09-14T03:51:19.798 INFO:tasks.rados.rados.0.smithi070.stdout:1030: finishing write tid 3 to smithi07085154-16 2024-09-14T03:51:19.798 INFO:tasks.rados.rados.0.smithi070.stdout:1030: finishing write tid 4 to smithi07085154-16 2024-09-14T03:51:19.798 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 860 (ObjNum 338 snap 112 seq_num 338) dirty exists 2024-09-14T03:51:19.798 INFO:tasks.rados.rados.0.smithi070.stdout:1030: left oid 16 (ObjNum 338 snap 112 seq_num 338) 2024-09-14T03:51:19.798 INFO:tasks.rados.rados.0.smithi070.stdout:1031: finishing copy_from racing read to smithi07085154-34 2024-09-14T03:51:19.798 INFO:tasks.rados.rados.0.smithi070.stdout:1031: finishing copy_from to smithi07085154-34 2024-09-14T03:51:19.798 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 402 (ObjNum 325 snap 107 seq_num 325) dirty exists 2024-09-14T03:51:19.798 INFO:tasks.rados.rados.0.smithi070.stdout:1027: done (3 left) 2024-09-14T03:51:19.798 INFO:tasks.rados.rados.0.smithi070.stdout:1029: done (2 left) 2024-09-14T03:51:19.799 INFO:tasks.rados.rados.0.smithi070.stdout:1030: done (1 left) 2024-09-14T03:51:19.799 INFO:tasks.rados.rados.0.smithi070.stdout:1031: done (0 left) 2024-09-14T03:51:19.799 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 10 current snap is 112 2024-09-14T03:51:19.799 INFO:tasks.rados.rados.0.smithi070.stdout:1032: seq_num 339 ranges {1056768=483328,1540096=401408} 2024-09-14T03:51:19.806 INFO:tasks.rados.rados.0.smithi070.stdout:1032: writing smithi07085154-10 from 1056768 to 1540096 tid 1 2024-09-14T03:51:19.808 INFO:tasks.rados.rados.0.smithi070.stdout:1032: writing smithi07085154-10 from 1540096 to 1941504 tid 2 2024-09-14T03:51:19.808 INFO:tasks.rados.rados.0.smithi070.stdout:1033: rollback oid 41 current snap is 112 2024-09-14T03:51:19.809 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 41 to 108 2024-09-14T03:51:19.809 INFO:tasks.rados.rados.0.smithi070.stdout:1034: delete oid 38 current snap is 112 2024-09-14T03:51:19.810 INFO:tasks.rados.rados.0.smithi070.stdout:1034: done (2 left) 2024-09-14T03:51:19.810 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 7 current snap is 112 2024-09-14T03:51:19.810 INFO:tasks.rados.rados.0.smithi070.stdout:1035: seq_num 341 ranges {0=516096,516096=401408} 2024-09-14T03:51:19.813 INFO:tasks.rados.rados.0.smithi070.stdout:1035: writing smithi07085154-7 from 0 to 516096 tid 1 2024-09-14T03:51:19.816 INFO:tasks.rados.rados.0.smithi070.stdout:1035: writing smithi07085154-7 from 516096 to 917504 tid 2 2024-09-14T03:51:19.816 INFO:tasks.rados.rados.0.smithi070.stdout:1036: read oid 21 snap -1 2024-09-14T03:51:19.816 INFO:tasks.rados.rados.0.smithi070.stdout:1036: expect (ObjNum 335 snap 111 seq_num 335) 2024-09-14T03:51:19.816 INFO:tasks.rados.rados.0.smithi070.stdout:1037: snap_remove snap 68 2024-09-14T03:51:20.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:19 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:20.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:19 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:20.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:19 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:20.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:19 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:51:20.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:19 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:20.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:19 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:20.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:19 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:20.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:19 smithi094 ceph-mon[31053]: Upgrade: Setting container_image for all mgr 2024-09-14T03:51:20.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:19 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:20.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:19 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-14T03:51:20.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:19 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-14T03:51:20.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:19 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-14T03:51:20.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:19 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-14T03:51:20.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:19 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-14T03:51:20.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:19 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-14T03:51:20.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:19 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-14T03:51:20.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:19 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-14T03:51:20.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:19 smithi094 ceph-mon[31053]: Upgrade: It appears safe to stop mon.a 2024-09-14T03:51:20.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:19 smithi094 ceph-mon[31053]: osdmap e267: 8 total, 8 up, 8 in 2024-09-14T03:51:20.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:20.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:20.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:20.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:51:20.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:20.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:20.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:20.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[27495]: Upgrade: Setting container_image for all mgr 2024-09-14T03:51:20.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:20.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-14T03:51:20.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-14T03:51:20.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-14T03:51:20.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-14T03:51:20.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-14T03:51:20.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-14T03:51:20.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-14T03:51:20.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-14T03:51:20.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[27495]: Upgrade: It appears safe to stop mon.a 2024-09-14T03:51:20.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[27495]: osdmap e267: 8 total, 8 up, 8 in 2024-09-14T03:51:20.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:20.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:20.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:20.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:51:20.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:20.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:20.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:51:20.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[39288]: Upgrade: Setting container_image for all mgr 2024-09-14T03:51:20.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:20.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-14T03:51:20.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-14T03:51:20.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-14T03:51:20.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-14T03:51:20.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-14T03:51:20.041 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-14T03:51:20.041 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-14T03:51:20.041 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-14T03:51:20.041 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[39288]: Upgrade: It appears safe to stop mon.a 2024-09-14T03:51:20.041 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:19 smithi070 ceph-mon[39288]: osdmap e267: 8 total, 8 up, 8 in 2024-09-14T03:51:20.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:20 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:51:20.549 INFO:tasks.rados.rados.0.smithi070.stdout:1037: done (4 left) 2024-09-14T03:51:20.549 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 23 current snap is 112 2024-09-14T03:51:20.549 INFO:tasks.rados.rados.0.smithi070.stdout:1032: finishing write tid 1 to smithi07085154-10 2024-09-14T03:51:20.549 INFO:tasks.rados.rados.0.smithi070.stdout:1032: finishing write tid 2 to smithi07085154-10 2024-09-14T03:51:20.549 INFO:tasks.rados.rados.0.smithi070.stdout:1032: finishing write tid 3 to smithi07085154-10 2024-09-14T03:51:20.549 INFO:tasks.rados.rados.0.smithi070.stdout:1033: finishing rollback tid 0 to smithi07085154-41 2024-09-14T03:51:20.549 INFO:tasks.rados.rados.0.smithi070.stdout:1033: finishing rollback tid 1 to smithi07085154-41 2024-09-14T03:51:20.549 INFO:tasks.rados.rados.0.smithi070.stdout:1033: finishing rollback tid 2 to smithi07085154-41 2024-09-14T03:51:20.549 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 798 (ObjNum 211 snap 60 seq_num 211) dirty exists 2024-09-14T03:51:20.550 INFO:tasks.rados.rados.0.smithi070.stdout:1032: finishing write tid 4 to smithi07085154-10 2024-09-14T03:51:20.550 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 864 (ObjNum 339 snap 112 seq_num 339) dirty exists 2024-09-14T03:51:20.550 INFO:tasks.rados.rados.0.smithi070.stdout:1032: left oid 10 (ObjNum 339 snap 112 seq_num 339) 2024-09-14T03:51:20.550 INFO:tasks.rados.rados.0.smithi070.stdout:1035: finishing write tid 1 to smithi07085154-7 2024-09-14T03:51:20.550 INFO:tasks.rados.rados.0.smithi070.stdout:1038: seq_num 342 ranges {6856704=532480,7389184=401408} 2024-09-14T03:51:20.589 INFO:tasks.rados.rados.0.smithi070.stdout:1038: writing smithi07085154-23 from 6856704 to 7389184 tid 1 2024-09-14T03:51:20.591 INFO:tasks.rados.rados.0.smithi070.stdout:1038: writing smithi07085154-23 from 7389184 to 7790592 tid 2 2024-09-14T03:51:20.591 INFO:tasks.rados.rados.0.smithi070.stdout:1032: done (4 left) 2024-09-14T03:51:20.592 INFO:tasks.rados.rados.0.smithi070.stdout:1033: done (3 left) 2024-09-14T03:51:20.592 INFO:tasks.rados.rados.0.smithi070.stdout:1039: snap_create 2024-09-14T03:51:20.592 INFO:tasks.rados.rados.0.smithi070.stdout:1035: finishing write tid 2 to smithi07085154-7 2024-09-14T03:51:20.592 INFO:tasks.rados.rados.0.smithi070.stdout:1035: finishing write tid 3 to smithi07085154-7 2024-09-14T03:51:20.592 INFO:tasks.rados.rados.0.smithi070.stdout:1035: finishing write tid 4 to smithi07085154-7 2024-09-14T03:51:20.592 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 390 (ObjNum 341 snap 112 seq_num 341) dirty exists 2024-09-14T03:51:20.592 INFO:tasks.rados.rados.0.smithi070.stdout:1035: left oid 7 (ObjNum 341 snap 112 seq_num 341) 2024-09-14T03:51:20.592 INFO:tasks.rados.rados.0.smithi070.stdout:1036: expect (ObjNum 335 snap 111 seq_num 335) 2024-09-14T03:51:20.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:20 smithi070 ceph-mon[39288]: pgmap v45: 121 pgs: 121 active+clean; 280 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 439 KiB/s rd, 970 KiB/s wr, 9 op/s 2024-09-14T03:51:20.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:20 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:20.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:20 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:51:20.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:20 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:51:20.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:20 smithi070 ceph-mon[39288]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:20.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:20 smithi070 ceph-mon[39288]: osdmap e268: 8 total, 8 up, 8 in 2024-09-14T03:51:20.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:20 smithi070 ceph-mon[27495]: pgmap v45: 121 pgs: 121 active+clean; 280 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 439 KiB/s rd, 970 KiB/s wr, 9 op/s 2024-09-14T03:51:20.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:20 smithi070 ceph-mon[27495]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:20.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:20 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:51:20.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:20 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:51:20.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:20 smithi070 ceph-mon[27495]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:20.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:20 smithi070 ceph-mon[27495]: osdmap e268: 8 total, 8 up, 8 in 2024-09-14T03:51:20.804 INFO:tasks.rados.rados.0.smithi070.stdout:1038: finishing write tid 1 to smithi07085154-23 2024-09-14T03:51:20.805 INFO:tasks.rados.rados.0.smithi070.stdout:1038: finishing write tid 2 to smithi07085154-23 2024-09-14T03:51:20.805 INFO:tasks.rados.rados.0.smithi070.stdout:1038: finishing write tid 3 to smithi07085154-23 2024-09-14T03:51:20.805 INFO:tasks.rados.rados.0.smithi070.stdout:1038: finishing write tid 4 to smithi07085154-23 2024-09-14T03:51:20.805 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 407 (ObjNum 342 snap 112 seq_num 342) dirty exists 2024-09-14T03:51:20.805 INFO:tasks.rados.rados.0.smithi070.stdout:1038: left oid 23 (ObjNum 342 snap 112 seq_num 342) 2024-09-14T03:51:21.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:20 smithi094 ceph-mon[31053]: pgmap v45: 121 pgs: 121 active+clean; 280 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 439 KiB/s rd, 970 KiB/s wr, 9 op/s 2024-09-14T03:51:21.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:20 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='mgr.y' 2024-09-14T03:51:21.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:20 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:51:21.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:20 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:51:21.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:20 smithi094 ceph-mon[31053]: from='mgr.15159 172.21.15.70:0/1490341208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:21.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:20 smithi094 ceph-mon[31053]: osdmap e268: 8 total, 8 up, 8 in 2024-09-14T03:51:21.551 INFO:tasks.rados.rados.0.smithi070.stdout:1035: done (3 left) 2024-09-14T03:51:21.551 INFO:tasks.rados.rados.0.smithi070.stdout:1036: done (2 left) 2024-09-14T03:51:21.551 INFO:tasks.rados.rados.0.smithi070.stdout:1038: done (1 left) 2024-09-14T03:51:21.551 INFO:tasks.rados.rados.0.smithi070.stdout:1039: done (0 left) 2024-09-14T03:51:21.551 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 19 current snap is 113 2024-09-14T03:51:21.551 INFO:tasks.rados.rados.0.smithi070.stdout:1040: seq_num 343 ranges {0=548864,548864=401408} 2024-09-14T03:51:21.553 INFO:tasks.rados.rados.0.smithi070.stdout:1040: writing smithi07085154-19 from 0 to 548864 tid 1 2024-09-14T03:51:21.556 INFO:tasks.rados.rados.0.smithi070.stdout:1040: writing smithi07085154-19 from 548864 to 950272 tid 2 2024-09-14T03:51:21.556 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 25 current snap is 113 2024-09-14T03:51:21.556 INFO:tasks.rados.rados.0.smithi070.stdout:1041: seq_num 344 ranges {7741440=565248,8306688=401408} 2024-09-14T03:51:21.600 INFO:tasks.rados.rados.0.smithi070.stdout:1041: writing smithi07085154-25 from 7741440 to 8306688 tid 1 2024-09-14T03:51:21.603 INFO:tasks.rados.rados.0.smithi070.stdout:1041: writing smithi07085154-25 from 8306688 to 8708096 tid 2 2024-09-14T03:51:21.603 INFO:tasks.rados.rados.0.smithi070.stdout:1042: rmattr oid 42 current snap is 113 2024-09-14T03:51:21.603 INFO:tasks.rados.rados.0.smithi070.stdout:1040: finishing write tid 1 to smithi07085154-19 2024-09-14T03:51:21.603 INFO:tasks.rados.rados.0.smithi070.stdout:1043: delete oid 10 current snap is 113 2024-09-14T03:51:21.603 INFO:tasks.rados.rados.0.smithi070.stdout:1040: finishing write tid 2 to smithi07085154-19 2024-09-14T03:51:21.603 INFO:tasks.rados.rados.0.smithi070.stdout:1040: finishing write tid 3 to smithi07085154-19 2024-09-14T03:51:21.603 INFO:tasks.rados.rados.0.smithi070.stdout:1040: finishing write tid 4 to smithi07085154-19 2024-09-14T03:51:21.604 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 19 v 562 (ObjNum 343 snap 113 seq_num 343) dirty exists 2024-09-14T03:51:21.604 INFO:tasks.rados.rados.0.smithi070.stdout:1040: left oid 19 (ObjNum 343 snap 113 seq_num 343) 2024-09-14T03:51:21.604 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 0 (ObjNum 280 snap 86 seq_num 280) dirty exists 2024-09-14T03:51:21.607 INFO:tasks.rados.rados.0.smithi070.stdout:1041: finishing write tid 1 to smithi07085154-25 2024-09-14T03:51:21.607 INFO:tasks.rados.rados.0.smithi070.stdout:1040: done (3 left) 2024-09-14T03:51:21.607 INFO:tasks.rados.rados.0.smithi070.stdout:1042: done (2 left) 2024-09-14T03:51:21.607 INFO:tasks.rados.rados.0.smithi070.stdout:1043: done (1 left) 2024-09-14T03:51:21.607 INFO:tasks.rados.rados.0.smithi070.stdout:1044: copy_from oid 21 from oid 5 current snap is 113 2024-09-14T03:51:21.607 INFO:tasks.rados.rados.0.smithi070.stdout:1045: snap_create 2024-09-14T03:51:21.607 INFO:tasks.rados.rados.0.smithi070.stdout:1041: finishing write tid 2 to smithi07085154-25 2024-09-14T03:51:21.608 INFO:tasks.rados.rados.0.smithi070.stdout:1041: finishing write tid 3 to smithi07085154-25 2024-09-14T03:51:21.608 INFO:tasks.rados.rados.0.smithi070.stdout:1041: finishing write tid 4 to smithi07085154-25 2024-09-14T03:51:21.608 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 436 (ObjNum 344 snap 113 seq_num 344) dirty exists 2024-09-14T03:51:21.608 INFO:tasks.rados.rados.0.smithi070.stdout:1041: left oid 25 (ObjNum 344 snap 113 seq_num 344) 2024-09-14T03:51:21.622 INFO:tasks.rados.rados.0.smithi070.stdout:1044: finishing copy_from racing read to smithi07085154-21 2024-09-14T03:51:21.626 INFO:tasks.rados.rados.0.smithi070.stdout:1044: finishing copy_from to smithi07085154-21 2024-09-14T03:51:21.626 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 542 (ObjNum 114 snap 24 seq_num 114) dirty exists 2024-09-14T03:51:22.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:21 smithi094 ceph-mon[31053]: Upgrade: Updating mon.a 2024-09-14T03:51:22.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:21 smithi094 ceph-mon[31053]: Deploying daemon mon.a on smithi070 2024-09-14T03:51:22.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:21 smithi094 ceph-mon[31053]: osdmap e269: 8 total, 8 up, 8 in 2024-09-14T03:51:22.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:21 smithi070 ceph-mon[27495]: Upgrade: Updating mon.a 2024-09-14T03:51:22.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:21 smithi070 ceph-mon[27495]: Deploying daemon mon.a on smithi070 2024-09-14T03:51:22.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:21 smithi070 ceph-mon[27495]: osdmap e269: 8 total, 8 up, 8 in 2024-09-14T03:51:22.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:21 smithi070 ceph-mon[39288]: Upgrade: Updating mon.a 2024-09-14T03:51:22.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:21 smithi070 ceph-mon[39288]: Deploying daemon mon.a on smithi070 2024-09-14T03:51:22.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:21 smithi070 ceph-mon[39288]: osdmap e269: 8 total, 8 up, 8 in 2024-09-14T03:51:22.329 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:22 smithi070 systemd[1]: Stopping Ceph mon.a for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:51:22.718 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:22 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-a[27472]: 2024-09-14T03:51:22.325+0000 7fd888fa9640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:51:22.719 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:22 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-a[27472]: 2024-09-14T03:51:22.325+0000 7fd888fa9640 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-09-14T03:51:22.719 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:22 smithi070 podman[96516]: 2024-09-14 03:51:22.429449941 +0000 UTC m=+0.246973299 container died 12d19db91f54d5f441d45c517e4322d8db7d960f73065890c8753bb54579fc25 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-a, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=quincy-eb16b97, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-14T03:51:23.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:22 smithi070 podman[96516]: 2024-09-14 03:51:22.816119375 +0000 UTC m=+0.633642722 container cleanup 12d19db91f54d5f441d45c517e4322d8db7d960f73065890c8753bb54579fc25 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=quincy-eb16b97, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-14T03:51:23.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:22 smithi070 bash[96516]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-a 2024-09-14T03:51:23.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 podman[96536]: 2024-09-14 03:51:23.101268177 +0000 UTC m=+0.668468753 container remove 12d19db91f54d5f441d45c517e4322d8db7d960f73065890c8753bb54579fc25 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-a, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=quincy-eb16b97) 2024-09-14T03:51:23.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@mon.a.service: Deactivated successfully. 2024-09-14T03:51:23.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 systemd[1]: Stopped Ceph mon.a for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:51:23.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@mon.a.service: Consumed 13.654s CPU time. 2024-09-14T03:51:23.667 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:51:23] "GET /metrics HTTP/1.1" 200 36199 "" "Prometheus/2.51.0" 2024-09-14T03:51:23.667 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 systemd[1]: Starting Ceph mon.a for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:51:23.923 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 podman[96666]: 2024-09-14 03:51:23.569232436 +0000 UTC m=+0.015723935 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:51:23.923 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 podman[96666]: 2024-09-14 03:51:23.669039012 +0000 UTC m=+0.115530517 container create 96542de31cac9c93f77613ee79c8ff2fe4ea7431d2292a5bb822697b7019365b (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-a, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-14T03:51:23.923 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 podman[96666]: 2024-09-14 03:51:23.921692679 +0000 UTC m=+0.368184178 container init 96542de31cac9c93f77613ee79c8ff2fe4ea7431d2292a5bb822697b7019365b (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-a, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True) 2024-09-14T03:51:24.270 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 podman[96666]: 2024-09-14 03:51:23.926510686 +0000 UTC m=+0.373002186 container start 96542de31cac9c93f77613ee79c8ff2fe4ea7431d2292a5bb822697b7019365b (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-a, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231) 2024-09-14T03:51:24.270 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: set uid:gid to 167:167 (ceph:ceph) 2024-09-14T03:51:24.270 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev), process ceph-mon, pid 7 2024-09-14T03:51:24.270 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: pidfile_write: ignore empty --pid-file 2024-09-14T03:51:24.270 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: load: jerasure load: lrc 2024-09-14T03:51:24.270 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: RocksDB version: 7.9.2 2024-09-14T03:51:24.270 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Git sha 0 2024-09-14T03:51:24.270 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Compile date 2024-09-12 20:09:22 2024-09-14T03:51:24.271 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: DB SUMMARY 2024-09-14T03:51:24.271 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: DB Session ID: MSEELCN6BINYCXJZBW7A 2024-09-14T03:51:24.271 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: CURRENT file: CURRENT 2024-09-14T03:51:24.271 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: IDENTITY file: IDENTITY 2024-09-14T03:51:24.271 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1173 Bytes 2024-09-14T03:51:24.271 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000033.sst 2024-09-14T03:51:24.271 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000031.log size: 5483534 ; 2024-09-14T03:51:24.271 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.error_if_exists: 0 2024-09-14T03:51:24.271 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.create_if_missing: 0 2024-09-14T03:51:24.271 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.paranoid_checks: 1 2024-09-14T03:51:24.271 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-14T03:51:24.271 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-14T03:51:24.271 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-14T03:51:24.272 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.env: 0x55d4656ce120 2024-09-14T03:51:24.272 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.fs: PosixFileSystem 2024-09-14T03:51:24.272 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.info_log: 0x55d467269700 2024-09-14T03:51:24.272 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.max_file_opening_threads: 16 2024-09-14T03:51:24.272 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.statistics: (nil) 2024-09-14T03:51:24.272 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.use_fsync: 0 2024-09-14T03:51:24.272 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.max_log_file_size: 0 2024-09-14T03:51:24.272 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-14T03:51:24.272 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-14T03:51:24.272 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.keep_log_file_num: 1000 2024-09-14T03:51:24.272 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.recycle_log_file_num: 0 2024-09-14T03:51:24.272 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.allow_fallocate: 1 2024-09-14T03:51:24.272 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.allow_mmap_reads: 0 2024-09-14T03:51:24.273 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.allow_mmap_writes: 0 2024-09-14T03:51:24.273 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.use_direct_reads: 0 2024-09-14T03:51:24.273 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-14T03:51:24.273 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.create_missing_column_families: 0 2024-09-14T03:51:24.273 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.db_log_dir: 2024-09-14T03:51:24.273 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.wal_dir: 2024-09-14T03:51:24.273 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-14T03:51:24.273 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-14T03:51:24.273 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-14T03:51:24.273 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-14T03:51:24.273 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-14T03:51:24.273 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-14T03:51:24.273 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.advise_random_on_open: 1 2024-09-14T03:51:24.273 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.db_write_buffer_size: 0 2024-09-14T03:51:24.274 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.write_buffer_manager: 0x55d46726db80 2024-09-14T03:51:24.274 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-14T03:51:24.274 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-14T03:51:24.274 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-14T03:51:24.274 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.rate_limiter: (nil) 2024-09-14T03:51:24.274 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-14T03:51:24.274 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.wal_recovery_mode: 2 2024-09-14T03:51:24.274 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.enable_thread_tracking: 0 2024-09-14T03:51:24.274 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.enable_pipelined_write: 0 2024-09-14T03:51:24.274 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.unordered_write: 0 2024-09-14T03:51:24.274 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-14T03:51:24.274 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-14T03:51:24.274 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-14T03:51:24.274 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-14T03:51:24.275 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.row_cache: None 2024-09-14T03:51:24.275 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.wal_filter: None 2024-09-14T03:51:24.275 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-14T03:51:24.275 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.allow_ingest_behind: 0 2024-09-14T03:51:24.275 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.two_write_queues: 0 2024-09-14T03:51:24.275 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.manual_wal_flush: 0 2024-09-14T03:51:24.275 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.wal_compression: 0 2024-09-14T03:51:24.275 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.atomic_flush: 0 2024-09-14T03:51:24.275 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-14T03:51:24.275 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-14T03:51:24.275 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-14T03:51:24.276 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.log_readahead_size: 0 2024-09-14T03:51:24.276 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-14T03:51:24.277 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.best_efforts_recovery: 0 2024-09-14T03:51:24.277 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-14T03:51:24.277 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-14T03:51:24.277 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.allow_data_in_errors: 0 2024-09-14T03:51:24.277 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.db_host_id: __hostname__ 2024-09-14T03:51:24.277 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-14T03:51:24.277 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.max_background_jobs: 2 2024-09-14T03:51:24.277 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.max_background_compactions: -1 2024-09-14T03:51:24.277 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.max_subcompactions: 1 2024-09-14T03:51:24.277 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-14T03:51:24.277 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-14T03:51:24.277 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-14T03:51:24.277 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.max_total_wal_size: 0 2024-09-14T03:51:24.278 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-14T03:51:24.278 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-14T03:51:24.278 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-14T03:51:24.278 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-14T03:51:24.278 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.max_open_files: -1 2024-09-14T03:51:24.278 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.bytes_per_sync: 0 2024-09-14T03:51:24.278 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-14T03:51:24.278 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-14T03:51:24.278 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.compaction_readahead_size: 0 2024-09-14T03:51:24.278 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.max_background_flushes: -1 2024-09-14T03:51:24.278 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Compression algorithms supported: 2024-09-14T03:51:24.278 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: kZSTD supported: 0 2024-09-14T03:51:24.278 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: kXpressCompression supported: 0 2024-09-14T03:51:24.279 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: kBZip2Compression supported: 0 2024-09-14T03:51:24.279 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-14T03:51:24.279 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: kLZ4Compression supported: 1 2024-09-14T03:51:24.279 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: kZlibCompression supported: 1 2024-09-14T03:51:24.279 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: kLZ4HCCompression supported: 1 2024-09-14T03:51:24.279 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: kSnappyCompression supported: 1 2024-09-14T03:51:24.279 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-14T03:51:24.279 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-14T03:51:24.279 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-09-14T03:51:24.279 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-14T03:51:24.279 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-14T03:51:24.279 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.merge_operator: 2024-09-14T03:51:24.279 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.compaction_filter: None 2024-09-14T03:51:24.279 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.compaction_filter_factory: None 2024-09-14T03:51:24.279 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.sst_partitioner_factory: None 2024-09-14T03:51:24.280 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-14T03:51:24.280 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-14T03:51:24.280 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55d467269320) 2024-09-14T03:51:24.280 INFO:journalctl@ceph.mon.a.smithi070.stdout: cache_index_and_filter_blocks: 1 2024-09-14T03:51:24.280 INFO:journalctl@ceph.mon.a.smithi070.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-14T03:51:24.280 INFO:journalctl@ceph.mon.a.smithi070.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-14T03:51:24.280 INFO:journalctl@ceph.mon.a.smithi070.stdout: pin_top_level_index_and_filter: 1 2024-09-14T03:51:24.280 INFO:journalctl@ceph.mon.a.smithi070.stdout: index_type: 0 2024-09-14T03:51:24.280 INFO:journalctl@ceph.mon.a.smithi070.stdout: data_block_index_type: 0 2024-09-14T03:51:24.280 INFO:journalctl@ceph.mon.a.smithi070.stdout: index_shortening: 1 2024-09-14T03:51:24.280 INFO:journalctl@ceph.mon.a.smithi070.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-14T03:51:24.280 INFO:journalctl@ceph.mon.a.smithi070.stdout: checksum: 4 2024-09-14T03:51:24.280 INFO:journalctl@ceph.mon.a.smithi070.stdout: no_block_cache: 0 2024-09-14T03:51:24.280 INFO:journalctl@ceph.mon.a.smithi070.stdout: block_cache: 0x55d46725d610 2024-09-14T03:51:24.281 INFO:journalctl@ceph.mon.a.smithi070.stdout: block_cache_name: BinnedLRUCache 2024-09-14T03:51:24.281 INFO:journalctl@ceph.mon.a.smithi070.stdout: block_cache_options: 2024-09-14T03:51:24.281 INFO:journalctl@ceph.mon.a.smithi070.stdout: capacity : 536870912 2024-09-14T03:51:24.281 INFO:journalctl@ceph.mon.a.smithi070.stdout: num_shard_bits : 4 2024-09-14T03:51:24.281 INFO:journalctl@ceph.mon.a.smithi070.stdout: strict_capacity_limit : 0 2024-09-14T03:51:24.281 INFO:journalctl@ceph.mon.a.smithi070.stdout: high_pri_pool_ratio: 0.000 2024-09-14T03:51:24.281 INFO:journalctl@ceph.mon.a.smithi070.stdout: block_cache_compressed: (nil) 2024-09-14T03:51:24.281 INFO:journalctl@ceph.mon.a.smithi070.stdout: persistent_cache: (nil) 2024-09-14T03:51:24.281 INFO:journalctl@ceph.mon.a.smithi070.stdout: block_size: 4096 2024-09-14T03:51:24.281 INFO:journalctl@ceph.mon.a.smithi070.stdout: block_size_deviation: 10 2024-09-14T03:51:24.281 INFO:journalctl@ceph.mon.a.smithi070.stdout: block_restart_interval: 16 2024-09-14T03:51:24.281 INFO:journalctl@ceph.mon.a.smithi070.stdout: index_block_restart_interval: 1 2024-09-14T03:51:24.282 INFO:journalctl@ceph.mon.a.smithi070.stdout: metadata_block_size: 4096 2024-09-14T03:51:24.282 INFO:journalctl@ceph.mon.a.smithi070.stdout: partition_filters: 0 2024-09-14T03:51:24.282 INFO:journalctl@ceph.mon.a.smithi070.stdout: use_delta_encoding: 1 2024-09-14T03:51:24.282 INFO:journalctl@ceph.mon.a.smithi070.stdout: filter_policy: bloomfilter 2024-09-14T03:51:24.282 INFO:journalctl@ceph.mon.a.smithi070.stdout: whole_key_filtering: 1 2024-09-14T03:51:24.282 INFO:journalctl@ceph.mon.a.smithi070.stdout: verify_compression: 0 2024-09-14T03:51:24.282 INFO:journalctl@ceph.mon.a.smithi070.stdout: read_amp_bytes_per_bit: 0 2024-09-14T03:51:24.282 INFO:journalctl@ceph.mon.a.smithi070.stdout: format_version: 5 2024-09-14T03:51:24.282 INFO:journalctl@ceph.mon.a.smithi070.stdout: enable_index_compression: 1 2024-09-14T03:51:24.282 INFO:journalctl@ceph.mon.a.smithi070.stdout: block_align: 0 2024-09-14T03:51:24.282 INFO:journalctl@ceph.mon.a.smithi070.stdout: max_auto_readahead_size: 262144 2024-09-14T03:51:24.282 INFO:journalctl@ceph.mon.a.smithi070.stdout: prepopulate_block_cache: 0 2024-09-14T03:51:24.283 INFO:journalctl@ceph.mon.a.smithi070.stdout: initial_auto_readahead_size: 8192 2024-09-14T03:51:24.283 INFO:journalctl@ceph.mon.a.smithi070.stdout: num_file_reads_for_auto_readahead: 2 2024-09-14T03:51:24.283 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.write_buffer_size: 33554432 2024-09-14T03:51:24.283 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.max_write_buffer_number: 2 2024-09-14T03:51:24.283 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.compression: NoCompression 2024-09-14T03:51:24.283 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.bottommost_compression: Disabled 2024-09-14T03:51:24.283 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.prefix_extractor: nullptr 2024-09-14T03:51:24.283 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-14T03:51:24.283 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.num_levels: 7 2024-09-14T03:51:24.283 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-14T03:51:24.283 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-14T03:51:24.284 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-14T03:51:24.284 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-14T03:51:24.284 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-14T03:51:24.284 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-14T03:51:24.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-14T03:51:24.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:51:24.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-14T03:51:24.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-14T03:51:24.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-14T03:51:24.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-14T03:51:24.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-14T03:51:24.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.compression_opts.level: 32767 2024-09-14T03:51:24.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.compression_opts.strategy: 0 2024-09-14T03:51:24.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-14T03:51:24.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:51:24.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-14T03:51:24.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-14T03:51:24.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.compression_opts.enabled: false 2024-09-14T03:51:24.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-14T03:51:24.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-14T03:51:24.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-14T03:51:24.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-14T03:51:24.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.target_file_size_base: 67108864 2024-09-14T03:51:24.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-14T03:51:24.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-14T03:51:24.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-14T03:51:24.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-14T03:51:24.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-14T03:51:24.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-14T03:51:24.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-14T03:51:24.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-14T03:51:24.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-14T03:51:24.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-14T03:51:24.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-14T03:51:24.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-14T03:51:24.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-14T03:51:24.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-14T03:51:24.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.arena_block_size: 1048576 2024-09-14T03:51:24.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-14T03:51:24.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-14T03:51:24.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.disable_auto_compactions: 0 2024-09-14T03:51:24.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-14T03:51:24.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-14T03:51:24.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-14T03:51:24.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-14T03:51:24.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-14T03:51:24.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-14T03:51:24.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-14T03:51:24.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-14T03:51:24.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-14T03:51:24.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-14T03:51:24.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-14T03:51:24.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.inplace_update_support: 0 2024-09-14T03:51:24.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-14T03:51:24.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-14T03:51:24.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-14T03:51:24.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-14T03:51:24.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.bloom_locality: 0 2024-09-14T03:51:24.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.max_successive_merges: 0 2024-09-14T03:51:24.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-14T03:51:24.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.paranoid_file_checks: 0 2024-09-14T03:51:24.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.force_consistency_checks: 1 2024-09-14T03:51:24.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.report_bg_io_stats: 0 2024-09-14T03:51:24.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.ttl: 2592000 2024-09-14T03:51:24.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-14T03:51:24.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-14T03:51:24.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-14T03:51:24.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.enable_blob_files: false 2024-09-14T03:51:24.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.min_blob_size: 0 2024-09-14T03:51:24.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.blob_file_size: 268435456 2024-09-14T03:51:24.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-14T03:51:24.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-14T03:51:24.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-14T03:51:24.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-14T03:51:24.290 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-14T03:51:24.290 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.blob_file_starting_level: 0 2024-09-14T03:51:24.290 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-14T03:51:24.290 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 33.sst 2024-09-14T03:51:24.290 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-14T03:51:24.290 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 35, last_sequence is 12820, log_number is 31,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-14T03:51:24.290 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 31 2024-09-14T03:51:24.290 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 29b140d3-74bf-45c9-9731-c0c64f5f6ae9 2024-09-14T03:51:24.290 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285883953553, "job": 1, "event": "recovery_started", "wal_files": [31]} 2024-09-14T03:51:24.290 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #31 mode 2 2024-09-14T03:51:24.290 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285883970742, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 36, "file_size": 5151528, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12821, "largest_seqno": 15895, "table_properties": {"data_size": 5136487, "index_size": 10282, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 3717, "raw_key_size": 33690, "raw_average_key_size": 22, "raw_value_size": 5105605, "raw_average_value_size": 3477, "num_data_blocks": 446, "num_entries": 1468, "num_filter_entries": 1468, "num_deletions": 294, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726285883, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "29b140d3-74bf-45c9-9731-c0c64f5f6ae9", "db_session_id": "MSEELCN6BINYCXJZBW7A", "orig_file_number": 36, "seqno_to_time_mapping": "N/A"}} 2024-09-14T03:51:24.290 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285883970904, "job": 1, "event": "recovery_finished"} 2024-09-14T03:51:24.290 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: [db/version_set.cc:5047] Creating manifest 38 2024-09-14T03:51:24.291 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-14T03:51:24.291 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000031.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-14T03:51:24.291 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55d46728ce00 2024-09-14T03:51:24.291 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: DB pointer 0x55d46738e000 2024-09-14T03:51:24.291 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-14T03:51:24.291 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-14T03:51:24.291 INFO:journalctl@ceph.mon.a.smithi070.stdout: ** DB Stats ** 2024-09-14T03:51:24.291 INFO:journalctl@ceph.mon.a.smithi070.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:51:24.291 INFO:journalctl@ceph.mon.a.smithi070.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-14T03:51:24.291 INFO:journalctl@ceph.mon.a.smithi070.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:51:24.291 INFO:journalctl@ceph.mon.a.smithi070.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:51:24.291 INFO:journalctl@ceph.mon.a.smithi070.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-14T03:51:24.291 INFO:journalctl@ceph.mon.a.smithi070.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:51:24.291 INFO:journalctl@ceph.mon.a.smithi070.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:51:24.292 INFO:journalctl@ceph.mon.a.smithi070.stdout: 2024-09-14T03:51:24.292 INFO:journalctl@ceph.mon.a.smithi070.stdout: ** Compaction Stats [default] ** 2024-09-14T03:51:24.292 INFO:journalctl@ceph.mon.a.smithi070.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-14T03:51:24.292 INFO:journalctl@ceph.mon.a.smithi070.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-14T03:51:24.292 INFO:journalctl@ceph.mon.a.smithi070.stdout: L0 1/0 4.91 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 677.4 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-14T03:51:24.292 INFO:journalctl@ceph.mon.a.smithi070.stdout: L6 1/0 7.63 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-14T03:51:24.292 INFO:journalctl@ceph.mon.a.smithi070.stdout: Sum 2/0 12.54 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 677.4 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-14T03:51:24.292 INFO:journalctl@ceph.mon.a.smithi070.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 677.4 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-14T03:51:24.292 INFO:journalctl@ceph.mon.a.smithi070.stdout: 2024-09-14T03:51:24.292 INFO:journalctl@ceph.mon.a.smithi070.stdout: ** Compaction Stats [default] ** 2024-09-14T03:51:24.292 INFO:journalctl@ceph.mon.a.smithi070.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-14T03:51:24.292 INFO:journalctl@ceph.mon.a.smithi070.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-14T03:51:24.292 INFO:journalctl@ceph.mon.a.smithi070.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 677.4 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-14T03:51:24.293 INFO:journalctl@ceph.mon.a.smithi070.stdout: 2024-09-14T03:51:24.293 INFO:journalctl@ceph.mon.a.smithi070.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-14T03:51:24.293 INFO:journalctl@ceph.mon.a.smithi070.stdout: 2024-09-14T03:51:24.293 INFO:journalctl@ceph.mon.a.smithi070.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:51:24.293 INFO:journalctl@ceph.mon.a.smithi070.stdout: Flush(GB): cumulative 0.005, interval 0.005 2024-09-14T03:51:24.293 INFO:journalctl@ceph.mon.a.smithi070.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-14T03:51:24.293 INFO:journalctl@ceph.mon.a.smithi070.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-14T03:51:24.293 INFO:journalctl@ceph.mon.a.smithi070.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-14T03:51:24.293 INFO:journalctl@ceph.mon.a.smithi070.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-14T03:51:24.293 INFO:journalctl@ceph.mon.a.smithi070.stdout: Cumulative compaction: 0.00 GB write, 240.52 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:51:24.293 INFO:journalctl@ceph.mon.a.smithi070.stdout: Interval compaction: 0.00 GB write, 240.52 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:51:24.293 INFO:journalctl@ceph.mon.a.smithi070.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-14T03:51:24.293 INFO:journalctl@ceph.mon.a.smithi070.stdout: Block cache BinnedLRUCache@0x55d46725d610#7 capacity: 512.00 MB usage: 16.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-14T03:51:24.293 INFO:journalctl@ceph.mon.a.smithi070.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,4.03 KB,0.0007689%) IndexBlock(1,12.08 KB,0.00230372%) Misc(1,0.00 KB,0%) 2024-09-14T03:51:24.294 INFO:journalctl@ceph.mon.a.smithi070.stdout: 2024-09-14T03:51:24.294 INFO:journalctl@ceph.mon.a.smithi070.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-14T03:51:24.294 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: starting mon.a rank 0 at public addrs [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] at bind addrs [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:51:24.294 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: mon.a@-1(???) e3 preinit fsid b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:51:24.294 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: mon.a@-1(???).mds e8 new map 2024-09-14T03:51:24.294 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: mon.a@-1(???).mds e8 print_map 2024-09-14T03:51:24.294 INFO:journalctl@ceph.mon.a.smithi070.stdout: e8 2024-09-14T03:51:24.294 INFO:journalctl@ceph.mon.a.smithi070.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-14T03:51:24.294 INFO:journalctl@ceph.mon.a.smithi070.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-14T03:51:24.294 INFO:journalctl@ceph.mon.a.smithi070.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-14T03:51:24.294 INFO:journalctl@ceph.mon.a.smithi070.stdout: legacy client fscid: 1 2024-09-14T03:51:24.294 INFO:journalctl@ceph.mon.a.smithi070.stdout: 2024-09-14T03:51:24.294 INFO:journalctl@ceph.mon.a.smithi070.stdout: Filesystem 'cephfs' (1) 2024-09-14T03:51:24.294 INFO:journalctl@ceph.mon.a.smithi070.stdout: fs_name cephfs 2024-09-14T03:51:24.295 INFO:journalctl@ceph.mon.a.smithi070.stdout: epoch 8 2024-09-14T03:51:24.295 INFO:journalctl@ceph.mon.a.smithi070.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-14T03:51:24.295 INFO:journalctl@ceph.mon.a.smithi070.stdout: created 2024-09-14T03:44:03.568995+0000 2024-09-14T03:51:24.295 INFO:journalctl@ceph.mon.a.smithi070.stdout: modified 2024-09-14T03:44:16.900718+0000 2024-09-14T03:51:24.295 INFO:journalctl@ceph.mon.a.smithi070.stdout: tableserver 0 2024-09-14T03:51:24.295 INFO:journalctl@ceph.mon.a.smithi070.stdout: root 0 2024-09-14T03:51:24.295 INFO:journalctl@ceph.mon.a.smithi070.stdout: session_timeout 60 2024-09-14T03:51:24.295 INFO:journalctl@ceph.mon.a.smithi070.stdout: session_autoclose 300 2024-09-14T03:51:24.295 INFO:journalctl@ceph.mon.a.smithi070.stdout: max_file_size 1099511627776 2024-09-14T03:51:24.295 INFO:journalctl@ceph.mon.a.smithi070.stdout: max_xattr_size 65536 2024-09-14T03:51:24.295 INFO:journalctl@ceph.mon.a.smithi070.stdout: required_client_features {} 2024-09-14T03:51:24.295 INFO:journalctl@ceph.mon.a.smithi070.stdout: last_failure 0 2024-09-14T03:51:24.296 INFO:journalctl@ceph.mon.a.smithi070.stdout: last_failure_osd_epoch 0 2024-09-14T03:51:24.296 INFO:journalctl@ceph.mon.a.smithi070.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-14T03:51:24.296 INFO:journalctl@ceph.mon.a.smithi070.stdout: max_mds 1 2024-09-14T03:51:24.296 INFO:journalctl@ceph.mon.a.smithi070.stdout: in 0 2024-09-14T03:51:24.296 INFO:journalctl@ceph.mon.a.smithi070.stdout: up {0=24323} 2024-09-14T03:51:24.296 INFO:journalctl@ceph.mon.a.smithi070.stdout: failed 2024-09-14T03:51:24.296 INFO:journalctl@ceph.mon.a.smithi070.stdout: damaged 2024-09-14T03:51:24.296 INFO:journalctl@ceph.mon.a.smithi070.stdout: stopped 2024-09-14T03:51:24.296 INFO:journalctl@ceph.mon.a.smithi070.stdout: data_pools [3] 2024-09-14T03:51:24.296 INFO:journalctl@ceph.mon.a.smithi070.stdout: metadata_pool 2 2024-09-14T03:51:24.296 INFO:journalctl@ceph.mon.a.smithi070.stdout: inline_data disabled 2024-09-14T03:51:24.297 INFO:journalctl@ceph.mon.a.smithi070.stdout: balancer 2024-09-14T03:51:24.297 INFO:journalctl@ceph.mon.a.smithi070.stdout: bal_rank_mask -1 2024-09-14T03:51:24.297 INFO:journalctl@ceph.mon.a.smithi070.stdout: standby_count_wanted 0 2024-09-14T03:51:24.297 INFO:journalctl@ceph.mon.a.smithi070.stdout: qdb_cluster leader: 0 members: 2024-09-14T03:51:24.297 INFO:journalctl@ceph.mon.a.smithi070.stdout: [mds.a{0:24323} state up:active seq 6 addr [v2:172.21.15.70:6834/421917097,v1:172.21.15.70:6835/421917097] compat {c=[1],r=[1],i=[7ff]}] 2024-09-14T03:51:24.297 INFO:journalctl@ceph.mon.a.smithi070.stdout: 2024-09-14T03:51:24.297 INFO:journalctl@ceph.mon.a.smithi070.stdout: 2024-09-14T03:51:24.297 INFO:journalctl@ceph.mon.a.smithi070.stdout: Standby daemons: 2024-09-14T03:51:24.297 INFO:journalctl@ceph.mon.a.smithi070.stdout: 2024-09-14T03:51:24.297 INFO:journalctl@ceph.mon.a.smithi070.stdout: [mds.b{-1:24304} state up:standby seq 1 addr [v2:172.21.15.94:6832/3620660683,v1:172.21.15.94:6833/3620660683] compat {c=[1],r=[1],i=[7ff]}] 2024-09-14T03:51:24.297 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: mon.a@-1(???).osd e269 crush map has features 3314933069573799936, adjusting msgr requires 2024-09-14T03:51:24.297 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: mon.a@-1(???).osd e269 crush map has features 432629308056666112, adjusting msgr requires 2024-09-14T03:51:24.297 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: mon.a@-1(???).osd e269 crush map has features 432629308056666112, adjusting msgr requires 2024-09-14T03:51:24.298 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: mon.a@-1(???).osd e269 crush map has features 432629308056666112, adjusting msgr requires 2024-09-14T03:51:24.298 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:23 smithi070 ceph-mon[96704]: mon.a@-1(???).paxosservice(auth 1..19) refresh upgraded, format 0 -> 3 2024-09-14T03:51:24.298 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:24 smithi070 bash[96666]: 96542de31cac9c93f77613ee79c8ff2fe4ea7431d2292a5bb822697b7019365b 2024-09-14T03:51:24.298 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:24 smithi070 systemd[1]: Started Ceph mon.a for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:51:24.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:24 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:24.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:51:26.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:25.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:51:26.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:25.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:51:27.060 INFO:tasks.rados.rados.0.smithi070.stdout:1041: done (2 left) 2024-09-14T03:51:27.060 INFO:tasks.rados.rados.0.smithi070.stdout:1044: done (1 left) 2024-09-14T03:51:27.060 INFO:tasks.rados.rados.0.smithi070.stdout:1045: done (0 left) 2024-09-14T03:51:27.060 INFO:tasks.rados.rados.0.smithi070.stdout:1046: rollback oid 44 current snap is 114 2024-09-14T03:51:27.060 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 44 to 51 2024-09-14T03:51:27.060 INFO:tasks.rados.rados.0.smithi070.stdout:1047: copy_from oid 18 from oid 38 current snap is 114 2024-09-14T03:51:27.060 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 3 current snap is 114 2024-09-14T03:51:27.060 INFO:tasks.rados.rados.0.smithi070.stdout:1048: seq_num 346 ranges {0=598016,598016=401408} 2024-09-14T03:51:27.063 INFO:tasks.rados.rados.0.smithi070.stdout:1048: writing smithi07085154-3 from 0 to 598016 tid 1 2024-09-14T03:51:27.066 INFO:tasks.rados.rados.0.smithi070.stdout:1048: writing smithi07085154-3 from 598016 to 999424 tid 2 2024-09-14T03:51:27.066 INFO:tasks.rados.rados.0.smithi070.stdout:1049: snap_remove snap 97 2024-09-14T03:51:27.066 INFO:tasks.rados.rados.0.smithi070.stdout:1046: finishing rollback tid 1 to smithi07085154-44 2024-09-14T03:51:27.066 INFO:tasks.rados.rados.0.smithi070.stdout:1046: finishing rollback tid 2 to smithi07085154-44 2024-09-14T03:51:27.066 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 347 (ObjNum 152 snap 45 seq_num 152) dirty exists 2024-09-14T03:51:27.066 INFO:tasks.rados.rados.0.smithi070.stdout:1047: finishing copy_from to smithi07085154-18 2024-09-14T03:51:27.066 INFO:tasks.rados.rados.0.smithi070.stdout:1047: got expected ENOENT (src dne) 2024-09-14T03:51:27.066 INFO:tasks.rados.rados.0.smithi070.stdout:1047: finishing copy_from racing read to smithi07085154-18 2024-09-14T03:51:27.276 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:51:27 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[78773]: ignoring --setuser ceph since I am not root 2024-09-14T03:51:27.277 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:51:27 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[78773]: ignoring --setgroup ceph since I am not root 2024-09-14T03:51:27.277 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:51:27 smithi094 ceph-mgr[78796]: -- 172.21.15.94:0/3186217549 <== mon.1 v2:172.21.15.94:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55abaea050e0 con 0x55abae9e2800 2024-09-14T03:51:27.277 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:26 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:26.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:51:27.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:27 smithi094 ceph-mon[31053]: pgmap v50: 121 pgs: 121 active+clean; 265 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.9 MiB/s rd, 2.9 MiB/s wr, 12 op/s 2024-09-14T03:51:27.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:27 smithi094 ceph-mon[31053]: pgmap v51: 121 pgs: 121 active+clean; 265 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 2.2 MiB/s wr, 9 op/s 2024-09-14T03:51:27.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:27 smithi094 ceph-mon[31053]: mon.a calling monitor election 2024-09-14T03:51:27.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:27 smithi094 ceph-mon[31053]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:51:27.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:27 smithi094 ceph-mon[31053]: monmap epoch 3 2024-09-14T03:51:27.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:27 smithi094 ceph-mon[31053]: fsid b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:51:27.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:27 smithi094 ceph-mon[31053]: last_changed 2024-09-14T03:40:08.852192+0000 2024-09-14T03:51:27.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:27 smithi094 ceph-mon[31053]: created 2024-09-14T03:38:26.272229+0000 2024-09-14T03:51:27.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:27 smithi094 ceph-mon[31053]: min_mon_release 17 (quincy) 2024-09-14T03:51:27.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:27 smithi094 ceph-mon[31053]: election_strategy: 1 2024-09-14T03:51:27.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:27 smithi094 ceph-mon[31053]: 0: [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] mon.a 2024-09-14T03:51:27.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:27 smithi094 ceph-mon[31053]: 1: [v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0] mon.b 2024-09-14T03:51:27.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:27 smithi094 ceph-mon[31053]: 2: [v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0] mon.c 2024-09-14T03:51:27.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:27 smithi094 ceph-mon[31053]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:51:27.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:27 smithi094 ceph-mon[31053]: osdmap e269: 8 total, 8 up, 8 in 2024-09-14T03:51:27.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:27 smithi094 ceph-mon[31053]: mgrmap e36: y(active, since 35s), standbys: x 2024-09-14T03:51:27.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:27 smithi094 ceph-mon[31053]: overall HEALTH_OK 2024-09-14T03:51:27.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:27 smithi094 ceph-mon[31053]: from='mgr.15159 ' entity='' 2024-09-14T03:51:27.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:27 smithi094 ceph-mon[31053]: mgrmap e37: y(active, since 35s), standbys: x 2024-09-14T03:51:27.535 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ignoring --setuser ceph since I am not root 2024-09-14T03:51:27.536 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ignoring --setgroup ceph since I am not root 2024-09-14T03:51:27.536 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mgr[88658]: -- 172.21.15.70:0/1533947750 <== mon.0 v2:172.21.15.70:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x555aca7390e0 con 0x555aca717400 2024-09-14T03:51:27.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[96704]: pgmap v50: 121 pgs: 121 active+clean; 265 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.9 MiB/s rd, 2.9 MiB/s wr, 12 op/s 2024-09-14T03:51:27.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[96704]: pgmap v51: 121 pgs: 121 active+clean; 265 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 2.2 MiB/s wr, 9 op/s 2024-09-14T03:51:27.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[96704]: mon.a calling monitor election 2024-09-14T03:51:27.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[96704]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:51:27.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[96704]: monmap epoch 3 2024-09-14T03:51:27.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[96704]: fsid b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:51:27.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[96704]: last_changed 2024-09-14T03:40:08.852192+0000 2024-09-14T03:51:27.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[96704]: created 2024-09-14T03:38:26.272229+0000 2024-09-14T03:51:27.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[96704]: min_mon_release 17 (quincy) 2024-09-14T03:51:27.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[96704]: election_strategy: 1 2024-09-14T03:51:27.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[96704]: 0: [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] mon.a 2024-09-14T03:51:27.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[96704]: 1: [v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0] mon.b 2024-09-14T03:51:27.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[96704]: 2: [v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0] mon.c 2024-09-14T03:51:27.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[96704]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:51:27.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[96704]: osdmap e269: 8 total, 8 up, 8 in 2024-09-14T03:51:27.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[96704]: mgrmap e36: y(active, since 35s), standbys: x 2024-09-14T03:51:27.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[96704]: overall HEALTH_OK 2024-09-14T03:51:27.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[96704]: from='mgr.15159 ' entity='' 2024-09-14T03:51:27.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[96704]: mgrmap e37: y(active, since 35s), standbys: x 2024-09-14T03:51:27.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[39288]: pgmap v50: 121 pgs: 121 active+clean; 265 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.9 MiB/s rd, 2.9 MiB/s wr, 12 op/s 2024-09-14T03:51:27.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[39288]: pgmap v51: 121 pgs: 121 active+clean; 265 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 2.2 MiB/s wr, 9 op/s 2024-09-14T03:51:27.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[39288]: mon.a calling monitor election 2024-09-14T03:51:27.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[39288]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:51:27.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[39288]: monmap epoch 3 2024-09-14T03:51:27.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[39288]: fsid b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:51:27.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[39288]: last_changed 2024-09-14T03:40:08.852192+0000 2024-09-14T03:51:27.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[39288]: created 2024-09-14T03:38:26.272229+0000 2024-09-14T03:51:27.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[39288]: min_mon_release 17 (quincy) 2024-09-14T03:51:27.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[39288]: election_strategy: 1 2024-09-14T03:51:27.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[39288]: 0: [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] mon.a 2024-09-14T03:51:27.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[39288]: 1: [v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0] mon.b 2024-09-14T03:51:27.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[39288]: 2: [v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0] mon.c 2024-09-14T03:51:27.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[39288]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:51:27.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[39288]: osdmap e269: 8 total, 8 up, 8 in 2024-09-14T03:51:27.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[39288]: mgrmap e36: y(active, since 35s), standbys: x 2024-09-14T03:51:27.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[39288]: overall HEALTH_OK 2024-09-14T03:51:27.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[39288]: from='mgr.15159 ' entity='' 2024-09-14T03:51:27.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:27 smithi070 ceph-mon[39288]: mgrmap e37: y(active, since 35s), standbys: x 2024-09-14T03:51:28.066 INFO:tasks.rados.rados.0.smithi070.stdout:1048: finishing write tid 1 to smithi07085154-3 2024-09-14T03:51:28.066 INFO:tasks.rados.rados.0.smithi070.stdout:1048: finishing write tid 2 to smithi07085154-3 2024-09-14T03:51:28.066 INFO:tasks.rados.rados.0.smithi070.stdout:1048: finishing write tid 3 to smithi07085154-3 2024-09-14T03:51:28.066 INFO:tasks.rados.rados.0.smithi070.stdout:1048: finishing write tid 4 to smithi07085154-3 2024-09-14T03:51:28.066 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 3 v 358 (ObjNum 346 snap 114 seq_num 346) dirty exists 2024-09-14T03:51:28.066 INFO:tasks.rados.rados.0.smithi070.stdout:1048: left oid 3 (ObjNum 346 snap 114 seq_num 346) 2024-09-14T03:51:28.066 INFO:tasks.rados.rados.0.smithi070.stdout:1046: done (3 left) 2024-09-14T03:51:28.066 INFO:tasks.rados.rados.0.smithi070.stdout:1047: done (2 left) 2024-09-14T03:51:28.067 INFO:tasks.rados.rados.0.smithi070.stdout:1048: done (1 left) 2024-09-14T03:51:28.067 INFO:tasks.rados.rados.0.smithi070.stdout:1049: done (0 left) 2024-09-14T03:51:28.067 INFO:tasks.rados.rados.0.smithi070.stdout:1050: snap_create 2024-09-14T03:51:28.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:28 smithi094 ceph-mon[31053]: osdmap e270: 8 total, 8 up, 8 in 2024-09-14T03:51:28.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:28 smithi070 ceph-mon[96704]: osdmap e270: 8 total, 8 up, 8 in 2024-09-14T03:51:28.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:28 smithi070 ceph-mon[39288]: osdmap e270: 8 total, 8 up, 8 in 2024-09-14T03:51:28.984 INFO:tasks.rados.rados.0.smithi070.stdout:1050: done (0 left) 2024-09-14T03:51:28.984 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 1 current snap is 115 2024-09-14T03:51:28.985 INFO:tasks.rados.rados.0.smithi070.stdout:1051: seq_num 347 ranges {2801664=614400,3416064=401408} 2024-09-14T03:51:29.003 INFO:tasks.rados.rados.0.smithi070.stdout:1051: writing smithi07085154-1 from 2801664 to 3416064 tid 1 2024-09-14T03:51:29.005 INFO:tasks.rados.rados.0.smithi070.stdout:1051: writing smithi07085154-1 from 3416064 to 3817472 tid 2 2024-09-14T03:51:29.005 INFO:tasks.rados.rados.0.smithi070.stdout:1052: read oid 35 snap -1 2024-09-14T03:51:29.005 INFO:tasks.rados.rados.0.smithi070.stdout:1052: expect deleted 2024-09-14T03:51:29.005 INFO:tasks.rados.rados.0.smithi070.stdout:1053: snap_create 2024-09-14T03:51:29.008 INFO:tasks.rados.rados.0.smithi070.stdout:1051: finishing write tid 1 to smithi07085154-1 2024-09-14T03:51:29.008 INFO:tasks.rados.rados.0.smithi070.stdout:1051: finishing write tid 2 to smithi07085154-1 2024-09-14T03:51:29.009 INFO:tasks.rados.rados.0.smithi070.stdout:1051: finishing write tid 3 to smithi07085154-1 2024-09-14T03:51:29.009 INFO:tasks.rados.rados.0.smithi070.stdout:1051: finishing write tid 4 to smithi07085154-1 2024-09-14T03:51:29.009 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 1 v 355 (ObjNum 347 snap 115 seq_num 347) dirty exists 2024-09-14T03:51:29.009 INFO:tasks.rados.rados.0.smithi070.stdout:1051: left oid 1 (ObjNum 347 snap 115 seq_num 347) 2024-09-14T03:51:29.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:29 smithi094 ceph-mon[31053]: osdmap e271: 8 total, 8 up, 8 in 2024-09-14T03:51:29.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:29 smithi094 ceph-mon[31053]: osdmap e272: 8 total, 8 up, 8 in 2024-09-14T03:51:29.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:29 smithi070 ceph-mon[96704]: osdmap e271: 8 total, 8 up, 8 in 2024-09-14T03:51:29.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:29 smithi070 ceph-mon[96704]: osdmap e272: 8 total, 8 up, 8 in 2024-09-14T03:51:29.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:29 smithi070 ceph-mon[39288]: osdmap e271: 8 total, 8 up, 8 in 2024-09-14T03:51:29.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:29 smithi070 ceph-mon[39288]: osdmap e272: 8 total, 8 up, 8 in 2024-09-14T03:51:29.985 INFO:tasks.rados.rados.0.smithi070.stdout:1051: done (2 left) 2024-09-14T03:51:29.986 INFO:tasks.rados.rados.0.smithi070.stdout:1052: done (1 left) 2024-09-14T03:51:29.986 INFO:tasks.rados.rados.0.smithi070.stdout:1053: done (0 left) 2024-09-14T03:51:29.986 INFO:tasks.rados.rados.0.smithi070.stdout:1054: rollback oid 37 current snap is 116 2024-09-14T03:51:29.986 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 37 to 108 2024-09-14T03:51:29.986 INFO:tasks.rados.rados.0.smithi070.stdout:1055: delete oid 44 current snap is 116 2024-09-14T03:51:29.988 INFO:tasks.rados.rados.0.smithi070.stdout:1055: done (1 left) 2024-09-14T03:51:29.988 INFO:tasks.rados.rados.0.smithi070.stdout:1056: copy_from oid 39 from oid 8 current snap is 116 2024-09-14T03:51:29.988 INFO:tasks.rados.rados.0.smithi070.stdout:1057: read oid 46 snap 73 2024-09-14T03:51:29.988 INFO:tasks.rados.rados.0.smithi070.stdout:1057: expect deleted 2024-09-14T03:51:29.988 INFO:tasks.rados.rados.0.smithi070.stdout:1058: rollback oid 29 current snap is 116 2024-09-14T03:51:29.988 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 29 to 57 2024-09-14T03:51:29.988 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 50 current snap is 116 2024-09-14T03:51:29.988 INFO:tasks.rados.rados.0.smithi070.stdout:1059: seq_num 349 ranges {614400=655360,1269760=401408} 2024-09-14T03:51:29.996 INFO:tasks.rados.rados.0.smithi070.stdout:1059: writing smithi07085154-50 from 614400 to 1269760 tid 1 2024-09-14T03:51:29.999 INFO:tasks.rados.rados.0.smithi070.stdout:1059: writing smithi07085154-50 from 1269760 to 1671168 tid 2 2024-09-14T03:51:30.000 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 24 current snap is 116 2024-09-14T03:51:30.000 INFO:tasks.rados.rados.0.smithi070.stdout:1058: finishing rollback tid 1 to smithi07085154-29 2024-09-14T03:51:30.000 INFO:tasks.rados.rados.0.smithi070.stdout:1060: seq_num 350 ranges {2367488=671744,3039232=401408} 2024-09-14T03:51:30.018 INFO:tasks.rados.rados.0.smithi070.stdout:1060: writing smithi07085154-24 from 2367488 to 3039232 tid 1 2024-09-14T03:51:30.021 INFO:tasks.rados.rados.0.smithi070.stdout:1060: writing smithi07085154-24 from 3039232 to 3440640 tid 2 2024-09-14T03:51:30.021 INFO:tasks.rados.rados.0.smithi070.stdout:1058: finishing rollback tid 2 to smithi07085154-29 2024-09-14T03:51:30.021 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 566 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-14T03:51:30.021 INFO:tasks.rados.rados.0.smithi070.stdout:1054: finishing rollback tid 0 to smithi07085154-37 2024-09-14T03:51:30.021 INFO:tasks.rados.rados.0.smithi070.stdout:1057: done (5 left) 2024-09-14T03:51:30.022 INFO:tasks.rados.rados.0.smithi070.stdout:1058: done (4 left) 2024-09-14T03:51:30.022 INFO:tasks.rados.rados.0.smithi070.stdout:1061: snap_remove snap 103 2024-09-14T03:51:30.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:51:30.990 INFO:tasks.rados.rados.0.smithi070.stdout:1061: done (4 left) 2024-09-14T03:51:30.990 INFO:tasks.rados.rados.0.smithi070.stdout:1062: rmattr oid 20 current snap is 116 2024-09-14T03:51:30.990 INFO:tasks.rados.rados.0.smithi070.stdout:1062: done (4 left) 2024-09-14T03:51:30.990 INFO:tasks.rados.rados.0.smithi070.stdout:1063: setattr oid 43 current snap is 116 2024-09-14T03:51:30.990 INFO:tasks.rados.rados.0.smithi070.stdout:1054: finishing rollback tid 1 to smithi07085154-37 2024-09-14T03:51:30.990 INFO:tasks.rados.rados.0.smithi070.stdout:1054: finishing rollback tid 2 to smithi07085154-37 2024-09-14T03:51:30.991 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 37 v 801 (ObjNum 312 snap 105 seq_num 312) dirty exists 2024-09-14T03:51:30.991 INFO:tasks.rados.rados.0.smithi070.stdout:1059: finishing write tid 1 to smithi07085154-50 2024-09-14T03:51:30.991 INFO:tasks.rados.rados.0.smithi070.stdout:1056: finishing copy_from racing read to smithi07085154-39 2024-09-14T03:51:30.991 INFO:tasks.rados.rados.0.smithi070.stdout:1056: finishing copy_from to smithi07085154-39 2024-09-14T03:51:30.991 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 39 v 307 (ObjNum 202 snap 56 seq_num 202) dirty exists 2024-09-14T03:51:30.991 INFO:tasks.rados.rados.0.smithi070.stdout:1059: finishing write tid 2 to smithi07085154-50 2024-09-14T03:51:30.991 INFO:tasks.rados.rados.0.smithi070.stdout:1059: finishing write tid 3 to smithi07085154-50 2024-09-14T03:51:30.991 INFO:tasks.rados.rados.0.smithi070.stdout:1059: finishing write tid 4 to smithi07085154-50 2024-09-14T03:51:30.991 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 570 (ObjNum 349 snap 116 seq_num 349) dirty exists 2024-09-14T03:51:30.991 INFO:tasks.rados.rados.0.smithi070.stdout:1059: left oid 50 (ObjNum 349 snap 116 seq_num 349) 2024-09-14T03:51:30.991 INFO:tasks.rados.rados.0.smithi070.stdout:1060: finishing write tid 1 to smithi07085154-24 2024-09-14T03:51:30.991 INFO:tasks.rados.rados.0.smithi070.stdout:1060: finishing write tid 2 to smithi07085154-24 2024-09-14T03:51:30.991 INFO:tasks.rados.rados.0.smithi070.stdout:1060: finishing write tid 3 to smithi07085154-24 2024-09-14T03:51:30.991 INFO:tasks.rados.rados.0.smithi070.stdout:1060: finishing write tid 4 to smithi07085154-24 2024-09-14T03:51:30.992 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 410 (ObjNum 350 snap 116 seq_num 350) dirty exists 2024-09-14T03:51:30.992 INFO:tasks.rados.rados.0.smithi070.stdout:1060: left oid 24 (ObjNum 350 snap 116 seq_num 350) 2024-09-14T03:51:30.993 INFO:tasks.rados.rados.0.smithi070.stdout:1054: done (4 left) 2024-09-14T03:51:30.993 INFO:tasks.rados.rados.0.smithi070.stdout:1056: done (3 left) 2024-09-14T03:51:30.993 INFO:tasks.rados.rados.0.smithi070.stdout:1059: done (2 left) 2024-09-14T03:51:30.994 INFO:tasks.rados.rados.0.smithi070.stdout:1060: done (1 left) 2024-09-14T03:51:30.994 INFO:tasks.rados.rados.0.smithi070.stdout:1064: snap_remove snap 92 2024-09-14T03:51:31.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:30 smithi094 ceph-mon[31053]: osdmap e273: 8 total, 8 up, 8 in 2024-09-14T03:51:31.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:30 smithi070 ceph-mon[96704]: osdmap e273: 8 total, 8 up, 8 in 2024-09-14T03:51:31.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:30 smithi070 ceph-mon[39288]: osdmap e273: 8 total, 8 up, 8 in 2024-09-14T03:51:31.998 INFO:tasks.rados.rados.0.smithi070.stdout:1064: done (1 left) 2024-09-14T03:51:31.998 INFO:tasks.rados.rados.0.smithi070.stdout:1065: rollback oid 16 current snap is 116 2024-09-14T03:51:31.999 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 802 (ObjNum 307 snap 101 seq_num 307) dirty exists 2024-09-14T03:51:31.999 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 16 to 79 2024-09-14T03:51:31.999 INFO:tasks.rados.rados.0.smithi070.stdout:1063: done (1 left) 2024-09-14T03:51:31.999 INFO:tasks.rados.rados.0.smithi070.stdout:1066: snap_remove snap 111 2024-09-14T03:51:32.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:31 smithi094 ceph-mon[31053]: osdmap e274: 8 total, 8 up, 8 in 2024-09-14T03:51:32.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:31 smithi094 ceph-mon[31053]: Standby manager daemon x restarted 2024-09-14T03:51:32.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:31 smithi094 ceph-mon[31053]: Standby manager daemon x started 2024-09-14T03:51:32.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:31 smithi094 ceph-mon[31053]: from='mgr.? 172.21.15.94:0/872687568' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:51:32.277 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:51:31 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[78773]: [14/Sep/2024:03:51:31] ENGINE Bus STARTING 2024-09-14T03:51:32.277 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:51:31 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[78773]: CherryPy Checker: 2024-09-14T03:51:32.277 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:51:31 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[78773]: The Application mounted at '' has an empty config. 2024-09-14T03:51:32.277 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:51:31 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[78773]: 2024-09-14T03:51:32.278 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[78773]: [14/Sep/2024:03:51:32] ENGINE Serving on http://:::9283 2024-09-14T03:51:32.278 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x[78773]: [14/Sep/2024:03:51:32] ENGINE Bus STARTED 2024-09-14T03:51:32.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:31 smithi070 ceph-mon[96704]: osdmap e274: 8 total, 8 up, 8 in 2024-09-14T03:51:32.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:31 smithi070 ceph-mon[96704]: Standby manager daemon x restarted 2024-09-14T03:51:32.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:31 smithi070 ceph-mon[96704]: Standby manager daemon x started 2024-09-14T03:51:32.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:31 smithi070 ceph-mon[96704]: from='mgr.? 172.21.15.94:0/872687568' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:51:32.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:31 smithi070 ceph-mon[39288]: osdmap e274: 8 total, 8 up, 8 in 2024-09-14T03:51:32.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:31 smithi070 ceph-mon[39288]: Standby manager daemon x restarted 2024-09-14T03:51:32.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:31 smithi070 ceph-mon[39288]: Standby manager daemon x started 2024-09-14T03:51:32.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:31 smithi070 ceph-mon[39288]: from='mgr.? 172.21.15.94:0/872687568' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:51:32.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:51:32.375+0000 7fa47ca87640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:51:32.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:51:32.375+0000 7fa47ca87640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:51:32.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:51:32.375+0000 7fa47ca87640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:51:32.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:51:32.375+0000 7fa47ca87640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:51:32.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:51:32.375+0000 7fa47ca87640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:51:32.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:51:32.378+0000 7fa48028e640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:51:32.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:51:32.378+0000 7fa48028e640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:51:32.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:51:32.378+0000 7fa48028e640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:51:32.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:51:32.378+0000 7fa48028e640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:51:32.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:51:32.378+0000 7fa48028e640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:51:33.010 INFO:tasks.rados.rados.0.smithi070.stdout:1066: done (1 left) 2024-09-14T03:51:33.010 INFO:tasks.rados.rados.0.smithi070.stdout:1067: read oid 46 snap -1 2024-09-14T03:51:33.010 INFO:tasks.rados.rados.0.smithi070.stdout:1067: expect (ObjNum 320 snap 107 seq_num 320) 2024-09-14T03:51:33.011 INFO:tasks.rados.rados.0.smithi070.stdout:1068: read oid 10 snap 83 2024-09-14T03:51:33.011 INFO:tasks.rados.rados.0.smithi070.stdout:1068: expect deleted 2024-09-14T03:51:33.011 INFO:tasks.rados.rados.0.smithi070.stdout:1069: rollback oid 23 current snap is 116 2024-09-14T03:51:33.011 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 23 to 113 2024-09-14T03:51:33.011 INFO:tasks.rados.rados.0.smithi070.stdout:1070: setattr oid 24 current snap is 116 2024-09-14T03:51:33.011 INFO:tasks.rados.rados.0.smithi070.stdout:1065: finishing rollback tid 0 to smithi07085154-16 2024-09-14T03:51:33.011 INFO:tasks.rados.rados.0.smithi070.stdout:1065: finishing rollback tid 1 to smithi07085154-16 2024-09-14T03:51:33.011 INFO:tasks.rados.rados.0.smithi070.stdout:1065: finishing rollback tid 2 to smithi07085154-16 2024-09-14T03:51:33.011 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 906 (ObjNum 114 snap 24 seq_num 114) dirty exists 2024-09-14T03:51:33.012 INFO:tasks.rados.rados.0.smithi070.stdout:1065: done (4 left) 2024-09-14T03:51:33.012 INFO:tasks.rados.rados.0.smithi070.stdout:1071: read oid 5 snap 79 2024-09-14T03:51:33.012 INFO:tasks.rados.rados.0.smithi070.stdout:1071: expect deleted 2024-09-14T03:51:33.012 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 4 current snap is 116 2024-09-14T03:51:33.012 INFO:tasks.rados.rados.0.smithi070.stdout:1072: seq_num 351 ranges {2531328=688128,3219456=401408} 2024-09-14T03:51:33.035 INFO:tasks.rados.rados.0.smithi070.stdout:1072: writing smithi07085154-4 from 2531328 to 3219456 tid 1 2024-09-14T03:51:33.038 INFO:tasks.rados.rados.0.smithi070.stdout:1072: writing smithi07085154-4 from 3219456 to 3620864 tid 2 2024-09-14T03:51:33.038 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 25 current snap is 116 2024-09-14T03:51:33.038 INFO:tasks.rados.rados.0.smithi070.stdout:1073: seq_num 352 ranges {8708096=704512,9412608=401408} 2024-09-14T03:51:33.089 INFO:tasks.rados.rados.0.smithi070.stdout:1073: writing smithi07085154-25 from 8708096 to 9412608 tid 1 2024-09-14T03:51:33.092 INFO:tasks.rados.rados.0.smithi070.stdout:1073: writing smithi07085154-25 from 9412608 to 9814016 tid 2 2024-09-14T03:51:33.092 INFO:tasks.rados.rados.0.smithi070.stdout:1074: copy_from oid 26 from oid 33 current snap is 116 2024-09-14T03:51:33.092 INFO:tasks.rados.rados.0.smithi070.stdout:1069: finishing rollback tid 0 to smithi07085154-23 2024-09-14T03:51:33.092 INFO:tasks.rados.rados.0.smithi070.stdout:1069: finishing rollback tid 1 to smithi07085154-23 2024-09-14T03:51:33.092 INFO:tasks.rados.rados.0.smithi070.stdout:1069: finishing rollback tid 2 to smithi07085154-23 2024-09-14T03:51:33.092 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 413 (ObjNum 342 snap 112 seq_num 342) dirty exists 2024-09-14T03:51:33.092 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 414 (ObjNum 350 snap 116 seq_num 350) dirty exists 2024-09-14T03:51:33.092 INFO:tasks.rados.rados.0.smithi070.stdout:1068: done (7 left) 2024-09-14T03:51:33.092 INFO:tasks.rados.rados.0.smithi070.stdout:1069: done (6 left) 2024-09-14T03:51:33.092 INFO:tasks.rados.rados.0.smithi070.stdout:1070: done (5 left) 2024-09-14T03:51:33.093 INFO:tasks.rados.rados.0.smithi070.stdout:1071: done (4 left) 2024-09-14T03:51:33.093 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 40 current snap is 116 2024-09-14T03:51:33.093 INFO:tasks.rados.rados.0.smithi070.stdout:1075: seq_num 353 ranges {6856704=720896,7577600=401408} 2024-09-14T03:51:33.132 INFO:tasks.rados.rados.0.smithi070.stdout:1075: writing smithi07085154-40 from 6856704 to 7577600 tid 1 2024-09-14T03:51:33.134 INFO:tasks.rados.rados.0.smithi070.stdout:1075: writing smithi07085154-40 from 7577600 to 7979008 tid 2 2024-09-14T03:51:33.135 INFO:tasks.rados.rados.0.smithi070.stdout:1076: read oid 49 snap 90 2024-09-14T03:51:33.135 INFO:tasks.rados.rados.0.smithi070.stdout:1076: expect deleted 2024-09-14T03:51:33.135 INFO:tasks.rados.rados.0.smithi070.stdout:1077: read oid 15 snap -1 2024-09-14T03:51:33.135 INFO:tasks.rados.rados.0.smithi070.stdout:1077: expect (ObjNum 332 snap 110 seq_num 332) 2024-09-14T03:51:33.135 INFO:tasks.rados.rados.0.smithi070.stdout:1078: read oid 32 snap 110 2024-09-14T03:51:33.135 INFO:tasks.rados.rados.0.smithi070.stdout:1078: expect deleted 2024-09-14T03:51:33.135 INFO:tasks.rados.rados.0.smithi070.stdout:1079: snap_create 2024-09-14T03:51:33.135 INFO:tasks.rados.rados.0.smithi070.stdout:1067: expect (ObjNum 320 snap 107 seq_num 320) 2024-09-14T03:51:33.205 INFO:tasks.rados.rados.0.smithi070.stdout:1072: finishing write tid 1 to smithi07085154-4 2024-09-14T03:51:33.205 INFO:tasks.rados.rados.0.smithi070.stdout:1072: finishing write tid 2 to smithi07085154-4 2024-09-14T03:51:33.206 INFO:tasks.rados.rados.0.smithi070.stdout:1072: finishing write tid 3 to smithi07085154-4 2024-09-14T03:51:33.206 INFO:tasks.rados.rados.0.smithi070.stdout:1072: finishing write tid 4 to smithi07085154-4 2024-09-14T03:51:33.206 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 404 (ObjNum 351 snap 116 seq_num 351) dirty exists 2024-09-14T03:51:33.206 INFO:tasks.rados.rados.0.smithi070.stdout:1072: left oid 4 (ObjNum 351 snap 116 seq_num 351) 2024-09-14T03:51:33.206 INFO:tasks.rados.rados.0.smithi070.stdout:1073: finishing write tid 1 to smithi07085154-25 2024-09-14T03:51:33.206 INFO:tasks.rados.rados.0.smithi070.stdout:1073: finishing write tid 2 to smithi07085154-25 2024-09-14T03:51:33.206 INFO:tasks.rados.rados.0.smithi070.stdout:1073: finishing write tid 3 to smithi07085154-25 2024-09-14T03:51:33.206 INFO:tasks.rados.rados.0.smithi070.stdout:1073: finishing write tid 4 to smithi07085154-25 2024-09-14T03:51:33.206 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 452 (ObjNum 352 snap 116 seq_num 352) dirty exists 2024-09-14T03:51:33.206 INFO:tasks.rados.rados.0.smithi070.stdout:1073: left oid 25 (ObjNum 352 snap 116 seq_num 352) 2024-09-14T03:51:33.206 INFO:tasks.rados.rados.0.smithi070.stdout:1074: finishing copy_from racing read to smithi07085154-26 2024-09-14T03:51:33.206 INFO:tasks.rados.rados.0.smithi070.stdout:1074: finishing copy_from to smithi07085154-26 2024-09-14T03:51:33.206 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 297 (ObjNum 319 snap 107 seq_num 319) dirty exists 2024-09-14T03:51:33.207 INFO:tasks.rados.rados.0.smithi070.stdout:1075: finishing write tid 1 to smithi07085154-40 2024-09-14T03:51:33.207 INFO:tasks.rados.rados.0.smithi070.stdout:1075: finishing write tid 2 to smithi07085154-40 2024-09-14T03:51:33.207 INFO:tasks.rados.rados.0.smithi070.stdout:1075: finishing write tid 3 to smithi07085154-40 2024-09-14T03:51:33.207 INFO:tasks.rados.rados.0.smithi070.stdout:1075: finishing write tid 4 to smithi07085154-40 2024-09-14T03:51:33.207 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 369 (ObjNum 353 snap 116 seq_num 353) dirty exists 2024-09-14T03:51:33.207 INFO:tasks.rados.rados.0.smithi070.stdout:1075: left oid 40 (ObjNum 353 snap 116 seq_num 353) 2024-09-14T03:51:33.207 INFO:tasks.rados.rados.0.smithi070.stdout:1077: expect (ObjNum 332 snap 110 seq_num 332) 2024-09-14T03:51:33.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-mon[31053]: from='mgr.? 172.21.15.94:0/872687568' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:51:33.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-mon[31053]: from='mgr.? 172.21.15.94:0/872687568' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:51:33.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-mon[31053]: Active manager daemon y restarted 2024-09-14T03:51:33.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-mon[31053]: mgrmap e38: y(active, since 40s), standbys: x 2024-09-14T03:51:33.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-mon[31053]: from='mgr.? 172.21.15.94:0/872687568' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:51:33.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-mon[31053]: Active manager daemon y restarted 2024-09-14T03:51:33.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-mon[31053]: Activating manager daemon y 2024-09-14T03:51:33.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-mon[31053]: osdmap e275: 8 total, 8 up, 8 in 2024-09-14T03:51:33.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-mon[31053]: osdmap e276: 8 total, 8 up, 8 in 2024-09-14T03:51:33.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-mon[31053]: mgrmap e39: y(active, starting, since 0.0123129s), standbys: x 2024-09-14T03:51:33.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:51:33.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:51:33.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:51:33.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "b"} : dispatch 2024-09-14T03:51:33.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-14T03:51:33.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "y", "id": "y"} : dispatch 2024-09-14T03:51:33.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "x", "id": "x"} : dispatch 2024-09-14T03:51:33.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-14T03:51:33.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-14T03:51:33.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-14T03:51:33.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-14T03:51:33.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-14T03:51:33.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-14T03:51:33.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-14T03:51:33.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-14T03:51:33.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mds metadata"} : dispatch 2024-09-14T03:51:33.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata"} : dispatch 2024-09-14T03:51:33.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata"} : dispatch 2024-09-14T03:51:33.280 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-mon[31053]: Manager daemon y is now available 2024-09-14T03:51:33.280 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:51:33.280 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:51:33.280 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-14T03:51:33.280 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-14T03:51:33.280 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:32 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-14T03:51:33.286 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:51:33.024+0000 7fa4af5ba640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-09-14T03:51:33.286 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: [14/Sep/2024:03:51:33] ENGINE Bus STARTING 2024-09-14T03:51:33.286 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: CherryPy Checker: 2024-09-14T03:51:33.286 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: The Application mounted at '' has an empty config. 2024-09-14T03:51:33.286 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:51:33.286 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: [14/Sep/2024:03:51:33] ENGINE Serving on http://:::9283 2024-09-14T03:51:33.286 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: [14/Sep/2024:03:51:33] ENGINE Bus STARTED 2024-09-14T03:51:33.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[39288]: from='mgr.? 172.21.15.94:0/872687568' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:51:33.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[39288]: from='mgr.? 172.21.15.94:0/872687568' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:51:33.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[39288]: Active manager daemon y restarted 2024-09-14T03:51:33.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[39288]: mgrmap e38: y(active, since 40s), standbys: x 2024-09-14T03:51:33.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[39288]: from='mgr.? 172.21.15.94:0/872687568' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:51:33.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[39288]: Active manager daemon y restarted 2024-09-14T03:51:33.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[39288]: Activating manager daemon y 2024-09-14T03:51:33.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[39288]: osdmap e275: 8 total, 8 up, 8 in 2024-09-14T03:51:33.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[39288]: osdmap e276: 8 total, 8 up, 8 in 2024-09-14T03:51:33.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[39288]: mgrmap e39: y(active, starting, since 0.0123129s), standbys: x 2024-09-14T03:51:33.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:51:33.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:51:33.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:51:33.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "b"} : dispatch 2024-09-14T03:51:33.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-14T03:51:33.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "y", "id": "y"} : dispatch 2024-09-14T03:51:33.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "x", "id": "x"} : dispatch 2024-09-14T03:51:33.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-14T03:51:33.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-14T03:51:33.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-14T03:51:33.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-14T03:51:33.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-14T03:51:33.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-14T03:51:33.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-14T03:51:33.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-14T03:51:33.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mds metadata"} : dispatch 2024-09-14T03:51:33.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata"} : dispatch 2024-09-14T03:51:33.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata"} : dispatch 2024-09-14T03:51:33.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[39288]: Manager daemon y is now available 2024-09-14T03:51:33.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:51:33.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:51:33.290 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-14T03:51:33.290 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-14T03:51:33.290 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-14T03:51:33.290 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[96704]: from='mgr.? 172.21.15.94:0/872687568' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:51:33.290 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[96704]: from='mgr.? 172.21.15.94:0/872687568' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:51:33.290 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[96704]: Active manager daemon y restarted 2024-09-14T03:51:33.290 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[96704]: mgrmap e38: y(active, since 40s), standbys: x 2024-09-14T03:51:33.291 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[96704]: from='mgr.? 172.21.15.94:0/872687568' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:51:33.291 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[96704]: Active manager daemon y restarted 2024-09-14T03:51:33.291 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[96704]: Activating manager daemon y 2024-09-14T03:51:33.291 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[96704]: osdmap e275: 8 total, 8 up, 8 in 2024-09-14T03:51:33.291 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[96704]: osdmap e276: 8 total, 8 up, 8 in 2024-09-14T03:51:33.291 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[96704]: mgrmap e39: y(active, starting, since 0.0123129s), standbys: x 2024-09-14T03:51:33.291 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:51:33.291 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:51:33.291 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:51:33.291 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "b"} : dispatch 2024-09-14T03:51:33.291 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-14T03:51:33.291 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "y", "id": "y"} : dispatch 2024-09-14T03:51:33.291 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "x", "id": "x"} : dispatch 2024-09-14T03:51:33.292 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-14T03:51:33.292 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-14T03:51:33.292 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-14T03:51:33.292 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-14T03:51:33.292 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-14T03:51:33.292 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-14T03:51:33.292 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-14T03:51:33.292 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-14T03:51:33.292 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mds metadata"} : dispatch 2024-09-14T03:51:33.292 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata"} : dispatch 2024-09-14T03:51:33.292 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata"} : dispatch 2024-09-14T03:51:33.292 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[96704]: Manager daemon y is now available 2024-09-14T03:51:33.292 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:51:33.293 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:51:33.293 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-14T03:51:33.293 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-14T03:51:33.293 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:32 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-14T03:51:33.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:51:33.368+0000 7fa45d88f640 -1 log_channel(cephadm) log [ERR] : [14/Sep/2024:03:51:33] ENGINE Error in HTTPServer.serve 2024-09-14T03:51:33.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: Traceback (most recent call last): 2024-09-14T03:51:33.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:51:33.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: self._connections.run(self.expiration_interval) 2024-09-14T03:51:33.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:51:33.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: self._run(expiration_interval) 2024-09-14T03:51:33.787 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:51:33.787 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:51:33.787 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:51:33.787 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:51:33.787 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:51:33.787 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: s = self.context.wrap_socket( 2024-09-14T03:51:33.787 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:51:33.787 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: return self.sslsocket_class._create( 2024-09-14T03:51:33.787 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:51:33.787 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: self.do_handshake() 2024-09-14T03:51:33.787 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:51:33.787 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: self._sslobj.do_handshake() 2024-09-14T03:51:33.787 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:51:33.787 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:51:33.788 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:51:33] "GET /metrics HTTP/1.1" 200 33445 "" "Prometheus/2.51.0" 2024-09-14T03:51:34.025 INFO:tasks.rados.rados.0.smithi070.stdout:1067: done (8 left) 2024-09-14T03:51:34.025 INFO:tasks.rados.rados.0.smithi070.stdout:1072: done (7 left) 2024-09-14T03:51:34.025 INFO:tasks.rados.rados.0.smithi070.stdout:1073: done (6 left) 2024-09-14T03:51:34.025 INFO:tasks.rados.rados.0.smithi070.stdout:1074: done (5 left) 2024-09-14T03:51:34.025 INFO:tasks.rados.rados.0.smithi070.stdout:1075: done (4 left) 2024-09-14T03:51:34.026 INFO:tasks.rados.rados.0.smithi070.stdout:1076: done (3 left) 2024-09-14T03:51:34.026 INFO:tasks.rados.rados.0.smithi070.stdout:1077: done (2 left) 2024-09-14T03:51:34.026 INFO:tasks.rados.rados.0.smithi070.stdout:1078: done (1 left) 2024-09-14T03:51:34.026 INFO:tasks.rados.rados.0.smithi070.stdout:1079: done (0 left) 2024-09-14T03:51:34.026 INFO:tasks.rados.rados.0.smithi070.stdout:1080: read oid 3 snap -1 2024-09-14T03:51:34.026 INFO:tasks.rados.rados.0.smithi070.stdout:1080: expect (ObjNum 346 snap 114 seq_num 346) 2024-09-14T03:51:34.026 INFO:tasks.rados.rados.0.smithi070.stdout:1081: copy_from oid 50 from oid 8 current snap is 117 2024-09-14T03:51:34.026 INFO:tasks.rados.rados.0.smithi070.stdout:1082: snap_remove snap 90 2024-09-14T03:51:34.266 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:34 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:34.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:51:34.266 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:34 smithi094 ceph-mon[31053]: osdmap e277: 8 total, 8 up, 8 in 2024-09-14T03:51:34.266 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:34 smithi094 ceph-mon[31053]: mgrmap e40: y(active, since 1.0203s), standbys: x 2024-09-14T03:51:34.266 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:34 smithi094 ceph-mon[31053]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:51:34.266 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:34 smithi094 ceph-mon[31053]: pgmap v4: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 272 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:34.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:34 smithi070 ceph-mon[39288]: osdmap e277: 8 total, 8 up, 8 in 2024-09-14T03:51:34.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:34 smithi070 ceph-mon[39288]: mgrmap e40: y(active, since 1.0203s), standbys: x 2024-09-14T03:51:34.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:34 smithi070 ceph-mon[39288]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:51:34.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:34 smithi070 ceph-mon[39288]: pgmap v4: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 272 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:34.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:34 smithi070 ceph-mon[96704]: osdmap e277: 8 total, 8 up, 8 in 2024-09-14T03:51:34.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:34 smithi070 ceph-mon[96704]: mgrmap e40: y(active, since 1.0203s), standbys: x 2024-09-14T03:51:34.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:34 smithi070 ceph-mon[96704]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:51:34.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:34 smithi070 ceph-mon[96704]: pgmap v4: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 272 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:35.028 INFO:tasks.rados.rados.0.smithi070.stdout:1082: done (2 left) 2024-09-14T03:51:35.028 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 38 current snap is 117 2024-09-14T03:51:35.028 INFO:tasks.rados.rados.0.smithi070.stdout:1083: seq_num 354 ranges {0=737280,737280=401408} 2024-09-14T03:51:35.032 INFO:tasks.rados.rados.0.smithi070.stdout:1083: writing smithi07085154-38 from 0 to 737280 tid 1 2024-09-14T03:51:35.035 INFO:tasks.rados.rados.0.smithi070.stdout:1083: writing smithi07085154-38 from 737280 to 1138688 tid 2 2024-09-14T03:51:35.036 INFO:tasks.rados.rados.0.smithi070.stdout:1080: expect (ObjNum 346 snap 114 seq_num 346) 2024-09-14T03:51:35.066 INFO:tasks.rados.rados.0.smithi070.stdout:1081: finishing copy_from racing read to smithi07085154-50 2024-09-14T03:51:35.066 INFO:tasks.rados.rados.0.smithi070.stdout:1081: finishing copy_from to smithi07085154-50 2024-09-14T03:51:35.066 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 307 (ObjNum 202 snap 56 seq_num 202) dirty exists 2024-09-14T03:51:35.066 INFO:tasks.rados.rados.0.smithi070.stdout:1080: done (2 left) 2024-09-14T03:51:35.066 INFO:tasks.rados.rados.0.smithi070.stdout:1081: done (1 left) 2024-09-14T03:51:35.066 INFO:tasks.rados.rados.0.smithi070.stdout:1084: read oid 41 snap -1 2024-09-14T03:51:35.066 INFO:tasks.rados.rados.0.smithi070.stdout:1084: expect (ObjNum 211 snap 60 seq_num 211) 2024-09-14T03:51:35.066 INFO:tasks.rados.rados.0.smithi070.stdout:1085: rollback oid 36 current snap is 117 2024-09-14T03:51:35.067 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 36 to 94 2024-09-14T03:51:35.067 INFO:tasks.rados.rados.0.smithi070.stdout:1083: finishing write tid 1 to smithi07085154-38 2024-09-14T03:51:35.067 INFO:tasks.rados.rados.0.smithi070.stdout:1083: finishing write tid 2 to smithi07085154-38 2024-09-14T03:51:35.067 INFO:tasks.rados.rados.0.smithi070.stdout:1083: finishing write tid 3 to smithi07085154-38 2024-09-14T03:51:35.067 INFO:tasks.rados.rados.0.smithi070.stdout:1083: finishing write tid 4 to smithi07085154-38 2024-09-14T03:51:35.067 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 38 v 377 (ObjNum 354 snap 117 seq_num 354) dirty exists 2024-09-14T03:51:35.067 INFO:tasks.rados.rados.0.smithi070.stdout:1083: left oid 38 (ObjNum 354 snap 117 seq_num 354) 2024-09-14T03:51:35.067 INFO:tasks.rados.rados.0.smithi070.stdout:1083: done (2 left) 2024-09-14T03:51:35.067 INFO:tasks.rados.rados.0.smithi070.stdout:1086: rollback oid 43 current snap is 117 2024-09-14T03:51:35.067 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 43 to 79 2024-09-14T03:51:35.067 INFO:tasks.rados.rados.0.smithi070.stdout:1087: read oid 18 snap -1 2024-09-14T03:51:35.067 INFO:tasks.rados.rados.0.smithi070.stdout:1087: expect deleted 2024-09-14T03:51:35.067 INFO:tasks.rados.rados.0.smithi070.stdout:1088: snap_create 2024-09-14T03:51:35.068 INFO:tasks.rados.rados.0.smithi070.stdout:1085: finishing rollback tid 0 to smithi07085154-36 2024-09-14T03:51:35.069 INFO:tasks.rados.rados.0.smithi070.stdout:1085: finishing rollback tid 1 to smithi07085154-36 2024-09-14T03:51:35.069 INFO:tasks.rados.rados.0.smithi070.stdout:1085: finishing rollback tid 2 to smithi07085154-36 2024-09-14T03:51:35.069 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 36 v 377 (ObjNum 241 snap 69 seq_num 241) dirty exists 2024-09-14T03:51:35.071 INFO:tasks.rados.rados.0.smithi070.stdout:1084: expect (ObjNum 211 snap 60 seq_num 211) 2024-09-14T03:51:35.145 INFO:tasks.rados.rados.0.smithi070.stdout:1086: finishing rollback tid 0 to smithi07085154-43 2024-09-14T03:51:35.146 INFO:tasks.rados.rados.0.smithi070.stdout:1086: finishing rollback tid 1 to smithi07085154-43 2024-09-14T03:51:35.146 INFO:tasks.rados.rados.0.smithi070.stdout:1086: finishing rollback tid 2 to smithi07085154-43 2024-09-14T03:51:35.146 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 805 (ObjNum 251 snap 72 seq_num 251) dirty exists 2024-09-14T03:51:35.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:35 smithi094 ceph-mon[31053]: [14/Sep/2024:03:51:33] ENGINE Bus STARTING 2024-09-14T03:51:35.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:35 smithi094 ceph-mon[31053]: [14/Sep/2024:03:51:33] ENGINE Serving on http://172.21.15.70:8765 2024-09-14T03:51:35.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:35 smithi094 ceph-mon[31053]: [14/Sep/2024:03:51:33] ENGINE Serving on https://172.21.15.70:7150 2024-09-14T03:51:35.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:35 smithi094 ceph-mon[31053]: [14/Sep/2024:03:51:33] ENGINE Bus STARTED 2024-09-14T03:51:35.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:35 smithi094 ceph-mon[31053]: [14/Sep/2024:03:51:33] ENGINE Error in HTTPServer.serve 2024-09-14T03:51:35.277 INFO:journalctl@ceph.mon.b.smithi094.stdout: Traceback (most recent call last): 2024-09-14T03:51:35.277 INFO:journalctl@ceph.mon.b.smithi094.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:51:35.277 INFO:journalctl@ceph.mon.b.smithi094.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:51:35.277 INFO:journalctl@ceph.mon.b.smithi094.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:51:35.277 INFO:journalctl@ceph.mon.b.smithi094.stdout: self._run(expiration_interval) 2024-09-14T03:51:35.277 INFO:journalctl@ceph.mon.b.smithi094.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:51:35.278 INFO:journalctl@ceph.mon.b.smithi094.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:51:35.278 INFO:journalctl@ceph.mon.b.smithi094.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:51:35.278 INFO:journalctl@ceph.mon.b.smithi094.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:51:35.278 INFO:journalctl@ceph.mon.b.smithi094.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:51:35.278 INFO:journalctl@ceph.mon.b.smithi094.stdout: s = self.context.wrap_socket( 2024-09-14T03:51:35.278 INFO:journalctl@ceph.mon.b.smithi094.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:51:35.278 INFO:journalctl@ceph.mon.b.smithi094.stdout: return self.sslsocket_class._create( 2024-09-14T03:51:35.278 INFO:journalctl@ceph.mon.b.smithi094.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:51:35.278 INFO:journalctl@ceph.mon.b.smithi094.stdout: self.do_handshake() 2024-09-14T03:51:35.278 INFO:journalctl@ceph.mon.b.smithi094.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:51:35.278 INFO:journalctl@ceph.mon.b.smithi094.stdout: self._sslobj.do_handshake() 2024-09-14T03:51:35.278 INFO:journalctl@ceph.mon.b.smithi094.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:51:35.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:35 smithi094 ceph-mon[31053]: mgrmap e41: y(active, since 2s), standbys: x 2024-09-14T03:51:35.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:35 smithi094 ceph-mon[31053]: osdmap e278: 8 total, 8 up, 8 in 2024-09-14T03:51:35.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:35 smithi094 ceph-mon[31053]: pgmap v6: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 272 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:35.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:35 smithi070 ceph-mon[96704]: [14/Sep/2024:03:51:33] ENGINE Bus STARTING 2024-09-14T03:51:35.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:35 smithi070 ceph-mon[96704]: [14/Sep/2024:03:51:33] ENGINE Serving on http://172.21.15.70:8765 2024-09-14T03:51:35.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:35 smithi070 ceph-mon[96704]: [14/Sep/2024:03:51:33] ENGINE Serving on https://172.21.15.70:7150 2024-09-14T03:51:35.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:35 smithi070 ceph-mon[96704]: [14/Sep/2024:03:51:33] ENGINE Bus STARTED 2024-09-14T03:51:35.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:35 smithi070 ceph-mon[96704]: [14/Sep/2024:03:51:33] ENGINE Error in HTTPServer.serve 2024-09-14T03:51:35.285 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-09-14T03:51:35.285 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:51:35.285 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:51:35.286 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:51:35.286 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._run(expiration_interval) 2024-09-14T03:51:35.286 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:51:35.286 INFO:journalctl@ceph.mon.a.smithi070.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:51:35.286 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:51:35.286 INFO:journalctl@ceph.mon.a.smithi070.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:51:35.286 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:51:35.286 INFO:journalctl@ceph.mon.a.smithi070.stdout: s = self.context.wrap_socket( 2024-09-14T03:51:35.286 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:51:35.286 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.sslsocket_class._create( 2024-09-14T03:51:35.286 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:51:35.286 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.do_handshake() 2024-09-14T03:51:35.286 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:51:35.286 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._sslobj.do_handshake() 2024-09-14T03:51:35.287 INFO:journalctl@ceph.mon.a.smithi070.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:51:35.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:35 smithi070 ceph-mon[96704]: mgrmap e41: y(active, since 2s), standbys: x 2024-09-14T03:51:35.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:35 smithi070 ceph-mon[96704]: osdmap e278: 8 total, 8 up, 8 in 2024-09-14T03:51:35.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:35 smithi070 ceph-mon[96704]: pgmap v6: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 272 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:35.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:35 smithi070 ceph-mon[39288]: [14/Sep/2024:03:51:33] ENGINE Bus STARTING 2024-09-14T03:51:35.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:35 smithi070 ceph-mon[39288]: [14/Sep/2024:03:51:33] ENGINE Serving on http://172.21.15.70:8765 2024-09-14T03:51:35.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:35 smithi070 ceph-mon[39288]: [14/Sep/2024:03:51:33] ENGINE Serving on https://172.21.15.70:7150 2024-09-14T03:51:35.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:35 smithi070 ceph-mon[39288]: [14/Sep/2024:03:51:33] ENGINE Bus STARTED 2024-09-14T03:51:35.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:35 smithi070 ceph-mon[39288]: [14/Sep/2024:03:51:33] ENGINE Error in HTTPServer.serve 2024-09-14T03:51:35.287 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-09-14T03:51:35.287 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:51:35.288 INFO:journalctl@ceph.mon.c.smithi070.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:51:35.288 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:51:35.288 INFO:journalctl@ceph.mon.c.smithi070.stdout: self._run(expiration_interval) 2024-09-14T03:51:35.288 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:51:35.288 INFO:journalctl@ceph.mon.c.smithi070.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:51:35.288 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:51:35.288 INFO:journalctl@ceph.mon.c.smithi070.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:51:35.288 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:51:35.288 INFO:journalctl@ceph.mon.c.smithi070.stdout: s = self.context.wrap_socket( 2024-09-14T03:51:35.288 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:51:35.288 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.sslsocket_class._create( 2024-09-14T03:51:35.288 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:51:35.288 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.do_handshake() 2024-09-14T03:51:35.288 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:51:35.289 INFO:journalctl@ceph.mon.c.smithi070.stdout: self._sslobj.do_handshake() 2024-09-14T03:51:35.289 INFO:journalctl@ceph.mon.c.smithi070.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:51:35.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:35 smithi070 ceph-mon[39288]: mgrmap e41: y(active, since 2s), standbys: x 2024-09-14T03:51:35.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:35 smithi070 ceph-mon[39288]: osdmap e278: 8 total, 8 up, 8 in 2024-09-14T03:51:35.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:35 smithi070 ceph-mon[39288]: pgmap v6: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 272 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:36.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:35 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:35.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:51:36.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:35 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:35.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:51:36.036 INFO:tasks.rados.rados.0.smithi070.stdout:1084: done (4 left) 2024-09-14T03:51:36.036 INFO:tasks.rados.rados.0.smithi070.stdout:1085: done (3 left) 2024-09-14T03:51:36.036 INFO:tasks.rados.rados.0.smithi070.stdout:1086: done (2 left) 2024-09-14T03:51:36.037 INFO:tasks.rados.rados.0.smithi070.stdout:1087: done (1 left) 2024-09-14T03:51:36.037 INFO:tasks.rados.rados.0.smithi070.stdout:1088: done (0 left) 2024-09-14T03:51:36.037 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 37 current snap is 118 2024-09-14T03:51:36.037 INFO:tasks.rados.rados.0.smithi070.stdout:1089: seq_num 355 ranges {794624=753664,1548288=401408} 2024-09-14T03:51:36.044 INFO:tasks.rados.rados.0.smithi070.stdout:1089: writing smithi07085154-37 from 794624 to 1548288 tid 1 2024-09-14T03:51:36.046 INFO:tasks.rados.rados.0.smithi070.stdout:1089: writing smithi07085154-37 from 1548288 to 1949696 tid 2 2024-09-14T03:51:36.046 INFO:tasks.rados.rados.0.smithi070.stdout:1090: rmattr oid 2 current snap is 118 2024-09-14T03:51:36.046 INFO:tasks.rados.rados.0.smithi070.stdout:1091: read oid 50 snap 94 2024-09-14T03:51:36.046 INFO:tasks.rados.rados.0.smithi070.stdout:1091: expect deleted 2024-09-14T03:51:36.046 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 3 current snap is 118 2024-09-14T03:51:36.046 INFO:tasks.rados.rados.0.smithi070.stdout:1092: seq_num 356 ranges {999424=770048,1769472=401408} 2024-09-14T03:51:36.056 INFO:tasks.rados.rados.0.smithi070.stdout:1092: writing smithi07085154-3 from 999424 to 1769472 tid 1 2024-09-14T03:51:36.058 INFO:tasks.rados.rados.0.smithi070.stdout:1092: writing smithi07085154-3 from 1769472 to 2170880 tid 2 2024-09-14T03:51:36.058 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 10 current snap is 118 2024-09-14T03:51:36.058 INFO:tasks.rados.rados.0.smithi070.stdout:1089: finishing write tid 1 to smithi07085154-37 2024-09-14T03:51:36.058 INFO:tasks.rados.rados.0.smithi070.stdout:1093: seq_num 357 ranges {0=786432,786432=401408} 2024-09-14T03:51:36.062 INFO:tasks.rados.rados.0.smithi070.stdout:1093: writing smithi07085154-10 from 0 to 786432 tid 1 2024-09-14T03:51:36.066 INFO:tasks.rados.rados.0.smithi070.stdout:1093: writing smithi07085154-10 from 786432 to 1187840 tid 2 2024-09-14T03:51:36.067 INFO:tasks.rados.rados.0.smithi070.stdout:1094: rmattr oid 27 current snap is 118 2024-09-14T03:51:36.067 INFO:tasks.rados.rados.0.smithi070.stdout:1094: done (5 left) 2024-09-14T03:51:36.067 INFO:tasks.rados.rados.0.smithi070.stdout:1095: setattr oid 17 current snap is 118 2024-09-14T03:51:36.067 INFO:tasks.rados.rados.0.smithi070.stdout:1089: finishing write tid 2 to smithi07085154-37 2024-09-14T03:51:36.067 INFO:tasks.rados.rados.0.smithi070.stdout:1089: finishing write tid 3 to smithi07085154-37 2024-09-14T03:51:36.068 INFO:tasks.rados.rados.0.smithi070.stdout:1089: finishing write tid 4 to smithi07085154-37 2024-09-14T03:51:36.068 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 37 v 808 (ObjNum 355 snap 118 seq_num 355) dirty exists 2024-09-14T03:51:36.068 INFO:tasks.rados.rados.0.smithi070.stdout:1089: left oid 37 (ObjNum 355 snap 118 seq_num 355) 2024-09-14T03:51:36.068 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 2 v 937 (ObjNum 307 snap 101 seq_num 307) dirty exists 2024-09-14T03:51:36.069 INFO:tasks.rados.rados.0.smithi070.stdout:1092: finishing write tid 1 to smithi07085154-3 2024-09-14T03:51:36.069 INFO:tasks.rados.rados.0.smithi070.stdout:1092: finishing write tid 2 to smithi07085154-3 2024-09-14T03:51:36.069 INFO:tasks.rados.rados.0.smithi070.stdout:1092: finishing write tid 3 to smithi07085154-3 2024-09-14T03:51:36.069 INFO:tasks.rados.rados.0.smithi070.stdout:1092: finishing write tid 4 to smithi07085154-3 2024-09-14T03:51:36.069 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 3 v 381 (ObjNum 356 snap 118 seq_num 356) dirty exists 2024-09-14T03:51:36.069 INFO:tasks.rados.rados.0.smithi070.stdout:1092: left oid 3 (ObjNum 356 snap 118 seq_num 356) 2024-09-14T03:51:36.070 INFO:tasks.rados.rados.0.smithi070.stdout:1089: done (5 left) 2024-09-14T03:51:36.070 INFO:tasks.rados.rados.0.smithi070.stdout:1090: done (4 left) 2024-09-14T03:51:36.070 INFO:tasks.rados.rados.0.smithi070.stdout:1091: done (3 left) 2024-09-14T03:51:36.070 INFO:tasks.rados.rados.0.smithi070.stdout:1092: done (2 left) 2024-09-14T03:51:36.070 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 27 current snap is 118 2024-09-14T03:51:36.071 INFO:tasks.rados.rados.0.smithi070.stdout:1096: seq_num 358 ranges {1236992=802816,2039808=401408} 2024-09-14T03:51:36.077 INFO:tasks.rados.rados.0.smithi070.stdout:1096: writing smithi07085154-27 from 1236992 to 2039808 tid 1 2024-09-14T03:51:36.079 INFO:tasks.rados.rados.0.smithi070.stdout:1096: writing smithi07085154-27 from 2039808 to 2441216 tid 2 2024-09-14T03:51:36.079 INFO:tasks.rados.rados.0.smithi070.stdout:1097: delete oid 31 current snap is 118 2024-09-14T03:51:36.080 INFO:tasks.rados.rados.0.smithi070.stdout:1093: finishing write tid 1 to smithi07085154-10 2024-09-14T03:51:36.080 INFO:tasks.rados.rados.0.smithi070.stdout:1093: finishing write tid 2 to smithi07085154-10 2024-09-14T03:51:36.080 INFO:tasks.rados.rados.0.smithi070.stdout:1093: finishing write tid 3 to smithi07085154-10 2024-09-14T03:51:36.080 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 458 (ObjNum 325 snap 107 seq_num 325) dirty exists 2024-09-14T03:51:36.080 INFO:tasks.rados.rados.0.smithi070.stdout:1093: finishing write tid 4 to smithi07085154-10 2024-09-14T03:51:36.080 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 940 (ObjNum 357 snap 118 seq_num 357) dirty exists 2024-09-14T03:51:36.080 INFO:tasks.rados.rados.0.smithi070.stdout:1093: left oid 10 (ObjNum 357 snap 118 seq_num 357) 2024-09-14T03:51:36.081 INFO:tasks.rados.rados.0.smithi070.stdout:1093: done (3 left) 2024-09-14T03:51:36.082 INFO:tasks.rados.rados.0.smithi070.stdout:1095: done (2 left) 2024-09-14T03:51:36.082 INFO:tasks.rados.rados.0.smithi070.stdout:1097: done (1 left) 2024-09-14T03:51:36.082 INFO:tasks.rados.rados.0.smithi070.stdout:1098: read oid 5 snap -1 2024-09-14T03:51:36.082 INFO:tasks.rados.rados.0.smithi070.stdout:1098: expect (ObjNum 114 snap 24 seq_num 114) 2024-09-14T03:51:36.082 INFO:tasks.rados.rados.0.smithi070.stdout:1099: rmattr oid 32 current snap is 118 2024-09-14T03:51:36.082 INFO:tasks.rados.rados.0.smithi070.stdout:1099: done (2 left) 2024-09-14T03:51:36.082 INFO:tasks.rados.rados.0.smithi070.stdout:1100: rmattr oid 47 current snap is 118 2024-09-14T03:51:36.082 INFO:tasks.rados.rados.0.smithi070.stdout:1100: done (2 left) 2024-09-14T03:51:36.082 INFO:tasks.rados.rados.0.smithi070.stdout:1101: setattr oid 47 current snap is 118 2024-09-14T03:51:36.083 INFO:tasks.rados.rados.0.smithi070.stdout:1096: finishing write tid 1 to smithi07085154-27 2024-09-14T03:51:36.084 INFO:tasks.rados.rados.0.smithi070.stdout:1096: finishing write tid 2 to smithi07085154-27 2024-09-14T03:51:36.084 INFO:tasks.rados.rados.0.smithi070.stdout:1096: finishing write tid 3 to smithi07085154-27 2024-09-14T03:51:36.084 INFO:tasks.rados.rados.0.smithi070.stdout:1102: read oid 33 snap 112 2024-09-14T03:51:36.084 INFO:tasks.rados.rados.0.smithi070.stdout:1102: expect (ObjNum 319 snap 107 seq_num 319) 2024-09-14T03:51:36.084 INFO:tasks.rados.rados.0.smithi070.stdout:1103: rollback oid 9 current snap is 118 2024-09-14T03:51:36.084 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 9 to 93 2024-09-14T03:51:36.084 INFO:tasks.rados.rados.0.smithi070.stdout:1104: setattr oid 39 current snap is 118 2024-09-14T03:51:36.088 INFO:tasks.rados.rados.0.smithi070.stdout:1105: rollback oid 31 current snap is 118 2024-09-14T03:51:36.089 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 31 to 107 2024-09-14T03:51:36.089 INFO:tasks.rados.rados.0.smithi070.stdout:1106: rollback oid 37 current snap is 118 2024-09-14T03:51:36.089 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 37 to 83 2024-09-14T03:51:36.089 INFO:tasks.rados.rados.0.smithi070.stdout:1107: read oid 28 snap -1 2024-09-14T03:51:36.089 INFO:tasks.rados.rados.0.smithi070.stdout:1107: expect (ObjNum 160 snap 47 seq_num 160) 2024-09-14T03:51:36.089 INFO:tasks.rados.rados.0.smithi070.stdout:1108: snap_create 2024-09-14T03:51:36.089 INFO:tasks.rados.rados.0.smithi070.stdout:1102: expect (ObjNum 319 snap 107 seq_num 319) 2024-09-14T03:51:36.112 INFO:tasks.rados.rados.0.smithi070.stdout:1105: finishing rollback tid 1 to smithi07085154-31 2024-09-14T03:51:36.112 INFO:tasks.rados.rados.0.smithi070.stdout:1105: finishing rollback tid 2 to smithi07085154-31 2024-09-14T03:51:36.112 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 461 (ObjNum 2257 snap 0 seq_num 2930879088) dirty exists 2024-09-14T03:51:36.112 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 39 v 308 (ObjNum 202 snap 56 seq_num 202) dirty exists 2024-09-14T03:51:36.112 INFO:tasks.rados.rados.0.smithi070.stdout:1106: finishing rollback tid 0 to smithi07085154-37 2024-09-14T03:51:36.112 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 47 v 383 (ObjNum 331 snap 109 seq_num 331) dirty exists 2024-09-14T03:51:36.112 INFO:tasks.rados.rados.0.smithi070.stdout:1106: finishing rollback tid 1 to smithi07085154-37 2024-09-14T03:51:36.112 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 37 v 810 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-14T03:51:36.113 INFO:tasks.rados.rados.0.smithi070.stdout:1096: finishing write tid 4 to smithi07085154-27 2024-09-14T03:51:36.113 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 944 (ObjNum 358 snap 118 seq_num 358) dirty exists 2024-09-14T03:51:36.113 INFO:tasks.rados.rados.0.smithi070.stdout:1096: left oid 27 (ObjNum 358 snap 118 seq_num 358) 2024-09-14T03:51:36.113 INFO:tasks.rados.rados.0.smithi070.stdout:1103: finishing rollback tid 0 to smithi07085154-9 2024-09-14T03:51:36.113 INFO:tasks.rados.rados.0.smithi070.stdout:1098: expect (ObjNum 114 snap 24 seq_num 114) 2024-09-14T03:51:36.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:36 smithi070 ceph-mon[39288]: osdmap e279: 8 total, 8 up, 8 in 2024-09-14T03:51:36.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:36 smithi070 ceph-mon[96704]: osdmap e279: 8 total, 8 up, 8 in 2024-09-14T03:51:36.315 INFO:tasks.rados.rados.0.smithi070.stdout:1103: finishing rollback tid 1 to smithi07085154-9 2024-09-14T03:51:36.315 INFO:tasks.rados.rados.0.smithi070.stdout:1103: finishing rollback tid 2 to smithi07085154-9 2024-09-14T03:51:36.315 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 948 (ObjNum 281 snap 86 seq_num 281) dirty exists 2024-09-14T03:51:36.315 INFO:tasks.rados.rados.0.smithi070.stdout:1107: expect (ObjNum 160 snap 47 seq_num 160) 2024-09-14T03:51:36.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:36 smithi094 ceph-mon[31053]: osdmap e279: 8 total, 8 up, 8 in 2024-09-14T03:51:37.078 INFO:tasks.rados.rados.0.smithi070.stdout:1096: done (9 left) 2024-09-14T03:51:37.078 INFO:tasks.rados.rados.0.smithi070.stdout:1098: done (8 left) 2024-09-14T03:51:37.078 INFO:tasks.rados.rados.0.smithi070.stdout:1101: done (7 left) 2024-09-14T03:51:37.078 INFO:tasks.rados.rados.0.smithi070.stdout:1102: done (6 left) 2024-09-14T03:51:37.078 INFO:tasks.rados.rados.0.smithi070.stdout:1103: done (5 left) 2024-09-14T03:51:37.078 INFO:tasks.rados.rados.0.smithi070.stdout:1104: done (4 left) 2024-09-14T03:51:37.078 INFO:tasks.rados.rados.0.smithi070.stdout:1105: done (3 left) 2024-09-14T03:51:37.078 INFO:tasks.rados.rados.0.smithi070.stdout:1106: done (2 left) 2024-09-14T03:51:37.078 INFO:tasks.rados.rados.0.smithi070.stdout:1107: done (1 left) 2024-09-14T03:51:37.078 INFO:tasks.rados.rados.0.smithi070.stdout:1108: done (0 left) 2024-09-14T03:51:37.079 INFO:tasks.rados.rados.0.smithi070.stdout:1109: copy_from oid 2 from oid 18 current snap is 119 2024-09-14T03:51:37.079 INFO:tasks.rados.rados.0.smithi070.stdout:1110: snap_create 2024-09-14T03:51:37.079 INFO:tasks.rados.rados.0.smithi070.stdout:1109: finishing copy_from to smithi07085154-2 2024-09-14T03:51:37.080 INFO:tasks.rados.rados.0.smithi070.stdout:1109: got expected ENOENT (src dne) 2024-09-14T03:51:37.080 INFO:tasks.rados.rados.0.smithi070.stdout:1109: finishing copy_from racing read to smithi07085154-2 2024-09-14T03:51:37.276 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:36 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:36.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:51:37.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:37 smithi094 ceph-mon[31053]: pgmap v8: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 272 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:37.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:37 smithi094 ceph-mon[31053]: mgrmap e42: y(active, since 4s), standbys: x 2024-09-14T03:51:37.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:37 smithi094 ceph-mon[31053]: osdmap e280: 8 total, 8 up, 8 in 2024-09-14T03:51:37.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:37 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:37.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:37 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:37.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:37 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:37.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:37 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:37.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:37 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:37.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:37 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:37.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:37 smithi070 ceph-mon[39288]: pgmap v8: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 272 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:37.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:37 smithi070 ceph-mon[39288]: mgrmap e42: y(active, since 4s), standbys: x 2024-09-14T03:51:37.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:37 smithi070 ceph-mon[39288]: osdmap e280: 8 total, 8 up, 8 in 2024-09-14T03:51:37.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:37 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:37.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:37 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:37.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:37 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:37.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:37 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:37.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:37 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:37.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:37 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:37.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:37 smithi070 ceph-mon[96704]: pgmap v8: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 272 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:37.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:37 smithi070 ceph-mon[96704]: mgrmap e42: y(active, since 4s), standbys: x 2024-09-14T03:51:37.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:37 smithi070 ceph-mon[96704]: osdmap e280: 8 total, 8 up, 8 in 2024-09-14T03:51:37.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:37 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:37.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:37 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:37.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:37 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:37.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:37 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:37.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:37 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:37.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:37 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:38.079 INFO:tasks.rados.rados.0.smithi070.stdout:1109: done (1 left) 2024-09-14T03:51:38.079 INFO:tasks.rados.rados.0.smithi070.stdout:1110: done (0 left) 2024-09-14T03:51:38.079 INFO:tasks.rados.rados.0.smithi070.stdout:1111: read oid 19 snap -1 2024-09-14T03:51:38.080 INFO:tasks.rados.rados.0.smithi070.stdout:1111: expect (ObjNum 343 snap 113 seq_num 343) 2024-09-14T03:51:38.080 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 44 current snap is 120 2024-09-14T03:51:38.080 INFO:tasks.rados.rados.0.smithi070.stdout:1112: seq_num 360 ranges {0=434176,434176=704512,1138688=98304} 2024-09-14T03:51:38.081 INFO:tasks.rados.rados.0.smithi070.stdout:1112: writing smithi07085154-44 from 0 to 434176 tid 1 2024-09-14T03:51:38.085 INFO:tasks.rados.rados.0.smithi070.stdout:1112: writing smithi07085154-44 from 434176 to 1138688 tid 2 2024-09-14T03:51:38.086 INFO:tasks.rados.rados.0.smithi070.stdout:1112: writing smithi07085154-44 from 1138688 to 1236992 tid 3 2024-09-14T03:51:38.086 INFO:tasks.rados.rados.0.smithi070.stdout:1113: snap_create 2024-09-14T03:51:38.089 INFO:tasks.rados.rados.0.smithi070.stdout:1112: finishing write tid 1 to smithi07085154-44 2024-09-14T03:51:38.089 INFO:tasks.rados.rados.0.smithi070.stdout:1111: expect (ObjNum 343 snap 113 seq_num 343) 2024-09-14T03:51:38.113 INFO:tasks.rados.rados.0.smithi070.stdout:1112: finishing write tid 2 to smithi07085154-44 2024-09-14T03:51:38.114 INFO:tasks.rados.rados.0.smithi070.stdout:1112: finishing write tid 3 to smithi07085154-44 2024-09-14T03:51:38.114 INFO:tasks.rados.rados.0.smithi070.stdout:1112: finishing write tid 4 to smithi07085154-44 2024-09-14T03:51:38.114 INFO:tasks.rados.rados.0.smithi070.stdout:1112: finishing write tid 5 to smithi07085154-44 2024-09-14T03:51:38.114 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 381 (ObjNum 360 snap 120 seq_num 360) dirty exists 2024-09-14T03:51:38.114 INFO:tasks.rados.rados.0.smithi070.stdout:1112: left oid 44 (ObjNum 360 snap 120 seq_num 360) 2024-09-14T03:51:38.200 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:38 smithi094 ceph-mon[31053]: osdmap e281: 8 total, 8 up, 8 in 2024-09-14T03:51:38.200 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:38 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:38.200 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:38 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:38.200 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:38 smithi094 ceph-mon[31053]: pgmap v11: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 272 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:38.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:38 smithi070 ceph-mon[39288]: osdmap e281: 8 total, 8 up, 8 in 2024-09-14T03:51:38.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:38 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:38.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:38 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:38.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:38 smithi070 ceph-mon[39288]: pgmap v11: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 272 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:38.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:38 smithi070 ceph-mon[96704]: osdmap e281: 8 total, 8 up, 8 in 2024-09-14T03:51:38.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:38 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:38.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:38 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:38.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:38 smithi070 ceph-mon[96704]: pgmap v11: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 272 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:38.987 INFO:tasks.rados.rados.0.smithi070.stdout:1111: done (2 left) 2024-09-14T03:51:38.988 INFO:tasks.rados.rados.0.smithi070.stdout:1112: done (1 left) 2024-09-14T03:51:38.988 INFO:tasks.rados.rados.0.smithi070.stdout:1113: done (0 left) 2024-09-14T03:51:38.988 INFO:tasks.rados.rados.0.smithi070.stdout:1114: copy_from oid 44 from oid 47 current snap is 121 2024-09-14T03:51:38.988 INFO:tasks.rados.rados.0.smithi070.stdout:1115: copy_from oid 18 from oid 21 current snap is 121 2024-09-14T03:51:38.988 INFO:tasks.rados.rados.0.smithi070.stdout:1116: snap_create 2024-09-14T03:51:39.004 INFO:tasks.rados.rados.0.smithi070.stdout:1114: finishing copy_from racing read to smithi07085154-44 2024-09-14T03:51:39.006 INFO:tasks.rados.rados.0.smithi070.stdout:1114: finishing copy_from to smithi07085154-44 2024-09-14T03:51:39.007 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 383 (ObjNum 331 snap 109 seq_num 331) dirty exists 2024-09-14T03:51:39.012 INFO:tasks.rados.rados.0.smithi070.stdout:1115: finishing copy_from racing read to smithi07085154-18 2024-09-14T03:51:39.014 INFO:tasks.rados.rados.0.smithi070.stdout:1115: finishing copy_from to smithi07085154-18 2024-09-14T03:51:39.014 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 542 (ObjNum 114 snap 24 seq_num 114) dirty exists 2024-09-14T03:51:39.207 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:39 smithi094 ceph-mon[31053]: osdmap e282: 8 total, 8 up, 8 in 2024-09-14T03:51:39.207 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:39 smithi094 ceph-mon[31053]: osdmap e283: 8 total, 8 up, 8 in 2024-09-14T03:51:39.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:39 smithi070 ceph-mon[96704]: osdmap e282: 8 total, 8 up, 8 in 2024-09-14T03:51:39.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:39 smithi070 ceph-mon[96704]: osdmap e283: 8 total, 8 up, 8 in 2024-09-14T03:51:39.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:39 smithi070 ceph-mon[39288]: osdmap e282: 8 total, 8 up, 8 in 2024-09-14T03:51:39.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:39 smithi070 ceph-mon[39288]: osdmap e283: 8 total, 8 up, 8 in 2024-09-14T03:51:39.989 INFO:tasks.rados.rados.0.smithi070.stdout:1114: done (2 left) 2024-09-14T03:51:39.989 INFO:tasks.rados.rados.0.smithi070.stdout:1115: done (1 left) 2024-09-14T03:51:39.989 INFO:tasks.rados.rados.0.smithi070.stdout:1116: done (0 left) 2024-09-14T03:51:39.989 INFO:tasks.rados.rados.0.smithi070.stdout:1117: read oid 16 snap 71 2024-09-14T03:51:39.989 INFO:tasks.rados.rados.0.smithi070.stdout:1117: expect (ObjNum 114 snap 24 seq_num 114) 2024-09-14T03:51:39.989 INFO:tasks.rados.rados.0.smithi070.stdout:1118: setattr oid 20 current snap is 122 2024-09-14T03:51:39.990 INFO:tasks.rados.rados.0.smithi070.stdout:1119: copy_from oid 23 from oid 36 current snap is 122 2024-09-14T03:51:39.990 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 17 current snap is 122 2024-09-14T03:51:39.990 INFO:tasks.rados.rados.0.smithi070.stdout:1120: seq_num 361 ranges {2154496=450560,2605056=581632,3186688=221184} 2024-09-14T03:51:40.005 INFO:tasks.rados.rados.0.smithi070.stdout:1120: writing smithi07085154-17 from 2154496 to 2605056 tid 1 2024-09-14T03:51:40.008 INFO:tasks.rados.rados.0.smithi070.stdout:1120: writing smithi07085154-17 from 2605056 to 3186688 tid 2 2024-09-14T03:51:40.010 INFO:tasks.rados.rados.0.smithi070.stdout:1120: writing smithi07085154-17 from 3186688 to 3407872 tid 3 2024-09-14T03:51:40.010 INFO:tasks.rados.rados.0.smithi070.stdout:1121: setattr oid 13 current snap is 122 2024-09-14T03:51:40.010 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 20 v 409 (ObjNum 305 snap 0 seq_num 203) dirty exists 2024-09-14T03:51:40.010 INFO:tasks.rados.rados.0.smithi070.stdout:1119: finishing copy_from racing read to smithi07085154-23 2024-09-14T03:51:40.010 INFO:tasks.rados.rados.0.smithi070.stdout:1119: finishing copy_from to smithi07085154-23 2024-09-14T03:51:40.010 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 377 (ObjNum 241 snap 69 seq_num 241) dirty exists 2024-09-14T03:51:40.010 INFO:tasks.rados.rados.0.smithi070.stdout:1117: expect (ObjNum 114 snap 24 seq_num 114) 2024-09-14T03:51:40.203 INFO:tasks.rados.rados.0.smithi070.stdout:1120: finishing write tid 1 to smithi07085154-17 2024-09-14T03:51:40.204 INFO:tasks.rados.rados.0.smithi070.stdout:1120: finishing write tid 2 to smithi07085154-17 2024-09-14T03:51:40.204 INFO:tasks.rados.rados.0.smithi070.stdout:1120: finishing write tid 3 to smithi07085154-17 2024-09-14T03:51:40.204 INFO:tasks.rados.rados.0.smithi070.stdout:1120: finishing write tid 4 to smithi07085154-17 2024-09-14T03:51:40.204 INFO:tasks.rados.rados.0.smithi070.stdout:1120: finishing write tid 5 to smithi07085154-17 2024-09-14T03:51:40.204 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 466 (ObjNum 361 snap 122 seq_num 361) dirty exists 2024-09-14T03:51:40.204 INFO:tasks.rados.rados.0.smithi070.stdout:1120: left oid 17 (ObjNum 361 snap 122 seq_num 361) 2024-09-14T03:51:40.204 INFO:tasks.rados.rados.0.smithi070.stdout:1117: done (4 left) 2024-09-14T03:51:40.204 INFO:tasks.rados.rados.0.smithi070.stdout:1118: done (3 left) 2024-09-14T03:51:40.204 INFO:tasks.rados.rados.0.smithi070.stdout:1119: done (2 left) 2024-09-14T03:51:40.204 INFO:tasks.rados.rados.0.smithi070.stdout:1120: done (1 left) 2024-09-14T03:51:40.204 INFO:tasks.rados.rados.0.smithi070.stdout:1122: snap_create 2024-09-14T03:51:40.206 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 399 (ObjNum 301 snap 93 seq_num 301) dirty exists 2024-09-14T03:51:40.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:40 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:51:40.993 INFO:tasks.rados.rados.0.smithi070.stdout:1121: done (1 left) 2024-09-14T03:51:40.993 INFO:tasks.rados.rados.0.smithi070.stdout:1122: done (0 left) 2024-09-14T03:51:40.993 INFO:tasks.rados.rados.0.smithi070.stdout:1123: snap_create 2024-09-14T03:51:41.266 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:40 smithi094 ceph-mon[31053]: osdmap e284: 8 total, 8 up, 8 in 2024-09-14T03:51:41.266 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:40 smithi094 ceph-mon[31053]: pgmap v15: 121 pgs: 121 active+clean; 287 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.9 MiB/s rd, 2.7 MiB/s wr, 52 op/s 2024-09-14T03:51:41.266 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:40 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:41.266 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:40 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:41.266 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:40 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:51:41.267 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:40 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:51:41.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:40 smithi070 ceph-mon[96704]: osdmap e284: 8 total, 8 up, 8 in 2024-09-14T03:51:41.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:40 smithi070 ceph-mon[96704]: pgmap v15: 121 pgs: 121 active+clean; 287 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.9 MiB/s rd, 2.7 MiB/s wr, 52 op/s 2024-09-14T03:51:41.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:40 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:41.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:40 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:41.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:40 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:51:41.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:40 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:51:41.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:40 smithi070 ceph-mon[39288]: osdmap e284: 8 total, 8 up, 8 in 2024-09-14T03:51:41.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:40 smithi070 ceph-mon[39288]: pgmap v15: 121 pgs: 121 active+clean; 287 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.9 MiB/s rd, 2.7 MiB/s wr, 52 op/s 2024-09-14T03:51:41.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:40 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:41.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:40 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:41.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:40 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:51:41.289 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:40 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:51:41.993 INFO:tasks.rados.rados.0.smithi070.stdout:1123: done (0 left) 2024-09-14T03:51:41.994 INFO:tasks.rados.rados.0.smithi070.stdout:1124: copy_from oid 10 from oid 16 current snap is 124 2024-09-14T03:51:41.994 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 5 current snap is 124 2024-09-14T03:51:41.994 INFO:tasks.rados.rados.0.smithi070.stdout:1125: seq_num 362 ranges {4784128=466944,5251072=466944,5718016=335872} 2024-09-14T03:51:42.022 INFO:tasks.rados.rados.0.smithi070.stdout:1125: writing smithi07085154-5 from 4784128 to 5251072 tid 1 2024-09-14T03:51:42.025 INFO:tasks.rados.rados.0.smithi070.stdout:1125: writing smithi07085154-5 from 5251072 to 5718016 tid 2 2024-09-14T03:51:42.028 INFO:tasks.rados.rados.0.smithi070.stdout:1125: writing smithi07085154-5 from 5718016 to 6053888 tid 3 2024-09-14T03:51:42.029 INFO:tasks.rados.rados.0.smithi070.stdout:1126: read oid 23 snap 98 2024-09-14T03:51:42.029 INFO:tasks.rados.rados.0.smithi070.stdout:1126: expect (ObjNum 274 snap 84 seq_num 274) 2024-09-14T03:51:42.029 INFO:tasks.rados.rados.0.smithi070.stdout:1127: read oid 35 snap 117 2024-09-14T03:51:42.029 INFO:tasks.rados.rados.0.smithi070.stdout:1127: expect deleted 2024-09-14T03:51:42.029 INFO:tasks.rados.rados.0.smithi070.stdout:1128: setattr oid 50 current snap is 124 2024-09-14T03:51:42.029 INFO:tasks.rados.rados.0.smithi070.stdout:1124: finishing copy_from racing read to smithi07085154-10 2024-09-14T03:51:42.030 INFO:tasks.rados.rados.0.smithi070.stdout:1124: finishing copy_from to smithi07085154-10 2024-09-14T03:51:42.030 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 906 (ObjNum 114 snap 24 seq_num 114) dirty exists 2024-09-14T03:51:42.030 INFO:tasks.rados.rados.0.smithi070.stdout:1125: finishing write tid 1 to smithi07085154-5 2024-09-14T03:51:42.030 INFO:tasks.rados.rados.0.smithi070.stdout:1125: finishing write tid 2 to smithi07085154-5 2024-09-14T03:51:42.030 INFO:tasks.rados.rados.0.smithi070.stdout:1124: done (4 left) 2024-09-14T03:51:42.030 INFO:tasks.rados.rados.0.smithi070.stdout:1127: done (3 left) 2024-09-14T03:51:42.031 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 1 current snap is 124 2024-09-14T03:51:42.031 INFO:tasks.rados.rados.0.smithi070.stdout:1129: seq_num 363 ranges {3817472=483328,4300800=753664,5054464=49152} 2024-09-14T03:51:42.052 INFO:tasks.rados.rados.0.smithi070.stdout:1129: writing smithi07085154-1 from 3817472 to 4300800 tid 1 2024-09-14T03:51:42.057 INFO:tasks.rados.rados.0.smithi070.stdout:1129: writing smithi07085154-1 from 4300800 to 5054464 tid 2 2024-09-14T03:51:42.058 INFO:tasks.rados.rados.0.smithi070.stdout:1129: writing smithi07085154-1 from 5054464 to 5103616 tid 3 2024-09-14T03:51:42.058 INFO:tasks.rados.rados.0.smithi070.stdout:1130: delete oid 31 current snap is 124 2024-09-14T03:51:42.058 INFO:tasks.rados.rados.0.smithi070.stdout:1125: finishing write tid 3 to smithi07085154-5 2024-09-14T03:51:42.058 INFO:tasks.rados.rados.0.smithi070.stdout:1125: finishing write tid 4 to smithi07085154-5 2024-09-14T03:51:42.058 INFO:tasks.rados.rados.0.smithi070.stdout:1125: finishing write tid 5 to smithi07085154-5 2024-09-14T03:51:42.058 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 546 (ObjNum 362 snap 124 seq_num 362) dirty exists 2024-09-14T03:51:42.058 INFO:tasks.rados.rados.0.smithi070.stdout:1125: left oid 5 (ObjNum 362 snap 124 seq_num 362) 2024-09-14T03:51:42.058 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 588 (ObjNum 202 snap 56 seq_num 202) dirty exists 2024-09-14T03:51:42.059 INFO:tasks.rados.rados.0.smithi070.stdout:1126: expect (ObjNum 274 snap 84 seq_num 274) 2024-09-14T03:51:42.185 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[96704]: Updating smithi070:/etc/ceph/ceph.conf 2024-09-14T03:51:42.185 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[96704]: Updating smithi094:/etc/ceph/ceph.conf 2024-09-14T03:51:42.185 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[96704]: Updating smithi070:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:51:42.185 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[96704]: Updating smithi094:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:51:42.185 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[96704]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:51:42.186 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[96704]: Updating smithi094:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:51:42.186 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[96704]: Updating smithi070:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:51:42.186 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[96704]: Updating smithi094:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:51:42.186 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[96704]: osdmap e285: 8 total, 8 up, 8 in 2024-09-14T03:51:42.186 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:42.186 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:42.186 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:42.186 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:42.186 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:42.186 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:51:42.187 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:51:42.187 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-14T03:51:42.187 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["c"]} : dispatch 2024-09-14T03:51:42.187 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[39288]: Updating smithi070:/etc/ceph/ceph.conf 2024-09-14T03:51:42.187 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[39288]: Updating smithi094:/etc/ceph/ceph.conf 2024-09-14T03:51:42.187 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[39288]: Updating smithi070:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:51:42.187 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[39288]: Updating smithi094:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:51:42.188 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[39288]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:51:42.188 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[39288]: Updating smithi094:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:51:42.188 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[39288]: Updating smithi070:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:51:42.188 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[39288]: Updating smithi094:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:51:42.188 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[39288]: osdmap e285: 8 total, 8 up, 8 in 2024-09-14T03:51:42.189 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:42.189 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:42.189 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:42.189 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:42.189 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:42.189 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:51:42.190 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:51:42.190 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-14T03:51:42.190 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:41 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["c"]} : dispatch 2024-09-14T03:51:42.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:41 smithi094 ceph-mon[31053]: Updating smithi070:/etc/ceph/ceph.conf 2024-09-14T03:51:42.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:41 smithi094 ceph-mon[31053]: Updating smithi094:/etc/ceph/ceph.conf 2024-09-14T03:51:42.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:41 smithi094 ceph-mon[31053]: Updating smithi070:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:51:42.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:41 smithi094 ceph-mon[31053]: Updating smithi094:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:51:42.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:41 smithi094 ceph-mon[31053]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:51:42.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:41 smithi094 ceph-mon[31053]: Updating smithi094:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:51:42.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:41 smithi094 ceph-mon[31053]: Updating smithi070:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:51:42.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:41 smithi094 ceph-mon[31053]: Updating smithi094:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:51:42.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:41 smithi094 ceph-mon[31053]: osdmap e285: 8 total, 8 up, 8 in 2024-09-14T03:51:42.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:41 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:42.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:41 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:42.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:41 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:42.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:41 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:42.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:41 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:42.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:41 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:51:42.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:41 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:51:42.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:41 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-14T03:51:42.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:41 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["c"]} : dispatch 2024-09-14T03:51:42.358 INFO:tasks.rados.rados.0.smithi070.stdout:1129: finishing write tid 1 to smithi07085154-1 2024-09-14T03:51:42.358 INFO:tasks.rados.rados.0.smithi070.stdout:1129: finishing write tid 2 to smithi07085154-1 2024-09-14T03:51:42.358 INFO:tasks.rados.rados.0.smithi070.stdout:1129: finishing write tid 3 to smithi07085154-1 2024-09-14T03:51:42.358 INFO:tasks.rados.rados.0.smithi070.stdout:1129: finishing write tid 4 to smithi07085154-1 2024-09-14T03:51:42.358 INFO:tasks.rados.rados.0.smithi070.stdout:1129: finishing write tid 5 to smithi07085154-1 2024-09-14T03:51:42.358 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 1 v 388 (ObjNum 363 snap 124 seq_num 363) dirty exists 2024-09-14T03:51:42.358 INFO:tasks.rados.rados.0.smithi070.stdout:1129: left oid 1 (ObjNum 363 snap 124 seq_num 363) 2024-09-14T03:51:42.359 INFO:tasks.rados.rados.0.smithi070.stdout:1125: done (4 left) 2024-09-14T03:51:42.359 INFO:tasks.rados.rados.0.smithi070.stdout:1126: done (3 left) 2024-09-14T03:51:42.359 INFO:tasks.rados.rados.0.smithi070.stdout:1128: done (2 left) 2024-09-14T03:51:42.359 INFO:tasks.rados.rados.0.smithi070.stdout:1129: done (1 left) 2024-09-14T03:51:42.359 INFO:tasks.rados.rados.0.smithi070.stdout:1130: done (0 left) 2024-09-14T03:51:42.359 INFO:tasks.rados.rados.0.smithi070.stdout:1131: setattr oid 29 current snap is 124 2024-09-14T03:51:42.359 INFO:tasks.rados.rados.0.smithi070.stdout:1132: rollback oid 3 current snap is 124 2024-09-14T03:51:42.360 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 3 to 107 2024-09-14T03:51:42.360 INFO:tasks.rados.rados.0.smithi070.stdout:1133: snap_create 2024-09-14T03:51:42.361 INFO:tasks.rados.rados.0.smithi070.stdout:1132: finishing rollback tid 0 to smithi07085154-3 2024-09-14T03:51:42.361 INFO:tasks.rados.rados.0.smithi070.stdout:1132: finishing rollback tid 1 to smithi07085154-3 2024-09-14T03:51:42.362 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 3 v 386 (ObjNum 33 snap 0 seq_num 4214920505) dirty dne 2024-09-14T03:51:42.362 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 590 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-14T03:51:43.002 INFO:tasks.rados.rados.0.smithi070.stdout:1131: done (2 left) 2024-09-14T03:51:43.002 INFO:tasks.rados.rados.0.smithi070.stdout:1132: done (1 left) 2024-09-14T03:51:43.002 INFO:tasks.rados.rados.0.smithi070.stdout:1133: done (0 left) 2024-09-14T03:51:43.002 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 36 current snap is 125 2024-09-14T03:51:43.002 INFO:tasks.rados.rados.0.smithi070.stdout:1134: seq_num 365 ranges {1646592=516096,2162688=573440,2736128=229376} 2024-09-14T03:51:43.013 INFO:tasks.rados.rados.0.smithi070.stdout:1134: writing smithi07085154-36 from 1646592 to 2162688 tid 1 2024-09-14T03:51:43.017 INFO:tasks.rados.rados.0.smithi070.stdout:1134: writing smithi07085154-36 from 2162688 to 2736128 tid 2 2024-09-14T03:51:43.019 INFO:tasks.rados.rados.0.smithi070.stdout:1134: writing smithi07085154-36 from 2736128 to 2965504 tid 3 2024-09-14T03:51:43.019 INFO:tasks.rados.rados.0.smithi070.stdout:1135: read oid 24 snap -1 2024-09-14T03:51:43.019 INFO:tasks.rados.rados.0.smithi070.stdout:1135: expect (ObjNum 350 snap 116 seq_num 350) 2024-09-14T03:51:43.019 INFO:tasks.rados.rados.0.smithi070.stdout:1136: read oid 30 snap -1 2024-09-14T03:51:43.019 INFO:tasks.rados.rados.0.smithi070.stdout:1136: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-14T03:51:43.019 INFO:tasks.rados.rados.0.smithi070.stdout:1137: rmattr oid 9 current snap is 125 2024-09-14T03:51:43.019 INFO:tasks.rados.rados.0.smithi070.stdout:1134: finishing write tid 1 to smithi07085154-36 2024-09-14T03:51:43.019 INFO:tasks.rados.rados.0.smithi070.stdout:1138: rollback oid 14 current snap is 125 2024-09-14T03:51:43.019 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 14 to 123 2024-09-14T03:51:43.019 INFO:tasks.rados.rados.0.smithi070.stdout:1139: delete oid 22 current snap is 125 2024-09-14T03:51:43.021 INFO:tasks.rados.rados.0.smithi070.stdout:1134: finishing write tid 2 to smithi07085154-36 2024-09-14T03:51:43.024 INFO:tasks.rados.rados.0.smithi070.stdout:1134: finishing write tid 3 to smithi07085154-36 2024-09-14T03:51:43.024 INFO:tasks.rados.rados.0.smithi070.stdout:1134: finishing write tid 4 to smithi07085154-36 2024-09-14T03:51:43.024 INFO:tasks.rados.rados.0.smithi070.stdout:1136: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-14T03:51:43.040 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 952 (ObjNum 281 snap 86 seq_num 281) dirty exists 2024-09-14T03:51:43.040 INFO:tasks.rados.rados.0.smithi070.stdout:1134: finishing write tid 5 to smithi07085154-36 2024-09-14T03:51:43.040 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 36 v 393 (ObjNum 365 snap 125 seq_num 365) dirty exists 2024-09-14T03:51:43.040 INFO:tasks.rados.rados.0.smithi070.stdout:1134: left oid 36 (ObjNum 365 snap 125 seq_num 365) 2024-09-14T03:51:43.040 INFO:tasks.rados.rados.0.smithi070.stdout:1138: finishing rollback tid 0 to smithi07085154-14 2024-09-14T03:51:43.040 INFO:tasks.rados.rados.0.smithi070.stdout:1138: finishing rollback tid 1 to smithi07085154-14 2024-09-14T03:51:43.040 INFO:tasks.rados.rados.0.smithi070.stdout:1135: expect (ObjNum 350 snap 116 seq_num 350) 2024-09-14T03:51:43.170 INFO:tasks.rados.rados.0.smithi070.stdout:1138: finishing rollback tid 2 to smithi07085154-14 2024-09-14T03:51:43.171 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 417 (ObjNum 313 snap 105 seq_num 313) dirty exists 2024-09-14T03:51:43.171 INFO:tasks.rados.rados.0.smithi070.stdout:1134: done (5 left) 2024-09-14T03:51:43.171 INFO:tasks.rados.rados.0.smithi070.stdout:1135: done (4 left) 2024-09-14T03:51:43.172 INFO:tasks.rados.rados.0.smithi070.stdout:1136: done (3 left) 2024-09-14T03:51:43.172 INFO:tasks.rados.rados.0.smithi070.stdout:1137: done (2 left) 2024-09-14T03:51:43.172 INFO:tasks.rados.rados.0.smithi070.stdout:1138: done (1 left) 2024-09-14T03:51:43.172 INFO:tasks.rados.rados.0.smithi070.stdout:1139: done (0 left) 2024-09-14T03:51:43.172 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 47 current snap is 125 2024-09-14T03:51:43.172 INFO:tasks.rados.rados.0.smithi070.stdout:1140: seq_num 367 ranges {3612672=548864,4161536=737280,4898816=65536} 2024-09-14T03:51:43.193 INFO:tasks.rados.rados.0.smithi070.stdout:1140: writing smithi07085154-47 from 3612672 to 4161536 tid 1 2024-09-14T03:51:43.197 INFO:tasks.rados.rados.0.smithi070.stdout:1140: writing smithi07085154-47 from 4161536 to 4898816 tid 2 2024-09-14T03:51:43.197 INFO:tasks.rados.rados.0.smithi070.stdout:1140: writing smithi07085154-47 from 4898816 to 4964352 tid 3 2024-09-14T03:51:43.197 INFO:tasks.rados.rados.0.smithi070.stdout:1141: read oid 18 snap -1 2024-09-14T03:51:43.198 INFO:tasks.rados.rados.0.smithi070.stdout:1141: expect (ObjNum 114 snap 24 seq_num 114) 2024-09-14T03:51:43.198 INFO:tasks.rados.rados.0.smithi070.stdout:1142: snap_remove snap 124 2024-09-14T03:51:43.222 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[39288]: Upgrade: It appears safe to stop mon.c 2024-09-14T03:51:43.222 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[39288]: osdmap e286: 8 total, 8 up, 8 in 2024-09-14T03:51:43.222 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[39288]: pgmap v18: 121 pgs: 121 active+clean; 287 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:43.222 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:51:43.222 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config-key dump", "format": "json"} : dispatch 2024-09-14T03:51:43.222 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:51:43.222 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:51:43.223 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:51:43.223 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:51:43.223 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:51:43.223 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:51:43.223 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:51:43.223 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:51:43.223 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:51:43.223 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:51:43.223 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:51:43.223 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:51:43.223 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:51:43.224 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:51:43.224 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:51:43.224 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:51:43.224 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:51:43.224 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:51:43.224 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:51:43.224 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:51:43.224 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[39288]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:51:43.224 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:51:43.224 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:51:43.224 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:51:43.224 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:43.224 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T03:51:43.225 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T03:51:43.225 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[39288]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:51:43.225 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[96704]: Upgrade: It appears safe to stop mon.c 2024-09-14T03:51:43.225 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[96704]: osdmap e286: 8 total, 8 up, 8 in 2024-09-14T03:51:43.225 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[96704]: pgmap v18: 121 pgs: 121 active+clean; 287 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:43.225 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:51:43.225 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config-key dump", "format": "json"} : dispatch 2024-09-14T03:51:43.225 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:51:43.226 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:51:43.226 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:51:43.226 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:51:43.226 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:51:43.226 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:51:43.226 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:51:43.226 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:51:43.226 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:51:43.226 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[96704]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:51:43.226 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[96704]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:51:43.226 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:51:43.226 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:51:43.226 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:51:43.226 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[96704]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:51:43.227 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[96704]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:51:43.227 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:51:43.227 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:51:43.227 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:51:43.227 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[96704]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:51:43.227 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[96704]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:51:43.227 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:51:43.227 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:51:43.227 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:51:43.227 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:43.227 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T03:51:43.227 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T03:51:43.227 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:42 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:51:43.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:42 smithi094 ceph-mon[31053]: Upgrade: It appears safe to stop mon.c 2024-09-14T03:51:43.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:42 smithi094 ceph-mon[31053]: osdmap e286: 8 total, 8 up, 8 in 2024-09-14T03:51:43.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:42 smithi094 ceph-mon[31053]: pgmap v18: 121 pgs: 121 active+clean; 287 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:51:43.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:42 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:51:43.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:42 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config-key dump", "format": "json"} : dispatch 2024-09-14T03:51:43.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:42 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:51:43.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:42 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:51:43.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:42 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:51:43.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:42 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:51:43.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:42 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:51:43.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:42 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:51:43.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:42 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:51:43.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:42 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:51:43.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:42 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:51:43.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:42 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:51:43.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:42 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:51:43.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:42 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:51:43.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:42 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:51:43.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:42 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:51:43.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:42 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:51:43.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:42 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:51:43.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:42 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:51:43.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:42 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:51:43.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:42 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:51:43.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:42 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-14T03:51:43.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:42 smithi094 ceph-mon[31053]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-14T03:51:43.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:42 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:51:43.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:42 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:51:43.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:42 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:51:43.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:42 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:43.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:42 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T03:51:43.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:42 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T03:51:43.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:42 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:51:43.536 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:43 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:51:43] "GET /metrics HTTP/1.1" 200 33445 "" "Prometheus/2.51.0" 2024-09-14T03:51:43.989 INFO:tasks.rados.rados.0.smithi070.stdout:1140: finishing write tid 1 to smithi07085154-47 2024-09-14T03:51:43.989 INFO:tasks.rados.rados.0.smithi070.stdout:1140: finishing write tid 2 to smithi07085154-47 2024-09-14T03:51:43.989 INFO:tasks.rados.rados.0.smithi070.stdout:1140: finishing write tid 3 to smithi07085154-47 2024-09-14T03:51:43.989 INFO:tasks.rados.rados.0.smithi070.stdout:1140: finishing write tid 4 to smithi07085154-47 2024-09-14T03:51:43.989 INFO:tasks.rados.rados.0.smithi070.stdout:1140: finishing write tid 5 to smithi07085154-47 2024-09-14T03:51:43.989 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 47 v 391 (ObjNum 367 snap 125 seq_num 367) dirty exists 2024-09-14T03:51:43.990 INFO:tasks.rados.rados.0.smithi070.stdout:1140: left oid 47 (ObjNum 367 snap 125 seq_num 367) 2024-09-14T03:51:43.990 INFO:tasks.rados.rados.0.smithi070.stdout:1141: expect (ObjNum 114 snap 24 seq_num 114) 2024-09-14T03:51:44.199 INFO:tasks.rados.rados.0.smithi070.stdout:1140: done (2 left) 2024-09-14T03:51:44.199 INFO:tasks.rados.rados.0.smithi070.stdout:1141: done (1 left) 2024-09-14T03:51:44.199 INFO:tasks.rados.rados.0.smithi070.stdout:1142: done (0 left) 2024-09-14T03:51:44.199 INFO:tasks.rados.rados.0.smithi070.stdout:1143: read oid 36 snap -1 2024-09-14T03:51:44.199 INFO:tasks.rados.rados.0.smithi070.stdout:1143: expect (ObjNum 365 snap 125 seq_num 365) 2024-09-14T03:51:44.199 INFO:tasks.rados.rados.0.smithi070.stdout:1144: rollback oid 10 current snap is 125 2024-09-14T03:51:44.199 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 10 to 107 2024-09-14T03:51:44.199 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 38 current snap is 125 2024-09-14T03:51:44.199 INFO:tasks.rados.rados.0.smithi070.stdout:1145: seq_num 368 ranges {1138688=573440,1712128=622592,2334720=180224} 2024-09-14T03:51:44.208 INFO:tasks.rados.rados.0.smithi070.stdout:1145: writing smithi07085154-38 from 1138688 to 1712128 tid 1 2024-09-14T03:51:44.212 INFO:tasks.rados.rados.0.smithi070.stdout:1145: writing smithi07085154-38 from 1712128 to 2334720 tid 2 2024-09-14T03:51:44.213 INFO:tasks.rados.rados.0.smithi070.stdout:1145: writing smithi07085154-38 from 2334720 to 2514944 tid 3 2024-09-14T03:51:44.213 INFO:tasks.rados.rados.0.smithi070.stdout:1146: setattr oid 27 current snap is 125 2024-09-14T03:51:44.213 INFO:tasks.rados.rados.0.smithi070.stdout:1144: finishing rollback tid 0 to smithi07085154-10 2024-09-14T03:51:44.213 INFO:tasks.rados.rados.0.smithi070.stdout:1144: finishing rollback tid 1 to smithi07085154-10 2024-09-14T03:51:44.213 INFO:tasks.rados.rados.0.smithi070.stdout:1144: finishing rollback tid 2 to smithi07085154-10 2024-09-14T03:51:44.213 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 957 (ObjNum 317 snap 107 seq_num 317) dirty exists 2024-09-14T03:51:44.213 INFO:tasks.rados.rados.0.smithi070.stdout:1143: expect (ObjNum 365 snap 125 seq_num 365) 2024-09-14T03:51:44.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:43 smithi094 ceph-mon[31053]: Upgrade: Updating mon.c 2024-09-14T03:51:44.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:43 smithi094 ceph-mon[31053]: Deploying daemon mon.c on smithi070 2024-09-14T03:51:44.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:43 smithi094 ceph-mon[31053]: osdmap e287: 8 total, 8 up, 8 in 2024-09-14T03:51:44.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:43 smithi094 ceph-mon[31053]: osdmap e288: 8 total, 8 up, 8 in 2024-09-14T03:51:44.277 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:44 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:44.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:51:44.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:43 smithi070 ceph-mon[96704]: Upgrade: Updating mon.c 2024-09-14T03:51:44.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:43 smithi070 ceph-mon[96704]: Deploying daemon mon.c on smithi070 2024-09-14T03:51:44.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:43 smithi070 ceph-mon[96704]: osdmap e287: 8 total, 8 up, 8 in 2024-09-14T03:51:44.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:44 smithi070 ceph-mon[96704]: osdmap e288: 8 total, 8 up, 8 in 2024-09-14T03:51:44.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:43 smithi070 ceph-mon[39288]: Upgrade: Updating mon.c 2024-09-14T03:51:44.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:43 smithi070 ceph-mon[39288]: Deploying daemon mon.c on smithi070 2024-09-14T03:51:44.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:43 smithi070 ceph-mon[39288]: osdmap e287: 8 total, 8 up, 8 in 2024-09-14T03:51:44.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:43 smithi070 ceph-mon[39288]: osdmap e288: 8 total, 8 up, 8 in 2024-09-14T03:51:44.320 INFO:tasks.rados.rados.0.smithi070.stdout:1145: finishing write tid 1 to smithi07085154-38 2024-09-14T03:51:44.320 INFO:tasks.rados.rados.0.smithi070.stdout:1145: finishing write tid 2 to smithi07085154-38 2024-09-14T03:51:44.320 INFO:tasks.rados.rados.0.smithi070.stdout:1145: finishing write tid 3 to smithi07085154-38 2024-09-14T03:51:44.320 INFO:tasks.rados.rados.0.smithi070.stdout:1145: finishing write tid 4 to smithi07085154-38 2024-09-14T03:51:44.321 INFO:tasks.rados.rados.0.smithi070.stdout:1145: finishing write tid 5 to smithi07085154-38 2024-09-14T03:51:44.321 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 38 v 398 (ObjNum 368 snap 125 seq_num 368) dirty exists 2024-09-14T03:51:44.321 INFO:tasks.rados.rados.0.smithi070.stdout:1145: left oid 38 (ObjNum 368 snap 125 seq_num 368) 2024-09-14T03:51:44.321 INFO:tasks.rados.rados.0.smithi070.stdout:1143: done (3 left) 2024-09-14T03:51:44.321 INFO:tasks.rados.rados.0.smithi070.stdout:1144: done (2 left) 2024-09-14T03:51:44.321 INFO:tasks.rados.rados.0.smithi070.stdout:1145: done (1 left) 2024-09-14T03:51:44.321 INFO:tasks.rados.rados.0.smithi070.stdout:1147: setattr oid 9 current snap is 125 2024-09-14T03:51:44.322 INFO:tasks.rados.rados.0.smithi070.stdout:1148: rmattr oid 11 current snap is 125 2024-09-14T03:51:44.322 INFO:tasks.rados.rados.0.smithi070.stdout:1149: read oid 43 snap -1 2024-09-14T03:51:44.322 INFO:tasks.rados.rados.0.smithi070.stdout:1149: expect (ObjNum 251 snap 72 seq_num 251) 2024-09-14T03:51:44.322 INFO:tasks.rados.rados.0.smithi070.stdout:1150: copy_from oid 7 from oid 34 current snap is 125 2024-09-14T03:51:44.322 INFO:tasks.rados.rados.0.smithi070.stdout:1151: snap_create 2024-09-14T03:51:44.324 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 959 (ObjNum 358 snap 118 seq_num 358) dirty exists 2024-09-14T03:51:44.325 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 960 (ObjNum 281 snap 86 seq_num 281) dirty exists 2024-09-14T03:51:44.326 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 11 v 401 (ObjNum 293 snap 91 seq_num 293) dirty exists 2024-09-14T03:51:44.335 INFO:tasks.rados.rados.0.smithi070.stdout:1150: finishing copy_from racing read to smithi07085154-7 2024-09-14T03:51:44.336 INFO:tasks.rados.rados.0.smithi070.stdout:1150: finishing copy_from to smithi07085154-7 2024-09-14T03:51:44.336 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 402 (ObjNum 325 snap 107 seq_num 325) dirty exists 2024-09-14T03:51:44.341 INFO:tasks.rados.rados.0.smithi070.stdout:1149: expect (ObjNum 251 snap 72 seq_num 251) 2024-09-14T03:51:44.624 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:44 smithi070 systemd[1]: Stopping Ceph mon.c for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:51:45.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:44 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-c[39265]: 2024-09-14T03:51:44.622+0000 7f02f8f44640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:51:45.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:44 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-c[39265]: 2024-09-14T03:51:44.622+0000 7f02f8f44640 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-09-14T03:51:45.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:44 smithi070 podman[98383]: 2024-09-14 03:51:44.806620647 +0000 UTC m=+0.324912996 container died 44ee80c29ee0962628f2ccebe8975350c32d044147afb5468ed661736e2ec58f (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-c, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=quincy-eb16b97, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-14T03:51:45.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:45 smithi070 podman[98383]: 2024-09-14 03:51:45.262252693 +0000 UTC m=+0.780545084 container cleanup 44ee80c29ee0962628f2ccebe8975350c32d044147afb5468ed661736e2ec58f (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-c, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-eb16b97, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-14T03:51:45.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:45 smithi070 bash[98383]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-c 2024-09-14T03:51:45.882 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-09-14T03:51:45.883 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:45 smithi070 podman[98400]: 2024-09-14 03:51:45.547307304 +0000 UTC m=+0.742803884 container remove 44ee80c29ee0962628f2ccebe8975350c32d044147afb5468ed661736e2ec58f (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-c, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=quincy-eb16b97, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-14T03:51:45.883 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:45 smithi070 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@mon.c.service: Deactivated successfully. 2024-09-14T03:51:45.883 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:45 smithi070 systemd[1]: Stopped Ceph mon.c for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:51:45.883 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:45 smithi070 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@mon.c.service: Consumed 8.001s CPU time. 2024-09-14T03:51:46.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:45 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:45.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:51:46.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:45 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:45.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:51:46.224 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:45 smithi070 systemd[1]: Starting Ceph mon.c for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:51:46.224 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 podman[98580]: 2024-09-14 03:51:45.999436166 +0000 UTC m=+0.016268208 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:51:46.224 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 podman[98580]: 2024-09-14 03:51:46.123303387 +0000 UTC m=+0.140135428 container create 5741e4fcf4b689cbaec23247721f6e1bb963f63973edd270d98aec5e1d955239 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-c, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-14T03:51:46.472 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:51:46.473 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (99s) 10s ago 6m 19.7M - 0.27.0 11f11916f8cd 45e79e51074c 2024-09-14T03:51:46.473 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi094 *:3000 running (76s) 9s ago 6m 80.6M - 9.4.7 954c08fa6188 2240810d0368 2024-09-14T03:51:46.473 INFO:teuthology.orchestra.run.smithi070.stdout:mds.a smithi070 running (8m) 10s ago 7m 19.9M - 17.2.7-1629-geb16b97b ab51d1bebd12 52e0dbb7be50 2024-09-14T03:51:46.473 INFO:teuthology.orchestra.run.smithi070.stdout:mds.b smithi094 running (7m) 9s ago 7m 15.2M - 17.2.7-1629-geb16b97b ab51d1bebd12 e3b5e3e00189 2024-09-14T03:51:46.473 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi094 *:8443,9283,8765 running (36s) 9s ago 11m 451M - 19.3.0-4907-ga2782315 8801d2529e43 569d9c7e0f06 2024-09-14T03:51:46.473 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (63s) 10s ago 12m 488M - 19.3.0-4907-ga2782315 8801d2529e43 0ef4d8e0cdbe 2024-09-14T03:51:46.473 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (22s) 10s ago 13m 32.2M 2048M 19.3.0-4907-ga2782315 8801d2529e43 96542de31cac 2024-09-14T03:51:46.473 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi094 running (11m) 9s ago 11m 58.5M 2048M 17.2.7-1629-geb16b97b ab51d1bebd12 90ea2e164909 2024-09-14T03:51:46.473 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 starting - - - 2048M 2024-09-14T03:51:46.473 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (93s) 10s ago 5m 9903k - 1.7.0 72c9c2088986 e316760a30b3 2024-09-14T03:51:46.473 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi094 *:9100 running (69s) 9s ago 5m 8204k - 1.7.0 72c9c2088986 767f8a093175 2024-09-14T03:51:46.473 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (11m) 10s ago 11m 160M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 741c06b6adbf 2024-09-14T03:51:46.473 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (10m) 10s ago 10m 171M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 af459ea7ef68 2024-09-14T03:51:46.474 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (10m) 10s ago 10m 143M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 109b66e4e60f 2024-09-14T03:51:46.474 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (9m) 10s ago 9m 116M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 2e4199b58392 2024-09-14T03:51:46.474 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi094 running (9m) 9s ago 9m 158M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 52b64527e24c 2024-09-14T03:51:46.474 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi094 running (9m) 9s ago 9m 151M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 b543f152ba87 2024-09-14T03:51:46.474 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi094 running (8m) 9s ago 8m 106M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 e66b592ac940 2024-09-14T03:51:46.474 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi094 running (8m) 9s ago 8m 139M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 037c6208e9ce 2024-09-14T03:51:46.474 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi094 *:9095 running (41s) 9s ago 6m 46.2M - 2.51.0 1d3b7f56885b ce1b353828dc 2024-09-14T03:51:46.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 podman[98580]: 2024-09-14 03:51:46.308934112 +0000 UTC m=+0.325766163 container init 5741e4fcf4b689cbaec23247721f6e1bb963f63973edd270d98aec5e1d955239 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-c, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-14T03:51:46.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 podman[98580]: 2024-09-14 03:51:46.314377632 +0000 UTC m=+0.331209673 container start 5741e4fcf4b689cbaec23247721f6e1bb963f63973edd270d98aec5e1d955239 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-c, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-14T03:51:46.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: set uid:gid to 167:167 (ceph:ceph) 2024-09-14T03:51:46.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev), process ceph-mon, pid 7 2024-09-14T03:51:46.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: pidfile_write: ignore empty --pid-file 2024-09-14T03:51:46.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: load: jerasure load: lrc 2024-09-14T03:51:46.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: RocksDB version: 7.9.2 2024-09-14T03:51:46.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Git sha 0 2024-09-14T03:51:46.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Compile date 2024-09-12 20:09:22 2024-09-14T03:51:46.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: DB SUMMARY 2024-09-14T03:51:46.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: DB Session ID: H5HZLGEUKTXPPUR8YS6Q 2024-09-14T03:51:46.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: CURRENT file: CURRENT 2024-09-14T03:51:46.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: IDENTITY file: IDENTITY 2024-09-14T03:51:46.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1097 Bytes 2024-09-14T03:51:46.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2024-09-14T03:51:46.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 11853666 ; 2024-09-14T03:51:46.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.error_if_exists: 0 2024-09-14T03:51:46.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.create_if_missing: 0 2024-09-14T03:51:46.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.paranoid_checks: 1 2024-09-14T03:51:46.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-14T03:51:46.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-14T03:51:46.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-14T03:51:46.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.env: 0x55ef6fc8f120 2024-09-14T03:51:46.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.fs: PosixFileSystem 2024-09-14T03:51:46.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.info_log: 0x55ef71aff700 2024-09-14T03:51:46.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.max_file_opening_threads: 16 2024-09-14T03:51:46.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.statistics: (nil) 2024-09-14T03:51:46.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.use_fsync: 0 2024-09-14T03:51:46.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.max_log_file_size: 0 2024-09-14T03:51:46.790 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-14T03:51:46.790 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-14T03:51:46.790 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.keep_log_file_num: 1000 2024-09-14T03:51:46.790 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.recycle_log_file_num: 0 2024-09-14T03:51:46.790 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.allow_fallocate: 1 2024-09-14T03:51:46.790 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.allow_mmap_reads: 0 2024-09-14T03:51:46.790 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.allow_mmap_writes: 0 2024-09-14T03:51:46.790 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.use_direct_reads: 0 2024-09-14T03:51:46.790 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-14T03:51:46.790 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.create_missing_column_families: 0 2024-09-14T03:51:46.790 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.db_log_dir: 2024-09-14T03:51:46.790 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.wal_dir: 2024-09-14T03:51:46.790 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-14T03:51:46.791 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-14T03:51:46.791 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-14T03:51:46.791 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-14T03:51:46.791 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-14T03:51:46.791 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-14T03:51:46.791 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.advise_random_on_open: 1 2024-09-14T03:51:46.791 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.db_write_buffer_size: 0 2024-09-14T03:51:46.791 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.write_buffer_manager: 0x55ef71b03b80 2024-09-14T03:51:46.791 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-14T03:51:46.791 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-14T03:51:46.791 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-14T03:51:46.791 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.rate_limiter: (nil) 2024-09-14T03:51:46.791 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-14T03:51:46.792 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.wal_recovery_mode: 2 2024-09-14T03:51:46.792 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.enable_thread_tracking: 0 2024-09-14T03:51:46.792 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.enable_pipelined_write: 0 2024-09-14T03:51:46.792 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.unordered_write: 0 2024-09-14T03:51:46.792 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-14T03:51:46.792 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-14T03:51:46.792 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-14T03:51:46.792 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-14T03:51:46.792 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.row_cache: None 2024-09-14T03:51:46.792 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.wal_filter: None 2024-09-14T03:51:46.792 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-14T03:51:46.792 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.allow_ingest_behind: 0 2024-09-14T03:51:46.793 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.two_write_queues: 0 2024-09-14T03:51:46.793 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.manual_wal_flush: 0 2024-09-14T03:51:46.793 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.wal_compression: 0 2024-09-14T03:51:46.793 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.atomic_flush: 0 2024-09-14T03:51:46.793 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-14T03:51:46.793 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-14T03:51:46.793 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-14T03:51:46.793 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.log_readahead_size: 0 2024-09-14T03:51:46.793 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-14T03:51:46.793 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.best_efforts_recovery: 0 2024-09-14T03:51:46.793 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-14T03:51:46.793 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-14T03:51:46.793 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.allow_data_in_errors: 0 2024-09-14T03:51:46.794 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.db_host_id: __hostname__ 2024-09-14T03:51:46.794 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-14T03:51:46.794 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.max_background_jobs: 2 2024-09-14T03:51:46.794 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.max_background_compactions: -1 2024-09-14T03:51:46.794 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.max_subcompactions: 1 2024-09-14T03:51:46.794 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-14T03:51:46.794 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-14T03:51:46.794 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-14T03:51:46.794 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.max_total_wal_size: 0 2024-09-14T03:51:46.794 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-14T03:51:46.794 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-14T03:51:46.794 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-14T03:51:46.794 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-14T03:51:46.795 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.max_open_files: -1 2024-09-14T03:51:46.795 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.bytes_per_sync: 0 2024-09-14T03:51:46.795 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-14T03:51:46.795 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-14T03:51:46.795 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.compaction_readahead_size: 0 2024-09-14T03:51:46.795 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.max_background_flushes: -1 2024-09-14T03:51:46.795 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Compression algorithms supported: 2024-09-14T03:51:46.795 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: kZSTD supported: 0 2024-09-14T03:51:46.795 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: kXpressCompression supported: 0 2024-09-14T03:51:46.795 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: kBZip2Compression supported: 0 2024-09-14T03:51:46.795 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-14T03:51:46.795 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: kLZ4Compression supported: 1 2024-09-14T03:51:46.795 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: kZlibCompression supported: 1 2024-09-14T03:51:46.795 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: kLZ4HCCompression supported: 1 2024-09-14T03:51:46.796 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: kSnappyCompression supported: 1 2024-09-14T03:51:46.796 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-14T03:51:46.796 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-14T03:51:46.796 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-09-14T03:51:46.796 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-14T03:51:46.796 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-14T03:51:46.796 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.merge_operator: 2024-09-14T03:51:46.796 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.compaction_filter: None 2024-09-14T03:51:46.796 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.compaction_filter_factory: None 2024-09-14T03:51:46.796 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.sst_partitioner_factory: None 2024-09-14T03:51:46.796 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-14T03:51:46.796 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-14T03:51:46.797 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ef71aff320) 2024-09-14T03:51:46.797 INFO:journalctl@ceph.mon.c.smithi070.stdout: cache_index_and_filter_blocks: 1 2024-09-14T03:51:46.797 INFO:journalctl@ceph.mon.c.smithi070.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-14T03:51:46.797 INFO:journalctl@ceph.mon.c.smithi070.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-14T03:51:46.797 INFO:journalctl@ceph.mon.c.smithi070.stdout: pin_top_level_index_and_filter: 1 2024-09-14T03:51:46.797 INFO:journalctl@ceph.mon.c.smithi070.stdout: index_type: 0 2024-09-14T03:51:46.797 INFO:journalctl@ceph.mon.c.smithi070.stdout: data_block_index_type: 0 2024-09-14T03:51:46.797 INFO:journalctl@ceph.mon.c.smithi070.stdout: index_shortening: 1 2024-09-14T03:51:46.797 INFO:journalctl@ceph.mon.c.smithi070.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-14T03:51:46.797 INFO:journalctl@ceph.mon.c.smithi070.stdout: checksum: 4 2024-09-14T03:51:46.797 INFO:journalctl@ceph.mon.c.smithi070.stdout: no_block_cache: 0 2024-09-14T03:51:46.797 INFO:journalctl@ceph.mon.c.smithi070.stdout: block_cache: 0x55ef71af3610 2024-09-14T03:51:46.797 INFO:journalctl@ceph.mon.c.smithi070.stdout: block_cache_name: BinnedLRUCache 2024-09-14T03:51:46.798 INFO:journalctl@ceph.mon.c.smithi070.stdout: block_cache_options: 2024-09-14T03:51:46.798 INFO:journalctl@ceph.mon.c.smithi070.stdout: capacity : 536870912 2024-09-14T03:51:46.798 INFO:journalctl@ceph.mon.c.smithi070.stdout: num_shard_bits : 4 2024-09-14T03:51:46.798 INFO:journalctl@ceph.mon.c.smithi070.stdout: strict_capacity_limit : 0 2024-09-14T03:51:46.798 INFO:journalctl@ceph.mon.c.smithi070.stdout: high_pri_pool_ratio: 0.000 2024-09-14T03:51:46.798 INFO:journalctl@ceph.mon.c.smithi070.stdout: block_cache_compressed: (nil) 2024-09-14T03:51:46.798 INFO:journalctl@ceph.mon.c.smithi070.stdout: persistent_cache: (nil) 2024-09-14T03:51:46.798 INFO:journalctl@ceph.mon.c.smithi070.stdout: block_size: 4096 2024-09-14T03:51:46.798 INFO:journalctl@ceph.mon.c.smithi070.stdout: block_size_deviation: 10 2024-09-14T03:51:46.798 INFO:journalctl@ceph.mon.c.smithi070.stdout: block_restart_interval: 16 2024-09-14T03:51:46.798 INFO:journalctl@ceph.mon.c.smithi070.stdout: index_block_restart_interval: 1 2024-09-14T03:51:46.798 INFO:journalctl@ceph.mon.c.smithi070.stdout: metadata_block_size: 4096 2024-09-14T03:51:46.798 INFO:journalctl@ceph.mon.c.smithi070.stdout: partition_filters: 0 2024-09-14T03:51:46.798 INFO:journalctl@ceph.mon.c.smithi070.stdout: use_delta_encoding: 1 2024-09-14T03:51:46.799 INFO:journalctl@ceph.mon.c.smithi070.stdout: filter_policy: bloomfilter 2024-09-14T03:51:46.799 INFO:journalctl@ceph.mon.c.smithi070.stdout: whole_key_filtering: 1 2024-09-14T03:51:46.799 INFO:journalctl@ceph.mon.c.smithi070.stdout: verify_compression: 0 2024-09-14T03:51:46.799 INFO:journalctl@ceph.mon.c.smithi070.stdout: read_amp_bytes_per_bit: 0 2024-09-14T03:51:46.799 INFO:journalctl@ceph.mon.c.smithi070.stdout: format_version: 5 2024-09-14T03:51:46.799 INFO:journalctl@ceph.mon.c.smithi070.stdout: enable_index_compression: 1 2024-09-14T03:51:46.799 INFO:journalctl@ceph.mon.c.smithi070.stdout: block_align: 0 2024-09-14T03:51:46.799 INFO:journalctl@ceph.mon.c.smithi070.stdout: max_auto_readahead_size: 262144 2024-09-14T03:51:46.799 INFO:journalctl@ceph.mon.c.smithi070.stdout: prepopulate_block_cache: 0 2024-09-14T03:51:46.799 INFO:journalctl@ceph.mon.c.smithi070.stdout: initial_auto_readahead_size: 8192 2024-09-14T03:51:46.799 INFO:journalctl@ceph.mon.c.smithi070.stdout: num_file_reads_for_auto_readahead: 2 2024-09-14T03:51:46.799 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.write_buffer_size: 33554432 2024-09-14T03:51:46.799 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.max_write_buffer_number: 2 2024-09-14T03:51:46.799 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.compression: NoCompression 2024-09-14T03:51:46.800 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.bottommost_compression: Disabled 2024-09-14T03:51:46.800 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.prefix_extractor: nullptr 2024-09-14T03:51:46.800 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-14T03:51:46.800 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.num_levels: 7 2024-09-14T03:51:46.800 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-14T03:51:46.800 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-14T03:51:46.800 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-14T03:51:46.800 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-14T03:51:46.800 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-14T03:51:46.800 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-14T03:51:46.800 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-14T03:51:46.800 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:51:46.800 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-14T03:51:46.801 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-14T03:51:46.801 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-14T03:51:46.801 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-14T03:51:46.801 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-14T03:51:46.801 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.compression_opts.level: 32767 2024-09-14T03:51:46.801 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.compression_opts.strategy: 0 2024-09-14T03:51:46.801 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-14T03:51:46.801 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:51:46.801 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-14T03:51:46.801 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-14T03:51:46.801 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.compression_opts.enabled: false 2024-09-14T03:51:46.801 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-14T03:51:46.801 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-14T03:51:46.802 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-14T03:51:46.802 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-14T03:51:46.802 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.target_file_size_base: 67108864 2024-09-14T03:51:46.802 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-14T03:51:46.802 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-14T03:51:46.802 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-14T03:51:46.802 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-14T03:51:46.802 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-14T03:51:46.802 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-14T03:51:46.802 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-14T03:51:46.802 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-14T03:51:46.802 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-14T03:51:46.802 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-14T03:51:46.803 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-14T03:51:46.803 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-14T03:51:46.803 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-14T03:51:46.803 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-14T03:51:46.803 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.arena_block_size: 1048576 2024-09-14T03:51:46.803 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-14T03:51:46.803 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-14T03:51:46.803 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.disable_auto_compactions: 0 2024-09-14T03:51:46.803 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-14T03:51:46.803 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-14T03:51:46.803 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-14T03:51:46.803 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-14T03:51:46.803 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-14T03:51:46.803 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-14T03:51:46.804 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-14T03:51:46.804 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-14T03:51:46.804 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-14T03:51:46.804 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-14T03:51:46.804 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-14T03:51:46.804 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.inplace_update_support: 0 2024-09-14T03:51:46.804 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-14T03:51:46.804 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-14T03:51:46.804 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-14T03:51:46.804 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-14T03:51:46.804 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.bloom_locality: 0 2024-09-14T03:51:46.804 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.max_successive_merges: 0 2024-09-14T03:51:46.804 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-14T03:51:46.805 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.paranoid_file_checks: 0 2024-09-14T03:51:46.805 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.force_consistency_checks: 1 2024-09-14T03:51:46.805 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.report_bg_io_stats: 0 2024-09-14T03:51:46.805 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.ttl: 2592000 2024-09-14T03:51:46.805 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-14T03:51:46.805 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-14T03:51:46.805 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-14T03:51:46.805 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.enable_blob_files: false 2024-09-14T03:51:46.805 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.min_blob_size: 0 2024-09-14T03:51:46.805 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.blob_file_size: 268435456 2024-09-14T03:51:46.805 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-14T03:51:46.809 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-14T03:51:46.809 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-14T03:51:46.809 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-14T03:51:46.809 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-14T03:51:46.809 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.blob_file_starting_level: 0 2024-09-14T03:51:46.809 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-14T03:51:46.809 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-09-14T03:51:46.809 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-14T03:51:46.810 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13412, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-14T03:51:46.810 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-09-14T03:51:46.810 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: cea3bb85-e0bf-4013-b011-cf1145605315 2024-09-14T03:51:46.810 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285906341101, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-09-14T03:51:46.810 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-09-14T03:51:46.810 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285906366332, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 7365943, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13417, "largest_seqno": 17470, "table_properties": {"data_size": 7347681, "index_size": 12735, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 4485, "raw_key_size": 40872, "raw_average_key_size": 23, "raw_value_size": 7310186, "raw_average_value_size": 4123, "num_data_blocks": 550, "num_entries": 1773, "num_filter_entries": 1773, "num_deletions": 298, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726285906, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "cea3bb85-e0bf-4013-b011-cf1145605315", "db_session_id": "H5HZLGEUKTXPPUR8YS6Q", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-09-14T03:51:46.810 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285906366456, "job": 1, "event": "recovery_finished"} 2024-09-14T03:51:46.810 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-09-14T03:51:46.810 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-14T03:51:46.810 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-14T03:51:46.810 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55ef71b22e00 2024-09-14T03:51:46.810 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: DB pointer 0x55ef71c24000 2024-09-14T03:51:46.811 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-14T03:51:46.811 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-14T03:51:46.811 INFO:journalctl@ceph.mon.c.smithi070.stdout: ** DB Stats ** 2024-09-14T03:51:46.811 INFO:journalctl@ceph.mon.c.smithi070.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:51:46.811 INFO:journalctl@ceph.mon.c.smithi070.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-14T03:51:46.811 INFO:journalctl@ceph.mon.c.smithi070.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:51:46.811 INFO:journalctl@ceph.mon.c.smithi070.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:51:46.811 INFO:journalctl@ceph.mon.c.smithi070.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-14T03:51:46.811 INFO:journalctl@ceph.mon.c.smithi070.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:51:46.811 INFO:journalctl@ceph.mon.c.smithi070.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:51:46.811 INFO:journalctl@ceph.mon.c.smithi070.stdout: 2024-09-14T03:51:46.811 INFO:journalctl@ceph.mon.c.smithi070.stdout: ** Compaction Stats [default] ** 2024-09-14T03:51:46.811 INFO:journalctl@ceph.mon.c.smithi070.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-14T03:51:46.812 INFO:journalctl@ceph.mon.c.smithi070.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-14T03:51:46.812 INFO:journalctl@ceph.mon.c.smithi070.stdout: L0 1/0 7.02 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 694.7 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-09-14T03:51:46.812 INFO:journalctl@ceph.mon.c.smithi070.stdout: L6 1/0 7.63 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-14T03:51:46.812 INFO:journalctl@ceph.mon.c.smithi070.stdout: Sum 2/0 14.65 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 694.7 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-09-14T03:51:46.812 INFO:journalctl@ceph.mon.c.smithi070.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 694.7 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-09-14T03:51:46.812 INFO:journalctl@ceph.mon.c.smithi070.stdout: 2024-09-14T03:51:46.812 INFO:journalctl@ceph.mon.c.smithi070.stdout: ** Compaction Stats [default] ** 2024-09-14T03:51:46.812 INFO:journalctl@ceph.mon.c.smithi070.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-14T03:51:46.812 INFO:journalctl@ceph.mon.c.smithi070.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-14T03:51:46.812 INFO:journalctl@ceph.mon.c.smithi070.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 694.7 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-09-14T03:51:46.812 INFO:journalctl@ceph.mon.c.smithi070.stdout: 2024-09-14T03:51:46.812 INFO:journalctl@ceph.mon.c.smithi070.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-14T03:51:46.812 INFO:journalctl@ceph.mon.c.smithi070.stdout: 2024-09-14T03:51:46.813 INFO:journalctl@ceph.mon.c.smithi070.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:51:46.813 INFO:journalctl@ceph.mon.c.smithi070.stdout: Flush(GB): cumulative 0.007, interval 0.007 2024-09-14T03:51:46.813 INFO:journalctl@ceph.mon.c.smithi070.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-14T03:51:46.813 INFO:journalctl@ceph.mon.c.smithi070.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-14T03:51:46.813 INFO:journalctl@ceph.mon.c.smithi070.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-14T03:51:46.813 INFO:journalctl@ceph.mon.c.smithi070.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-14T03:51:46.813 INFO:journalctl@ceph.mon.c.smithi070.stdout: Cumulative compaction: 0.01 GB write, 248.95 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:51:46.813 INFO:journalctl@ceph.mon.c.smithi070.stdout: Interval compaction: 0.01 GB write, 248.95 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:51:46.813 INFO:journalctl@ceph.mon.c.smithi070.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-14T03:51:46.813 INFO:journalctl@ceph.mon.c.smithi070.stdout: Block cache BinnedLRUCache@0x55ef71af3610#7 capacity: 512.00 MB usage: 17.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-14T03:51:46.813 INFO:journalctl@ceph.mon.c.smithi070.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,4.53 KB,0.000864267%) IndexBlock(1,13.08 KB,0.00249445%) Misc(1,0.00 KB,0%) 2024-09-14T03:51:46.813 INFO:journalctl@ceph.mon.c.smithi070.stdout: 2024-09-14T03:51:46.814 INFO:journalctl@ceph.mon.c.smithi070.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-14T03:51:46.814 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: starting mon.c rank 2 at public addrs [v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0] at bind addrs [v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:51:46.814 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: mon.c@-1(???) e3 preinit fsid b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:51:46.814 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: mon.c@-1(???).mds e8 new map 2024-09-14T03:51:46.814 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: mon.c@-1(???).mds e8 print_map 2024-09-14T03:51:46.814 INFO:journalctl@ceph.mon.c.smithi070.stdout: e8 2024-09-14T03:51:46.814 INFO:journalctl@ceph.mon.c.smithi070.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-14T03:51:46.814 INFO:journalctl@ceph.mon.c.smithi070.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-14T03:51:46.814 INFO:journalctl@ceph.mon.c.smithi070.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-14T03:51:46.814 INFO:journalctl@ceph.mon.c.smithi070.stdout: legacy client fscid: 1 2024-09-14T03:51:46.814 INFO:journalctl@ceph.mon.c.smithi070.stdout: 2024-09-14T03:51:46.814 INFO:journalctl@ceph.mon.c.smithi070.stdout: Filesystem 'cephfs' (1) 2024-09-14T03:51:46.814 INFO:journalctl@ceph.mon.c.smithi070.stdout: fs_name cephfs 2024-09-14T03:51:46.814 INFO:journalctl@ceph.mon.c.smithi070.stdout: epoch 8 2024-09-14T03:51:46.815 INFO:journalctl@ceph.mon.c.smithi070.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-14T03:51:46.815 INFO:journalctl@ceph.mon.c.smithi070.stdout: created 2024-09-14T03:44:03.568995+0000 2024-09-14T03:51:46.815 INFO:journalctl@ceph.mon.c.smithi070.stdout: modified 2024-09-14T03:44:16.900718+0000 2024-09-14T03:51:46.815 INFO:journalctl@ceph.mon.c.smithi070.stdout: tableserver 0 2024-09-14T03:51:46.815 INFO:journalctl@ceph.mon.c.smithi070.stdout: root 0 2024-09-14T03:51:46.815 INFO:journalctl@ceph.mon.c.smithi070.stdout: session_timeout 60 2024-09-14T03:51:46.815 INFO:journalctl@ceph.mon.c.smithi070.stdout: session_autoclose 300 2024-09-14T03:51:46.815 INFO:journalctl@ceph.mon.c.smithi070.stdout: max_file_size 1099511627776 2024-09-14T03:51:46.815 INFO:journalctl@ceph.mon.c.smithi070.stdout: max_xattr_size 65536 2024-09-14T03:51:46.815 INFO:journalctl@ceph.mon.c.smithi070.stdout: required_client_features {} 2024-09-14T03:51:46.815 INFO:journalctl@ceph.mon.c.smithi070.stdout: last_failure 0 2024-09-14T03:51:46.815 INFO:journalctl@ceph.mon.c.smithi070.stdout: last_failure_osd_epoch 0 2024-09-14T03:51:46.815 INFO:journalctl@ceph.mon.c.smithi070.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-14T03:51:46.816 INFO:journalctl@ceph.mon.c.smithi070.stdout: max_mds 1 2024-09-14T03:51:46.816 INFO:journalctl@ceph.mon.c.smithi070.stdout: in 0 2024-09-14T03:51:46.816 INFO:journalctl@ceph.mon.c.smithi070.stdout: up {0=24323} 2024-09-14T03:51:46.816 INFO:journalctl@ceph.mon.c.smithi070.stdout: failed 2024-09-14T03:51:46.816 INFO:journalctl@ceph.mon.c.smithi070.stdout: damaged 2024-09-14T03:51:46.816 INFO:journalctl@ceph.mon.c.smithi070.stdout: stopped 2024-09-14T03:51:46.816 INFO:journalctl@ceph.mon.c.smithi070.stdout: data_pools [3] 2024-09-14T03:51:46.816 INFO:journalctl@ceph.mon.c.smithi070.stdout: metadata_pool 2 2024-09-14T03:51:46.816 INFO:journalctl@ceph.mon.c.smithi070.stdout: inline_data disabled 2024-09-14T03:51:46.816 INFO:journalctl@ceph.mon.c.smithi070.stdout: balancer 2024-09-14T03:51:46.816 INFO:journalctl@ceph.mon.c.smithi070.stdout: bal_rank_mask -1 2024-09-14T03:51:46.816 INFO:journalctl@ceph.mon.c.smithi070.stdout: standby_count_wanted 0 2024-09-14T03:51:46.816 INFO:journalctl@ceph.mon.c.smithi070.stdout: qdb_cluster leader: 0 members: 2024-09-14T03:51:46.817 INFO:journalctl@ceph.mon.c.smithi070.stdout: [mds.a{0:24323} state up:active seq 6 addr [v2:172.21.15.70:6834/421917097,v1:172.21.15.70:6835/421917097] compat {c=[1],r=[1],i=[7ff]}] 2024-09-14T03:51:46.817 INFO:journalctl@ceph.mon.c.smithi070.stdout: 2024-09-14T03:51:46.817 INFO:journalctl@ceph.mon.c.smithi070.stdout: 2024-09-14T03:51:46.817 INFO:journalctl@ceph.mon.c.smithi070.stdout: Standby daemons: 2024-09-14T03:51:46.817 INFO:journalctl@ceph.mon.c.smithi070.stdout: 2024-09-14T03:51:46.817 INFO:journalctl@ceph.mon.c.smithi070.stdout: [mds.b{-1:24304} state up:standby seq 1 addr [v2:172.21.15.94:6832/3620660683,v1:172.21.15.94:6833/3620660683] compat {c=[1],r=[1],i=[7ff]}] 2024-09-14T03:51:46.817 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: mon.c@-1(???).osd e288 crush map has features 3314933069573799936, adjusting msgr requires 2024-09-14T03:51:46.817 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: mon.c@-1(???).osd e288 crush map has features 432629308056666112, adjusting msgr requires 2024-09-14T03:51:46.817 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: mon.c@-1(???).osd e288 crush map has features 432629308056666112, adjusting msgr requires 2024-09-14T03:51:46.817 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: mon.c@-1(???).osd e288 crush map has features 432629308056666112, adjusting msgr requires 2024-09-14T03:51:46.817 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 ceph-mon[98640]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-09-14T03:51:46.817 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 bash[98580]: 5741e4fcf4b689cbaec23247721f6e1bb963f63973edd270d98aec5e1d955239 2024-09-14T03:51:46.817 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:46 smithi070 systemd[1]: Started Ceph mon.c for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:51:46.851 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:51:46.851 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-09-14T03:51:46.851 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 1, 2024-09-14T03:51:46.851 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:51:46.851 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:51:46.851 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-09-14T03:51:46.851 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:51:46.851 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:51:46.851 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-09-14T03:51:46.851 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 8 2024-09-14T03:51:46.851 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:51:46.851 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": { 2024-09-14T03:51:46.852 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:51:46.852 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:51:46.852 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-09-14T03:51:46.852 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 11, 2024-09-14T03:51:46.852 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 4 2024-09-14T03:51:46.852 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-09-14T03:51:46.852 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:51:47.141 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:51:47.141 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:51:47.141 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-09-14T03:51:47.142 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:51:47.142 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [ 2024-09-14T03:51:47.142 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr" 2024-09-14T03:51:47.142 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2024-09-14T03:51:47.142 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "3/20 daemons upgraded", 2024-09-14T03:51:47.143 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "Currently upgrading mon daemons", 2024-09-14T03:51:47.143 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-09-14T03:51:47.143 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:51:47.277 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:46 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:46.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:51:47.402 INFO:tasks.rados.rados.0.smithi070.stdout:1146: done (5 left) 2024-09-14T03:51:47.402 INFO:tasks.rados.rados.0.smithi070.stdout:1147: done (4 left) 2024-09-14T03:51:47.402 INFO:tasks.rados.rados.0.smithi070.stdout:1148: done (3 left) 2024-09-14T03:51:47.402 INFO:tasks.rados.rados.0.smithi070.stdout:1149: done (2 left) 2024-09-14T03:51:47.402 INFO:tasks.rados.rados.0.smithi070.stdout:1150: done (1 left) 2024-09-14T03:51:47.402 INFO:tasks.rados.rados.0.smithi070.stdout:1151: done (0 left) 2024-09-14T03:51:47.402 INFO:tasks.rados.rados.0.smithi070.stdout:1152: read oid 48 snap -1 2024-09-14T03:51:47.402 INFO:tasks.rados.rados.0.smithi070.stdout:1152: expect (ObjNum 326 snap 108 seq_num 326) 2024-09-14T03:51:47.403 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 17 current snap is 126 2024-09-14T03:51:47.403 INFO:tasks.rados.rados.0.smithi070.stdout:1153: seq_num 369 ranges {3407872=589824,3997696=507904,4505600=294912} 2024-09-14T03:51:47.403 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[98640]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:47.404 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[96704]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:47.404 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[96704]: pgmap v22: 121 pgs: 121 active+clean; 308 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.5 MiB/s rd, 4.0 MiB/s wr, 36 op/s 2024-09-14T03:51:47.404 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[96704]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:47.404 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[96704]: mon.c calling monitor election 2024-09-14T03:51:47.404 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[96704]: mon.a calling monitor election 2024-09-14T03:51:47.404 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[96704]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:51:47.404 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[96704]: monmap epoch 3 2024-09-14T03:51:47.404 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[96704]: fsid b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:51:47.404 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[96704]: last_changed 2024-09-14T03:40:08.852192+0000 2024-09-14T03:51:47.405 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[96704]: created 2024-09-14T03:38:26.272229+0000 2024-09-14T03:51:47.405 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[96704]: min_mon_release 17 (quincy) 2024-09-14T03:51:47.405 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[96704]: election_strategy: 1 2024-09-14T03:51:47.405 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[96704]: 0: [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] mon.a 2024-09-14T03:51:47.405 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[96704]: 1: [v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0] mon.b 2024-09-14T03:51:47.405 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[96704]: 2: [v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0] mon.c 2024-09-14T03:51:47.405 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[96704]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:51:47.405 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[96704]: osdmap e288: 8 total, 8 up, 8 in 2024-09-14T03:51:47.405 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[96704]: mgrmap e43: y(active, since 14s), standbys: x 2024-09-14T03:51:47.405 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[96704]: overall HEALTH_OK 2024-09-14T03:51:47.405 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:47.405 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:47.406 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:51:47.406 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[96704]: from='client.34170 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:47.406 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/2800101495' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:51:47.406 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:51:47.426 INFO:tasks.rados.rados.0.smithi070.stdout:1153: writing smithi07085154-17 from 3407872 to 3997696 tid 1 2024-09-14T03:51:47.429 INFO:tasks.rados.rados.0.smithi070.stdout:1153: writing smithi07085154-17 from 3997696 to 4505600 tid 2 2024-09-14T03:51:47.432 INFO:tasks.rados.rados.0.smithi070.stdout:1153: writing smithi07085154-17 from 4505600 to 4800512 tid 3 2024-09-14T03:51:47.433 INFO:tasks.rados.rados.0.smithi070.stdout:1154: copy_from oid 9 from oid 10 current snap is 126 2024-09-14T03:51:47.433 INFO:tasks.rados.rados.0.smithi070.stdout:1152: expect (ObjNum 326 snap 108 seq_num 326) 2024-09-14T03:51:47.761 INFO:tasks.rados.rados.0.smithi070.stdout:1153: finishing write tid 1 to smithi07085154-17 2024-09-14T03:51:47.761 INFO:tasks.rados.rados.0.smithi070.stdout:1153: finishing write tid 2 to smithi07085154-17 2024-09-14T03:51:47.761 INFO:tasks.rados.rados.0.smithi070.stdout:1153: finishing write tid 3 to smithi07085154-17 2024-09-14T03:51:47.761 INFO:tasks.rados.rados.0.smithi070.stdout:1153: finishing write tid 4 to smithi07085154-17 2024-09-14T03:51:47.761 INFO:tasks.rados.rados.0.smithi070.stdout:1153: finishing write tid 5 to smithi07085154-17 2024-09-14T03:51:47.761 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 473 (ObjNum 369 snap 126 seq_num 369) dirty exists 2024-09-14T03:51:47.761 INFO:tasks.rados.rados.0.smithi070.stdout:1153: left oid 17 (ObjNum 369 snap 126 seq_num 369) 2024-09-14T03:51:47.761 INFO:tasks.rados.rados.0.smithi070.stdout:1154: finishing copy_from racing read to smithi07085154-9 2024-09-14T03:51:47.761 INFO:tasks.rados.rados.0.smithi070.stdout:1154: finishing copy_from to smithi07085154-9 2024-09-14T03:51:47.761 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 957 (ObjNum 317 snap 107 seq_num 317) dirty exists 2024-09-14T03:51:47.762 INFO:tasks.rados.rados.0.smithi070.stdout:1152: done (2 left) 2024-09-14T03:51:47.762 INFO:tasks.rados.rados.0.smithi070.stdout:1153: done (1 left) 2024-09-14T03:51:47.762 INFO:tasks.rados.rados.0.smithi070.stdout:1154: done (0 left) 2024-09-14T03:51:47.762 INFO:tasks.rados.rados.0.smithi070.stdout:1155: copy_from oid 49 from oid 9 current snap is 126 2024-09-14T03:51:47.762 INFO:tasks.rados.rados.0.smithi070.stdout:1156: read oid 33 snap -1 2024-09-14T03:51:47.762 INFO:tasks.rados.rados.0.smithi070.stdout:1156: expect (ObjNum 319 snap 107 seq_num 319) 2024-09-14T03:51:47.762 INFO:tasks.rados.rados.0.smithi070.stdout:1157: setattr oid 18 current snap is 126 2024-09-14T03:51:47.762 INFO:tasks.rados.rados.0.smithi070.stdout:1158: rollback oid 30 current snap is 126 2024-09-14T03:51:47.762 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 30 to 119 2024-09-14T03:51:47.762 INFO:tasks.rados.rados.0.smithi070.stdout:1159: rollback oid 21 current snap is 126 2024-09-14T03:51:47.762 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 21 to 122 2024-09-14T03:51:47.762 INFO:tasks.rados.rados.0.smithi070.stdout:1160: read oid 28 snap -1 2024-09-14T03:51:47.762 INFO:tasks.rados.rados.0.smithi070.stdout:1160: expect (ObjNum 160 snap 47 seq_num 160) 2024-09-14T03:51:47.763 INFO:tasks.rados.rados.0.smithi070.stdout:1161: rollback oid 39 current snap is 126 2024-09-14T03:51:47.763 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 39 to 73 2024-09-14T03:51:47.763 INFO:tasks.rados.rados.0.smithi070.stdout:1162: delete oid 38 current snap is 126 2024-09-14T03:51:47.766 INFO:tasks.rados.rados.0.smithi070.stdout:1156: expect (ObjNum 319 snap 107 seq_num 319) 2024-09-14T03:51:47.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:47 smithi094 ceph-mon[31053]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:47.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:47 smithi094 ceph-mon[31053]: pgmap v22: 121 pgs: 121 active+clean; 308 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.5 MiB/s rd, 4.0 MiB/s wr, 36 op/s 2024-09-14T03:51:47.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:47 smithi094 ceph-mon[31053]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:47.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:47 smithi094 ceph-mon[31053]: mon.c calling monitor election 2024-09-14T03:51:47.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:47 smithi094 ceph-mon[31053]: mon.a calling monitor election 2024-09-14T03:51:47.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:47 smithi094 ceph-mon[31053]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:51:47.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:47 smithi094 ceph-mon[31053]: monmap epoch 3 2024-09-14T03:51:47.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:47 smithi094 ceph-mon[31053]: fsid b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:51:47.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:47 smithi094 ceph-mon[31053]: last_changed 2024-09-14T03:40:08.852192+0000 2024-09-14T03:51:47.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:47 smithi094 ceph-mon[31053]: created 2024-09-14T03:38:26.272229+0000 2024-09-14T03:51:47.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:47 smithi094 ceph-mon[31053]: min_mon_release 17 (quincy) 2024-09-14T03:51:47.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:47 smithi094 ceph-mon[31053]: election_strategy: 1 2024-09-14T03:51:47.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:47 smithi094 ceph-mon[31053]: 0: [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] mon.a 2024-09-14T03:51:47.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:47 smithi094 ceph-mon[31053]: 1: [v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0] mon.b 2024-09-14T03:51:47.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:47 smithi094 ceph-mon[31053]: 2: [v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0] mon.c 2024-09-14T03:51:47.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:47 smithi094 ceph-mon[31053]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:51:47.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:47 smithi094 ceph-mon[31053]: osdmap e288: 8 total, 8 up, 8 in 2024-09-14T03:51:47.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:47 smithi094 ceph-mon[31053]: mgrmap e43: y(active, since 14s), standbys: x 2024-09-14T03:51:47.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:47 smithi094 ceph-mon[31053]: overall HEALTH_OK 2024-09-14T03:51:47.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:47 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:47.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:47 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:47.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:47 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:51:47.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:47 smithi094 ceph-mon[31053]: from='client.34170 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:47.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:47 smithi094 ceph-mon[31053]: from='client.? 172.21.15.70:0/2800101495' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:51:47.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:47 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:51:47.783 INFO:tasks.rados.rados.0.smithi070.stdout:1158: finishing rollback tid 0 to smithi07085154-30 2024-09-14T03:51:47.783 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 811 (ObjNum 114 snap 24 seq_num 114) dirty exists 2024-09-14T03:51:47.783 INFO:tasks.rados.rados.0.smithi070.stdout:1161: finishing rollback tid 0 to smithi07085154-39 2024-09-14T03:51:47.783 INFO:tasks.rados.rados.0.smithi070.stdout:1158: finishing rollback tid 1 to smithi07085154-30 2024-09-14T03:51:47.783 INFO:tasks.rados.rados.0.smithi070.stdout:1159: finishing rollback tid 0 to smithi07085154-21 2024-09-14T03:51:47.783 INFO:tasks.rados.rados.0.smithi070.stdout:1159: finishing rollback tid 1 to smithi07085154-21 2024-09-14T03:51:47.783 INFO:tasks.rados.rados.0.smithi070.stdout:1156: done (7 left) 2024-09-14T03:51:47.783 INFO:tasks.rados.rados.0.smithi070.stdout:1157: done (6 left) 2024-09-14T03:51:47.783 INFO:tasks.rados.rados.0.smithi070.stdout:1162: done (5 left) 2024-09-14T03:51:47.783 INFO:tasks.rados.rados.0.smithi070.stdout:1163: copy_from oid 15 from oid 50 current snap is 126 2024-09-14T03:51:47.783 INFO:tasks.rados.rados.0.smithi070.stdout:1159: finishing rollback tid 2 to smithi07085154-21 2024-09-14T03:51:47.783 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 545 (ObjNum 114 snap 24 seq_num 114) dirty exists 2024-09-14T03:51:47.784 INFO:tasks.rados.rados.0.smithi070.stdout:1158: finishing rollback tid 2 to smithi07085154-30 2024-09-14T03:51:47.784 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 30 v 402 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-09-14T03:51:47.784 INFO:tasks.rados.rados.0.smithi070.stdout:1161: finishing rollback tid 1 to smithi07085154-39 2024-09-14T03:51:47.784 INFO:tasks.rados.rados.0.smithi070.stdout:1161: finishing rollback tid 2 to smithi07085154-39 2024-09-14T03:51:47.784 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 39 v 311 (ObjNum 211 snap 60 seq_num 211) dirty exists 2024-09-14T03:51:47.784 INFO:tasks.rados.rados.0.smithi070.stdout:1155: finishing copy_from racing read to smithi07085154-49 2024-09-14T03:51:47.784 INFO:tasks.rados.rados.0.smithi070.stdout:1160: expect (ObjNum 160 snap 47 seq_num 160) 2024-09-14T03:51:47.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[98640]: pgmap v22: 121 pgs: 121 active+clean; 308 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.5 MiB/s rd, 4.0 MiB/s wr, 36 op/s 2024-09-14T03:51:47.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[98640]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:47.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[98640]: mon.c calling monitor election 2024-09-14T03:51:47.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[98640]: mon.a calling monitor election 2024-09-14T03:51:47.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[98640]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:51:47.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[98640]: monmap epoch 3 2024-09-14T03:51:47.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[98640]: fsid b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:51:47.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[98640]: last_changed 2024-09-14T03:40:08.852192+0000 2024-09-14T03:51:47.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[98640]: created 2024-09-14T03:38:26.272229+0000 2024-09-14T03:51:47.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[98640]: min_mon_release 17 (quincy) 2024-09-14T03:51:47.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[98640]: election_strategy: 1 2024-09-14T03:51:47.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[98640]: 0: [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] mon.a 2024-09-14T03:51:47.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[98640]: 1: [v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0] mon.b 2024-09-14T03:51:47.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[98640]: 2: [v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0] mon.c 2024-09-14T03:51:47.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[98640]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:51:47.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[98640]: osdmap e288: 8 total, 8 up, 8 in 2024-09-14T03:51:47.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[98640]: mgrmap e43: y(active, since 14s), standbys: x 2024-09-14T03:51:47.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[98640]: overall HEALTH_OK 2024-09-14T03:51:47.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:47.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:47.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:51:47.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[98640]: from='client.34170 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:47.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/2800101495' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:51:47.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:47 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:51:47.918 INFO:tasks.rados.rados.0.smithi070.stdout:1155: finishing copy_from to smithi07085154-49 2024-09-14T03:51:47.918 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 49 v 957 (ObjNum 317 snap 107 seq_num 317) dirty exists 2024-09-14T03:51:47.918 INFO:tasks.rados.rados.0.smithi070.stdout:1163: finishing copy_from racing read to smithi07085154-15 2024-09-14T03:51:47.918 INFO:tasks.rados.rados.0.smithi070.stdout:1155: done (5 left) 2024-09-14T03:51:47.918 INFO:tasks.rados.rados.0.smithi070.stdout:1158: done (4 left) 2024-09-14T03:51:47.918 INFO:tasks.rados.rados.0.smithi070.stdout:1159: done (3 left) 2024-09-14T03:51:47.918 INFO:tasks.rados.rados.0.smithi070.stdout:1160: done (2 left) 2024-09-14T03:51:47.918 INFO:tasks.rados.rados.0.smithi070.stdout:1161: done (1 left) 2024-09-14T03:51:47.918 INFO:tasks.rados.rados.0.smithi070.stdout:1164: setattr oid 3 current snap is 126 2024-09-14T03:51:47.919 INFO:tasks.rados.rados.0.smithi070.stdout:1163: finishing copy_from to smithi07085154-15 2024-09-14T03:51:47.919 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 15 v 588 (ObjNum 202 snap 56 seq_num 202) dirty exists 2024-09-14T03:51:47.919 INFO:tasks.rados.rados.0.smithi070.stdout:1163: done (1 left) 2024-09-14T03:51:47.919 INFO:tasks.rados.rados.0.smithi070.stdout:1165: snap_create 2024-09-14T03:51:47.921 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 3 v 405 (ObjNum 225 snap 0 seq_num 4194278832) dirty exists 2024-09-14T03:51:48.412 INFO:tasks.rados.rados.0.smithi070.stdout:1164: done (1 left) 2024-09-14T03:51:48.412 INFO:tasks.rados.rados.0.smithi070.stdout:1165: done (0 left) 2024-09-14T03:51:48.412 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 6 current snap is 127 2024-09-14T03:51:48.412 INFO:tasks.rados.rados.0.smithi070.stdout:1166: seq_num 371 ranges {1794048=622592,2416640=679936,3096576=122880} 2024-09-14T03:51:48.424 INFO:tasks.rados.rados.0.smithi070.stdout:1166: writing smithi07085154-6 from 1794048 to 2416640 tid 1 2024-09-14T03:51:48.429 INFO:tasks.rados.rados.0.smithi070.stdout:1166: writing smithi07085154-6 from 2416640 to 3096576 tid 2 2024-09-14T03:51:48.431 INFO:tasks.rados.rados.0.smithi070.stdout:1166: writing smithi07085154-6 from 3096576 to 3219456 tid 3 2024-09-14T03:51:48.431 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 44 current snap is 127 2024-09-14T03:51:48.431 INFO:tasks.rados.rados.0.smithi070.stdout:1166: finishing write tid 1 to smithi07085154-6 2024-09-14T03:51:48.431 INFO:tasks.rados.rados.0.smithi070.stdout:1167: seq_num 372 ranges {3612672=638976,4251648=565248,4816896=237568} 2024-09-14T03:51:48.454 INFO:tasks.rados.rados.0.smithi070.stdout:1167: writing smithi07085154-44 from 3612672 to 4251648 tid 1 2024-09-14T03:51:48.457 INFO:tasks.rados.rados.0.smithi070.stdout:1167: writing smithi07085154-44 from 4251648 to 4816896 tid 2 2024-09-14T03:51:48.459 INFO:tasks.rados.rados.0.smithi070.stdout:1167: writing smithi07085154-44 from 4816896 to 5054464 tid 3 2024-09-14T03:51:48.459 INFO:tasks.rados.rados.0.smithi070.stdout:1168: read oid 50 snap -1 2024-09-14T03:51:48.459 INFO:tasks.rados.rados.0.smithi070.stdout:1168: expect (ObjNum 202 snap 56 seq_num 202) 2024-09-14T03:51:48.459 INFO:tasks.rados.rados.0.smithi070.stdout:1169: delete oid 31 current snap is 127 2024-09-14T03:51:48.459 INFO:tasks.rados.rados.0.smithi070.stdout:1166: finishing write tid 2 to smithi07085154-6 2024-09-14T03:51:48.459 INFO:tasks.rados.rados.0.smithi070.stdout:1166: finishing write tid 3 to smithi07085154-6 2024-09-14T03:51:48.460 INFO:tasks.rados.rados.0.smithi070.stdout:1166: finishing write tid 4 to smithi07085154-6 2024-09-14T03:51:48.460 INFO:tasks.rados.rados.0.smithi070.stdout:1166: finishing write tid 5 to smithi07085154-6 2024-09-14T03:51:48.460 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 478 (ObjNum 371 snap 127 seq_num 371) dirty exists 2024-09-14T03:51:48.460 INFO:tasks.rados.rados.0.smithi070.stdout:1166: left oid 6 (ObjNum 371 snap 127 seq_num 371) 2024-09-14T03:51:48.460 INFO:tasks.rados.rados.0.smithi070.stdout:1167: finishing write tid 1 to smithi07085154-44 2024-09-14T03:51:48.461 INFO:tasks.rados.rados.0.smithi070.stdout:1166: done (3 left) 2024-09-14T03:51:48.462 INFO:tasks.rados.rados.0.smithi070.stdout:1169: done (2 left) 2024-09-14T03:51:48.462 INFO:tasks.rados.rados.0.smithi070.stdout:1170: rollback oid 29 current snap is 127 2024-09-14T03:51:48.462 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 29 to 105 2024-09-14T03:51:48.462 INFO:tasks.rados.rados.0.smithi070.stdout:1171: delete oid 37 current snap is 127 2024-09-14T03:51:48.464 INFO:tasks.rados.rados.0.smithi070.stdout:1171: done (3 left) 2024-09-14T03:51:48.464 INFO:tasks.rados.rados.0.smithi070.stdout:1172: snap_create 2024-09-14T03:51:48.465 INFO:tasks.rados.rados.0.smithi070.stdout:1167: finishing write tid 2 to smithi07085154-44 2024-09-14T03:51:48.467 INFO:tasks.rados.rados.0.smithi070.stdout:1167: finishing write tid 3 to smithi07085154-44 2024-09-14T03:51:48.467 INFO:tasks.rados.rados.0.smithi070.stdout:1167: finishing write tid 4 to smithi07085154-44 2024-09-14T03:51:48.468 INFO:tasks.rados.rados.0.smithi070.stdout:1168: expect (ObjNum 202 snap 56 seq_num 202) 2024-09-14T03:51:48.536 INFO:tasks.rados.rados.0.smithi070.stdout:1170: finishing rollback tid 0 to smithi07085154-29 2024-09-14T03:51:48.536 INFO:tasks.rados.rados.0.smithi070.stdout:1167: finishing write tid 5 to smithi07085154-44 2024-09-14T03:51:48.537 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 400 (ObjNum 372 snap 127 seq_num 372) dirty exists 2024-09-14T03:51:48.537 INFO:tasks.rados.rados.0.smithi070.stdout:1167: left oid 44 (ObjNum 372 snap 127 seq_num 372) 2024-09-14T03:51:48.537 INFO:tasks.rados.rados.0.smithi070.stdout:1170: finishing rollback tid 1 to smithi07085154-29 2024-09-14T03:51:48.537 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 597 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-14T03:51:48.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:48 smithi094 ceph-mon[31053]: from='client.34182 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:48.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:48 smithi094 ceph-mon[31053]: osdmap e289: 8 total, 8 up, 8 in 2024-09-14T03:51:48.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:48 smithi094 ceph-mon[31053]: pgmap v24: 121 pgs: 121 active+clean; 308 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.3 MiB/s rd, 3.4 MiB/s wr, 30 op/s 2024-09-14T03:51:48.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:48 smithi070 ceph-mon[96704]: from='client.34182 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:48.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:48 smithi070 ceph-mon[96704]: osdmap e289: 8 total, 8 up, 8 in 2024-09-14T03:51:48.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:48 smithi070 ceph-mon[96704]: pgmap v24: 121 pgs: 121 active+clean; 308 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.3 MiB/s rd, 3.4 MiB/s wr, 30 op/s 2024-09-14T03:51:48.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:48 smithi070 ceph-mon[98640]: from='client.34182 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:48.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:48 smithi070 ceph-mon[98640]: osdmap e289: 8 total, 8 up, 8 in 2024-09-14T03:51:48.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:48 smithi070 ceph-mon[98640]: pgmap v24: 121 pgs: 121 active+clean; 308 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.3 MiB/s rd, 3.4 MiB/s wr, 30 op/s 2024-09-14T03:51:49.412 INFO:tasks.rados.rados.0.smithi070.stdout:1167: done (3 left) 2024-09-14T03:51:49.412 INFO:tasks.rados.rados.0.smithi070.stdout:1168: done (2 left) 2024-09-14T03:51:49.412 INFO:tasks.rados.rados.0.smithi070.stdout:1170: done (1 left) 2024-09-14T03:51:49.413 INFO:tasks.rados.rados.0.smithi070.stdout:1172: done (0 left) 2024-09-14T03:51:49.413 INFO:tasks.rados.rados.0.smithi070.stdout:1173: read oid 12 snap -1 2024-09-14T03:51:49.413 INFO:tasks.rados.rados.0.smithi070.stdout:1173: expect (ObjNum 305 snap 97 seq_num 305) 2024-09-14T03:51:49.413 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 25 current snap is 128 2024-09-14T03:51:49.413 INFO:tasks.rados.rados.0.smithi070.stdout:1174: seq_num 375 ranges {9814016=688128,10502144=663552,11165696=139264} 2024-09-14T03:51:49.467 INFO:tasks.rados.rados.0.smithi070.stdout:1174: writing smithi07085154-25 from 9814016 to 10502144 tid 1 2024-09-14T03:51:49.470 INFO:tasks.rados.rados.0.smithi070.stdout:1174: writing smithi07085154-25 from 10502144 to 11165696 tid 2 2024-09-14T03:51:49.471 INFO:tasks.rados.rados.0.smithi070.stdout:1174: writing smithi07085154-25 from 11165696 to 11304960 tid 3 2024-09-14T03:51:49.472 INFO:tasks.rados.rados.0.smithi070.stdout:1175: rmattr oid 5 current snap is 128 2024-09-14T03:51:49.472 INFO:tasks.rados.rados.0.smithi070.stdout:1173: expect (ObjNum 305 snap 97 seq_num 305) 2024-09-14T03:51:49.481 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 0 (ObjNum 362 snap 124 seq_num 362) dirty exists 2024-09-14T03:51:49.481 INFO:tasks.rados.rados.0.smithi070.stdout:1174: finishing write tid 1 to smithi07085154-25 2024-09-14T03:51:49.481 INFO:tasks.rados.rados.0.smithi070.stdout:1173: done (2 left) 2024-09-14T03:51:49.481 INFO:tasks.rados.rados.0.smithi070.stdout:1175: done (1 left) 2024-09-14T03:51:49.482 INFO:tasks.rados.rados.0.smithi070.stdout:1176: snap_remove snap 71 2024-09-14T03:51:49.759 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:49 smithi070 ceph-mon[96704]: osdmap e290: 8 total, 8 up, 8 in 2024-09-14T03:51:49.759 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:49 smithi070 ceph-mon[98640]: osdmap e290: 8 total, 8 up, 8 in 2024-09-14T03:51:49.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:49 smithi094 ceph-mon[31053]: osdmap e290: 8 total, 8 up, 8 in 2024-09-14T03:51:50.423 INFO:tasks.rados.rados.0.smithi070.stdout:1176: done (1 left) 2024-09-14T03:51:50.423 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 30 current snap is 128 2024-09-14T03:51:50.423 INFO:tasks.rados.rados.0.smithi070.stdout:1177: seq_num 376 ranges {491520=704512,1196032=548864,1744896=253952} 2024-09-14T03:51:50.430 INFO:tasks.rados.rados.0.smithi070.stdout:1177: writing smithi07085154-30 from 491520 to 1196032 tid 1 2024-09-14T03:51:50.434 INFO:tasks.rados.rados.0.smithi070.stdout:1177: writing smithi07085154-30 from 1196032 to 1744896 tid 2 2024-09-14T03:51:50.436 INFO:tasks.rados.rados.0.smithi070.stdout:1177: writing smithi07085154-30 from 1744896 to 1998848 tid 3 2024-09-14T03:51:50.436 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 29 current snap is 128 2024-09-14T03:51:50.436 INFO:tasks.rados.rados.0.smithi070.stdout:1174: finishing write tid 2 to smithi07085154-25 2024-09-14T03:51:50.437 INFO:tasks.rados.rados.0.smithi070.stdout:1174: finishing write tid 3 to smithi07085154-25 2024-09-14T03:51:50.437 INFO:tasks.rados.rados.0.smithi070.stdout:1174: finishing write tid 4 to smithi07085154-25 2024-09-14T03:51:50.437 INFO:tasks.rados.rados.0.smithi070.stdout:1174: finishing write tid 5 to smithi07085154-25 2024-09-14T03:51:50.437 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 484 (ObjNum 375 snap 128 seq_num 375) dirty exists 2024-09-14T03:51:50.437 INFO:tasks.rados.rados.0.smithi070.stdout:1174: left oid 25 (ObjNum 375 snap 128 seq_num 375) 2024-09-14T03:51:50.437 INFO:tasks.rados.rados.0.smithi070.stdout:1178: seq_num 377 ranges {0=720896,720896=434176,1155072=368640} 2024-09-14T03:51:50.441 INFO:tasks.rados.rados.0.smithi070.stdout:1178: writing smithi07085154-29 from 0 to 720896 tid 1 2024-09-14T03:51:50.445 INFO:tasks.rados.rados.0.smithi070.stdout:1178: writing smithi07085154-29 from 720896 to 1155072 tid 2 2024-09-14T03:51:50.448 INFO:tasks.rados.rados.0.smithi070.stdout:1178: writing smithi07085154-29 from 1155072 to 1523712 tid 3 2024-09-14T03:51:50.448 INFO:tasks.rados.rados.0.smithi070.stdout:1174: done (2 left) 2024-09-14T03:51:50.448 INFO:tasks.rados.rados.0.smithi070.stdout:1179: snap_create 2024-09-14T03:51:50.448 INFO:tasks.rados.rados.0.smithi070.stdout:1177: finishing write tid 1 to smithi07085154-30 2024-09-14T03:51:50.448 INFO:tasks.rados.rados.0.smithi070.stdout:1177: finishing write tid 2 to smithi07085154-30 2024-09-14T03:51:50.448 INFO:tasks.rados.rados.0.smithi070.stdout:1177: finishing write tid 3 to smithi07085154-30 2024-09-14T03:51:50.448 INFO:tasks.rados.rados.0.smithi070.stdout:1177: finishing write tid 4 to smithi07085154-30 2024-09-14T03:51:50.448 INFO:tasks.rados.rados.0.smithi070.stdout:1177: finishing write tid 5 to smithi07085154-30 2024-09-14T03:51:50.448 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 30 v 416 (ObjNum 376 snap 128 seq_num 376) dirty exists 2024-09-14T03:51:50.448 INFO:tasks.rados.rados.0.smithi070.stdout:1177: left oid 30 (ObjNum 376 snap 128 seq_num 376) 2024-09-14T03:51:50.448 INFO:tasks.rados.rados.0.smithi070.stdout:1178: finishing write tid 1 to smithi07085154-29 2024-09-14T03:51:50.449 INFO:tasks.rados.rados.0.smithi070.stdout:1178: finishing write tid 2 to smithi07085154-29 2024-09-14T03:51:50.451 INFO:tasks.rados.rados.0.smithi070.stdout:1178: finishing write tid 3 to smithi07085154-29 2024-09-14T03:51:50.451 INFO:tasks.rados.rados.0.smithi070.stdout:1178: finishing write tid 4 to smithi07085154-29 2024-09-14T03:51:50.451 INFO:tasks.rados.rados.0.smithi070.stdout:1178: finishing write tid 5 to smithi07085154-29 2024-09-14T03:51:50.452 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 609 (ObjNum 377 snap 128 seq_num 377) dirty exists 2024-09-14T03:51:50.452 INFO:tasks.rados.rados.0.smithi070.stdout:1178: left oid 29 (ObjNum 377 snap 128 seq_num 377) 2024-09-14T03:51:50.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:50 smithi094 ceph-mon[31053]: osdmap e291: 8 total, 8 up, 8 in 2024-09-14T03:51:50.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:50 smithi094 ceph-mon[31053]: pgmap v27: 121 pgs: 121 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 1.6 MiB/s wr, 24 op/s 2024-09-14T03:51:50.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:50 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:50.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:50 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:50.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:50 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:51:50.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:50 smithi070 ceph-mon[96704]: osdmap e291: 8 total, 8 up, 8 in 2024-09-14T03:51:50.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:50 smithi070 ceph-mon[96704]: pgmap v27: 121 pgs: 121 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 1.6 MiB/s wr, 24 op/s 2024-09-14T03:51:50.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:50 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:50.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:50 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:50.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:50 smithi070 ceph-mon[98640]: osdmap e291: 8 total, 8 up, 8 in 2024-09-14T03:51:50.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:50 smithi070 ceph-mon[98640]: pgmap v27: 121 pgs: 121 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 1.6 MiB/s wr, 24 op/s 2024-09-14T03:51:50.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:50 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:50.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:50 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:51.424 INFO:tasks.rados.rados.0.smithi070.stdout:1177: done (2 left) 2024-09-14T03:51:51.424 INFO:tasks.rados.rados.0.smithi070.stdout:1178: done (1 left) 2024-09-14T03:51:51.425 INFO:tasks.rados.rados.0.smithi070.stdout:1179: done (0 left) 2024-09-14T03:51:51.425 INFO:tasks.rados.rados.0.smithi070.stdout:1180: read oid 5 snap 104 2024-09-14T03:51:51.425 INFO:tasks.rados.rados.0.smithi070.stdout:1180: expect (ObjNum 114 snap 24 seq_num 114) 2024-09-14T03:51:51.425 INFO:tasks.rados.rados.0.smithi070.stdout:1181: read oid 47 snap 89 2024-09-14T03:51:51.425 INFO:tasks.rados.rados.0.smithi070.stdout:1181: expect (ObjNum 112 snap 23 seq_num 112) 2024-09-14T03:51:51.425 INFO:tasks.rados.rados.0.smithi070.stdout:1182: copy_from oid 23 from oid 15 current snap is 129 2024-09-14T03:51:51.425 INFO:tasks.rados.rados.0.smithi070.stdout:1183: delete oid 9 current snap is 129 2024-09-14T03:51:51.433 INFO:tasks.rados.rados.0.smithi070.stdout:1183: done (3 left) 2024-09-14T03:51:51.433 INFO:tasks.rados.rados.0.smithi070.stdout:1184: setattr oid 21 current snap is 129 2024-09-14T03:51:51.435 INFO:tasks.rados.rados.0.smithi070.stdout:1185: rollback oid 31 current snap is 129 2024-09-14T03:51:51.435 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 31 to 89 2024-09-14T03:51:51.435 INFO:tasks.rados.rados.0.smithi070.stdout:1186: delete oid 29 current snap is 129 2024-09-14T03:51:51.439 INFO:tasks.rados.rados.0.smithi070.stdout:1186: done (5 left) 2024-09-14T03:51:51.439 INFO:tasks.rados.rados.0.smithi070.stdout:1187: copy_from oid 48 from oid 37 current snap is 129 2024-09-14T03:51:51.440 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 16 current snap is 129 2024-09-14T03:51:51.440 INFO:tasks.rados.rados.0.smithi070.stdout:1188: seq_num 380 ranges {4784128=770048,5554176=483328,6037504=319488} 2024-09-14T03:51:51.469 INFO:tasks.rados.rados.0.smithi070.stdout:1188: writing smithi07085154-16 from 4784128 to 5554176 tid 1 2024-09-14T03:51:51.472 INFO:tasks.rados.rados.0.smithi070.stdout:1188: writing smithi07085154-16 from 5554176 to 6037504 tid 2 2024-09-14T03:51:51.475 INFO:tasks.rados.rados.0.smithi070.stdout:1188: writing smithi07085154-16 from 6037504 to 6356992 tid 3 2024-09-14T03:51:51.475 INFO:tasks.rados.rados.0.smithi070.stdout:1189: delete oid 43 current snap is 129 2024-09-14T03:51:51.475 INFO:tasks.rados.rados.0.smithi070.stdout:1180: expect (ObjNum 114 snap 24 seq_num 114) 2024-09-14T03:51:51.670 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 546 (ObjNum 114 snap 24 seq_num 114) dirty exists 2024-09-14T03:51:51.670 INFO:tasks.rados.rados.0.smithi070.stdout:1187: finishing copy_from to smithi07085154-48 2024-09-14T03:51:51.670 INFO:tasks.rados.rados.0.smithi070.stdout:1187: got expected ENOENT (src dne) 2024-09-14T03:51:51.670 INFO:tasks.rados.rados.0.smithi070.stdout:1182: finishing copy_from racing read to smithi07085154-23 2024-09-14T03:51:51.671 INFO:tasks.rados.rados.0.smithi070.stdout:1187: finishing copy_from racing read to smithi07085154-48 2024-09-14T03:51:51.671 INFO:tasks.rados.rados.0.smithi070.stdout:1185: finishing rollback tid 1 to smithi07085154-31 2024-09-14T03:51:51.671 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 468 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-14T03:51:51.671 INFO:tasks.rados.rados.0.smithi070.stdout:1182: finishing copy_from to smithi07085154-23 2024-09-14T03:51:51.671 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 588 (ObjNum 202 snap 56 seq_num 202) dirty exists 2024-09-14T03:51:51.671 INFO:tasks.rados.rados.0.smithi070.stdout:1181: expect (ObjNum 112 snap 23 seq_num 112) 2024-09-14T03:51:51.726 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:51 smithi094 ceph-mon[31053]: osdmap e292: 8 total, 8 up, 8 in 2024-09-14T03:51:51.726 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:51 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:51.726 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:51 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:51.751 INFO:tasks.rados.rados.0.smithi070.stdout:1188: finishing write tid 1 to smithi07085154-16 2024-09-14T03:51:51.751 INFO:tasks.rados.rados.0.smithi070.stdout:1188: finishing write tid 2 to smithi07085154-16 2024-09-14T03:51:51.751 INFO:tasks.rados.rados.0.smithi070.stdout:1188: finishing write tid 3 to smithi07085154-16 2024-09-14T03:51:51.751 INFO:tasks.rados.rados.0.smithi070.stdout:1188: finishing write tid 4 to smithi07085154-16 2024-09-14T03:51:51.751 INFO:tasks.rados.rados.0.smithi070.stdout:1188: finishing write tid 5 to smithi07085154-16 2024-09-14T03:51:51.751 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 979 (ObjNum 380 snap 129 seq_num 380) dirty exists 2024-09-14T03:51:51.751 INFO:tasks.rados.rados.0.smithi070.stdout:1188: left oid 16 (ObjNum 380 snap 129 seq_num 380) 2024-09-14T03:51:51.751 INFO:tasks.rados.rados.0.smithi070.stdout:1180: done (7 left) 2024-09-14T03:51:51.751 INFO:tasks.rados.rados.0.smithi070.stdout:1181: done (6 left) 2024-09-14T03:51:51.751 INFO:tasks.rados.rados.0.smithi070.stdout:1182: done (5 left) 2024-09-14T03:51:51.751 INFO:tasks.rados.rados.0.smithi070.stdout:1184: done (4 left) 2024-09-14T03:51:51.751 INFO:tasks.rados.rados.0.smithi070.stdout:1185: done (3 left) 2024-09-14T03:51:51.752 INFO:tasks.rados.rados.0.smithi070.stdout:1187: done (2 left) 2024-09-14T03:51:51.752 INFO:tasks.rados.rados.0.smithi070.stdout:1188: done (1 left) 2024-09-14T03:51:51.752 INFO:tasks.rados.rados.0.smithi070.stdout:1189: done (0 left) 2024-09-14T03:51:51.752 INFO:tasks.rados.rados.0.smithi070.stdout:1190: snap_create 2024-09-14T03:51:51.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:51 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:51:51.376+0000 7fa4af5ba640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-09-14T03:51:51.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:51 smithi070 ceph-mon[96704]: osdmap e292: 8 total, 8 up, 8 in 2024-09-14T03:51:51.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:51 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:51.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:51 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:51.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:51 smithi070 ceph-mon[98640]: osdmap e292: 8 total, 8 up, 8 in 2024-09-14T03:51:51.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:51 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:51.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:51 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:52.429 INFO:tasks.rados.rados.0.smithi070.stdout:1190: done (0 left) 2024-09-14T03:51:52.429 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 47 current snap is 130 2024-09-14T03:51:52.429 INFO:tasks.rados.rados.0.smithi070.stdout:1191: seq_num 382 ranges {4964352=802816,5767168=704512,6471680=98304} 2024-09-14T03:51:52.459 INFO:tasks.rados.rados.0.smithi070.stdout:1191: writing smithi07085154-47 from 4964352 to 5767168 tid 1 2024-09-14T03:51:52.464 INFO:tasks.rados.rados.0.smithi070.stdout:1191: writing smithi07085154-47 from 5767168 to 6471680 tid 2 2024-09-14T03:51:52.464 INFO:tasks.rados.rados.0.smithi070.stdout:1191: writing smithi07085154-47 from 6471680 to 6569984 tid 3 2024-09-14T03:51:52.464 INFO:tasks.rados.rados.0.smithi070.stdout:1192: snap_remove snap 51 2024-09-14T03:51:52.468 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:52 smithi070 ceph-mon[96704]: osdmap e293: 8 total, 8 up, 8 in 2024-09-14T03:51:52.469 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:52 smithi070 ceph-mon[96704]: pgmap v30: 121 pgs: 121 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s rd, 2.4 MiB/s wr, 37 op/s 2024-09-14T03:51:52.469 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:52 smithi070 ceph-mon[98640]: osdmap e293: 8 total, 8 up, 8 in 2024-09-14T03:51:52.469 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:52 smithi070 ceph-mon[98640]: pgmap v30: 121 pgs: 121 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s rd, 2.4 MiB/s wr, 37 op/s 2024-09-14T03:51:52.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:52 smithi094 ceph-mon[31053]: osdmap e293: 8 total, 8 up, 8 in 2024-09-14T03:51:52.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:52 smithi094 ceph-mon[31053]: pgmap v30: 121 pgs: 121 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s rd, 2.4 MiB/s wr, 37 op/s 2024-09-14T03:51:53.427 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:51:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:51:53] "GET /metrics HTTP/1.1" 200 36130 "" "Prometheus/2.51.0" 2024-09-14T03:51:53.435 INFO:tasks.rados.rados.0.smithi070.stdout:1192: done (1 left) 2024-09-14T03:51:53.435 INFO:tasks.rados.rados.0.smithi070.stdout:1193: delete oid 9 current snap is 130 2024-09-14T03:51:53.436 INFO:tasks.rados.rados.0.smithi070.stdout:1191: finishing write tid 1 to smithi07085154-47 2024-09-14T03:51:53.436 INFO:tasks.rados.rados.0.smithi070.stdout:1191: finishing write tid 2 to smithi07085154-47 2024-09-14T03:51:53.436 INFO:tasks.rados.rados.0.smithi070.stdout:1191: finishing write tid 3 to smithi07085154-47 2024-09-14T03:51:53.436 INFO:tasks.rados.rados.0.smithi070.stdout:1191: finishing write tid 4 to smithi07085154-47 2024-09-14T03:51:53.436 INFO:tasks.rados.rados.0.smithi070.stdout:1191: finishing write tid 5 to smithi07085154-47 2024-09-14T03:51:53.436 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 47 v 421 (ObjNum 382 snap 130 seq_num 382) dirty exists 2024-09-14T03:51:53.436 INFO:tasks.rados.rados.0.smithi070.stdout:1191: left oid 47 (ObjNum 382 snap 130 seq_num 382) 2024-09-14T03:51:53.450 INFO:tasks.rados.rados.0.smithi070.stdout:1191: done (1 left) 2024-09-14T03:51:53.450 INFO:tasks.rados.rados.0.smithi070.stdout:1193: done (0 left) 2024-09-14T03:51:53.450 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 46 current snap is 130 2024-09-14T03:51:53.450 INFO:tasks.rados.rados.0.smithi070.stdout:1194: seq_num 384 ranges {1605632=434176,2039808=475136,2514944=630784,3145728=98304} 2024-09-14T03:51:53.461 INFO:tasks.rados.rados.0.smithi070.stdout:1194: writing smithi07085154-46 from 1605632 to 2039808 tid 1 2024-09-14T03:51:53.465 INFO:tasks.rados.rados.0.smithi070.stdout:1194: writing smithi07085154-46 from 2039808 to 2514944 tid 2 2024-09-14T03:51:53.469 INFO:tasks.rados.rados.0.smithi070.stdout:1194: writing smithi07085154-46 from 2514944 to 3145728 tid 3 2024-09-14T03:51:53.472 INFO:tasks.rados.rados.0.smithi070.stdout:1194: writing smithi07085154-46 from 3145728 to 3244032 tid 4 2024-09-14T03:51:53.472 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 15 current snap is 130 2024-09-14T03:51:53.472 INFO:tasks.rados.rados.0.smithi070.stdout:1195: seq_num 385 ranges {1769472=450560,2220032=761856,2981888=442368} 2024-09-14T03:51:53.483 INFO:tasks.rados.rados.0.smithi070.stdout:1195: writing smithi07085154-15 from 1769472 to 2220032 tid 1 2024-09-14T03:51:53.488 INFO:tasks.rados.rados.0.smithi070.stdout:1195: writing smithi07085154-15 from 2220032 to 2981888 tid 2 2024-09-14T03:51:53.490 INFO:tasks.rados.rados.0.smithi070.stdout:1195: writing smithi07085154-15 from 2981888 to 3424256 tid 3 2024-09-14T03:51:53.490 INFO:tasks.rados.rados.0.smithi070.stdout:1196: read oid 31 snap -1 2024-09-14T03:51:53.490 INFO:tasks.rados.rados.0.smithi070.stdout:1196: expect deleted 2024-09-14T03:51:53.491 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 23 current snap is 130 2024-09-14T03:51:53.491 INFO:tasks.rados.rados.0.smithi070.stdout:1194: finishing write tid 1 to smithi07085154-46 2024-09-14T03:51:53.491 INFO:tasks.rados.rados.0.smithi070.stdout:1197: seq_num 386 ranges {1769472=466944,2236416=638976,2875392=565248} 2024-09-14T03:51:53.503 INFO:tasks.rados.rados.0.smithi070.stdout:1197: writing smithi07085154-23 from 1769472 to 2236416 tid 1 2024-09-14T03:51:53.506 INFO:tasks.rados.rados.0.smithi070.stdout:1197: writing smithi07085154-23 from 2236416 to 2875392 tid 2 2024-09-14T03:51:53.509 INFO:tasks.rados.rados.0.smithi070.stdout:1197: writing smithi07085154-23 from 2875392 to 3440640 tid 3 2024-09-14T03:51:53.509 INFO:tasks.rados.rados.0.smithi070.stdout:1198: snap_remove snap 105 2024-09-14T03:51:53.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:53 smithi094 ceph-mon[31053]: osdmap e294: 8 total, 8 up, 8 in 2024-09-14T03:51:53.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:53 smithi070 ceph-mon[96704]: osdmap e294: 8 total, 8 up, 8 in 2024-09-14T03:51:53.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:53 smithi070 ceph-mon[98640]: osdmap e294: 8 total, 8 up, 8 in 2024-09-14T03:51:53.991 INFO:tasks.rados.rados.0.smithi070.stdout:1198: done (4 left) 2024-09-14T03:51:53.991 INFO:tasks.rados.rados.0.smithi070.stdout:1199: rmattr oid 47 current snap is 130 2024-09-14T03:51:53.992 INFO:tasks.rados.rados.0.smithi070.stdout:1200: snap_remove snap 118 2024-09-14T03:51:54.411 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:54 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:54.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:51:54.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:54 smithi094 ceph-mon[31053]: osdmap e295: 8 total, 8 up, 8 in 2024-09-14T03:51:54.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:54 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:54.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:54 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:54.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:54 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:51:54.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:54 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:51:54.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:54 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:54.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:54 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:51:54.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:54 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:51:54.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:54 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-14T03:51:54.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:54 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-09-14T03:51:54.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:54 smithi094 ceph-mon[31053]: Upgrade: It appears safe to stop mon.b 2024-09-14T03:51:54.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:54 smithi094 ceph-mon[31053]: osdmap e296: 8 total, 8 up, 8 in 2024-09-14T03:51:54.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:54 smithi094 ceph-mon[31053]: pgmap v34: 121 pgs: 121 active+clean; 327 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.6 MiB/s rd, 3.3 MiB/s wr, 22 op/s 2024-09-14T03:51:54.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:54 smithi070 ceph-mon[96704]: osdmap e295: 8 total, 8 up, 8 in 2024-09-14T03:51:54.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:54 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:54.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:54 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:54.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:54 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:51:54.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:54 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:51:54.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:54 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:54.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:54 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:51:54.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:54 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:51:54.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:54 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-14T03:51:54.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:54 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-09-14T03:51:54.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:54 smithi070 ceph-mon[96704]: Upgrade: It appears safe to stop mon.b 2024-09-14T03:51:54.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:54 smithi070 ceph-mon[96704]: osdmap e296: 8 total, 8 up, 8 in 2024-09-14T03:51:54.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:54 smithi070 ceph-mon[96704]: pgmap v34: 121 pgs: 121 active+clean; 327 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.6 MiB/s rd, 3.3 MiB/s wr, 22 op/s 2024-09-14T03:51:54.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:54 smithi070 ceph-mon[98640]: osdmap e295: 8 total, 8 up, 8 in 2024-09-14T03:51:54.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:54 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:54.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:54 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:54.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:54 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:51:54.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:54 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:51:54.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:54 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:54.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:54 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:51:54.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:54 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:51:54.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:54 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-14T03:51:54.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:54 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-09-14T03:51:54.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:54 smithi070 ceph-mon[98640]: Upgrade: It appears safe to stop mon.b 2024-09-14T03:51:54.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:54 smithi070 ceph-mon[98640]: osdmap e296: 8 total, 8 up, 8 in 2024-09-14T03:51:54.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:54 smithi070 ceph-mon[98640]: pgmap v34: 121 pgs: 121 active+clean; 327 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.6 MiB/s rd, 3.3 MiB/s wr, 22 op/s 2024-09-14T03:51:54.997 INFO:tasks.rados.rados.0.smithi070.stdout:1200: done (5 left) 2024-09-14T03:51:54.997 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 50 current snap is 130 2024-09-14T03:51:54.998 INFO:tasks.rados.rados.0.smithi070.stdout:1194: finishing write tid 2 to smithi07085154-46 2024-09-14T03:51:54.998 INFO:tasks.rados.rados.0.smithi070.stdout:1194: finishing write tid 3 to smithi07085154-46 2024-09-14T03:51:54.998 INFO:tasks.rados.rados.0.smithi070.stdout:1194: finishing write tid 4 to smithi07085154-46 2024-09-14T03:51:54.998 INFO:tasks.rados.rados.0.smithi070.stdout:1194: finishing write tid 5 to smithi07085154-46 2024-09-14T03:51:54.998 INFO:tasks.rados.rados.0.smithi070.stdout:1194: finishing write tid 6 to smithi07085154-46 2024-09-14T03:51:54.998 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 46 v 996 (ObjNum 384 snap 130 seq_num 384) dirty exists 2024-09-14T03:51:54.998 INFO:tasks.rados.rados.0.smithi070.stdout:1194: left oid 46 (ObjNum 384 snap 130 seq_num 384) 2024-09-14T03:51:54.998 INFO:tasks.rados.rados.0.smithi070.stdout:1195: finishing write tid 1 to smithi07085154-15 2024-09-14T03:51:54.998 INFO:tasks.rados.rados.0.smithi070.stdout:1195: finishing write tid 2 to smithi07085154-15 2024-09-14T03:51:54.999 INFO:tasks.rados.rados.0.smithi070.stdout:1195: finishing write tid 3 to smithi07085154-15 2024-09-14T03:51:54.999 INFO:tasks.rados.rados.0.smithi070.stdout:1201: seq_num 387 ranges {1769472=483328,2252800=524288,2777088=679936} 2024-09-14T03:51:55.011 INFO:tasks.rados.rados.0.smithi070.stdout:1201: writing smithi07085154-50 from 1769472 to 2252800 tid 1 2024-09-14T03:51:55.014 INFO:tasks.rados.rados.0.smithi070.stdout:1201: writing smithi07085154-50 from 2252800 to 2777088 tid 2 2024-09-14T03:51:55.019 INFO:tasks.rados.rados.0.smithi070.stdout:1201: writing smithi07085154-50 from 2777088 to 3457024 tid 3 2024-09-14T03:51:55.019 INFO:tasks.rados.rados.0.smithi070.stdout:1194: done (5 left) 2024-09-14T03:51:55.020 INFO:tasks.rados.rados.0.smithi070.stdout:1196: done (4 left) 2024-09-14T03:51:55.020 INFO:tasks.rados.rados.0.smithi070.stdout:1202: read oid 11 snap -1 2024-09-14T03:51:55.020 INFO:tasks.rados.rados.0.smithi070.stdout:1202: expect (ObjNum 293 snap 91 seq_num 293) 2024-09-14T03:51:55.020 INFO:tasks.rados.rados.0.smithi070.stdout:1203: snap_create 2024-09-14T03:51:55.020 INFO:tasks.rados.rados.0.smithi070.stdout:1195: finishing write tid 4 to smithi07085154-15 2024-09-14T03:51:55.020 INFO:tasks.rados.rados.0.smithi070.stdout:1195: finishing write tid 5 to smithi07085154-15 2024-09-14T03:51:55.020 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 15 v 816 (ObjNum 385 snap 130 seq_num 385) dirty exists 2024-09-14T03:51:55.020 INFO:tasks.rados.rados.0.smithi070.stdout:1195: left oid 15 (ObjNum 385 snap 130 seq_num 385) 2024-09-14T03:51:55.020 INFO:tasks.rados.rados.0.smithi070.stdout:1197: finishing write tid 1 to smithi07085154-23 2024-09-14T03:51:55.020 INFO:tasks.rados.rados.0.smithi070.stdout:1197: finishing write tid 2 to smithi07085154-23 2024-09-14T03:51:55.020 INFO:tasks.rados.rados.0.smithi070.stdout:1197: finishing write tid 3 to smithi07085154-23 2024-09-14T03:51:55.021 INFO:tasks.rados.rados.0.smithi070.stdout:1197: finishing write tid 4 to smithi07085154-23 2024-09-14T03:51:55.021 INFO:tasks.rados.rados.0.smithi070.stdout:1197: finishing write tid 5 to smithi07085154-23 2024-09-14T03:51:55.021 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 592 (ObjNum 386 snap 130 seq_num 386) dirty exists 2024-09-14T03:51:55.021 INFO:tasks.rados.rados.0.smithi070.stdout:1197: left oid 23 (ObjNum 386 snap 130 seq_num 386) 2024-09-14T03:51:55.021 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 47 v 426 (ObjNum 382 snap 130 seq_num 382) dirty exists 2024-09-14T03:51:55.021 INFO:tasks.rados.rados.0.smithi070.stdout:1201: finishing write tid 1 to smithi07085154-50 2024-09-14T03:51:55.021 INFO:tasks.rados.rados.0.smithi070.stdout:1201: finishing write tid 2 to smithi07085154-50 2024-09-14T03:51:55.022 INFO:tasks.rados.rados.0.smithi070.stdout:1201: finishing write tid 3 to smithi07085154-50 2024-09-14T03:51:55.022 INFO:tasks.rados.rados.0.smithi070.stdout:1201: finishing write tid 4 to smithi07085154-50 2024-09-14T03:51:55.023 INFO:tasks.rados.rados.0.smithi070.stdout:1201: finishing write tid 5 to smithi07085154-50 2024-09-14T03:51:55.023 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 632 (ObjNum 387 snap 130 seq_num 387) dirty exists 2024-09-14T03:51:55.023 INFO:tasks.rados.rados.0.smithi070.stdout:1201: left oid 50 (ObjNum 387 snap 130 seq_num 387) 2024-09-14T03:51:55.026 INFO:tasks.rados.rados.0.smithi070.stdout:1202: expect (ObjNum 293 snap 91 seq_num 293) 2024-09-14T03:51:55.944 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:55 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:55.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:51:55.944 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:55 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:55.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:51:56.000 INFO:tasks.rados.rados.0.smithi070.stdout:1195: done (5 left) 2024-09-14T03:51:56.001 INFO:tasks.rados.rados.0.smithi070.stdout:1197: done (4 left) 2024-09-14T03:51:56.001 INFO:tasks.rados.rados.0.smithi070.stdout:1199: done (3 left) 2024-09-14T03:51:56.001 INFO:tasks.rados.rados.0.smithi070.stdout:1201: done (2 left) 2024-09-14T03:51:56.001 INFO:tasks.rados.rados.0.smithi070.stdout:1202: done (1 left) 2024-09-14T03:51:56.001 INFO:tasks.rados.rados.0.smithi070.stdout:1203: done (0 left) 2024-09-14T03:51:56.001 INFO:tasks.rados.rados.0.smithi070.stdout:1204: snap_create 2024-09-14T03:51:56.121 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:55 smithi070 ceph-mon[96704]: osdmap e297: 8 total, 8 up, 8 in 2024-09-14T03:51:56.121 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:55 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:56.121 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:55 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T03:51:56.121 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:55 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T03:51:56.121 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:55 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:51:56.121 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:55 smithi070 ceph-mon[98640]: osdmap e297: 8 total, 8 up, 8 in 2024-09-14T03:51:56.121 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:55 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:56.121 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:55 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T03:51:56.122 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:55 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T03:51:56.122 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:55 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:51:56.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:55 smithi094 ceph-mon[31053]: osdmap e297: 8 total, 8 up, 8 in 2024-09-14T03:51:56.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:55 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:51:56.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:55 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T03:51:56.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:55 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T03:51:56.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:55 smithi094 ceph-mon[31053]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:51:56.999 INFO:tasks.rados.rados.0.smithi070.stdout:1204: done (0 left) 2024-09-14T03:51:56.999 INFO:tasks.rados.rados.0.smithi070.stdout:1205: snap_remove snap 128 2024-09-14T03:51:57.212 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:56 smithi094 systemd[1]: Stopping Ceph mon.b for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:51:57.212 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:57 smithi094 ceph-mon[31053]: Upgrade: Updating mon.b 2024-09-14T03:51:57.213 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:57 smithi094 ceph-mon[31053]: Deploying daemon mon.b on smithi094 2024-09-14T03:51:57.213 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:57 smithi094 ceph-mon[31053]: osdmap e298: 8 total, 8 up, 8 in 2024-09-14T03:51:57.213 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:57 smithi094 ceph-mon[31053]: pgmap v37: 121 pgs: 121 active+clean; 327 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.6 MiB/s rd, 3.3 MiB/s wr, 22 op/s 2024-09-14T03:51:57.213 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:57 smithi094 ceph-mon[31053]: osdmap e299: 8 total, 8 up, 8 in 2024-09-14T03:51:57.213 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:51:56 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:51:56.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:51:57.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:57 smithi070 ceph-mon[96704]: Upgrade: Updating mon.b 2024-09-14T03:51:57.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:57 smithi070 ceph-mon[96704]: Deploying daemon mon.b on smithi094 2024-09-14T03:51:57.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:57 smithi070 ceph-mon[96704]: osdmap e298: 8 total, 8 up, 8 in 2024-09-14T03:51:57.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:57 smithi070 ceph-mon[96704]: pgmap v37: 121 pgs: 121 active+clean; 327 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.6 MiB/s rd, 3.3 MiB/s wr, 22 op/s 2024-09-14T03:51:57.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:51:57 smithi070 ceph-mon[96704]: osdmap e299: 8 total, 8 up, 8 in 2024-09-14T03:51:57.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:57 smithi070 ceph-mon[98640]: Upgrade: Updating mon.b 2024-09-14T03:51:57.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:57 smithi070 ceph-mon[98640]: Deploying daemon mon.b on smithi094 2024-09-14T03:51:57.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:57 smithi070 ceph-mon[98640]: osdmap e298: 8 total, 8 up, 8 in 2024-09-14T03:51:57.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:57 smithi070 ceph-mon[98640]: pgmap v37: 121 pgs: 121 active+clean; 327 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.6 MiB/s rd, 3.3 MiB/s wr, 22 op/s 2024-09-14T03:51:57.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:51:57 smithi070 ceph-mon[98640]: osdmap e299: 8 total, 8 up, 8 in 2024-09-14T03:51:57.478 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:57 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-b[31028]: 2024-09-14T03:51:57.210+0000 7efd04a76640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:51:57.478 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:57 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-b[31028]: 2024-09-14T03:51:57.210+0000 7efd04a76640 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-09-14T03:51:57.735 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:57 smithi094 podman[86409]: 2024-09-14 03:51:57.477051376 +0000 UTC m=+0.361258410 container died 90ea2e16490969668a54b75d50384cc1565b48a36a16b2f8f08aacaf4191b8a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-b, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=quincy-eb16b97, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:51:58.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:57 smithi094 podman[86409]: 2024-09-14 03:51:57.825542552 +0000 UTC m=+0.709749573 container cleanup 90ea2e16490969668a54b75d50384cc1565b48a36a16b2f8f08aacaf4191b8a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-b, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, RELEASE=quincy-eb16b97, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-14T03:51:58.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:57 smithi094 bash[86409]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-b 2024-09-14T03:51:58.469 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:58 smithi094 podman[86426]: 2024-09-14 03:51:58.143535022 +0000 UTC m=+0.660603713 container remove 90ea2e16490969668a54b75d50384cc1565b48a36a16b2f8f08aacaf4191b8a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, RELEASE=quincy-eb16b97, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-14T03:51:58.469 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:58 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@mon.b.service: Deactivated successfully. 2024-09-14T03:51:58.470 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:58 smithi094 systemd[1]: Stopped Ceph mon.b for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:51:58.470 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:58 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@mon.b.service: Consumed 9.114s CPU time. 2024-09-14T03:51:58.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:58 smithi094 systemd[1]: Starting Ceph mon.b for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:51:58.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:58 smithi094 podman[86559]: 2024-09-14 03:51:58.57893673 +0000 UTC m=+0.015677575 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:51:59.131 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:58 smithi094 podman[86559]: 2024-09-14 03:51:58.861665538 +0000 UTC m=+0.298406428 container create 86827be986590181cb74d8eebe7278c077280c14de375cb751fbdbf912ab1df7 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-b, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-14T03:51:59.417 INFO:tasks.rados.rados.0.smithi070.stdout:1205: done (0 left) 2024-09-14T03:51:59.417 INFO:tasks.rados.rados.0.smithi070.stdout:1206: snap_remove snap 96 2024-09-14T03:51:59.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 podman[86559]: 2024-09-14 03:51:59.130753496 +0000 UTC m=+0.567494352 container init 86827be986590181cb74d8eebe7278c077280c14de375cb751fbdbf912ab1df7 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-b, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, RELEASE=main-a278231) 2024-09-14T03:51:59.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 podman[86559]: 2024-09-14 03:51:59.135196762 +0000 UTC m=+0.571937605 container start 86827be986590181cb74d8eebe7278c077280c14de375cb751fbdbf912ab1df7 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0, RELEASE=main-a278231, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-14T03:51:59.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: set uid:gid to 167:167 (ceph:ceph) 2024-09-14T03:51:59.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev), process ceph-mon, pid 7 2024-09-14T03:51:59.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: pidfile_write: ignore empty --pid-file 2024-09-14T03:51:59.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: load: jerasure load: lrc 2024-09-14T03:51:59.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: RocksDB version: 7.9.2 2024-09-14T03:51:59.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Git sha 0 2024-09-14T03:51:59.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Compile date 2024-09-12 20:09:22 2024-09-14T03:51:59.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: DB SUMMARY 2024-09-14T03:51:59.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: DB Session ID: N88WLD5C1RO6DBJHL3RC 2024-09-14T03:51:59.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: CURRENT file: CURRENT 2024-09-14T03:51:59.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: IDENTITY file: IDENTITY 2024-09-14T03:51:59.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1097 Bytes 2024-09-14T03:51:59.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2024-09-14T03:51:59.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 13101330 ; 2024-09-14T03:51:59.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.error_if_exists: 0 2024-09-14T03:51:59.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.create_if_missing: 0 2024-09-14T03:51:59.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.paranoid_checks: 1 2024-09-14T03:51:59.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-14T03:51:59.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-14T03:51:59.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-14T03:51:59.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.env: 0x55f67f090120 2024-09-14T03:51:59.529 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.fs: PosixFileSystem 2024-09-14T03:51:59.529 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.info_log: 0x55f6810f7700 2024-09-14T03:51:59.529 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.max_file_opening_threads: 16 2024-09-14T03:51:59.529 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.statistics: (nil) 2024-09-14T03:51:59.530 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.use_fsync: 0 2024-09-14T03:51:59.530 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.max_log_file_size: 0 2024-09-14T03:51:59.530 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-14T03:51:59.531 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-14T03:51:59.531 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.keep_log_file_num: 1000 2024-09-14T03:51:59.531 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.recycle_log_file_num: 0 2024-09-14T03:51:59.531 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.allow_fallocate: 1 2024-09-14T03:51:59.531 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.allow_mmap_reads: 0 2024-09-14T03:51:59.531 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.allow_mmap_writes: 0 2024-09-14T03:51:59.531 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.use_direct_reads: 0 2024-09-14T03:51:59.531 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-14T03:51:59.531 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.create_missing_column_families: 0 2024-09-14T03:51:59.531 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.db_log_dir: 2024-09-14T03:51:59.531 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.wal_dir: 2024-09-14T03:51:59.531 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-14T03:51:59.531 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-14T03:51:59.532 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-14T03:51:59.532 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-14T03:51:59.532 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-14T03:51:59.532 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-14T03:51:59.532 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.advise_random_on_open: 1 2024-09-14T03:51:59.532 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.db_write_buffer_size: 0 2024-09-14T03:51:59.532 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.write_buffer_manager: 0x55f6810fbb80 2024-09-14T03:51:59.532 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-14T03:51:59.532 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-14T03:51:59.532 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-14T03:51:59.532 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.rate_limiter: (nil) 2024-09-14T03:51:59.532 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-14T03:51:59.532 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.wal_recovery_mode: 2 2024-09-14T03:51:59.533 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.enable_thread_tracking: 0 2024-09-14T03:51:59.533 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.enable_pipelined_write: 0 2024-09-14T03:51:59.533 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.unordered_write: 0 2024-09-14T03:51:59.533 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-14T03:51:59.533 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-14T03:51:59.533 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-14T03:51:59.533 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-14T03:51:59.533 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.row_cache: None 2024-09-14T03:51:59.533 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.wal_filter: None 2024-09-14T03:51:59.533 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-14T03:51:59.533 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.allow_ingest_behind: 0 2024-09-14T03:51:59.533 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.two_write_queues: 0 2024-09-14T03:51:59.534 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.manual_wal_flush: 0 2024-09-14T03:51:59.534 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.wal_compression: 0 2024-09-14T03:51:59.534 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.atomic_flush: 0 2024-09-14T03:51:59.534 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-14T03:51:59.534 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-14T03:51:59.534 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-14T03:51:59.534 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.log_readahead_size: 0 2024-09-14T03:51:59.534 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-14T03:51:59.534 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.best_efforts_recovery: 0 2024-09-14T03:51:59.534 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-14T03:51:59.534 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-14T03:51:59.534 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.allow_data_in_errors: 0 2024-09-14T03:51:59.534 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.db_host_id: __hostname__ 2024-09-14T03:51:59.534 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-14T03:51:59.535 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.max_background_jobs: 2 2024-09-14T03:51:59.535 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.max_background_compactions: -1 2024-09-14T03:51:59.535 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.max_subcompactions: 1 2024-09-14T03:51:59.535 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-14T03:51:59.535 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-14T03:51:59.535 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-14T03:51:59.535 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.max_total_wal_size: 0 2024-09-14T03:51:59.535 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-14T03:51:59.535 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-14T03:51:59.535 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-14T03:51:59.535 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-14T03:51:59.535 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.max_open_files: -1 2024-09-14T03:51:59.535 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.bytes_per_sync: 0 2024-09-14T03:51:59.535 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-14T03:51:59.536 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-14T03:51:59.536 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.compaction_readahead_size: 0 2024-09-14T03:51:59.536 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.max_background_flushes: -1 2024-09-14T03:51:59.536 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Compression algorithms supported: 2024-09-14T03:51:59.536 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: kZSTD supported: 0 2024-09-14T03:51:59.536 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: kXpressCompression supported: 0 2024-09-14T03:51:59.536 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: kBZip2Compression supported: 0 2024-09-14T03:51:59.536 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-14T03:51:59.536 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: kLZ4Compression supported: 1 2024-09-14T03:51:59.536 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: kZlibCompression supported: 1 2024-09-14T03:51:59.536 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: kLZ4HCCompression supported: 1 2024-09-14T03:51:59.536 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: kSnappyCompression supported: 1 2024-09-14T03:51:59.536 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-14T03:51:59.537 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-14T03:51:59.537 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-09-14T03:51:59.537 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-14T03:51:59.537 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-14T03:51:59.537 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.merge_operator: 2024-09-14T03:51:59.537 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.compaction_filter: None 2024-09-14T03:51:59.537 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.compaction_filter_factory: None 2024-09-14T03:51:59.537 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.sst_partitioner_factory: None 2024-09-14T03:51:59.537 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-14T03:51:59.537 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-14T03:51:59.537 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f6810f7320) 2024-09-14T03:51:59.537 INFO:journalctl@ceph.mon.b.smithi094.stdout: cache_index_and_filter_blocks: 1 2024-09-14T03:51:59.537 INFO:journalctl@ceph.mon.b.smithi094.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-14T03:51:59.537 INFO:journalctl@ceph.mon.b.smithi094.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-14T03:51:59.538 INFO:journalctl@ceph.mon.b.smithi094.stdout: pin_top_level_index_and_filter: 1 2024-09-14T03:51:59.538 INFO:journalctl@ceph.mon.b.smithi094.stdout: index_type: 0 2024-09-14T03:51:59.538 INFO:journalctl@ceph.mon.b.smithi094.stdout: data_block_index_type: 0 2024-09-14T03:51:59.538 INFO:journalctl@ceph.mon.b.smithi094.stdout: index_shortening: 1 2024-09-14T03:51:59.538 INFO:journalctl@ceph.mon.b.smithi094.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-14T03:51:59.538 INFO:journalctl@ceph.mon.b.smithi094.stdout: checksum: 4 2024-09-14T03:51:59.538 INFO:journalctl@ceph.mon.b.smithi094.stdout: no_block_cache: 0 2024-09-14T03:51:59.538 INFO:journalctl@ceph.mon.b.smithi094.stdout: block_cache: 0x55f6810eb610 2024-09-14T03:51:59.538 INFO:journalctl@ceph.mon.b.smithi094.stdout: block_cache_name: BinnedLRUCache 2024-09-14T03:51:59.538 INFO:journalctl@ceph.mon.b.smithi094.stdout: block_cache_options: 2024-09-14T03:51:59.538 INFO:journalctl@ceph.mon.b.smithi094.stdout: capacity : 536870912 2024-09-14T03:51:59.538 INFO:journalctl@ceph.mon.b.smithi094.stdout: num_shard_bits : 4 2024-09-14T03:51:59.538 INFO:journalctl@ceph.mon.b.smithi094.stdout: strict_capacity_limit : 0 2024-09-14T03:51:59.538 INFO:journalctl@ceph.mon.b.smithi094.stdout: high_pri_pool_ratio: 0.000 2024-09-14T03:51:59.539 INFO:journalctl@ceph.mon.b.smithi094.stdout: block_cache_compressed: (nil) 2024-09-14T03:51:59.539 INFO:journalctl@ceph.mon.b.smithi094.stdout: persistent_cache: (nil) 2024-09-14T03:51:59.539 INFO:journalctl@ceph.mon.b.smithi094.stdout: block_size: 4096 2024-09-14T03:51:59.539 INFO:journalctl@ceph.mon.b.smithi094.stdout: block_size_deviation: 10 2024-09-14T03:51:59.539 INFO:journalctl@ceph.mon.b.smithi094.stdout: block_restart_interval: 16 2024-09-14T03:51:59.539 INFO:journalctl@ceph.mon.b.smithi094.stdout: index_block_restart_interval: 1 2024-09-14T03:51:59.539 INFO:journalctl@ceph.mon.b.smithi094.stdout: metadata_block_size: 4096 2024-09-14T03:51:59.539 INFO:journalctl@ceph.mon.b.smithi094.stdout: partition_filters: 0 2024-09-14T03:51:59.539 INFO:journalctl@ceph.mon.b.smithi094.stdout: use_delta_encoding: 1 2024-09-14T03:51:59.539 INFO:journalctl@ceph.mon.b.smithi094.stdout: filter_policy: bloomfilter 2024-09-14T03:51:59.539 INFO:journalctl@ceph.mon.b.smithi094.stdout: whole_key_filtering: 1 2024-09-14T03:51:59.539 INFO:journalctl@ceph.mon.b.smithi094.stdout: verify_compression: 0 2024-09-14T03:51:59.539 INFO:journalctl@ceph.mon.b.smithi094.stdout: read_amp_bytes_per_bit: 0 2024-09-14T03:51:59.539 INFO:journalctl@ceph.mon.b.smithi094.stdout: format_version: 5 2024-09-14T03:51:59.540 INFO:journalctl@ceph.mon.b.smithi094.stdout: enable_index_compression: 1 2024-09-14T03:51:59.540 INFO:journalctl@ceph.mon.b.smithi094.stdout: block_align: 0 2024-09-14T03:51:59.540 INFO:journalctl@ceph.mon.b.smithi094.stdout: max_auto_readahead_size: 262144 2024-09-14T03:51:59.540 INFO:journalctl@ceph.mon.b.smithi094.stdout: prepopulate_block_cache: 0 2024-09-14T03:51:59.540 INFO:journalctl@ceph.mon.b.smithi094.stdout: initial_auto_readahead_size: 8192 2024-09-14T03:51:59.540 INFO:journalctl@ceph.mon.b.smithi094.stdout: num_file_reads_for_auto_readahead: 2 2024-09-14T03:51:59.540 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.write_buffer_size: 33554432 2024-09-14T03:51:59.540 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.max_write_buffer_number: 2 2024-09-14T03:51:59.540 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.compression: NoCompression 2024-09-14T03:51:59.540 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.bottommost_compression: Disabled 2024-09-14T03:51:59.541 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.prefix_extractor: nullptr 2024-09-14T03:51:59.541 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-14T03:51:59.541 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.num_levels: 7 2024-09-14T03:51:59.541 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-14T03:51:59.541 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-14T03:51:59.541 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-14T03:51:59.541 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-14T03:51:59.541 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-14T03:51:59.541 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-14T03:51:59.541 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-14T03:51:59.541 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:51:59.541 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-14T03:51:59.541 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-14T03:51:59.541 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-14T03:51:59.542 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-14T03:51:59.542 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-14T03:51:59.542 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.compression_opts.level: 32767 2024-09-14T03:51:59.542 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.compression_opts.strategy: 0 2024-09-14T03:51:59.542 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-14T03:51:59.542 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:51:59.542 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-14T03:51:59.542 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-14T03:51:59.542 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.compression_opts.enabled: false 2024-09-14T03:51:59.542 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-14T03:51:59.542 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-14T03:51:59.543 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-14T03:51:59.543 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-14T03:51:59.543 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.target_file_size_base: 67108864 2024-09-14T03:51:59.543 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-14T03:51:59.543 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-14T03:51:59.543 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-14T03:51:59.543 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-14T03:51:59.543 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-14T03:51:59.543 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-14T03:51:59.543 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-14T03:51:59.543 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-14T03:51:59.544 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-14T03:51:59.544 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-14T03:51:59.544 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-14T03:51:59.544 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-14T03:51:59.544 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-14T03:51:59.544 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-14T03:51:59.544 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.arena_block_size: 1048576 2024-09-14T03:51:59.544 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-14T03:51:59.544 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-14T03:51:59.544 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.disable_auto_compactions: 0 2024-09-14T03:51:59.545 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-14T03:51:59.545 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-14T03:51:59.545 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-14T03:51:59.545 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-14T03:51:59.545 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-14T03:51:59.545 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-14T03:51:59.545 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-14T03:51:59.545 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-14T03:51:59.545 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-14T03:51:59.545 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-14T03:51:59.545 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-14T03:51:59.545 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.inplace_update_support: 0 2024-09-14T03:51:59.545 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-14T03:51:59.546 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-14T03:51:59.546 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-14T03:51:59.546 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-14T03:51:59.546 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.bloom_locality: 0 2024-09-14T03:51:59.546 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.max_successive_merges: 0 2024-09-14T03:51:59.546 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-14T03:51:59.546 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.paranoid_file_checks: 0 2024-09-14T03:51:59.546 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.force_consistency_checks: 1 2024-09-14T03:51:59.546 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.report_bg_io_stats: 0 2024-09-14T03:51:59.546 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.ttl: 2592000 2024-09-14T03:51:59.546 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-14T03:51:59.546 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-14T03:51:59.546 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-14T03:51:59.547 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.enable_blob_files: false 2024-09-14T03:51:59.547 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.min_blob_size: 0 2024-09-14T03:51:59.547 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.blob_file_size: 268435456 2024-09-14T03:51:59.547 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-14T03:51:59.547 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-14T03:51:59.547 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-14T03:51:59.547 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-14T03:51:59.547 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-14T03:51:59.547 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.blob_file_starting_level: 0 2024-09-14T03:51:59.547 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-14T03:51:59.547 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-09-14T03:51:59.547 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-14T03:51:59.547 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13488, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-14T03:51:59.548 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-09-14T03:51:59.548 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: ed9d50d3-ba7f-4e4f-9206-0c1f0b3802ac 2024-09-14T03:51:59.548 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285919162375, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-09-14T03:51:59.548 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-09-14T03:51:59.548 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285919192133, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 8069809, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13494, "largest_seqno": 17954, "table_properties": {"data_size": 8049854, "index_size": 14044, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 4869, "raw_key_size": 44400, "raw_average_key_size": 23, "raw_value_size": 8008930, "raw_average_value_size": 4158, "num_data_blocks": 606, "num_entries": 1926, "num_filter_entries": 1926, "num_deletions": 299, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726285919, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "ed9d50d3-ba7f-4e4f-9206-0c1f0b3802ac", "db_session_id": "N88WLD5C1RO6DBJHL3RC", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-09-14T03:51:59.548 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285919192239, "job": 1, "event": "recovery_finished"} 2024-09-14T03:51:59.548 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-09-14T03:51:59.548 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-14T03:51:59.548 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-14T03:51:59.548 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55f68111ae00 2024-09-14T03:51:59.548 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: DB pointer 0x55f68121c000 2024-09-14T03:51:59.548 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-14T03:51:59.548 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-14T03:51:59.549 INFO:journalctl@ceph.mon.b.smithi094.stdout: ** DB Stats ** 2024-09-14T03:51:59.549 INFO:journalctl@ceph.mon.b.smithi094.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:51:59.549 INFO:journalctl@ceph.mon.b.smithi094.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-14T03:51:59.549 INFO:journalctl@ceph.mon.b.smithi094.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:51:59.549 INFO:journalctl@ceph.mon.b.smithi094.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:51:59.549 INFO:journalctl@ceph.mon.b.smithi094.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-14T03:51:59.549 INFO:journalctl@ceph.mon.b.smithi094.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:51:59.549 INFO:journalctl@ceph.mon.b.smithi094.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:51:59.549 INFO:journalctl@ceph.mon.b.smithi094.stdout: 2024-09-14T03:51:59.549 INFO:journalctl@ceph.mon.b.smithi094.stdout: ** Compaction Stats [default] ** 2024-09-14T03:51:59.549 INFO:journalctl@ceph.mon.b.smithi094.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-14T03:51:59.549 INFO:journalctl@ceph.mon.b.smithi094.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-14T03:51:59.549 INFO:journalctl@ceph.mon.b.smithi094.stdout: L0 1/0 7.70 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 683.4 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-09-14T03:51:59.550 INFO:journalctl@ceph.mon.b.smithi094.stdout: L6 1/0 7.63 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-14T03:51:59.550 INFO:journalctl@ceph.mon.b.smithi094.stdout: Sum 2/0 15.32 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 683.4 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-09-14T03:51:59.550 INFO:journalctl@ceph.mon.b.smithi094.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 683.4 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-09-14T03:51:59.550 INFO:journalctl@ceph.mon.b.smithi094.stdout: 2024-09-14T03:51:59.550 INFO:journalctl@ceph.mon.b.smithi094.stdout: ** Compaction Stats [default] ** 2024-09-14T03:51:59.550 INFO:journalctl@ceph.mon.b.smithi094.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-14T03:51:59.550 INFO:journalctl@ceph.mon.b.smithi094.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-14T03:51:59.550 INFO:journalctl@ceph.mon.b.smithi094.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 683.4 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-09-14T03:51:59.550 INFO:journalctl@ceph.mon.b.smithi094.stdout: 2024-09-14T03:51:59.550 INFO:journalctl@ceph.mon.b.smithi094.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-14T03:51:59.550 INFO:journalctl@ceph.mon.b.smithi094.stdout: 2024-09-14T03:51:59.551 INFO:journalctl@ceph.mon.b.smithi094.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:51:59.551 INFO:journalctl@ceph.mon.b.smithi094.stdout: Flush(GB): cumulative 0.008, interval 0.008 2024-09-14T03:51:59.551 INFO:journalctl@ceph.mon.b.smithi094.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-14T03:51:59.551 INFO:journalctl@ceph.mon.b.smithi094.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-14T03:51:59.551 INFO:journalctl@ceph.mon.b.smithi094.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-14T03:51:59.551 INFO:journalctl@ceph.mon.b.smithi094.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-14T03:51:59.551 INFO:journalctl@ceph.mon.b.smithi094.stdout: Cumulative compaction: 0.01 GB write, 230.47 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:51:59.551 INFO:journalctl@ceph.mon.b.smithi094.stdout: Interval compaction: 0.01 GB write, 230.47 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:51:59.551 INFO:journalctl@ceph.mon.b.smithi094.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-14T03:51:59.551 INFO:journalctl@ceph.mon.b.smithi094.stdout: Block cache BinnedLRUCache@0x55f6810eb610#7 capacity: 512.00 MB usage: 21.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-14T03:51:59.551 INFO:journalctl@ceph.mon.b.smithi094.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,5.03 KB,0.000959635%) IndexBlock(1,16.08 KB,0.00306666%) Misc(1,0.00 KB,0%) 2024-09-14T03:51:59.551 INFO:journalctl@ceph.mon.b.smithi094.stdout: 2024-09-14T03:51:59.551 INFO:journalctl@ceph.mon.b.smithi094.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-14T03:51:59.552 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: starting mon.b rank 1 at public addrs [v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0] at bind addrs [v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:51:59.552 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: mon.b@-1(???) e3 preinit fsid b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:51:59.552 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: mon.b@-1(???).mds e8 new map 2024-09-14T03:51:59.552 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: mon.b@-1(???).mds e8 print_map 2024-09-14T03:51:59.552 INFO:journalctl@ceph.mon.b.smithi094.stdout: e8 2024-09-14T03:51:59.552 INFO:journalctl@ceph.mon.b.smithi094.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-14T03:51:59.552 INFO:journalctl@ceph.mon.b.smithi094.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-14T03:51:59.552 INFO:journalctl@ceph.mon.b.smithi094.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-14T03:51:59.552 INFO:journalctl@ceph.mon.b.smithi094.stdout: legacy client fscid: 1 2024-09-14T03:51:59.552 INFO:journalctl@ceph.mon.b.smithi094.stdout: 2024-09-14T03:51:59.552 INFO:journalctl@ceph.mon.b.smithi094.stdout: Filesystem 'cephfs' (1) 2024-09-14T03:51:59.552 INFO:journalctl@ceph.mon.b.smithi094.stdout: fs_name cephfs 2024-09-14T03:51:59.553 INFO:journalctl@ceph.mon.b.smithi094.stdout: epoch 8 2024-09-14T03:51:59.553 INFO:journalctl@ceph.mon.b.smithi094.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-14T03:51:59.553 INFO:journalctl@ceph.mon.b.smithi094.stdout: created 2024-09-14T03:44:03.568995+0000 2024-09-14T03:51:59.553 INFO:journalctl@ceph.mon.b.smithi094.stdout: modified 2024-09-14T03:44:16.900718+0000 2024-09-14T03:51:59.553 INFO:journalctl@ceph.mon.b.smithi094.stdout: tableserver 0 2024-09-14T03:51:59.553 INFO:journalctl@ceph.mon.b.smithi094.stdout: root 0 2024-09-14T03:51:59.553 INFO:journalctl@ceph.mon.b.smithi094.stdout: session_timeout 60 2024-09-14T03:51:59.553 INFO:journalctl@ceph.mon.b.smithi094.stdout: session_autoclose 300 2024-09-14T03:51:59.553 INFO:journalctl@ceph.mon.b.smithi094.stdout: max_file_size 1099511627776 2024-09-14T03:51:59.553 INFO:journalctl@ceph.mon.b.smithi094.stdout: max_xattr_size 65536 2024-09-14T03:51:59.553 INFO:journalctl@ceph.mon.b.smithi094.stdout: required_client_features {} 2024-09-14T03:51:59.553 INFO:journalctl@ceph.mon.b.smithi094.stdout: last_failure 0 2024-09-14T03:51:59.553 INFO:journalctl@ceph.mon.b.smithi094.stdout: last_failure_osd_epoch 0 2024-09-14T03:51:59.553 INFO:journalctl@ceph.mon.b.smithi094.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-14T03:51:59.554 INFO:journalctl@ceph.mon.b.smithi094.stdout: max_mds 1 2024-09-14T03:51:59.554 INFO:journalctl@ceph.mon.b.smithi094.stdout: in 0 2024-09-14T03:51:59.554 INFO:journalctl@ceph.mon.b.smithi094.stdout: up {0=24323} 2024-09-14T03:51:59.554 INFO:journalctl@ceph.mon.b.smithi094.stdout: failed 2024-09-14T03:51:59.554 INFO:journalctl@ceph.mon.b.smithi094.stdout: damaged 2024-09-14T03:51:59.554 INFO:journalctl@ceph.mon.b.smithi094.stdout: stopped 2024-09-14T03:51:59.554 INFO:journalctl@ceph.mon.b.smithi094.stdout: data_pools [3] 2024-09-14T03:51:59.554 INFO:journalctl@ceph.mon.b.smithi094.stdout: metadata_pool 2 2024-09-14T03:51:59.554 INFO:journalctl@ceph.mon.b.smithi094.stdout: inline_data disabled 2024-09-14T03:51:59.554 INFO:journalctl@ceph.mon.b.smithi094.stdout: balancer 2024-09-14T03:51:59.554 INFO:journalctl@ceph.mon.b.smithi094.stdout: bal_rank_mask -1 2024-09-14T03:51:59.554 INFO:journalctl@ceph.mon.b.smithi094.stdout: standby_count_wanted 0 2024-09-14T03:51:59.554 INFO:journalctl@ceph.mon.b.smithi094.stdout: qdb_cluster leader: 0 members: 2024-09-14T03:51:59.554 INFO:journalctl@ceph.mon.b.smithi094.stdout: [mds.a{0:24323} state up:active seq 6 addr [v2:172.21.15.70:6834/421917097,v1:172.21.15.70:6835/421917097] compat {c=[1],r=[1],i=[7ff]}] 2024-09-14T03:51:59.555 INFO:journalctl@ceph.mon.b.smithi094.stdout: 2024-09-14T03:51:59.555 INFO:journalctl@ceph.mon.b.smithi094.stdout: 2024-09-14T03:51:59.555 INFO:journalctl@ceph.mon.b.smithi094.stdout: Standby daemons: 2024-09-14T03:51:59.555 INFO:journalctl@ceph.mon.b.smithi094.stdout: 2024-09-14T03:51:59.555 INFO:journalctl@ceph.mon.b.smithi094.stdout: [mds.b{-1:24304} state up:standby seq 1 addr [v2:172.21.15.94:6832/3620660683,v1:172.21.15.94:6833/3620660683] compat {c=[1],r=[1],i=[7ff]}] 2024-09-14T03:51:59.555 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: mon.b@-1(???).osd e299 crush map has features 3314933069573799936, adjusting msgr requires 2024-09-14T03:51:59.555 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: mon.b@-1(???).osd e299 crush map has features 432629308056666112, adjusting msgr requires 2024-09-14T03:51:59.555 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: mon.b@-1(???).osd e299 crush map has features 432629308056666112, adjusting msgr requires 2024-09-14T03:51:59.555 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: mon.b@-1(???).osd e299 crush map has features 432629308056666112, adjusting msgr requires 2024-09-14T03:51:59.555 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 ceph-mon[86597]: mon.b@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2024-09-14T03:51:59.555 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 bash[86559]: 86827be986590181cb74d8eebe7278c077280c14de375cb751fbdbf912ab1df7 2024-09-14T03:51:59.555 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:51:59 smithi094 systemd[1]: Started Ceph mon.b for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:52:00.419 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:52:00 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:52:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:00.437 INFO:tasks.rados.rados.0.smithi070.stdout:1206: done (0 left) 2024-09-14T03:52:00.437 INFO:tasks.rados.rados.0.smithi070.stdout:1207: delete oid 6 current snap is 132 2024-09-14T03:52:00.450 INFO:tasks.rados.rados.0.smithi070.stdout:1207: done (0 left) 2024-09-14T03:52:00.450 INFO:tasks.rados.rados.0.smithi070.stdout:1208: read oid 23 snap 131 2024-09-14T03:52:00.450 INFO:tasks.rados.rados.0.smithi070.stdout:1208: expect (ObjNum 386 snap 130 seq_num 386) 2024-09-14T03:52:00.450 INFO:tasks.rados.rados.0.smithi070.stdout:1209: snap_remove snap 117 2024-09-14T03:52:00.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:00 smithi094 ceph-mon[86597]: osdmap e300: 8 total, 8 up, 8 in 2024-09-14T03:52:00.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:00 smithi094 ceph-mon[86597]: mgrmap e43: y(active, since 27s), standbys: x 2024-09-14T03:52:00.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:00 smithi094 ceph-mon[86597]: overall HEALTH_OK 2024-09-14T03:52:00.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:00 smithi094 ceph-mon[86597]: mon.a calling monitor election 2024-09-14T03:52:00.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:00 smithi094 ceph-mon[86597]: mon.b calling monitor election 2024-09-14T03:52:00.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:00 smithi094 ceph-mon[86597]: mon.c calling monitor election 2024-09-14T03:52:00.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:00 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:52:00.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:00 smithi094 ceph-mon[86597]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:52:00.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:00 smithi094 ceph-mon[86597]: monmap epoch 4 2024-09-14T03:52:00.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:00 smithi094 ceph-mon[86597]: fsid b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:52:00.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:00 smithi094 ceph-mon[86597]: last_changed 2024-09-14T03:51:59.413110+0000 2024-09-14T03:52:00.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:00 smithi094 ceph-mon[86597]: created 2024-09-14T03:38:26.272229+0000 2024-09-14T03:52:00.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:00 smithi094 ceph-mon[86597]: min_mon_release 19 (squid) 2024-09-14T03:52:00.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:00 smithi094 ceph-mon[86597]: election_strategy: 1 2024-09-14T03:52:00.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:00 smithi094 ceph-mon[86597]: 0: [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] mon.a 2024-09-14T03:52:00.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:00 smithi094 ceph-mon[86597]: 1: [v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0] mon.b 2024-09-14T03:52:00.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:00 smithi094 ceph-mon[86597]: 2: [v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0] mon.c 2024-09-14T03:52:00.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:00 smithi094 ceph-mon[86597]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:52:00.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:00 smithi094 ceph-mon[86597]: osdmap e300: 8 total, 8 up, 8 in 2024-09-14T03:52:00.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:00 smithi094 ceph-mon[86597]: mgrmap e43: y(active, since 27s), standbys: x 2024-09-14T03:52:00.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:00 smithi094 ceph-mon[86597]: overall HEALTH_OK 2024-09-14T03:52:00.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:00 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:00.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:00 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:52:00.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:00 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:52:00.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:00 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:00.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:00 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:52:00.779 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:00 smithi094 ceph-mon[86597]: pgmap v41: 121 pgs: 121 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.3 MiB/s wr, 7 op/s 2024-09-14T03:52:00.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[96704]: osdmap e300: 8 total, 8 up, 8 in 2024-09-14T03:52:00.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[96704]: mgrmap e43: y(active, since 27s), standbys: x 2024-09-14T03:52:00.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[96704]: overall HEALTH_OK 2024-09-14T03:52:00.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[96704]: mon.a calling monitor election 2024-09-14T03:52:00.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[96704]: mon.b calling monitor election 2024-09-14T03:52:00.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[96704]: mon.c calling monitor election 2024-09-14T03:52:00.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:52:00.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[96704]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:52:00.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[96704]: monmap epoch 4 2024-09-14T03:52:00.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[96704]: fsid b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:52:00.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[96704]: last_changed 2024-09-14T03:51:59.413110+0000 2024-09-14T03:52:00.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[96704]: created 2024-09-14T03:38:26.272229+0000 2024-09-14T03:52:00.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[96704]: min_mon_release 19 (squid) 2024-09-14T03:52:00.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[96704]: election_strategy: 1 2024-09-14T03:52:00.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[96704]: 0: [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] mon.a 2024-09-14T03:52:00.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[96704]: 1: [v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0] mon.b 2024-09-14T03:52:00.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[96704]: 2: [v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0] mon.c 2024-09-14T03:52:00.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[96704]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:52:00.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[96704]: osdmap e300: 8 total, 8 up, 8 in 2024-09-14T03:52:00.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[96704]: mgrmap e43: y(active, since 27s), standbys: x 2024-09-14T03:52:00.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[96704]: overall HEALTH_OK 2024-09-14T03:52:00.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:00.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:52:00.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:52:00.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:00.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:52:00.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[96704]: pgmap v41: 121 pgs: 121 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.3 MiB/s wr, 7 op/s 2024-09-14T03:52:00.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[98640]: osdmap e300: 8 total, 8 up, 8 in 2024-09-14T03:52:00.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[98640]: mgrmap e43: y(active, since 27s), standbys: x 2024-09-14T03:52:00.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[98640]: overall HEALTH_OK 2024-09-14T03:52:00.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[98640]: mon.a calling monitor election 2024-09-14T03:52:00.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[98640]: mon.b calling monitor election 2024-09-14T03:52:00.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[98640]: mon.c calling monitor election 2024-09-14T03:52:00.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T03:52:00.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[98640]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:52:00.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[98640]: monmap epoch 4 2024-09-14T03:52:00.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[98640]: fsid b6380d42-724a-11ef-bceb-c7b262605968 2024-09-14T03:52:00.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[98640]: last_changed 2024-09-14T03:51:59.413110+0000 2024-09-14T03:52:00.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[98640]: created 2024-09-14T03:38:26.272229+0000 2024-09-14T03:52:00.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[98640]: min_mon_release 19 (squid) 2024-09-14T03:52:00.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[98640]: election_strategy: 1 2024-09-14T03:52:00.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[98640]: 0: [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] mon.a 2024-09-14T03:52:00.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[98640]: 1: [v2:172.21.15.94:3300/0,v1:172.21.15.94:6789/0] mon.b 2024-09-14T03:52:00.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[98640]: 2: [v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0] mon.c 2024-09-14T03:52:00.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[98640]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:52:00.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[98640]: osdmap e300: 8 total, 8 up, 8 in 2024-09-14T03:52:00.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[98640]: mgrmap e43: y(active, since 27s), standbys: x 2024-09-14T03:52:00.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[98640]: overall HEALTH_OK 2024-09-14T03:52:00.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:00.790 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T03:52:00.790 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T03:52:00.790 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:00.790 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:52:00.790 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:00 smithi070 ceph-mon[98640]: pgmap v41: 121 pgs: 121 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.3 MiB/s wr, 7 op/s 2024-09-14T03:52:01.446 INFO:tasks.rados.rados.0.smithi070.stdout:1209: done (1 left) 2024-09-14T03:52:01.446 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 34 current snap is 132 2024-09-14T03:52:01.446 INFO:tasks.rados.rados.0.smithi070.stdout:1210: seq_num 389 ranges {2154496=524288,2678784=745472,3424256=458752} 2024-09-14T03:52:01.461 INFO:tasks.rados.rados.0.smithi070.stdout:1210: writing smithi07085154-34 from 2154496 to 2678784 tid 1 2024-09-14T03:52:01.466 INFO:tasks.rados.rados.0.smithi070.stdout:1210: writing smithi07085154-34 from 2678784 to 3424256 tid 2 2024-09-14T03:52:01.469 INFO:tasks.rados.rados.0.smithi070.stdout:1210: writing smithi07085154-34 from 3424256 to 3883008 tid 3 2024-09-14T03:52:01.469 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 4 current snap is 132 2024-09-14T03:52:01.469 INFO:tasks.rados.rados.0.smithi070.stdout:1211: seq_num 390 ranges {3620864=540672,4161536=630784,4792320=573440} 2024-09-14T03:52:01.491 INFO:tasks.rados.rados.0.smithi070.stdout:1211: writing smithi07085154-4 from 3620864 to 4161536 tid 1 2024-09-14T03:52:01.495 INFO:tasks.rados.rados.0.smithi070.stdout:1211: writing smithi07085154-4 from 4161536 to 4792320 tid 2 2024-09-14T03:52:01.499 INFO:tasks.rados.rados.0.smithi070.stdout:1211: writing smithi07085154-4 from 4792320 to 5365760 tid 3 2024-09-14T03:52:01.499 INFO:tasks.rados.rados.0.smithi070.stdout:1212: copy_from oid 7 from oid 1 current snap is 132 2024-09-14T03:52:01.499 INFO:tasks.rados.rados.0.smithi070.stdout:1208: expect (ObjNum 386 snap 130 seq_num 386) 2024-09-14T03:52:01.631 INFO:tasks.rados.rados.0.smithi070.stdout:1210: finishing write tid 1 to smithi07085154-34 2024-09-14T03:52:01.631 INFO:tasks.rados.rados.0.smithi070.stdout:1210: finishing write tid 2 to smithi07085154-34 2024-09-14T03:52:01.632 INFO:tasks.rados.rados.0.smithi070.stdout:1208: done (3 left) 2024-09-14T03:52:01.632 INFO:tasks.rados.rados.0.smithi070.stdout:1213: snap_remove snap 113 2024-09-14T03:52:01.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:01 smithi094 ceph-mon[86597]: osdmap e301: 8 total, 8 up, 8 in 2024-09-14T03:52:01.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:01 smithi070 ceph-mon[98640]: osdmap e301: 8 total, 8 up, 8 in 2024-09-14T03:52:01.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:01 smithi070 ceph-mon[96704]: osdmap e301: 8 total, 8 up, 8 in 2024-09-14T03:52:02.450 INFO:tasks.rados.rados.0.smithi070.stdout:1213: done (3 left) 2024-09-14T03:52:02.450 INFO:tasks.rados.rados.0.smithi070.stdout:1214: read oid 47 snap 66 2024-09-14T03:52:02.450 INFO:tasks.rados.rados.0.smithi070.stdout:1214: expect (ObjNum 151 snap 45 seq_num 151) 2024-09-14T03:52:02.450 INFO:tasks.rados.rados.0.smithi070.stdout:1215: snap_remove snap 115 2024-09-14T03:52:02.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:02 smithi094 ceph-mon[86597]: osdmap e302: 8 total, 8 up, 8 in 2024-09-14T03:52:02.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:02 smithi094 ceph-mon[86597]: pgmap v44: 121 pgs: 121 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.3 MiB/s wr, 7 op/s 2024-09-14T03:52:02.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:02 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:52:02.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:02 smithi070 ceph-mon[96704]: osdmap e302: 8 total, 8 up, 8 in 2024-09-14T03:52:02.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:02 smithi070 ceph-mon[96704]: pgmap v44: 121 pgs: 121 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.3 MiB/s wr, 7 op/s 2024-09-14T03:52:02.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:02 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:52:02.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:02 smithi070 ceph-mon[98640]: osdmap e302: 8 total, 8 up, 8 in 2024-09-14T03:52:02.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:02 smithi070 ceph-mon[98640]: pgmap v44: 121 pgs: 121 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.3 MiB/s wr, 7 op/s 2024-09-14T03:52:02.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:02 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:52:03.458 INFO:tasks.rados.rados.0.smithi070.stdout:1215: done (4 left) 2024-09-14T03:52:03.458 INFO:tasks.rados.rados.0.smithi070.stdout:1216: copy_from oid 30 from oid 16 current snap is 132 2024-09-14T03:52:03.458 INFO:tasks.rados.rados.0.smithi070.stdout:1210: finishing write tid 3 to smithi07085154-34 2024-09-14T03:52:03.458 INFO:tasks.rados.rados.0.smithi070.stdout:1210: finishing write tid 4 to smithi07085154-34 2024-09-14T03:52:03.458 INFO:tasks.rados.rados.0.smithi070.stdout:1210: finishing write tid 5 to smithi07085154-34 2024-09-14T03:52:03.458 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 649 (ObjNum 389 snap 132 seq_num 389) dirty exists 2024-09-14T03:52:03.458 INFO:tasks.rados.rados.0.smithi070.stdout:1210: left oid 34 (ObjNum 389 snap 132 seq_num 389) 2024-09-14T03:52:03.458 INFO:tasks.rados.rados.0.smithi070.stdout:1211: finishing write tid 1 to smithi07085154-4 2024-09-14T03:52:03.458 INFO:tasks.rados.rados.0.smithi070.stdout:1211: finishing write tid 2 to smithi07085154-4 2024-09-14T03:52:03.459 INFO:tasks.rados.rados.0.smithi070.stdout:1211: finishing write tid 3 to smithi07085154-4 2024-09-14T03:52:03.459 INFO:tasks.rados.rados.0.smithi070.stdout:1211: finishing write tid 4 to smithi07085154-4 2024-09-14T03:52:03.459 INFO:tasks.rados.rados.0.smithi070.stdout:1211: finishing write tid 5 to smithi07085154-4 2024-09-14T03:52:03.459 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 436 (ObjNum 390 snap 132 seq_num 390) dirty exists 2024-09-14T03:52:03.459 INFO:tasks.rados.rados.0.smithi070.stdout:1211: left oid 4 (ObjNum 390 snap 132 seq_num 390) 2024-09-14T03:52:03.459 INFO:tasks.rados.rados.0.smithi070.stdout:1212: finishing copy_from racing read to smithi07085154-7 2024-09-14T03:52:03.459 INFO:tasks.rados.rados.0.smithi070.stdout:1212: finishing copy_from to smithi07085154-7 2024-09-14T03:52:03.459 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 388 (ObjNum 363 snap 124 seq_num 363) dirty exists 2024-09-14T03:52:03.459 INFO:tasks.rados.rados.0.smithi070.stdout:1210: done (4 left) 2024-09-14T03:52:03.459 INFO:tasks.rados.rados.0.smithi070.stdout:1211: done (3 left) 2024-09-14T03:52:03.459 INFO:tasks.rados.rados.0.smithi070.stdout:1212: done (2 left) 2024-09-14T03:52:03.460 INFO:tasks.rados.rados.0.smithi070.stdout:1217: read oid 25 snap -1 2024-09-14T03:52:03.460 INFO:tasks.rados.rados.0.smithi070.stdout:1217: expect (ObjNum 375 snap 128 seq_num 375) 2024-09-14T03:52:03.460 INFO:tasks.rados.rados.0.smithi070.stdout:1218: delete oid 5 current snap is 132 2024-09-14T03:52:03.460 INFO:tasks.rados.rados.0.smithi070.stdout:1214: expect (ObjNum 151 snap 45 seq_num 151) 2024-09-14T03:52:03.484 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:03 smithi094 ceph-mon[86597]: osdmap e303: 8 total, 8 up, 8 in 2024-09-14T03:52:03.591 INFO:tasks.rados.rados.0.smithi070.stdout:1216: finishing copy_from racing read to smithi07085154-30 2024-09-14T03:52:03.591 INFO:tasks.rados.rados.0.smithi070.stdout:1216: finishing copy_from to smithi07085154-30 2024-09-14T03:52:03.591 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 30 v 979 (ObjNum 380 snap 129 seq_num 380) dirty exists 2024-09-14T03:52:03.591 INFO:tasks.rados.rados.0.smithi070.stdout:1217: expect (ObjNum 375 snap 128 seq_num 375) 2024-09-14T03:52:03.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:03 smithi070 ceph-mon[96704]: osdmap e303: 8 total, 8 up, 8 in 2024-09-14T03:52:03.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:03 smithi070 ceph-mon[98640]: osdmap e303: 8 total, 8 up, 8 in 2024-09-14T03:52:03.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:52:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:52:03] "GET /metrics HTTP/1.1" 200 36121 "" "Prometheus/2.51.0" 2024-09-14T03:52:04.453 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:52:04 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:52:04.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:04.495 INFO:tasks.rados.rados.0.smithi070.stdout:1214: done (3 left) 2024-09-14T03:52:04.495 INFO:tasks.rados.rados.0.smithi070.stdout:1216: done (2 left) 2024-09-14T03:52:04.496 INFO:tasks.rados.rados.0.smithi070.stdout:1217: done (1 left) 2024-09-14T03:52:04.496 INFO:tasks.rados.rados.0.smithi070.stdout:1218: done (0 left) 2024-09-14T03:52:04.496 INFO:tasks.rados.rados.0.smithi070.stdout:1219: delete oid 49 current snap is 132 2024-09-14T03:52:04.497 INFO:tasks.rados.rados.0.smithi070.stdout:1219: done (0 left) 2024-09-14T03:52:04.497 INFO:tasks.rados.rados.0.smithi070.stdout:1220: snap_create 2024-09-14T03:52:04.536 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:52:04 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:52:04.203+0000 7fa4af5ba640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-09-14T03:52:04.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:04 smithi070 ceph-mon[96704]: osdmap e304: 8 total, 8 up, 8 in 2024-09-14T03:52:04.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:04 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:04.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:04 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:04.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:04 smithi070 ceph-mon[96704]: osdmap e305: 8 total, 8 up, 8 in 2024-09-14T03:52:04.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:04 smithi070 ceph-mon[96704]: pgmap v48: 121 pgs: 121 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.0 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-14T03:52:04.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:04 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:04.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:04 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:04.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:04 smithi070 ceph-mon[98640]: osdmap e304: 8 total, 8 up, 8 in 2024-09-14T03:52:04.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:04 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:04.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:04 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:04.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:04 smithi070 ceph-mon[98640]: osdmap e305: 8 total, 8 up, 8 in 2024-09-14T03:52:04.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:04 smithi070 ceph-mon[98640]: pgmap v48: 121 pgs: 121 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.0 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-14T03:52:04.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:04 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:04.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:04 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:04.766 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:04 smithi094 ceph-mon[86597]: osdmap e304: 8 total, 8 up, 8 in 2024-09-14T03:52:04.767 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:04 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:04.767 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:04 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:04.767 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:04 smithi094 ceph-mon[86597]: osdmap e305: 8 total, 8 up, 8 in 2024-09-14T03:52:04.767 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:04 smithi094 ceph-mon[86597]: pgmap v48: 121 pgs: 121 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.0 MiB/s rd, 2.3 MiB/s wr, 5 op/s 2024-09-14T03:52:04.767 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:04 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:04.767 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:04 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:05.458 INFO:tasks.rados.rados.0.smithi070.stdout:1220: done (0 left) 2024-09-14T03:52:05.458 INFO:tasks.rados.rados.0.smithi070.stdout:1221: rmattr oid 12 current snap is 133 2024-09-14T03:52:05.458 INFO:tasks.rados.rados.0.smithi070.stdout:1222: rollback oid 16 current snap is 133 2024-09-14T03:52:05.458 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 16 to 76 2024-09-14T03:52:05.458 INFO:tasks.rados.rados.0.smithi070.stdout:1223: snap_remove snap 80 2024-09-14T03:52:05.944 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:52:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:52:05.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:05.945 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:52:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:52:05.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:06.465 INFO:tasks.rados.rados.0.smithi070.stdout:1223: done (2 left) 2024-09-14T03:52:06.465 INFO:tasks.rados.rados.0.smithi070.stdout:1224: snap_create 2024-09-14T03:52:06.465 INFO:tasks.rados.rados.0.smithi070.stdout:1222: finishing rollback tid 0 to smithi07085154-16 2024-09-14T03:52:06.465 INFO:tasks.rados.rados.0.smithi070.stdout:1222: finishing rollback tid 1 to smithi07085154-16 2024-09-14T03:52:06.465 INFO:tasks.rados.rados.0.smithi070.stdout:1222: finishing rollback tid 2 to smithi07085154-16 2024-09-14T03:52:06.465 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 1074 (ObjNum 114 snap 24 seq_num 114) dirty exists 2024-09-14T03:52:06.465 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 534 (ObjNum 305 snap 97 seq_num 305) dirty exists 2024-09-14T03:52:06.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:06 smithi094 ceph-mon[86597]: osdmap e306: 8 total, 8 up, 8 in 2024-09-14T03:52:06.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:06 smithi094 ceph-mon[86597]: pgmap v50: 121 pgs: 121 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.5 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-14T03:52:06.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:06 smithi070 ceph-mon[98640]: osdmap e306: 8 total, 8 up, 8 in 2024-09-14T03:52:06.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:06 smithi070 ceph-mon[98640]: pgmap v50: 121 pgs: 121 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.5 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-14T03:52:06.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:06 smithi070 ceph-mon[96704]: osdmap e306: 8 total, 8 up, 8 in 2024-09-14T03:52:06.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:06 smithi070 ceph-mon[96704]: pgmap v50: 121 pgs: 121 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.5 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-14T03:52:07.257 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:52:06 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:52:06.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:07.467 INFO:tasks.rados.rados.0.smithi070.stdout:1221: done (2 left) 2024-09-14T03:52:07.467 INFO:tasks.rados.rados.0.smithi070.stdout:1222: done (1 left) 2024-09-14T03:52:07.467 INFO:tasks.rados.rados.0.smithi070.stdout:1224: done (0 left) 2024-09-14T03:52:07.467 INFO:tasks.rados.rados.0.smithi070.stdout:1225: copy_from oid 23 from oid 38 current snap is 134 2024-09-14T03:52:07.467 INFO:tasks.rados.rados.0.smithi070.stdout:1226: snap_remove snap 132 2024-09-14T03:52:07.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:07 smithi094 ceph-mon[86597]: osdmap e307: 8 total, 8 up, 8 in 2024-09-14T03:52:07.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:07 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:07.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:07 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:07.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:07 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:07.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:07 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:52:07.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:07 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:07.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:07 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T03:52:07.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:07 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T03:52:07.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:07 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:07.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:07 smithi070 ceph-mon[96704]: osdmap e307: 8 total, 8 up, 8 in 2024-09-14T03:52:07.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:07 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:07.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:07 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:07.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:07 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:07.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:07 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:52:07.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:07 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:07.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:07 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T03:52:07.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:07 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T03:52:07.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:07 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:07.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:07 smithi070 ceph-mon[98640]: osdmap e307: 8 total, 8 up, 8 in 2024-09-14T03:52:07.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:07 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:07.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:07 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:07.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:07 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:07.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:07 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:52:07.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:07 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:07.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:07 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T03:52:07.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:07 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T03:52:07.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:07 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:08.472 INFO:tasks.rados.rados.0.smithi070.stdout:1226: done (1 left) 2024-09-14T03:52:08.472 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 12 current snap is 134 2024-09-14T03:52:08.472 INFO:tasks.rados.rados.0.smithi070.stdout:1225: finishing copy_from to smithi07085154-23 2024-09-14T03:52:08.472 INFO:tasks.rados.rados.0.smithi070.stdout:1225: got expected ENOENT (src dne) 2024-09-14T03:52:08.472 INFO:tasks.rados.rados.0.smithi070.stdout:1227: seq_num 393 ranges {311296=589824,901120=679936,1581056=524288} 2024-09-14T03:52:08.476 INFO:tasks.rados.rados.0.smithi070.stdout:1227: writing smithi07085154-12 from 311296 to 901120 tid 1 2024-09-14T03:52:08.480 INFO:tasks.rados.rados.0.smithi070.stdout:1227: writing smithi07085154-12 from 901120 to 1581056 tid 2 2024-09-14T03:52:08.485 INFO:tasks.rados.rados.0.smithi070.stdout:1227: writing smithi07085154-12 from 1581056 to 2105344 tid 3 2024-09-14T03:52:08.485 INFO:tasks.rados.rados.0.smithi070.stdout:1228: copy_from oid 44 from oid 4 current snap is 134 2024-09-14T03:52:08.485 INFO:tasks.rados.rados.0.smithi070.stdout:1229: rmattr oid 8 current snap is 134 2024-09-14T03:52:08.485 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 31 current snap is 134 2024-09-14T03:52:08.485 INFO:tasks.rados.rados.0.smithi070.stdout:1225: finishing copy_from racing read to smithi07085154-23 2024-09-14T03:52:08.485 INFO:tasks.rados.rados.0.smithi070.stdout:1230: seq_num 394 ranges {0=606208,606208=565248,1171456=557056,1728512=81920} 2024-09-14T03:52:08.489 INFO:tasks.rados.rados.0.smithi070.stdout:1230: writing smithi07085154-31 from 0 to 606208 tid 1 2024-09-14T03:52:08.493 INFO:tasks.rados.rados.0.smithi070.stdout:1230: writing smithi07085154-31 from 606208 to 1171456 tid 2 2024-09-14T03:52:08.497 INFO:tasks.rados.rados.0.smithi070.stdout:1230: writing smithi07085154-31 from 1171456 to 1728512 tid 3 2024-09-14T03:52:08.498 INFO:tasks.rados.rados.0.smithi070.stdout:1230: writing smithi07085154-31 from 1728512 to 1810432 tid 4 2024-09-14T03:52:08.498 INFO:tasks.rados.rados.0.smithi070.stdout:1225: done (4 left) 2024-09-14T03:52:08.498 INFO:tasks.rados.rados.0.smithi070.stdout:1231: snap_create 2024-09-14T03:52:08.498 INFO:tasks.rados.rados.0.smithi070.stdout:1227: finishing write tid 1 to smithi07085154-12 2024-09-14T03:52:08.498 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 8 v 817 (ObjNum 202 snap 56 seq_num 202) dirty exists 2024-09-14T03:52:08.498 INFO:tasks.rados.rados.0.smithi070.stdout:1227: finishing write tid 2 to smithi07085154-12 2024-09-14T03:52:08.498 INFO:tasks.rados.rados.0.smithi070.stdout:1227: finishing write tid 3 to smithi07085154-12 2024-09-14T03:52:08.498 INFO:tasks.rados.rados.0.smithi070.stdout:1227: finishing write tid 4 to smithi07085154-12 2024-09-14T03:52:08.498 INFO:tasks.rados.rados.0.smithi070.stdout:1227: finishing write tid 5 to smithi07085154-12 2024-09-14T03:52:08.499 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 538 (ObjNum 393 snap 134 seq_num 393) dirty exists 2024-09-14T03:52:08.499 INFO:tasks.rados.rados.0.smithi070.stdout:1227: left oid 12 (ObjNum 393 snap 134 seq_num 393) 2024-09-14T03:52:08.499 INFO:tasks.rados.rados.0.smithi070.stdout:1230: finishing write tid 1 to smithi07085154-31 2024-09-14T03:52:08.499 INFO:tasks.rados.rados.0.smithi070.stdout:1230: finishing write tid 2 to smithi07085154-31 2024-09-14T03:52:08.501 INFO:tasks.rados.rados.0.smithi070.stdout:1230: finishing write tid 3 to smithi07085154-31 2024-09-14T03:52:08.501 INFO:tasks.rados.rados.0.smithi070.stdout:1230: finishing write tid 4 to smithi07085154-31 2024-09-14T03:52:08.501 INFO:tasks.rados.rados.0.smithi070.stdout:1230: finishing write tid 5 to smithi07085154-31 2024-09-14T03:52:08.502 INFO:tasks.rados.rados.0.smithi070.stdout:1230: finishing write tid 6 to smithi07085154-31 2024-09-14T03:52:08.502 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 550 (ObjNum 394 snap 134 seq_num 394) dirty exists 2024-09-14T03:52:08.503 INFO:tasks.rados.rados.0.smithi070.stdout:1230: left oid 31 (ObjNum 394 snap 134 seq_num 394) 2024-09-14T03:52:08.514 INFO:tasks.rados.rados.0.smithi070.stdout:1228: finishing copy_from racing read to smithi07085154-44 2024-09-14T03:52:08.517 INFO:tasks.rados.rados.0.smithi070.stdout:1228: finishing copy_from to smithi07085154-44 2024-09-14T03:52:08.517 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 436 (ObjNum 390 snap 132 seq_num 390) dirty exists 2024-09-14T03:52:08.748 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:08 smithi070 ceph-mon[96704]: Reconfiguring mon.a (monmap changed)... 2024-09-14T03:52:08.748 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:08 smithi070 ceph-mon[96704]: Reconfiguring daemon mon.a on smithi070 2024-09-14T03:52:08.748 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:08 smithi070 ceph-mon[96704]: osdmap e308: 8 total, 8 up, 8 in 2024-09-14T03:52:08.748 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:08 smithi070 ceph-mon[96704]: pgmap v53: 121 pgs: 121 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:08.748 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:08 smithi070 ceph-mon[98640]: Reconfiguring mon.a (monmap changed)... 2024-09-14T03:52:08.748 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:08 smithi070 ceph-mon[98640]: Reconfiguring daemon mon.a on smithi070 2024-09-14T03:52:08.749 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:08 smithi070 ceph-mon[98640]: osdmap e308: 8 total, 8 up, 8 in 2024-09-14T03:52:08.749 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:08 smithi070 ceph-mon[98640]: pgmap v53: 121 pgs: 121 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:08.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:08 smithi094 ceph-mon[86597]: Reconfiguring mon.a (monmap changed)... 2024-09-14T03:52:08.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:08 smithi094 ceph-mon[86597]: Reconfiguring daemon mon.a on smithi070 2024-09-14T03:52:08.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:08 smithi094 ceph-mon[86597]: osdmap e308: 8 total, 8 up, 8 in 2024-09-14T03:52:08.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:08 smithi094 ceph-mon[86597]: pgmap v53: 121 pgs: 121 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:09.476 INFO:tasks.rados.rados.0.smithi070.stdout:1227: done (4 left) 2024-09-14T03:52:09.477 INFO:tasks.rados.rados.0.smithi070.stdout:1228: done (3 left) 2024-09-14T03:52:09.477 INFO:tasks.rados.rados.0.smithi070.stdout:1229: done (2 left) 2024-09-14T03:52:09.477 INFO:tasks.rados.rados.0.smithi070.stdout:1230: done (1 left) 2024-09-14T03:52:09.477 INFO:tasks.rados.rados.0.smithi070.stdout:1231: done (0 left) 2024-09-14T03:52:09.477 INFO:tasks.rados.rados.0.smithi070.stdout:1232: rollback oid 26 current snap is 135 2024-09-14T03:52:09.477 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 26 to 134 2024-09-14T03:52:09.477 INFO:tasks.rados.rados.0.smithi070.stdout:1233: copy_from oid 48 from oid 14 current snap is 135 2024-09-14T03:52:09.477 INFO:tasks.rados.rados.0.smithi070.stdout:1234: snap_create 2024-09-14T03:52:09.480 INFO:tasks.rados.rados.0.smithi070.stdout:1232: finishing rollback tid 0 to smithi07085154-26 2024-09-14T03:52:09.482 INFO:tasks.rados.rados.0.smithi070.stdout:1232: finishing rollback tid 1 to smithi07085154-26 2024-09-14T03:52:09.482 INFO:tasks.rados.rados.0.smithi070.stdout:1232: finishing rollback tid 2 to smithi07085154-26 2024-09-14T03:52:09.482 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 1090 (ObjNum 319 snap 107 seq_num 319) dirty exists 2024-09-14T03:52:09.486 INFO:tasks.rados.rados.0.smithi070.stdout:1233: finishing copy_from racing read to smithi07085154-48 2024-09-14T03:52:09.487 INFO:tasks.rados.rados.0.smithi070.stdout:1233: finishing copy_from to smithi07085154-48 2024-09-14T03:52:09.487 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 417 (ObjNum 313 snap 105 seq_num 313) dirty exists 2024-09-14T03:52:09.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:09 smithi094 ceph-mon[86597]: osdmap e309: 8 total, 8 up, 8 in 2024-09-14T03:52:09.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:09 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:09.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:09 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:09.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:09 smithi094 ceph-mon[86597]: Reconfiguring mgr.y (monmap changed)... 2024-09-14T03:52:09.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:09 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T03:52:09.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:09 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-14T03:52:09.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:09 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:09.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:09 smithi094 ceph-mon[86597]: Reconfiguring daemon mgr.y on smithi070 2024-09-14T03:52:09.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:09 smithi070 ceph-mon[96704]: osdmap e309: 8 total, 8 up, 8 in 2024-09-14T03:52:09.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:09 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:09.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:09 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:09.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:09 smithi070 ceph-mon[96704]: Reconfiguring mgr.y (monmap changed)... 2024-09-14T03:52:09.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:09 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T03:52:09.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:09 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-14T03:52:09.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:09 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:09.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:09 smithi070 ceph-mon[96704]: Reconfiguring daemon mgr.y on smithi070 2024-09-14T03:52:09.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:09 smithi070 ceph-mon[98640]: osdmap e309: 8 total, 8 up, 8 in 2024-09-14T03:52:09.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:09 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:09.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:09 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:09.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:09 smithi070 ceph-mon[98640]: Reconfiguring mgr.y (monmap changed)... 2024-09-14T03:52:09.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:09 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T03:52:09.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:09 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-14T03:52:09.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:09 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:09.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:09 smithi070 ceph-mon[98640]: Reconfiguring daemon mgr.y on smithi070 2024-09-14T03:52:10.477 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:52:10 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:52:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:10.480 INFO:tasks.rados.rados.0.smithi070.stdout:1232: done (2 left) 2024-09-14T03:52:10.480 INFO:tasks.rados.rados.0.smithi070.stdout:1233: done (1 left) 2024-09-14T03:52:10.480 INFO:tasks.rados.rados.0.smithi070.stdout:1234: done (0 left) 2024-09-14T03:52:10.480 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 23 current snap is 136 2024-09-14T03:52:10.480 INFO:tasks.rados.rados.0.smithi070.stdout:1235: seq_num 395 ranges {3440640=622592,4063232=450560,4513792=753664} 2024-09-14T03:52:10.502 INFO:tasks.rados.rados.0.smithi070.stdout:1235: writing smithi07085154-23 from 3440640 to 4063232 tid 1 2024-09-14T03:52:10.505 INFO:tasks.rados.rados.0.smithi070.stdout:1235: writing smithi07085154-23 from 4063232 to 4513792 tid 2 2024-09-14T03:52:10.509 INFO:tasks.rados.rados.0.smithi070.stdout:1235: writing smithi07085154-23 from 4513792 to 5267456 tid 3 2024-09-14T03:52:10.509 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 36 current snap is 136 2024-09-14T03:52:10.509 INFO:tasks.rados.rados.0.smithi070.stdout:1235: finishing write tid 1 to smithi07085154-23 2024-09-14T03:52:10.509 INFO:tasks.rados.rados.0.smithi070.stdout:1236: seq_num 396 ranges {2965504=638976,3604480=786432,4390912=417792} 2024-09-14T03:52:10.529 INFO:tasks.rados.rados.0.smithi070.stdout:1236: writing smithi07085154-36 from 2965504 to 3604480 tid 1 2024-09-14T03:52:10.534 INFO:tasks.rados.rados.0.smithi070.stdout:1236: writing smithi07085154-36 from 3604480 to 4390912 tid 2 2024-09-14T03:52:10.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:10 smithi070 ceph-mon[96704]: osdmap e310: 8 total, 8 up, 8 in 2024-09-14T03:52:10.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:10 smithi070 ceph-mon[96704]: pgmap v56: 121 pgs: 121 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.1 MiB/s rd, 1.5 MiB/s wr, 3 op/s 2024-09-14T03:52:10.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:10 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:10.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:10 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:10.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:10 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T03:52:10.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:10 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T03:52:10.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:10 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:10.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:10 smithi070 ceph-mon[98640]: osdmap e310: 8 total, 8 up, 8 in 2024-09-14T03:52:10.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:10 smithi070 ceph-mon[98640]: pgmap v56: 121 pgs: 121 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.1 MiB/s rd, 1.5 MiB/s wr, 3 op/s 2024-09-14T03:52:10.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:10 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:10.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:10 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:10.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:10 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T03:52:10.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:10 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T03:52:10.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:10 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:10.537 INFO:tasks.rados.rados.0.smithi070.stdout:1236: writing smithi07085154-36 from 4390912 to 4808704 tid 3 2024-09-14T03:52:10.537 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 12 current snap is 136 2024-09-14T03:52:10.537 INFO:tasks.rados.rados.0.smithi070.stdout:1235: finishing write tid 2 to smithi07085154-23 2024-09-14T03:52:10.537 INFO:tasks.rados.rados.0.smithi070.stdout:1235: finishing write tid 3 to smithi07085154-23 2024-09-14T03:52:10.537 INFO:tasks.rados.rados.0.smithi070.stdout:1237: seq_num 397 ranges {2105344=655360,2760704=671744,3432448=532480} 2024-09-14T03:52:10.551 INFO:tasks.rados.rados.0.smithi070.stdout:1237: writing smithi07085154-12 from 2105344 to 2760704 tid 1 2024-09-14T03:52:10.556 INFO:tasks.rados.rados.0.smithi070.stdout:1237: writing smithi07085154-12 from 2760704 to 3432448 tid 2 2024-09-14T03:52:10.559 INFO:tasks.rados.rados.0.smithi070.stdout:1237: writing smithi07085154-12 from 3432448 to 3964928 tid 3 2024-09-14T03:52:10.560 INFO:tasks.rados.rados.0.smithi070.stdout:1238: snap_create 2024-09-14T03:52:10.560 INFO:tasks.rados.rados.0.smithi070.stdout:1235: finishing write tid 4 to smithi07085154-23 2024-09-14T03:52:10.560 INFO:tasks.rados.rados.0.smithi070.stdout:1235: finishing write tid 5 to smithi07085154-23 2024-09-14T03:52:10.560 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 596 (ObjNum 395 snap 136 seq_num 395) dirty exists 2024-09-14T03:52:10.560 INFO:tasks.rados.rados.0.smithi070.stdout:1235: left oid 23 (ObjNum 395 snap 136 seq_num 395) 2024-09-14T03:52:10.560 INFO:tasks.rados.rados.0.smithi070.stdout:1236: finishing write tid 1 to smithi07085154-36 2024-09-14T03:52:10.560 INFO:tasks.rados.rados.0.smithi070.stdout:1236: finishing write tid 2 to smithi07085154-36 2024-09-14T03:52:10.560 INFO:tasks.rados.rados.0.smithi070.stdout:1236: finishing write tid 3 to smithi07085154-36 2024-09-14T03:52:10.560 INFO:tasks.rados.rados.0.smithi070.stdout:1236: finishing write tid 4 to smithi07085154-36 2024-09-14T03:52:10.560 INFO:tasks.rados.rados.0.smithi070.stdout:1236: finishing write tid 5 to smithi07085154-36 2024-09-14T03:52:10.560 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 36 v 459 (ObjNum 396 snap 136 seq_num 396) dirty exists 2024-09-14T03:52:10.560 INFO:tasks.rados.rados.0.smithi070.stdout:1236: left oid 36 (ObjNum 396 snap 136 seq_num 396) 2024-09-14T03:52:10.560 INFO:tasks.rados.rados.0.smithi070.stdout:1237: finishing write tid 1 to smithi07085154-12 2024-09-14T03:52:10.561 INFO:tasks.rados.rados.0.smithi070.stdout:1237: finishing write tid 2 to smithi07085154-12 2024-09-14T03:52:10.562 INFO:tasks.rados.rados.0.smithi070.stdout:1237: finishing write tid 3 to smithi07085154-12 2024-09-14T03:52:10.562 INFO:tasks.rados.rados.0.smithi070.stdout:1237: finishing write tid 4 to smithi07085154-12 2024-09-14T03:52:10.563 INFO:tasks.rados.rados.0.smithi070.stdout:1237: finishing write tid 5 to smithi07085154-12 2024-09-14T03:52:10.563 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 542 (ObjNum 397 snap 136 seq_num 397) dirty exists 2024-09-14T03:52:10.563 INFO:tasks.rados.rados.0.smithi070.stdout:1237: left oid 12 (ObjNum 397 snap 136 seq_num 397) 2024-09-14T03:52:10.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:10 smithi094 ceph-mon[86597]: osdmap e310: 8 total, 8 up, 8 in 2024-09-14T03:52:10.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:10 smithi094 ceph-mon[86597]: pgmap v56: 121 pgs: 121 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.1 MiB/s rd, 1.5 MiB/s wr, 3 op/s 2024-09-14T03:52:10.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:10 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:10.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:10 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:10.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:10 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T03:52:10.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:10 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T03:52:10.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:10 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:11.486 INFO:tasks.rados.rados.0.smithi070.stdout:1235: done (3 left) 2024-09-14T03:52:11.486 INFO:tasks.rados.rados.0.smithi070.stdout:1236: done (2 left) 2024-09-14T03:52:11.486 INFO:tasks.rados.rados.0.smithi070.stdout:1237: done (1 left) 2024-09-14T03:52:11.486 INFO:tasks.rados.rados.0.smithi070.stdout:1238: done (0 left) 2024-09-14T03:52:11.486 INFO:tasks.rados.rados.0.smithi070.stdout:1239: read oid 16 snap -1 2024-09-14T03:52:11.486 INFO:tasks.rados.rados.0.smithi070.stdout:1239: expect (ObjNum 114 snap 24 seq_num 114) 2024-09-14T03:52:11.486 INFO:tasks.rados.rados.0.smithi070.stdout:1240: read oid 47 snap -1 2024-09-14T03:52:11.486 INFO:tasks.rados.rados.0.smithi070.stdout:1240: expect (ObjNum 382 snap 130 seq_num 382) 2024-09-14T03:52:11.486 INFO:tasks.rados.rados.0.smithi070.stdout:1241: read oid 26 snap 101 2024-09-14T03:52:11.487 INFO:tasks.rados.rados.0.smithi070.stdout:1241: expect deleted 2024-09-14T03:52:11.487 INFO:tasks.rados.rados.0.smithi070.stdout:1242: snap_create 2024-09-14T03:52:11.500 INFO:tasks.rados.rados.0.smithi070.stdout:1239: expect (ObjNum 114 snap 24 seq_num 114) 2024-09-14T03:52:11.697 INFO:tasks.rados.rados.0.smithi070.stdout:1240: expect (ObjNum 382 snap 130 seq_num 382) 2024-09-14T03:52:11.737 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:11 smithi070 ceph-mon[96704]: Reconfiguring mon.c (monmap changed)... 2024-09-14T03:52:11.738 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:11 smithi070 ceph-mon[96704]: Reconfiguring daemon mon.c on smithi070 2024-09-14T03:52:11.738 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:11 smithi070 ceph-mon[96704]: osdmap e311: 8 total, 8 up, 8 in 2024-09-14T03:52:11.738 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:11 smithi070 ceph-mon[98640]: Reconfiguring mon.c (monmap changed)... 2024-09-14T03:52:11.738 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:11 smithi070 ceph-mon[98640]: Reconfiguring daemon mon.c on smithi070 2024-09-14T03:52:11.738 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:11 smithi070 ceph-mon[98640]: osdmap e311: 8 total, 8 up, 8 in 2024-09-14T03:52:11.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:11 smithi094 ceph-mon[86597]: Reconfiguring mon.c (monmap changed)... 2024-09-14T03:52:11.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:11 smithi094 ceph-mon[86597]: Reconfiguring daemon mon.c on smithi070 2024-09-14T03:52:11.744 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:11 smithi094 ceph-mon[86597]: osdmap e311: 8 total, 8 up, 8 in 2024-09-14T03:52:12.487 INFO:tasks.rados.rados.0.smithi070.stdout:1239: done (3 left) 2024-09-14T03:52:12.487 INFO:tasks.rados.rados.0.smithi070.stdout:1240: done (2 left) 2024-09-14T03:52:12.487 INFO:tasks.rados.rados.0.smithi070.stdout:1241: done (1 left) 2024-09-14T03:52:12.487 INFO:tasks.rados.rados.0.smithi070.stdout:1242: done (0 left) 2024-09-14T03:52:12.487 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 14 current snap is 138 2024-09-14T03:52:12.487 INFO:tasks.rados.rados.0.smithi070.stdout:1243: seq_num 398 ranges {450560=671744,1122304=557056,1679360=532480,2211840=114688} 2024-09-14T03:52:12.493 INFO:tasks.rados.rados.0.smithi070.stdout:1243: writing smithi07085154-14 from 450560 to 1122304 tid 1 2024-09-14T03:52:12.496 INFO:tasks.rados.rados.0.smithi070.stdout:1243: writing smithi07085154-14 from 1122304 to 1679360 tid 2 2024-09-14T03:52:12.499 INFO:tasks.rados.rados.0.smithi070.stdout:1243: writing smithi07085154-14 from 1679360 to 2211840 tid 3 2024-09-14T03:52:12.500 INFO:tasks.rados.rados.0.smithi070.stdout:1243: writing smithi07085154-14 from 2211840 to 2326528 tid 4 2024-09-14T03:52:12.500 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 39 current snap is 138 2024-09-14T03:52:12.500 INFO:tasks.rados.rados.0.smithi070.stdout:1243: finishing write tid 1 to smithi07085154-14 2024-09-14T03:52:12.500 INFO:tasks.rados.rados.0.smithi070.stdout:1243: finishing write tid 2 to smithi07085154-14 2024-09-14T03:52:12.500 INFO:tasks.rados.rados.0.smithi070.stdout:1244: seq_num 399 ranges {2195456=688128,2883584=434176,3317760=745472,4063232=24576} 2024-09-14T03:52:12.516 INFO:tasks.rados.rados.0.smithi070.stdout:1244: writing smithi07085154-39 from 2195456 to 2883584 tid 1 2024-09-14T03:52:12.518 INFO:tasks.rados.rados.0.smithi070.stdout:1244: writing smithi07085154-39 from 2883584 to 3317760 tid 2 2024-09-14T03:52:12.523 INFO:tasks.rados.rados.0.smithi070.stdout:1244: writing smithi07085154-39 from 3317760 to 4063232 tid 3 2024-09-14T03:52:12.523 INFO:tasks.rados.rados.0.smithi070.stdout:1244: writing smithi07085154-39 from 4063232 to 4087808 tid 4 2024-09-14T03:52:12.523 INFO:tasks.rados.rados.0.smithi070.stdout:1245: read oid 9 snap -1 2024-09-14T03:52:12.523 INFO:tasks.rados.rados.0.smithi070.stdout:1245: expect deleted 2024-09-14T03:52:12.523 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 28 current snap is 138 2024-09-14T03:52:12.523 INFO:tasks.rados.rados.0.smithi070.stdout:1243: finishing write tid 3 to smithi07085154-14 2024-09-14T03:52:12.523 INFO:tasks.rados.rados.0.smithi070.stdout:1246: seq_num 400 ranges {3031040=704512,3735552=720896,4456448=483328} 2024-09-14T03:52:12.542 INFO:tasks.rados.rados.0.smithi070.stdout:1246: writing smithi07085154-28 from 3031040 to 3735552 tid 1 2024-09-14T03:52:12.547 INFO:tasks.rados.rados.0.smithi070.stdout:1246: writing smithi07085154-28 from 3735552 to 4456448 tid 2 2024-09-14T03:52:12.551 INFO:tasks.rados.rados.0.smithi070.stdout:1246: writing smithi07085154-28 from 4456448 to 4939776 tid 3 2024-09-14T03:52:12.551 INFO:tasks.rados.rados.0.smithi070.stdout:1247: snap_create 2024-09-14T03:52:12.552 INFO:tasks.rados.rados.0.smithi070.stdout:1243: finishing write tid 4 to smithi07085154-14 2024-09-14T03:52:12.552 INFO:tasks.rados.rados.0.smithi070.stdout:1243: finishing write tid 5 to smithi07085154-14 2024-09-14T03:52:12.552 INFO:tasks.rados.rados.0.smithi070.stdout:1243: finishing write tid 6 to smithi07085154-14 2024-09-14T03:52:12.552 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 601 (ObjNum 398 snap 138 seq_num 398) dirty exists 2024-09-14T03:52:12.552 INFO:tasks.rados.rados.0.smithi070.stdout:1243: left oid 14 (ObjNum 398 snap 138 seq_num 398) 2024-09-14T03:52:12.552 INFO:tasks.rados.rados.0.smithi070.stdout:1244: finishing write tid 1 to smithi07085154-39 2024-09-14T03:52:12.552 INFO:tasks.rados.rados.0.smithi070.stdout:1244: finishing write tid 2 to smithi07085154-39 2024-09-14T03:52:12.552 INFO:tasks.rados.rados.0.smithi070.stdout:1244: finishing write tid 3 to smithi07085154-39 2024-09-14T03:52:12.552 INFO:tasks.rados.rados.0.smithi070.stdout:1244: finishing write tid 4 to smithi07085154-39 2024-09-14T03:52:12.552 INFO:tasks.rados.rados.0.smithi070.stdout:1244: finishing write tid 5 to smithi07085154-39 2024-09-14T03:52:12.552 INFO:tasks.rados.rados.0.smithi070.stdout:1244: finishing write tid 6 to smithi07085154-39 2024-09-14T03:52:12.553 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 39 v 316 (ObjNum 399 snap 138 seq_num 399) dirty exists 2024-09-14T03:52:12.553 INFO:tasks.rados.rados.0.smithi070.stdout:1244: left oid 39 (ObjNum 399 snap 138 seq_num 399) 2024-09-14T03:52:12.553 INFO:tasks.rados.rados.0.smithi070.stdout:1246: finishing write tid 1 to smithi07085154-28 2024-09-14T03:52:12.553 INFO:tasks.rados.rados.0.smithi070.stdout:1246: finishing write tid 2 to smithi07085154-28 2024-09-14T03:52:12.553 INFO:tasks.rados.rados.0.smithi070.stdout:1246: finishing write tid 3 to smithi07085154-28 2024-09-14T03:52:12.554 INFO:tasks.rados.rados.0.smithi070.stdout:1246: finishing write tid 4 to smithi07085154-28 2024-09-14T03:52:12.554 INFO:tasks.rados.rados.0.smithi070.stdout:1246: finishing write tid 5 to smithi07085154-28 2024-09-14T03:52:12.554 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 28 v 457 (ObjNum 400 snap 138 seq_num 400) dirty exists 2024-09-14T03:52:12.554 INFO:tasks.rados.rados.0.smithi070.stdout:1246: left oid 28 (ObjNum 400 snap 138 seq_num 400) 2024-09-14T03:52:12.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:12 smithi094 ceph-mon[86597]: osdmap e312: 8 total, 8 up, 8 in 2024-09-14T03:52:12.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:12 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:12.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:12 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:12.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:12 smithi094 ceph-mon[86597]: Reconfiguring osd.0 (monmap changed)... 2024-09-14T03:52:12.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:12 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-14T03:52:12.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:12 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:12.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:12 smithi094 ceph-mon[86597]: Reconfiguring daemon osd.0 on smithi070 2024-09-14T03:52:12.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:12 smithi094 ceph-mon[86597]: pgmap v59: 121 pgs: 121 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.1 MiB/s rd, 1.5 MiB/s wr, 3 op/s 2024-09-14T03:52:12.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:12 smithi070 ceph-mon[96704]: osdmap e312: 8 total, 8 up, 8 in 2024-09-14T03:52:12.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:12 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:12.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:12 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:12.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:12 smithi070 ceph-mon[96704]: Reconfiguring osd.0 (monmap changed)... 2024-09-14T03:52:12.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:12 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-14T03:52:12.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:12 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:12.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:12 smithi070 ceph-mon[96704]: Reconfiguring daemon osd.0 on smithi070 2024-09-14T03:52:12.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:12 smithi070 ceph-mon[96704]: pgmap v59: 121 pgs: 121 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.1 MiB/s rd, 1.5 MiB/s wr, 3 op/s 2024-09-14T03:52:12.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:12 smithi070 ceph-mon[98640]: osdmap e312: 8 total, 8 up, 8 in 2024-09-14T03:52:12.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:12 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:12.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:12 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:12.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:12 smithi070 ceph-mon[98640]: Reconfiguring osd.0 (monmap changed)... 2024-09-14T03:52:12.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:12 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-14T03:52:12.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:12 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:12.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:12 smithi070 ceph-mon[98640]: Reconfiguring daemon osd.0 on smithi070 2024-09-14T03:52:12.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:12 smithi070 ceph-mon[98640]: pgmap v59: 121 pgs: 121 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.1 MiB/s rd, 1.5 MiB/s wr, 3 op/s 2024-09-14T03:52:13.488 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:52:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:52:13] "GET /metrics HTTP/1.1" 200 36121 "" "Prometheus/2.51.0" 2024-09-14T03:52:13.489 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:13 smithi070 ceph-mon[96704]: osdmap e313: 8 total, 8 up, 8 in 2024-09-14T03:52:13.489 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:13 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:13.489 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:13 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:13.489 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:13 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-14T03:52:13.495 INFO:tasks.rados.rados.0.smithi070.stdout:1243: done (4 left) 2024-09-14T03:52:13.495 INFO:tasks.rados.rados.0.smithi070.stdout:1244: done (3 left) 2024-09-14T03:52:13.495 INFO:tasks.rados.rados.0.smithi070.stdout:1245: done (2 left) 2024-09-14T03:52:13.495 INFO:tasks.rados.rados.0.smithi070.stdout:1246: done (1 left) 2024-09-14T03:52:13.495 INFO:tasks.rados.rados.0.smithi070.stdout:1247: done (0 left) 2024-09-14T03:52:13.495 INFO:tasks.rados.rados.0.smithi070.stdout:1248: snap_create 2024-09-14T03:52:13.760 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:13 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:13.761 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:13 smithi070 ceph-mon[98640]: osdmap e313: 8 total, 8 up, 8 in 2024-09-14T03:52:13.761 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:13 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:13.761 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:13 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:13.761 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:13 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-14T03:52:13.761 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:13 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:13.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:13 smithi094 ceph-mon[86597]: osdmap e313: 8 total, 8 up, 8 in 2024-09-14T03:52:13.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:13 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:13.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:13 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:13.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:13 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-14T03:52:13.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:13 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:13.995 INFO:tasks.rados.rados.0.smithi070.stdout:1248: done (0 left) 2024-09-14T03:52:13.995 INFO:tasks.rados.rados.0.smithi070.stdout:1249: delete oid 18 current snap is 140 2024-09-14T03:52:14.001 INFO:tasks.rados.rados.0.smithi070.stdout:1249: done (0 left) 2024-09-14T03:52:14.001 INFO:tasks.rados.rados.0.smithi070.stdout:1250: read oid 32 snap -1 2024-09-14T03:52:14.001 INFO:tasks.rados.rados.0.smithi070.stdout:1250: expect deleted 2024-09-14T03:52:14.001 INFO:tasks.rados.rados.0.smithi070.stdout:1251: read oid 48 snap -1 2024-09-14T03:52:14.001 INFO:tasks.rados.rados.0.smithi070.stdout:1251: expect (ObjNum 313 snap 105 seq_num 313) 2024-09-14T03:52:14.001 INFO:tasks.rados.rados.0.smithi070.stdout:1252: read oid 14 snap 122 2024-09-14T03:52:14.002 INFO:tasks.rados.rados.0.smithi070.stdout:1252: expect (ObjNum 313 snap 105 seq_num 313) 2024-09-14T03:52:14.002 INFO:tasks.rados.rados.0.smithi070.stdout:1253: copy_from oid 30 from oid 43 current snap is 140 2024-09-14T03:52:14.002 INFO:tasks.rados.rados.0.smithi070.stdout:1254: read oid 39 snap -1 2024-09-14T03:52:14.002 INFO:tasks.rados.rados.0.smithi070.stdout:1254: expect (ObjNum 399 snap 138 seq_num 399) 2024-09-14T03:52:14.002 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 8 current snap is 140 2024-09-14T03:52:14.002 INFO:tasks.rados.rados.0.smithi070.stdout:1255: seq_num 402 ranges {1769472=737280,2506752=491520,2998272=516096,3514368=196608} 2024-09-14T03:52:14.014 INFO:tasks.rados.rados.0.smithi070.stdout:1255: writing smithi07085154-8 from 1769472 to 2506752 tid 1 2024-09-14T03:52:14.018 INFO:tasks.rados.rados.0.smithi070.stdout:1255: writing smithi07085154-8 from 2506752 to 2998272 tid 2 2024-09-14T03:52:14.021 INFO:tasks.rados.rados.0.smithi070.stdout:1255: writing smithi07085154-8 from 2998272 to 3514368 tid 3 2024-09-14T03:52:14.022 INFO:tasks.rados.rados.0.smithi070.stdout:1255: writing smithi07085154-8 from 3514368 to 3710976 tid 4 2024-09-14T03:52:14.023 INFO:tasks.rados.rados.0.smithi070.stdout:1256: rollback oid 49 current snap is 140 2024-09-14T03:52:14.023 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 49 to 121 2024-09-14T03:52:14.023 INFO:tasks.rados.rados.0.smithi070.stdout:1253: finishing copy_from to smithi07085154-30 2024-09-14T03:52:14.023 INFO:tasks.rados.rados.0.smithi070.stdout:1253: got expected ENOENT (src dne) 2024-09-14T03:52:14.023 INFO:tasks.rados.rados.0.smithi070.stdout:1253: finishing copy_from racing read to smithi07085154-30 2024-09-14T03:52:14.023 INFO:tasks.rados.rados.0.smithi070.stdout:1250: done (6 left) 2024-09-14T03:52:14.023 INFO:tasks.rados.rados.0.smithi070.stdout:1253: done (5 left) 2024-09-14T03:52:14.023 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 13 current snap is 140 2024-09-14T03:52:14.023 INFO:tasks.rados.rados.0.smithi070.stdout:1251: expect (ObjNum 313 snap 105 seq_num 313) 2024-09-14T03:52:14.036 INFO:tasks.rados.rados.0.smithi070.stdout:1252: expect (ObjNum 313 snap 105 seq_num 313) 2024-09-14T03:52:14.048 INFO:tasks.rados.rados.0.smithi070.stdout:1255: finishing write tid 1 to smithi07085154-8 2024-09-14T03:52:14.048 INFO:tasks.rados.rados.0.smithi070.stdout:1255: finishing write tid 2 to smithi07085154-8 2024-09-14T03:52:14.048 INFO:tasks.rados.rados.0.smithi070.stdout:1257: seq_num 403 ranges {7151616=753664,7905280=425984,8331264=720896,9052160=57344} 2024-09-14T03:52:14.089 INFO:tasks.rados.rados.0.smithi070.stdout:1257: writing smithi07085154-13 from 7151616 to 7905280 tid 1 2024-09-14T03:52:14.091 INFO:tasks.rados.rados.0.smithi070.stdout:1257: writing smithi07085154-13 from 7905280 to 8331264 tid 2 2024-09-14T03:52:14.096 INFO:tasks.rados.rados.0.smithi070.stdout:1257: writing smithi07085154-13 from 8331264 to 9052160 tid 3 2024-09-14T03:52:14.096 INFO:tasks.rados.rados.0.smithi070.stdout:1257: writing smithi07085154-13 from 9052160 to 9109504 tid 4 2024-09-14T03:52:14.097 INFO:tasks.rados.rados.0.smithi070.stdout:1251: done (5 left) 2024-09-14T03:52:14.097 INFO:tasks.rados.rados.0.smithi070.stdout:1252: done (4 left) 2024-09-14T03:52:14.097 INFO:tasks.rados.rados.0.smithi070.stdout:1258: snap_remove snap 57 2024-09-14T03:52:14.494 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:52:14 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:52:14.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:14.635 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:14 smithi070 ceph-mon[98640]: Reconfiguring osd.1 (monmap changed)... 2024-09-14T03:52:14.636 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:14 smithi070 ceph-mon[98640]: Reconfiguring daemon osd.1 on smithi070 2024-09-14T03:52:14.636 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:14 smithi070 ceph-mon[98640]: osdmap e314: 8 total, 8 up, 8 in 2024-09-14T03:52:14.636 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:14 smithi070 ceph-mon[98640]: osdmap e315: 8 total, 8 up, 8 in 2024-09-14T03:52:14.636 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:14 smithi070 ceph-mon[98640]: pgmap v63: 121 pgs: 121 active+clean; 363 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.2 MiB/s rd, 5.5 MiB/s wr, 18 op/s 2024-09-14T03:52:14.636 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:14 smithi070 ceph-mon[96704]: Reconfiguring osd.1 (monmap changed)... 2024-09-14T03:52:14.636 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:14 smithi070 ceph-mon[96704]: Reconfiguring daemon osd.1 on smithi070 2024-09-14T03:52:14.636 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:14 smithi070 ceph-mon[96704]: osdmap e314: 8 total, 8 up, 8 in 2024-09-14T03:52:14.636 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:14 smithi070 ceph-mon[96704]: osdmap e315: 8 total, 8 up, 8 in 2024-09-14T03:52:14.636 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:14 smithi070 ceph-mon[96704]: pgmap v63: 121 pgs: 121 active+clean; 363 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.2 MiB/s rd, 5.5 MiB/s wr, 18 op/s 2024-09-14T03:52:14.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:14 smithi094 ceph-mon[86597]: Reconfiguring osd.1 (monmap changed)... 2024-09-14T03:52:14.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:14 smithi094 ceph-mon[86597]: Reconfiguring daemon osd.1 on smithi070 2024-09-14T03:52:14.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:14 smithi094 ceph-mon[86597]: osdmap e314: 8 total, 8 up, 8 in 2024-09-14T03:52:14.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:14 smithi094 ceph-mon[86597]: osdmap e315: 8 total, 8 up, 8 in 2024-09-14T03:52:14.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:14 smithi094 ceph-mon[86597]: pgmap v63: 121 pgs: 121 active+clean; 363 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.2 MiB/s rd, 5.5 MiB/s wr, 18 op/s 2024-09-14T03:52:14.999 INFO:tasks.rados.rados.0.smithi070.stdout:1258: done (4 left) 2024-09-14T03:52:15.000 INFO:tasks.rados.rados.0.smithi070.stdout:1259: snap_remove snap 67 2024-09-14T03:52:16.004 INFO:tasks.rados.rados.0.smithi070.stdout:1259: done (4 left) 2024-09-14T03:52:16.004 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 16 current snap is 140 2024-09-14T03:52:16.004 INFO:tasks.rados.rados.0.smithi070.stdout:1260: seq_num 404 ranges {4784128=770048,5554176=712704,6266880=491520} 2024-09-14T03:52:16.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:15 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:16.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:15 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:16.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:15 smithi094 ceph-mon[86597]: Reconfiguring osd.2 (monmap changed)... 2024-09-14T03:52:16.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:15 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-14T03:52:16.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:15 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:16.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:15 smithi094 ceph-mon[86597]: Reconfiguring daemon osd.2 on smithi070 2024-09-14T03:52:16.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:15 smithi094 ceph-mon[86597]: osdmap e316: 8 total, 8 up, 8 in 2024-09-14T03:52:16.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:52:15 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:52:15.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:16.028 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:52:15 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:52:15.742Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:16.035 INFO:tasks.rados.rados.0.smithi070.stdout:1260: writing smithi07085154-16 from 4784128 to 5554176 tid 1 2024-09-14T03:52:16.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:15 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:16.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:15 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:16.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:15 smithi070 ceph-mon[96704]: Reconfiguring osd.2 (monmap changed)... 2024-09-14T03:52:16.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:15 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-14T03:52:16.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:15 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:16.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:15 smithi070 ceph-mon[96704]: Reconfiguring daemon osd.2 on smithi070 2024-09-14T03:52:16.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:15 smithi070 ceph-mon[96704]: osdmap e316: 8 total, 8 up, 8 in 2024-09-14T03:52:16.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:15 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:16.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:15 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:16.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:15 smithi070 ceph-mon[98640]: Reconfiguring osd.2 (monmap changed)... 2024-09-14T03:52:16.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:15 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-14T03:52:16.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:15 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:16.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:15 smithi070 ceph-mon[98640]: Reconfiguring daemon osd.2 on smithi070 2024-09-14T03:52:16.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:15 smithi070 ceph-mon[98640]: osdmap e316: 8 total, 8 up, 8 in 2024-09-14T03:52:16.039 INFO:tasks.rados.rados.0.smithi070.stdout:1260: writing smithi07085154-16 from 5554176 to 6266880 tid 2 2024-09-14T03:52:16.043 INFO:tasks.rados.rados.0.smithi070.stdout:1260: writing smithi07085154-16 from 6266880 to 6758400 tid 3 2024-09-14T03:52:16.043 INFO:tasks.rados.rados.0.smithi070.stdout:1261: read oid 23 snap -1 2024-09-14T03:52:16.043 INFO:tasks.rados.rados.0.smithi070.stdout:1261: expect (ObjNum 395 snap 136 seq_num 395) 2024-09-14T03:52:16.043 INFO:tasks.rados.rados.0.smithi070.stdout:1262: rollback oid 27 current snap is 140 2024-09-14T03:52:16.043 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 27 to 125 2024-09-14T03:52:16.043 INFO:tasks.rados.rados.0.smithi070.stdout:1254: expect (ObjNum 399 snap 138 seq_num 399) 2024-09-14T03:52:16.217 INFO:tasks.rados.rados.0.smithi070.stdout:1256: finishing rollback tid 1 to smithi07085154-49 2024-09-14T03:52:16.217 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 49 v 958 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-14T03:52:16.217 INFO:tasks.rados.rados.0.smithi070.stdout:1255: finishing write tid 3 to smithi07085154-8 2024-09-14T03:52:16.217 INFO:tasks.rados.rados.0.smithi070.stdout:1255: finishing write tid 4 to smithi07085154-8 2024-09-14T03:52:16.217 INFO:tasks.rados.rados.0.smithi070.stdout:1255: finishing write tid 5 to smithi07085154-8 2024-09-14T03:52:16.217 INFO:tasks.rados.rados.0.smithi070.stdout:1255: finishing write tid 6 to smithi07085154-8 2024-09-14T03:52:16.218 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 8 v 823 (ObjNum 402 snap 140 seq_num 402) dirty exists 2024-09-14T03:52:16.218 INFO:tasks.rados.rados.0.smithi070.stdout:1255: left oid 8 (ObjNum 402 snap 140 seq_num 402) 2024-09-14T03:52:16.218 INFO:tasks.rados.rados.0.smithi070.stdout:1257: finishing write tid 1 to smithi07085154-13 2024-09-14T03:52:16.218 INFO:tasks.rados.rados.0.smithi070.stdout:1257: finishing write tid 2 to smithi07085154-13 2024-09-14T03:52:16.218 INFO:tasks.rados.rados.0.smithi070.stdout:1257: finishing write tid 3 to smithi07085154-13 2024-09-14T03:52:16.218 INFO:tasks.rados.rados.0.smithi070.stdout:1257: finishing write tid 4 to smithi07085154-13 2024-09-14T03:52:16.218 INFO:tasks.rados.rados.0.smithi070.stdout:1257: finishing write tid 5 to smithi07085154-13 2024-09-14T03:52:16.218 INFO:tasks.rados.rados.0.smithi070.stdout:1257: finishing write tid 6 to smithi07085154-13 2024-09-14T03:52:16.218 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 449 (ObjNum 403 snap 140 seq_num 403) dirty exists 2024-09-14T03:52:16.218 INFO:tasks.rados.rados.0.smithi070.stdout:1257: left oid 13 (ObjNum 403 snap 140 seq_num 403) 2024-09-14T03:52:16.218 INFO:tasks.rados.rados.0.smithi070.stdout:1254: done (6 left) 2024-09-14T03:52:16.218 INFO:tasks.rados.rados.0.smithi070.stdout:1255: done (5 left) 2024-09-14T03:52:16.218 INFO:tasks.rados.rados.0.smithi070.stdout:1256: done (4 left) 2024-09-14T03:52:16.218 INFO:tasks.rados.rados.0.smithi070.stdout:1257: done (3 left) 2024-09-14T03:52:16.219 INFO:tasks.rados.rados.0.smithi070.stdout:1263: read oid 35 snap -1 2024-09-14T03:52:16.219 INFO:tasks.rados.rados.0.smithi070.stdout:1263: expect deleted 2024-09-14T03:52:16.219 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 26 current snap is 140 2024-09-14T03:52:16.219 INFO:tasks.rados.rados.0.smithi070.stdout:1260: finishing write tid 1 to smithi07085154-16 2024-09-14T03:52:16.219 INFO:tasks.rados.rados.0.smithi070.stdout:1260: finishing write tid 2 to smithi07085154-16 2024-09-14T03:52:16.219 INFO:tasks.rados.rados.0.smithi070.stdout:1260: finishing write tid 3 to smithi07085154-16 2024-09-14T03:52:16.219 INFO:tasks.rados.rados.0.smithi070.stdout:1260: finishing write tid 4 to smithi07085154-16 2024-09-14T03:52:16.219 INFO:tasks.rados.rados.0.smithi070.stdout:1262: finishing rollback tid 0 to smithi07085154-27 2024-09-14T03:52:16.219 INFO:tasks.rados.rados.0.smithi070.stdout:1262: finishing rollback tid 1 to smithi07085154-27 2024-09-14T03:52:16.219 INFO:tasks.rados.rados.0.smithi070.stdout:1260: finishing write tid 5 to smithi07085154-16 2024-09-14T03:52:16.219 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 1117 (ObjNum 404 snap 140 seq_num 404) dirty exists 2024-09-14T03:52:16.219 INFO:tasks.rados.rados.0.smithi070.stdout:1260: left oid 16 (ObjNum 404 snap 140 seq_num 404) 2024-09-14T03:52:16.219 INFO:tasks.rados.rados.0.smithi070.stdout:1262: finishing rollback tid 2 to smithi07085154-27 2024-09-14T03:52:16.220 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 1121 (ObjNum 358 snap 118 seq_num 358) dirty exists 2024-09-14T03:52:16.220 INFO:tasks.rados.rados.0.smithi070.stdout:1261: expect (ObjNum 395 snap 136 seq_num 395) 2024-09-14T03:52:16.469 INFO:tasks.rados.rados.0.smithi070.stdout:1264: seq_num 405 ranges {548864=786432,1335296=589824,1925120=614400} 2024-09-14T03:52:16.475 INFO:tasks.rados.rados.0.smithi070.stdout:1264: writing smithi07085154-26 from 548864 to 1335296 tid 1 2024-09-14T03:52:16.479 INFO:tasks.rados.rados.0.smithi070.stdout:1264: writing smithi07085154-26 from 1335296 to 1925120 tid 2 2024-09-14T03:52:16.483 INFO:tasks.rados.rados.0.smithi070.stdout:1264: writing smithi07085154-26 from 1925120 to 2539520 tid 3 2024-09-14T03:52:16.483 INFO:tasks.rados.rados.0.smithi070.stdout:1260: done (4 left) 2024-09-14T03:52:16.483 INFO:tasks.rados.rados.0.smithi070.stdout:1261: done (3 left) 2024-09-14T03:52:16.483 INFO:tasks.rados.rados.0.smithi070.stdout:1262: done (2 left) 2024-09-14T03:52:16.483 INFO:tasks.rados.rados.0.smithi070.stdout:1263: done (1 left) 2024-09-14T03:52:16.484 INFO:tasks.rados.rados.0.smithi070.stdout:1265: delete oid 2 current snap is 140 2024-09-14T03:52:16.484 INFO:tasks.rados.rados.0.smithi070.stdout:1264: finishing write tid 1 to smithi07085154-26 2024-09-14T03:52:16.484 INFO:tasks.rados.rados.0.smithi070.stdout:1264: finishing write tid 2 to smithi07085154-26 2024-09-14T03:52:16.487 INFO:tasks.rados.rados.0.smithi070.stdout:1264: finishing write tid 3 to smithi07085154-26 2024-09-14T03:52:16.487 INFO:tasks.rados.rados.0.smithi070.stdout:1264: finishing write tid 4 to smithi07085154-26 2024-09-14T03:52:16.487 INFO:tasks.rados.rados.0.smithi070.stdout:1265: done (1 left) 2024-09-14T03:52:16.487 INFO:tasks.rados.rados.0.smithi070.stdout:1266: snap_remove snap 89 2024-09-14T03:52:17.255 INFO:tasks.rados.rados.0.smithi070.stdout:1266: done (1 left) 2024-09-14T03:52:17.256 INFO:tasks.rados.rados.0.smithi070.stdout:1267: copy_from oid 43 from oid 4 current snap is 140 2024-09-14T03:52:17.256 INFO:tasks.rados.rados.0.smithi070.stdout:1264: finishing write tid 5 to smithi07085154-26 2024-09-14T03:52:17.256 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 1126 (ObjNum 405 snap 140 seq_num 405) dirty exists 2024-09-14T03:52:17.256 INFO:tasks.rados.rados.0.smithi070.stdout:1264: left oid 26 (ObjNum 405 snap 140 seq_num 405) 2024-09-14T03:52:17.256 INFO:tasks.rados.rados.0.smithi070.stdout:1264: done (1 left) 2024-09-14T03:52:17.256 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 49 current snap is 140 2024-09-14T03:52:17.256 INFO:tasks.rados.rados.0.smithi070.stdout:1268: seq_num 407 ranges {0=425984,425984=761856,1187840=704512,1892352=139264} 2024-09-14T03:52:17.265 INFO:tasks.rados.rados.0.smithi070.stdout:1268: writing smithi07085154-49 from 0 to 425984 tid 1 2024-09-14T03:52:17.266 INFO:tasks.rados.rados.0.smithi070.stdout:1268: writing smithi07085154-49 from 425984 to 1187840 tid 2 2024-09-14T03:52:17.272 INFO:tasks.rados.rados.0.smithi070.stdout:1268: writing smithi07085154-49 from 1187840 to 1892352 tid 3 2024-09-14T03:52:17.274 INFO:tasks.rados.rados.0.smithi070.stdout:1268: writing smithi07085154-49 from 1892352 to 2031616 tid 4 2024-09-14T03:52:17.274 INFO:tasks.rados.rados.0.smithi070.stdout:1268: finishing write tid 1 to smithi07085154-49 2024-09-14T03:52:17.274 INFO:tasks.rados.rados.0.smithi070.stdout:1269: delete oid 26 current snap is 140 2024-09-14T03:52:17.276 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:52:16 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:52:16.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:17.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:17 smithi094 ceph-mon[86597]: osdmap e317: 8 total, 8 up, 8 in 2024-09-14T03:52:17.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:17 smithi094 ceph-mon[86597]: pgmap v66: 121 pgs: 121 active+clean; 363 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.2 MiB/s rd, 5.5 MiB/s wr, 18 op/s 2024-09-14T03:52:17.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:17 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:17.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:17 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:17.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:17 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-14T03:52:17.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:17 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:17.278 INFO:tasks.rados.rados.0.smithi070.stdout:1268: finishing write tid 2 to smithi07085154-49 2024-09-14T03:52:17.278 INFO:tasks.rados.rados.0.smithi070.stdout:1268: finishing write tid 3 to smithi07085154-49 2024-09-14T03:52:17.278 INFO:tasks.rados.rados.0.smithi070.stdout:1268: finishing write tid 4 to smithi07085154-49 2024-09-14T03:52:17.279 INFO:tasks.rados.rados.0.smithi070.stdout:1268: finishing write tid 5 to smithi07085154-49 2024-09-14T03:52:17.282 INFO:tasks.rados.rados.0.smithi070.stdout:1269: done (2 left) 2024-09-14T03:52:17.282 INFO:tasks.rados.rados.0.smithi070.stdout:1270: read oid 32 snap 137 2024-09-14T03:52:17.282 INFO:tasks.rados.rados.0.smithi070.stdout:1270: expect deleted 2024-09-14T03:52:17.282 INFO:tasks.rados.rados.0.smithi070.stdout:1271: rollback oid 35 current snap is 140 2024-09-14T03:52:17.282 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 35 to 138 2024-09-14T03:52:17.282 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 24 current snap is 140 2024-09-14T03:52:17.282 INFO:tasks.rados.rados.0.smithi070.stdout:1272: seq_num 409 ranges {3440640=458752,3899392=532480,4431872=720896,5152768=352256} 2024-09-14T03:52:17.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:17 smithi070 ceph-mon[96704]: osdmap e317: 8 total, 8 up, 8 in 2024-09-14T03:52:17.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:17 smithi070 ceph-mon[96704]: pgmap v66: 121 pgs: 121 active+clean; 363 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.2 MiB/s rd, 5.5 MiB/s wr, 18 op/s 2024-09-14T03:52:17.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:17 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:17.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:17 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:17.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:17 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-14T03:52:17.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:17 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:17.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:17 smithi070 ceph-mon[98640]: osdmap e317: 8 total, 8 up, 8 in 2024-09-14T03:52:17.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:17 smithi070 ceph-mon[98640]: pgmap v66: 121 pgs: 121 active+clean; 363 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.2 MiB/s rd, 5.5 MiB/s wr, 18 op/s 2024-09-14T03:52:17.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:17 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:17.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:17 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:17.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:17 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-14T03:52:17.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:17 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:17.303 INFO:tasks.rados.rados.0.smithi070.stdout:1272: writing smithi07085154-24 from 3440640 to 3899392 tid 1 2024-09-14T03:52:17.306 INFO:tasks.rados.rados.0.smithi070.stdout:1272: writing smithi07085154-24 from 3899392 to 4431872 tid 2 2024-09-14T03:52:17.310 INFO:tasks.rados.rados.0.smithi070.stdout:1272: writing smithi07085154-24 from 4431872 to 5152768 tid 3 2024-09-14T03:52:17.312 INFO:tasks.rados.rados.0.smithi070.stdout:1272: writing smithi07085154-24 from 5152768 to 5505024 tid 4 2024-09-14T03:52:17.313 INFO:tasks.rados.rados.0.smithi070.stdout:1273: read oid 2 snap -1 2024-09-14T03:52:17.313 INFO:tasks.rados.rados.0.smithi070.stdout:1273: expect deleted 2024-09-14T03:52:17.313 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 25 current snap is 140 2024-09-14T03:52:17.313 INFO:tasks.rados.rados.0.smithi070.stdout:1274: seq_num 410 ranges {11304960=475136,11780096=417792,12197888=475136,12673024=425984,13099008=286720} 2024-09-14T03:52:17.376 INFO:tasks.rados.rados.0.smithi070.stdout:1274: writing smithi07085154-25 from 11304960 to 11780096 tid 1 2024-09-14T03:52:17.379 INFO:tasks.rados.rados.0.smithi070.stdout:1274: writing smithi07085154-25 from 11780096 to 12197888 tid 2 2024-09-14T03:52:17.381 INFO:tasks.rados.rados.0.smithi070.stdout:1274: writing smithi07085154-25 from 12197888 to 12673024 tid 3 2024-09-14T03:52:17.384 INFO:tasks.rados.rados.0.smithi070.stdout:1274: writing smithi07085154-25 from 12673024 to 13099008 tid 4 2024-09-14T03:52:17.386 INFO:tasks.rados.rados.0.smithi070.stdout:1274: writing smithi07085154-25 from 13099008 to 13385728 tid 5 2024-09-14T03:52:17.386 INFO:tasks.rados.rados.0.smithi070.stdout:1268: finishing write tid 6 to smithi07085154-49 2024-09-14T03:52:17.386 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 49 v 963 (ObjNum 407 snap 140 seq_num 407) dirty exists 2024-09-14T03:52:17.387 INFO:tasks.rados.rados.0.smithi070.stdout:1268: left oid 49 (ObjNum 407 snap 140 seq_num 407) 2024-09-14T03:52:17.387 INFO:tasks.rados.rados.0.smithi070.stdout:1271: finishing rollback tid 1 to smithi07085154-35 2024-09-14T03:52:17.387 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 35 v 24 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-14T03:52:17.387 INFO:tasks.rados.rados.0.smithi070.stdout:1267: finishing copy_from racing read to smithi07085154-43 2024-09-14T03:52:17.387 INFO:tasks.rados.rados.0.smithi070.stdout:1267: finishing copy_from to smithi07085154-43 2024-09-14T03:52:17.387 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 436 (ObjNum 390 snap 132 seq_num 390) dirty exists 2024-09-14T03:52:17.387 INFO:tasks.rados.rados.0.smithi070.stdout:1272: finishing write tid 1 to smithi07085154-24 2024-09-14T03:52:17.387 INFO:tasks.rados.rados.0.smithi070.stdout:1272: finishing write tid 2 to smithi07085154-24 2024-09-14T03:52:17.387 INFO:tasks.rados.rados.0.smithi070.stdout:1275: read oid 22 snap -1 2024-09-14T03:52:17.387 INFO:tasks.rados.rados.0.smithi070.stdout:1275: expect deleted 2024-09-14T03:52:17.387 INFO:tasks.rados.rados.0.smithi070.stdout:1267: done (7 left) 2024-09-14T03:52:17.387 INFO:tasks.rados.rados.0.smithi070.stdout:1268: done (6 left) 2024-09-14T03:52:17.387 INFO:tasks.rados.rados.0.smithi070.stdout:1270: done (5 left) 2024-09-14T03:52:17.388 INFO:tasks.rados.rados.0.smithi070.stdout:1271: done (4 left) 2024-09-14T03:52:17.388 INFO:tasks.rados.rados.0.smithi070.stdout:1276: snap_create 2024-09-14T03:52:17.388 INFO:tasks.rados.rados.0.smithi070.stdout:1272: finishing write tid 3 to smithi07085154-24 2024-09-14T03:52:17.388 INFO:tasks.rados.rados.0.smithi070.stdout:1272: finishing write tid 4 to smithi07085154-24 2024-09-14T03:52:17.388 INFO:tasks.rados.rados.0.smithi070.stdout:1272: finishing write tid 5 to smithi07085154-24 2024-09-14T03:52:17.388 INFO:tasks.rados.rados.0.smithi070.stdout:1272: finishing write tid 6 to smithi07085154-24 2024-09-14T03:52:17.388 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 606 (ObjNum 409 snap 140 seq_num 409) dirty exists 2024-09-14T03:52:17.388 INFO:tasks.rados.rados.0.smithi070.stdout:1272: left oid 24 (ObjNum 409 snap 140 seq_num 409) 2024-09-14T03:52:17.388 INFO:tasks.rados.rados.0.smithi070.stdout:1274: finishing write tid 1 to smithi07085154-25 2024-09-14T03:52:17.388 INFO:tasks.rados.rados.0.smithi070.stdout:1274: finishing write tid 2 to smithi07085154-25 2024-09-14T03:52:17.388 INFO:tasks.rados.rados.0.smithi070.stdout:1274: finishing write tid 3 to smithi07085154-25 2024-09-14T03:52:17.389 INFO:tasks.rados.rados.0.smithi070.stdout:1274: finishing write tid 4 to smithi07085154-25 2024-09-14T03:52:17.389 INFO:tasks.rados.rados.0.smithi070.stdout:1274: finishing write tid 5 to smithi07085154-25 2024-09-14T03:52:17.389 INFO:tasks.rados.rados.0.smithi070.stdout:1274: finishing write tid 6 to smithi07085154-25 2024-09-14T03:52:17.390 INFO:tasks.rados.rados.0.smithi070.stdout:1274: finishing write tid 7 to smithi07085154-25 2024-09-14T03:52:17.390 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 577 (ObjNum 410 snap 140 seq_num 410) dirty exists 2024-09-14T03:52:17.390 INFO:tasks.rados.rados.0.smithi070.stdout:1274: left oid 25 (ObjNum 410 snap 140 seq_num 410) 2024-09-14T03:52:17.465 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-09-14T03:52:18.005 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:18 smithi070 ceph-mon[96704]: Reconfiguring osd.3 (monmap changed)... 2024-09-14T03:52:18.005 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:18 smithi070 ceph-mon[96704]: Reconfiguring daemon osd.3 on smithi070 2024-09-14T03:52:18.005 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:18 smithi070 ceph-mon[96704]: osdmap e318: 8 total, 8 up, 8 in 2024-09-14T03:52:18.085 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:52:18.086 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (2m) 27s ago 6m 19.7M - 0.27.0 11f11916f8cd 45e79e51074c 2024-09-14T03:52:18.086 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi094 *:3000 running (107s) 14s ago 6m 79.5M - 9.4.7 954c08fa6188 2240810d0368 2024-09-14T03:52:18.086 INFO:teuthology.orchestra.run.smithi070.stdout:mds.a smithi070 running (8m) 27s ago 8m 20.1M - 17.2.7-1629-geb16b97b ab51d1bebd12 52e0dbb7be50 2024-09-14T03:52:18.086 INFO:teuthology.orchestra.run.smithi070.stdout:mds.b smithi094 running (8m) 14s ago 8m 15.4M - 17.2.7-1629-geb16b97b ab51d1bebd12 e3b5e3e00189 2024-09-14T03:52:18.086 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi094 *:8443,9283,8765 running (68s) 14s ago 11m 452M - 19.3.0-4907-ga2782315 8801d2529e43 569d9c7e0f06 2024-09-14T03:52:18.086 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (95s) 27s ago 13m 513M - 19.3.0-4907-ga2782315 8801d2529e43 0ef4d8e0cdbe 2024-09-14T03:52:18.087 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (54s) 27s ago 13m 44.0M 2048M 19.3.0-4907-ga2782315 8801d2529e43 96542de31cac 2024-09-14T03:52:18.087 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi094 running (19s) 14s ago 12m 21.3M 2048M 19.3.0-4907-ga2782315 8801d2529e43 86827be98659 2024-09-14T03:52:18.087 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (32s) 27s ago 12m 28.2M 2048M 19.3.0-4907-ga2782315 8801d2529e43 5741e4fcf4b6 2024-09-14T03:52:18.087 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (2m) 27s ago 6m 9613k - 1.7.0 72c9c2088986 e316760a30b3 2024-09-14T03:52:18.087 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi094 *:9100 running (101s) 14s ago 6m 7750k - 1.7.0 72c9c2088986 767f8a093175 2024-09-14T03:52:18.087 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (11m) 27s ago 11m 166M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 741c06b6adbf 2024-09-14T03:52:18.087 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (11m) 27s ago 11m 179M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 af459ea7ef68 2024-09-14T03:52:18.087 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (10m) 27s ago 10m 149M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 109b66e4e60f 2024-09-14T03:52:18.087 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (10m) 27s ago 10m 123M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 2e4199b58392 2024-09-14T03:52:18.087 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi094 running (10m) 14s ago 10m 180M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 52b64527e24c 2024-09-14T03:52:18.087 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi094 running (9m) 14s ago 9m 168M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 b543f152ba87 2024-09-14T03:52:18.087 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi094 running (9m) 14s ago 9m 114M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 e66b592ac940 2024-09-14T03:52:18.087 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi094 running (8m) 14s ago 8m 152M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 037c6208e9ce 2024-09-14T03:52:18.087 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi094 *:9095 running (73s) 14s ago 7m 46.3M - 2.51.0 1d3b7f56885b ce1b353828dc 2024-09-14T03:52:18.257 INFO:tasks.rados.rados.0.smithi070.stdout:1272: done (4 left) 2024-09-14T03:52:18.257 INFO:tasks.rados.rados.0.smithi070.stdout:1273: done (3 left) 2024-09-14T03:52:18.257 INFO:tasks.rados.rados.0.smithi070.stdout:1274: done (2 left) 2024-09-14T03:52:18.257 INFO:tasks.rados.rados.0.smithi070.stdout:1275: done (1 left) 2024-09-14T03:52:18.257 INFO:tasks.rados.rados.0.smithi070.stdout:1276: done (0 left) 2024-09-14T03:52:18.257 INFO:tasks.rados.rados.0.smithi070.stdout:1277: read oid 18 snap -1 2024-09-14T03:52:18.257 INFO:tasks.rados.rados.0.smithi070.stdout:1277: expect deleted 2024-09-14T03:52:18.257 INFO:tasks.rados.rados.0.smithi070.stdout:1278: setattr oid 1 current snap is 141 2024-09-14T03:52:18.258 INFO:tasks.rados.rados.0.smithi070.stdout:1277: done (1 left) 2024-09-14T03:52:18.259 INFO:tasks.rados.rados.0.smithi070.stdout:1279: snap_create 2024-09-14T03:52:18.264 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 1 v 477 (ObjNum 363 snap 124 seq_num 363) dirty exists 2024-09-14T03:52:18.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:18 smithi094 ceph-mon[86597]: Reconfiguring osd.3 (monmap changed)... 2024-09-14T03:52:18.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:18 smithi094 ceph-mon[86597]: Reconfiguring daemon osd.3 on smithi070 2024-09-14T03:52:18.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:18 smithi094 ceph-mon[86597]: osdmap e318: 8 total, 8 up, 8 in 2024-09-14T03:52:18.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:18 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:52:18.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:18 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:18.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:18 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:18.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:18 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:52:18.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:18 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:18.278 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:18 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:52:18.278 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:18 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:18.278 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:18 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:18.278 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:18 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:52:18.279 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:18 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:18.279 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:18 smithi070 ceph-mon[98640]: Reconfiguring osd.3 (monmap changed)... 2024-09-14T03:52:18.279 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:18 smithi070 ceph-mon[98640]: Reconfiguring daemon osd.3 on smithi070 2024-09-14T03:52:18.279 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:18 smithi070 ceph-mon[98640]: osdmap e318: 8 total, 8 up, 8 in 2024-09-14T03:52:18.279 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:18 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:52:18.279 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:18 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:18.279 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:18 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:18.279 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:18 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:52:18.279 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:18 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:18.445 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:52:18.445 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-09-14T03:52:18.446 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:52:18.446 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:52:18.446 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-09-14T03:52:18.446 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:52:18.446 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:52:18.446 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-09-14T03:52:18.446 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 8 2024-09-14T03:52:18.446 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:52:18.446 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": { 2024-09-14T03:52:18.446 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:52:18.446 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:52:18.446 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-09-14T03:52:18.447 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 10, 2024-09-14T03:52:18.447 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 5 2024-09-14T03:52:18.447 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-09-14T03:52:18.447 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:52:18.734 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:52:18.734 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:52:18.735 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-09-14T03:52:18.735 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:52:18.735 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [ 2024-09-14T03:52:18.735 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr", 2024-09-14T03:52:18.735 INFO:teuthology.orchestra.run.smithi070.stdout: "mon" 2024-09-14T03:52:18.735 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2024-09-14T03:52:18.735 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "5/20 daemons upgraded", 2024-09-14T03:52:18.735 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "Currently upgrading mon daemons", 2024-09-14T03:52:18.735 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-09-14T03:52:18.735 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:52:19.257 INFO:tasks.rados.rados.0.smithi070.stdout:1278: done (1 left) 2024-09-14T03:52:19.257 INFO:tasks.rados.rados.0.smithi070.stdout:1279: done (0 left) 2024-09-14T03:52:19.257 INFO:tasks.rados.rados.0.smithi070.stdout:1280: setattr oid 40 current snap is 142 2024-09-14T03:52:19.258 INFO:tasks.rados.rados.0.smithi070.stdout:1281: read oid 8 snap -1 2024-09-14T03:52:19.259 INFO:tasks.rados.rados.0.smithi070.stdout:1281: expect (ObjNum 402 snap 140 seq_num 402) 2024-09-14T03:52:19.259 INFO:tasks.rados.rados.0.smithi070.stdout:1282: rollback oid 27 current snap is 142 2024-09-14T03:52:19.259 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 27 to 99 2024-09-14T03:52:19.259 INFO:tasks.rados.rados.0.smithi070.stdout:1283: snap_create 2024-09-14T03:52:19.263 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 479 (ObjNum 353 snap 116 seq_num 353) dirty exists 2024-09-14T03:52:19.266 INFO:tasks.rados.rados.0.smithi070.stdout:1282: finishing rollback tid 0 to smithi07085154-27 2024-09-14T03:52:19.266 INFO:tasks.rados.rados.0.smithi070.stdout:1282: finishing rollback tid 1 to smithi07085154-27 2024-09-14T03:52:19.266 INFO:tasks.rados.rados.0.smithi070.stdout:1282: finishing rollback tid 2 to smithi07085154-27 2024-09-14T03:52:19.266 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 1141 (ObjNum 289 snap 91 seq_num 289) dirty exists 2024-09-14T03:52:19.270 INFO:tasks.rados.rados.0.smithi070.stdout:1281: expect (ObjNum 402 snap 140 seq_num 402) 2024-09-14T03:52:19.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:19 smithi094 ceph-mon[86597]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:52:19.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:19 smithi094 ceph-mon[86597]: from='client.34194 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:52:19.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:19 smithi094 ceph-mon[86597]: Reconfiguring mds.a (monmap changed)... 2024-09-14T03:52:19.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:19 smithi094 ceph-mon[86597]: Reconfiguring daemon mds.a on smithi070 2024-09-14T03:52:19.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:19 smithi094 ceph-mon[86597]: pgmap v68: 121 pgs: 121 active+clean; 363 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:19.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:19 smithi094 ceph-mon[86597]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:52:19.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:19 smithi094 ceph-mon[86597]: osdmap e319: 8 total, 8 up, 8 in 2024-09-14T03:52:19.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:19 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/774629083' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:52:19.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:19 smithi070 ceph-mon[98640]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:52:19.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:19 smithi070 ceph-mon[98640]: from='client.34194 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:52:19.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:19 smithi070 ceph-mon[98640]: Reconfiguring mds.a (monmap changed)... 2024-09-14T03:52:19.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:19 smithi070 ceph-mon[98640]: Reconfiguring daemon mds.a on smithi070 2024-09-14T03:52:19.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:19 smithi070 ceph-mon[98640]: pgmap v68: 121 pgs: 121 active+clean; 363 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:19.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:19 smithi070 ceph-mon[98640]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:52:19.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:19 smithi070 ceph-mon[98640]: osdmap e319: 8 total, 8 up, 8 in 2024-09-14T03:52:19.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:19 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/774629083' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:52:19.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:19 smithi070 ceph-mon[96704]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:52:19.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:19 smithi070 ceph-mon[96704]: from='client.34194 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:52:19.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:19 smithi070 ceph-mon[96704]: Reconfiguring mds.a (monmap changed)... 2024-09-14T03:52:19.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:19 smithi070 ceph-mon[96704]: Reconfiguring daemon mds.a on smithi070 2024-09-14T03:52:19.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:19 smithi070 ceph-mon[96704]: pgmap v68: 121 pgs: 121 active+clean; 363 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:52:19.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:19 smithi070 ceph-mon[96704]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:52:19.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:19 smithi070 ceph-mon[96704]: osdmap e319: 8 total, 8 up, 8 in 2024-09-14T03:52:19.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:19 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/774629083' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:52:20.214 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:20 smithi094 ceph-mon[86597]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:52:20.214 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:20 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:20.214 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:20 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:20.214 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:20 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T03:52:20.215 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:20 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T03:52:20.215 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:20 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:20.215 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:20 smithi094 ceph-mon[86597]: osdmap e320: 8 total, 8 up, 8 in 2024-09-14T03:52:20.260 INFO:tasks.rados.rados.0.smithi070.stdout:1280: done (3 left) 2024-09-14T03:52:20.260 INFO:tasks.rados.rados.0.smithi070.stdout:1281: done (2 left) 2024-09-14T03:52:20.260 INFO:tasks.rados.rados.0.smithi070.stdout:1282: done (1 left) 2024-09-14T03:52:20.260 INFO:tasks.rados.rados.0.smithi070.stdout:1283: done (0 left) 2024-09-14T03:52:20.260 INFO:tasks.rados.rados.0.smithi070.stdout:1284: rmattr oid 37 current snap is 143 2024-09-14T03:52:20.260 INFO:tasks.rados.rados.0.smithi070.stdout:1284: done (0 left) 2024-09-14T03:52:20.260 INFO:tasks.rados.rados.0.smithi070.stdout:1285: rmattr oid 26 current snap is 143 2024-09-14T03:52:20.260 INFO:tasks.rados.rados.0.smithi070.stdout:1285: done (0 left) 2024-09-14T03:52:20.260 INFO:tasks.rados.rados.0.smithi070.stdout:1286: copy_from oid 29 from oid 50 current snap is 143 2024-09-14T03:52:20.261 INFO:tasks.rados.rados.0.smithi070.stdout:1287: snap_create 2024-09-14T03:52:20.272 INFO:tasks.rados.rados.0.smithi070.stdout:1286: finishing copy_from racing read to smithi07085154-29 2024-09-14T03:52:20.276 INFO:tasks.rados.rados.0.smithi070.stdout:1286: finishing copy_from to smithi07085154-29 2024-09-14T03:52:20.276 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 632 (ObjNum 387 snap 130 seq_num 387) dirty exists 2024-09-14T03:52:20.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:20 smithi070 ceph-mon[96704]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:52:20.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:20 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:20.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:20 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:20.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:20 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T03:52:20.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:20 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T03:52:20.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:20 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:20.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:20 smithi070 ceph-mon[96704]: osdmap e320: 8 total, 8 up, 8 in 2024-09-14T03:52:20.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:20 smithi070 ceph-mon[98640]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:52:20.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:20 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:20.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:20 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:20.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:20 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T03:52:20.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:20 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T03:52:20.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:20 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:20.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:20 smithi070 ceph-mon[98640]: osdmap e320: 8 total, 8 up, 8 in 2024-09-14T03:52:20.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:52:20 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:52:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:21.260 INFO:tasks.rados.rados.0.smithi070.stdout:1286: done (1 left) 2024-09-14T03:52:21.261 INFO:tasks.rados.rados.0.smithi070.stdout:1287: done (0 left) 2024-09-14T03:52:21.261 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 28 current snap is 144 2024-09-14T03:52:21.261 INFO:tasks.rados.rados.0.smithi070.stdout:1288: seq_num 411 ranges {4939776=491520,5431296=753664,6184960=688128,6873088=163840} 2024-09-14T03:52:21.269 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:21 smithi094 ceph-mon[86597]: Reconfiguring mon.b (monmap changed)... 2024-09-14T03:52:21.269 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:21 smithi094 ceph-mon[86597]: Reconfiguring daemon mon.b on smithi094 2024-09-14T03:52:21.269 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:21 smithi094 ceph-mon[86597]: pgmap v71: 121 pgs: 121 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.3 MiB/s rd, 4.6 MiB/s wr, 16 op/s 2024-09-14T03:52:21.269 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:21 smithi094 ceph-mon[86597]: osdmap e321: 8 total, 8 up, 8 in 2024-09-14T03:52:21.269 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:21 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:21.269 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:21 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:21.269 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:21 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T03:52:21.270 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:21 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-14T03:52:21.270 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:21 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:21.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:21 smithi070 ceph-mon[96704]: Reconfiguring mon.b (monmap changed)... 2024-09-14T03:52:21.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:21 smithi070 ceph-mon[96704]: Reconfiguring daemon mon.b on smithi094 2024-09-14T03:52:21.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:21 smithi070 ceph-mon[96704]: pgmap v71: 121 pgs: 121 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.3 MiB/s rd, 4.6 MiB/s wr, 16 op/s 2024-09-14T03:52:21.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:21 smithi070 ceph-mon[96704]: osdmap e321: 8 total, 8 up, 8 in 2024-09-14T03:52:21.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:21 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:21.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:21 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:21.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:21 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T03:52:21.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:21 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-14T03:52:21.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:21 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:21.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:21 smithi070 ceph-mon[98640]: Reconfiguring mon.b (monmap changed)... 2024-09-14T03:52:21.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:21 smithi070 ceph-mon[98640]: Reconfiguring daemon mon.b on smithi094 2024-09-14T03:52:21.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:21 smithi070 ceph-mon[98640]: pgmap v71: 121 pgs: 121 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.3 MiB/s rd, 4.6 MiB/s wr, 16 op/s 2024-09-14T03:52:21.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:21 smithi070 ceph-mon[98640]: osdmap e321: 8 total, 8 up, 8 in 2024-09-14T03:52:21.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:21 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:21.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:21 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:21.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:21 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T03:52:21.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:21 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-14T03:52:21.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:21 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:21.289 INFO:tasks.rados.rados.0.smithi070.stdout:1288: writing smithi07085154-28 from 4939776 to 5431296 tid 1 2024-09-14T03:52:21.293 INFO:tasks.rados.rados.0.smithi070.stdout:1288: writing smithi07085154-28 from 5431296 to 6184960 tid 2 2024-09-14T03:52:21.298 INFO:tasks.rados.rados.0.smithi070.stdout:1288: writing smithi07085154-28 from 6184960 to 6873088 tid 3 2024-09-14T03:52:21.299 INFO:tasks.rados.rados.0.smithi070.stdout:1288: writing smithi07085154-28 from 6873088 to 7036928 tid 4 2024-09-14T03:52:21.299 INFO:tasks.rados.rados.0.smithi070.stdout:1289: rmattr oid 39 current snap is 144 2024-09-14T03:52:21.300 INFO:tasks.rados.rados.0.smithi070.stdout:1288: finishing write tid 1 to smithi07085154-28 2024-09-14T03:52:21.300 INFO:tasks.rados.rados.0.smithi070.stdout:1288: finishing write tid 2 to smithi07085154-28 2024-09-14T03:52:21.300 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 4 current snap is 144 2024-09-14T03:52:21.300 INFO:tasks.rados.rados.0.smithi070.stdout:1290: seq_num 412 ranges {5365760=507904,5873664=630784,6504448=491520,6995968=483328} 2024-09-14T03:52:21.331 INFO:tasks.rados.rados.0.smithi070.stdout:1290: writing smithi07085154-4 from 5365760 to 5873664 tid 1 2024-09-14T03:52:21.335 INFO:tasks.rados.rados.0.smithi070.stdout:1290: writing smithi07085154-4 from 5873664 to 6504448 tid 2 2024-09-14T03:52:21.338 INFO:tasks.rados.rados.0.smithi070.stdout:1290: writing smithi07085154-4 from 6504448 to 6995968 tid 3 2024-09-14T03:52:21.342 INFO:tasks.rados.rados.0.smithi070.stdout:1290: writing smithi07085154-4 from 6995968 to 7479296 tid 4 2024-09-14T03:52:21.342 INFO:tasks.rados.rados.0.smithi070.stdout:1291: copy_from oid 37 from oid 50 current snap is 144 2024-09-14T03:52:21.342 INFO:tasks.rados.rados.0.smithi070.stdout:1288: finishing write tid 3 to smithi07085154-28 2024-09-14T03:52:21.342 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 39 v 317 (ObjNum 399 snap 138 seq_num 399) dirty exists 2024-09-14T03:52:21.343 INFO:tasks.rados.rados.0.smithi070.stdout:1288: finishing write tid 4 to smithi07085154-28 2024-09-14T03:52:21.343 INFO:tasks.rados.rados.0.smithi070.stdout:1288: finishing write tid 5 to smithi07085154-28 2024-09-14T03:52:21.343 INFO:tasks.rados.rados.0.smithi070.stdout:1288: finishing write tid 6 to smithi07085154-28 2024-09-14T03:52:21.343 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 28 v 481 (ObjNum 411 snap 144 seq_num 411) dirty exists 2024-09-14T03:52:21.343 INFO:tasks.rados.rados.0.smithi070.stdout:1288: left oid 28 (ObjNum 411 snap 144 seq_num 411) 2024-09-14T03:52:21.343 INFO:tasks.rados.rados.0.smithi070.stdout:1290: finishing write tid 1 to smithi07085154-4 2024-09-14T03:52:21.343 INFO:tasks.rados.rados.0.smithi070.stdout:1288: done (3 left) 2024-09-14T03:52:21.343 INFO:tasks.rados.rados.0.smithi070.stdout:1289: done (2 left) 2024-09-14T03:52:21.343 INFO:tasks.rados.rados.0.smithi070.stdout:1292: copy_from oid 49 from oid 44 current snap is 144 2024-09-14T03:52:21.343 INFO:tasks.rados.rados.0.smithi070.stdout:1290: finishing write tid 2 to smithi07085154-4 2024-09-14T03:52:21.343 INFO:tasks.rados.rados.0.smithi070.stdout:1290: finishing write tid 3 to smithi07085154-4 2024-09-14T03:52:21.343 INFO:tasks.rados.rados.0.smithi070.stdout:1293: snap_create 2024-09-14T03:52:21.345 INFO:tasks.rados.rados.0.smithi070.stdout:1290: finishing write tid 4 to smithi07085154-4 2024-09-14T03:52:21.346 INFO:tasks.rados.rados.0.smithi070.stdout:1290: finishing write tid 5 to smithi07085154-4 2024-09-14T03:52:21.347 INFO:tasks.rados.rados.0.smithi070.stdout:1290: finishing write tid 6 to smithi07085154-4 2024-09-14T03:52:21.347 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 487 (ObjNum 412 snap 144 seq_num 412) dirty exists 2024-09-14T03:52:21.347 INFO:tasks.rados.rados.0.smithi070.stdout:1290: left oid 4 (ObjNum 412 snap 144 seq_num 412) 2024-09-14T03:52:21.358 INFO:tasks.rados.rados.0.smithi070.stdout:1291: finishing copy_from racing read to smithi07085154-37 2024-09-14T03:52:21.359 INFO:tasks.rados.rados.0.smithi070.stdout:1291: finishing copy_from to smithi07085154-37 2024-09-14T03:52:21.359 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 37 v 632 (ObjNum 387 snap 130 seq_num 387) dirty exists 2024-09-14T03:52:21.368 INFO:tasks.rados.rados.0.smithi070.stdout:1292: finishing copy_from racing read to smithi07085154-49 2024-09-14T03:52:21.369 INFO:tasks.rados.rados.0.smithi070.stdout:1292: finishing copy_from to smithi07085154-49 2024-09-14T03:52:21.369 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 49 v 436 (ObjNum 390 snap 132 seq_num 390) dirty exists 2024-09-14T03:52:22.262 INFO:tasks.rados.rados.0.smithi070.stdout:1290: done (3 left) 2024-09-14T03:52:22.262 INFO:tasks.rados.rados.0.smithi070.stdout:1291: done (2 left) 2024-09-14T03:52:22.262 INFO:tasks.rados.rados.0.smithi070.stdout:1292: done (1 left) 2024-09-14T03:52:22.263 INFO:tasks.rados.rados.0.smithi070.stdout:1293: done (0 left) 2024-09-14T03:52:22.263 INFO:tasks.rados.rados.0.smithi070.stdout:1294: snap_create 2024-09-14T03:52:22.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:22 smithi094 ceph-mon[86597]: Reconfiguring mgr.x (monmap changed)... 2024-09-14T03:52:22.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:22 smithi094 ceph-mon[86597]: Reconfiguring daemon mgr.x on smithi094 2024-09-14T03:52:22.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:22 smithi094 ceph-mon[86597]: osdmap e322: 8 total, 8 up, 8 in 2024-09-14T03:52:22.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:22 smithi070 ceph-mon[96704]: Reconfiguring mgr.x (monmap changed)... 2024-09-14T03:52:22.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:22 smithi070 ceph-mon[96704]: Reconfiguring daemon mgr.x on smithi094 2024-09-14T03:52:22.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:22 smithi070 ceph-mon[96704]: osdmap e322: 8 total, 8 up, 8 in 2024-09-14T03:52:22.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:22 smithi070 ceph-mon[98640]: Reconfiguring mgr.x (monmap changed)... 2024-09-14T03:52:22.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:22 smithi070 ceph-mon[98640]: Reconfiguring daemon mgr.x on smithi094 2024-09-14T03:52:22.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:22 smithi070 ceph-mon[98640]: osdmap e322: 8 total, 8 up, 8 in 2024-09-14T03:52:23.264 INFO:tasks.rados.rados.0.smithi070.stdout:1294: done (0 left) 2024-09-14T03:52:23.264 INFO:tasks.rados.rados.0.smithi070.stdout:1295: snap_remove snap 110 2024-09-14T03:52:23.393 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:23 smithi094 ceph-mon[86597]: pgmap v74: 121 pgs: 121 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.4 MiB/s rd, 4.6 MiB/s wr, 16 op/s 2024-09-14T03:52:23.393 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:23 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:23.393 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:23 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:23.393 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:23 smithi094 ceph-mon[86597]: Reconfiguring osd.4 (monmap changed)... 2024-09-14T03:52:23.393 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:23 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-14T03:52:23.394 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:23 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:23.394 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:23 smithi094 ceph-mon[86597]: Reconfiguring daemon osd.4 on smithi094 2024-09-14T03:52:23.394 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:23 smithi094 ceph-mon[86597]: osdmap e323: 8 total, 8 up, 8 in 2024-09-14T03:52:23.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:23 smithi070 ceph-mon[98640]: pgmap v74: 121 pgs: 121 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.4 MiB/s rd, 4.6 MiB/s wr, 16 op/s 2024-09-14T03:52:23.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:23 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:23.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:23 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:23.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:23 smithi070 ceph-mon[98640]: Reconfiguring osd.4 (monmap changed)... 2024-09-14T03:52:23.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:23 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-14T03:52:23.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:23 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:23.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:23 smithi070 ceph-mon[98640]: Reconfiguring daemon osd.4 on smithi094 2024-09-14T03:52:23.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:23 smithi070 ceph-mon[98640]: osdmap e323: 8 total, 8 up, 8 in 2024-09-14T03:52:23.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:23 smithi070 ceph-mon[96704]: pgmap v74: 121 pgs: 121 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.4 MiB/s rd, 4.6 MiB/s wr, 16 op/s 2024-09-14T03:52:23.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:23 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:23.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:23 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:23.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:23 smithi070 ceph-mon[96704]: Reconfiguring osd.4 (monmap changed)... 2024-09-14T03:52:23.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:23 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-14T03:52:23.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:23 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:23.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:23 smithi070 ceph-mon[96704]: Reconfiguring daemon osd.4 on smithi094 2024-09-14T03:52:23.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:23 smithi070 ceph-mon[96704]: osdmap e323: 8 total, 8 up, 8 in 2024-09-14T03:52:23.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:52:23 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:52:23] "GET /metrics HTTP/1.1" 200 36191 "" "Prometheus/2.51.0" 2024-09-14T03:52:24.000 INFO:tasks.rados.rados.0.smithi070.stdout:1295: done (0 left) 2024-09-14T03:52:24.001 INFO:tasks.rados.rados.0.smithi070.stdout:1296: rollback oid 5 current snap is 146 2024-09-14T03:52:24.001 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 5 to 120 2024-09-14T03:52:24.001 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 29 current snap is 146 2024-09-14T03:52:24.001 INFO:tasks.rados.rados.0.smithi070.stdout:1297: seq_num 413 ranges {3457024=524288,3981312=516096,4497408=696320,5193728=393216} 2024-09-14T03:52:24.024 INFO:tasks.rados.rados.0.smithi070.stdout:1297: writing smithi07085154-29 from 3457024 to 3981312 tid 1 2024-09-14T03:52:24.027 INFO:tasks.rados.rados.0.smithi070.stdout:1297: writing smithi07085154-29 from 3981312 to 4497408 tid 2 2024-09-14T03:52:24.030 INFO:tasks.rados.rados.0.smithi070.stdout:1297: writing smithi07085154-29 from 4497408 to 5193728 tid 3 2024-09-14T03:52:24.033 INFO:tasks.rados.rados.0.smithi070.stdout:1297: writing smithi07085154-29 from 5193728 to 5586944 tid 4 2024-09-14T03:52:24.033 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 10 current snap is 146 2024-09-14T03:52:24.033 INFO:tasks.rados.rados.0.smithi070.stdout:1296: finishing rollback tid 1 to smithi07085154-5 2024-09-14T03:52:24.034 INFO:tasks.rados.rados.0.smithi070.stdout:1296: finishing rollback tid 2 to smithi07085154-5 2024-09-14T03:52:24.034 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 549 (ObjNum 114 snap 24 seq_num 114) dirty exists 2024-09-14T03:52:24.034 INFO:tasks.rados.rados.0.smithi070.stdout:1297: finishing write tid 1 to smithi07085154-29 2024-09-14T03:52:24.034 INFO:tasks.rados.rados.0.smithi070.stdout:1297: finishing write tid 2 to smithi07085154-29 2024-09-14T03:52:24.034 INFO:tasks.rados.rados.0.smithi070.stdout:1298: seq_num 414 ranges {1056768=540672,1597440=802816,2400256=458752,2859008=344064} 2024-09-14T03:52:24.042 INFO:tasks.rados.rados.0.smithi070.stdout:1298: writing smithi07085154-10 from 1056768 to 1597440 tid 1 2024-09-14T03:52:24.046 INFO:tasks.rados.rados.0.smithi070.stdout:1298: writing smithi07085154-10 from 1597440 to 2400256 tid 2 2024-09-14T03:52:24.049 INFO:tasks.rados.rados.0.smithi070.stdout:1298: writing smithi07085154-10 from 2400256 to 2859008 tid 3 2024-09-14T03:52:24.052 INFO:tasks.rados.rados.0.smithi070.stdout:1298: writing smithi07085154-10 from 2859008 to 3203072 tid 4 2024-09-14T03:52:24.052 INFO:tasks.rados.rados.0.smithi070.stdout:1296: done (2 left) 2024-09-14T03:52:24.052 INFO:tasks.rados.rados.0.smithi070.stdout:1299: delete oid 17 current snap is 146 2024-09-14T03:52:24.052 INFO:tasks.rados.rados.0.smithi070.stdout:1297: finishing write tid 3 to smithi07085154-29 2024-09-14T03:52:24.052 INFO:tasks.rados.rados.0.smithi070.stdout:1297: finishing write tid 4 to smithi07085154-29 2024-09-14T03:52:24.052 INFO:tasks.rados.rados.0.smithi070.stdout:1297: finishing write tid 5 to smithi07085154-29 2024-09-14T03:52:24.052 INFO:tasks.rados.rados.0.smithi070.stdout:1297: finishing write tid 6 to smithi07085154-29 2024-09-14T03:52:24.053 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 704 (ObjNum 413 snap 146 seq_num 413) dirty exists 2024-09-14T03:52:24.053 INFO:tasks.rados.rados.0.smithi070.stdout:1297: left oid 29 (ObjNum 413 snap 146 seq_num 413) 2024-09-14T03:52:24.053 INFO:tasks.rados.rados.0.smithi070.stdout:1298: finishing write tid 1 to smithi07085154-10 2024-09-14T03:52:24.053 INFO:tasks.rados.rados.0.smithi070.stdout:1298: finishing write tid 2 to smithi07085154-10 2024-09-14T03:52:24.054 INFO:tasks.rados.rados.0.smithi070.stdout:1298: finishing write tid 3 to smithi07085154-10 2024-09-14T03:52:24.055 INFO:tasks.rados.rados.0.smithi070.stdout:1298: finishing write tid 4 to smithi07085154-10 2024-09-14T03:52:24.055 INFO:tasks.rados.rados.0.smithi070.stdout:1297: done (2 left) 2024-09-14T03:52:24.055 INFO:tasks.rados.rados.0.smithi070.stdout:1299: done (1 left) 2024-09-14T03:52:24.055 INFO:tasks.rados.rados.0.smithi070.stdout:1300: read oid 39 snap -1 2024-09-14T03:52:24.055 INFO:tasks.rados.rados.0.smithi070.stdout:1300: expect (ObjNum 399 snap 138 seq_num 399) 2024-09-14T03:52:24.055 INFO:tasks.rados.rados.0.smithi070.stdout:1301: read oid 21 snap 104 2024-09-14T03:52:24.055 INFO:tasks.rados.rados.0.smithi070.stdout:1301: expect (ObjNum 242 snap 69 seq_num 242) 2024-09-14T03:52:24.055 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 17 current snap is 146 2024-09-14T03:52:24.055 INFO:tasks.rados.rados.0.smithi070.stdout:1298: finishing write tid 5 to smithi07085154-10 2024-09-14T03:52:24.055 INFO:tasks.rados.rados.0.smithi070.stdout:1302: seq_num 416 ranges {0=573440,573440=573440,1146880=475136,1622016=557056} 2024-09-14T03:52:24.058 INFO:tasks.rados.rados.0.smithi070.stdout:1302: writing smithi07085154-17 from 0 to 573440 tid 1 2024-09-14T03:52:24.062 INFO:tasks.rados.rados.0.smithi070.stdout:1302: writing smithi07085154-17 from 573440 to 1146880 tid 2 2024-09-14T03:52:24.064 INFO:tasks.rados.rados.0.smithi070.stdout:1302: writing smithi07085154-17 from 1146880 to 1622016 tid 3 2024-09-14T03:52:24.068 INFO:tasks.rados.rados.0.smithi070.stdout:1302: writing smithi07085154-17 from 1622016 to 2179072 tid 4 2024-09-14T03:52:24.068 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 42 current snap is 146 2024-09-14T03:52:24.068 INFO:tasks.rados.rados.0.smithi070.stdout:1298: finishing write tid 6 to smithi07085154-10 2024-09-14T03:52:24.068 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 1159 (ObjNum 414 snap 146 seq_num 414) dirty exists 2024-09-14T03:52:24.068 INFO:tasks.rados.rados.0.smithi070.stdout:1298: left oid 10 (ObjNum 414 snap 146 seq_num 414) 2024-09-14T03:52:24.068 INFO:tasks.rados.rados.0.smithi070.stdout:1303: seq_num 417 ranges {5332992=589824,5922816=458752,6381568=679936,7061504=466944} 2024-09-14T03:52:24.099 INFO:tasks.rados.rados.0.smithi070.stdout:1303: writing smithi07085154-42 from 5332992 to 5922816 tid 1 2024-09-14T03:52:24.102 INFO:tasks.rados.rados.0.smithi070.stdout:1303: writing smithi07085154-42 from 5922816 to 6381568 tid 2 2024-09-14T03:52:24.105 INFO:tasks.rados.rados.0.smithi070.stdout:1303: writing smithi07085154-42 from 6381568 to 7061504 tid 3 2024-09-14T03:52:24.108 INFO:tasks.rados.rados.0.smithi070.stdout:1303: writing smithi07085154-42 from 7061504 to 7528448 tid 4 2024-09-14T03:52:24.108 INFO:tasks.rados.rados.0.smithi070.stdout:1298: done (4 left) 2024-09-14T03:52:24.108 INFO:tasks.rados.rados.0.smithi070.stdout:1304: snap_remove snap 112 2024-09-14T03:52:24.512 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:24 smithi094 ceph-mon[86597]: osdmap e324: 8 total, 8 up, 8 in 2024-09-14T03:52:24.512 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:24 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:24.512 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:24 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:24.512 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:24 smithi094 ceph-mon[86597]: Reconfiguring osd.5 (monmap changed)... 2024-09-14T03:52:24.512 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:24 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-14T03:52:24.512 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:24 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:24.512 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:24 smithi094 ceph-mon[86597]: Reconfiguring daemon osd.5 on smithi094 2024-09-14T03:52:24.512 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:24 smithi094 ceph-mon[86597]: osdmap e325: 8 total, 8 up, 8 in 2024-09-14T03:52:24.512 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:24 smithi094 ceph-mon[86597]: pgmap v78: 121 pgs: 121 active+clean; 362 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.8 MiB/s rd, 4.2 MiB/s wr, 19 op/s 2024-09-14T03:52:24.513 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:52:24 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:52:24.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:24.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:24 smithi070 ceph-mon[96704]: osdmap e324: 8 total, 8 up, 8 in 2024-09-14T03:52:24.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:24 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:24.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:24 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:24.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:24 smithi070 ceph-mon[96704]: Reconfiguring osd.5 (monmap changed)... 2024-09-14T03:52:24.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:24 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-14T03:52:24.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:24 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:24.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:24 smithi070 ceph-mon[96704]: Reconfiguring daemon osd.5 on smithi094 2024-09-14T03:52:24.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:24 smithi070 ceph-mon[96704]: osdmap e325: 8 total, 8 up, 8 in 2024-09-14T03:52:24.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:24 smithi070 ceph-mon[96704]: pgmap v78: 121 pgs: 121 active+clean; 362 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.8 MiB/s rd, 4.2 MiB/s wr, 19 op/s 2024-09-14T03:52:24.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:24 smithi070 ceph-mon[98640]: osdmap e324: 8 total, 8 up, 8 in 2024-09-14T03:52:24.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:24 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:24.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:24 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:24.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:24 smithi070 ceph-mon[98640]: Reconfiguring osd.5 (monmap changed)... 2024-09-14T03:52:24.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:24 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-14T03:52:24.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:24 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:24.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:24 smithi070 ceph-mon[98640]: Reconfiguring daemon osd.5 on smithi094 2024-09-14T03:52:24.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:24 smithi070 ceph-mon[98640]: osdmap e325: 8 total, 8 up, 8 in 2024-09-14T03:52:24.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:24 smithi070 ceph-mon[98640]: pgmap v78: 121 pgs: 121 active+clean; 362 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.8 MiB/s rd, 4.2 MiB/s wr, 19 op/s 2024-09-14T03:52:25.004 INFO:tasks.rados.rados.0.smithi070.stdout:1304: done (4 left) 2024-09-14T03:52:25.004 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 48 current snap is 146 2024-09-14T03:52:25.004 INFO:tasks.rados.rados.0.smithi070.stdout:1305: seq_num 418 ranges {450560=606208,1056768=737280,1794048=434176,2228224=434176} 2024-09-14T03:52:25.010 INFO:tasks.rados.rados.0.smithi070.stdout:1305: writing smithi07085154-48 from 450560 to 1056768 tid 1 2024-09-14T03:52:25.016 INFO:tasks.rados.rados.0.smithi070.stdout:1305: writing smithi07085154-48 from 1056768 to 1794048 tid 2 2024-09-14T03:52:25.019 INFO:tasks.rados.rados.0.smithi070.stdout:1305: writing smithi07085154-48 from 1794048 to 2228224 tid 3 2024-09-14T03:52:25.022 INFO:tasks.rados.rados.0.smithi070.stdout:1305: writing smithi07085154-48 from 2228224 to 2662400 tid 4 2024-09-14T03:52:25.022 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 32 current snap is 146 2024-09-14T03:52:25.022 INFO:tasks.rados.rados.0.smithi070.stdout:1302: finishing write tid 1 to smithi07085154-17 2024-09-14T03:52:25.022 INFO:tasks.rados.rados.0.smithi070.stdout:1301: expect (ObjNum 242 snap 69 seq_num 242) 2024-09-14T03:52:25.103 INFO:tasks.rados.rados.0.smithi070.stdout:1302: finishing write tid 2 to smithi07085154-17 2024-09-14T03:52:25.103 INFO:tasks.rados.rados.0.smithi070.stdout:1306: seq_num 419 ranges {0=622592,622592=671744,1294336=647168,1941504=286720} 2024-09-14T03:52:25.107 INFO:tasks.rados.rados.0.smithi070.stdout:1306: writing smithi07085154-32 from 0 to 622592 tid 1 2024-09-14T03:52:25.111 INFO:tasks.rados.rados.0.smithi070.stdout:1306: writing smithi07085154-32 from 622592 to 1294336 tid 2 2024-09-14T03:52:25.114 INFO:tasks.rados.rados.0.smithi070.stdout:1306: writing smithi07085154-32 from 1294336 to 1941504 tid 3 2024-09-14T03:52:25.116 INFO:tasks.rados.rados.0.smithi070.stdout:1306: writing smithi07085154-32 from 1941504 to 2228224 tid 4 2024-09-14T03:52:25.116 INFO:tasks.rados.rados.0.smithi070.stdout:1301: done (5 left) 2024-09-14T03:52:25.116 INFO:tasks.rados.rados.0.smithi070.stdout:1307: snap_remove snap 101 2024-09-14T03:52:25.989 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:52:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:52:25.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:25.989 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:52:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:52:25.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:26.003 INFO:tasks.rados.rados.0.smithi070.stdout:1307: done (5 left) 2024-09-14T03:52:26.003 INFO:tasks.rados.rados.0.smithi070.stdout:1308: rollback oid 1 current snap is 146 2024-09-14T03:52:26.004 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 1 to 93 2024-09-14T03:52:26.004 INFO:tasks.rados.rados.0.smithi070.stdout:1302: finishing write tid 3 to smithi07085154-17 2024-09-14T03:52:26.004 INFO:tasks.rados.rados.0.smithi070.stdout:1302: finishing write tid 4 to smithi07085154-17 2024-09-14T03:52:26.004 INFO:tasks.rados.rados.0.smithi070.stdout:1302: finishing write tid 5 to smithi07085154-17 2024-09-14T03:52:26.004 INFO:tasks.rados.rados.0.smithi070.stdout:1302: finishing write tid 6 to smithi07085154-17 2024-09-14T03:52:26.004 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 590 (ObjNum 416 snap 146 seq_num 416) dirty exists 2024-09-14T03:52:26.004 INFO:tasks.rados.rados.0.smithi070.stdout:1302: left oid 17 (ObjNum 416 snap 146 seq_num 416) 2024-09-14T03:52:26.004 INFO:tasks.rados.rados.0.smithi070.stdout:1300: expect (ObjNum 399 snap 138 seq_num 399) 2024-09-14T03:52:26.166 INFO:tasks.rados.rados.0.smithi070.stdout:1303: finishing write tid 1 to smithi07085154-42 2024-09-14T03:52:26.166 INFO:tasks.rados.rados.0.smithi070.stdout:1303: finishing write tid 2 to smithi07085154-42 2024-09-14T03:52:26.166 INFO:tasks.rados.rados.0.smithi070.stdout:1303: finishing write tid 3 to smithi07085154-42 2024-09-14T03:52:26.166 INFO:tasks.rados.rados.0.smithi070.stdout:1303: finishing write tid 4 to smithi07085154-42 2024-09-14T03:52:26.166 INFO:tasks.rados.rados.0.smithi070.stdout:1303: finishing write tid 5 to smithi07085154-42 2024-09-14T03:52:26.166 INFO:tasks.rados.rados.0.smithi070.stdout:1303: finishing write tid 6 to smithi07085154-42 2024-09-14T03:52:26.166 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 1165 (ObjNum 417 snap 146 seq_num 417) dirty exists 2024-09-14T03:52:26.166 INFO:tasks.rados.rados.0.smithi070.stdout:1303: left oid 42 (ObjNum 417 snap 146 seq_num 417) 2024-09-14T03:52:26.166 INFO:tasks.rados.rados.0.smithi070.stdout:1305: finishing write tid 1 to smithi07085154-48 2024-09-14T03:52:26.166 INFO:tasks.rados.rados.0.smithi070.stdout:1300: done (5 left) 2024-09-14T03:52:26.166 INFO:tasks.rados.rados.0.smithi070.stdout:1302: done (4 left) 2024-09-14T03:52:26.166 INFO:tasks.rados.rados.0.smithi070.stdout:1303: done (3 left) 2024-09-14T03:52:26.167 INFO:tasks.rados.rados.0.smithi070.stdout:1309: read oid 24 snap -1 2024-09-14T03:52:26.167 INFO:tasks.rados.rados.0.smithi070.stdout:1309: expect (ObjNum 409 snap 140 seq_num 409) 2024-09-14T03:52:26.167 INFO:tasks.rados.rados.0.smithi070.stdout:1310: read oid 29 snap -1 2024-09-14T03:52:26.167 INFO:tasks.rados.rados.0.smithi070.stdout:1310: expect (ObjNum 413 snap 146 seq_num 413) 2024-09-14T03:52:26.167 INFO:tasks.rados.rados.0.smithi070.stdout:1311: delete oid 34 current snap is 146 2024-09-14T03:52:26.167 INFO:tasks.rados.rados.0.smithi070.stdout:1305: finishing write tid 2 to smithi07085154-48 2024-09-14T03:52:26.167 INFO:tasks.rados.rados.0.smithi070.stdout:1305: finishing write tid 3 to smithi07085154-48 2024-09-14T03:52:26.167 INFO:tasks.rados.rados.0.smithi070.stdout:1305: finishing write tid 4 to smithi07085154-48 2024-09-14T03:52:26.167 INFO:tasks.rados.rados.0.smithi070.stdout:1305: finishing write tid 5 to smithi07085154-48 2024-09-14T03:52:26.167 INFO:tasks.rados.rados.0.smithi070.stdout:1305: finishing write tid 6 to smithi07085154-48 2024-09-14T03:52:26.167 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 547 (ObjNum 418 snap 146 seq_num 418) dirty exists 2024-09-14T03:52:26.167 INFO:tasks.rados.rados.0.smithi070.stdout:1305: left oid 48 (ObjNum 418 snap 146 seq_num 418) 2024-09-14T03:52:26.167 INFO:tasks.rados.rados.0.smithi070.stdout:1306: finishing write tid 1 to smithi07085154-32 2024-09-14T03:52:26.167 INFO:tasks.rados.rados.0.smithi070.stdout:1306: finishing write tid 2 to smithi07085154-32 2024-09-14T03:52:26.168 INFO:tasks.rados.rados.0.smithi070.stdout:1306: finishing write tid 3 to smithi07085154-32 2024-09-14T03:52:26.168 INFO:tasks.rados.rados.0.smithi070.stdout:1306: finishing write tid 4 to smithi07085154-32 2024-09-14T03:52:26.168 INFO:tasks.rados.rados.0.smithi070.stdout:1306: finishing write tid 5 to smithi07085154-32 2024-09-14T03:52:26.168 INFO:tasks.rados.rados.0.smithi070.stdout:1306: finishing write tid 6 to smithi07085154-32 2024-09-14T03:52:26.168 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 478 (ObjNum 419 snap 146 seq_num 419) dirty exists 2024-09-14T03:52:26.168 INFO:tasks.rados.rados.0.smithi070.stdout:1306: left oid 32 (ObjNum 419 snap 146 seq_num 419) 2024-09-14T03:52:26.168 INFO:tasks.rados.rados.0.smithi070.stdout:1308: finishing rollback tid 0 to smithi07085154-1 2024-09-14T03:52:26.168 INFO:tasks.rados.rados.0.smithi070.stdout:1308: finishing rollback tid 1 to smithi07085154-1 2024-09-14T03:52:26.168 INFO:tasks.rados.rados.0.smithi070.stdout:1308: finishing rollback tid 2 to smithi07085154-1 2024-09-14T03:52:26.168 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 1 v 499 (ObjNum 285 snap 89 seq_num 285) dirty exists 2024-09-14T03:52:26.206 INFO:tasks.rados.rados.0.smithi070.stdout:1310: expect (ObjNum 413 snap 146 seq_num 413) 2024-09-14T03:52:26.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:26.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:26.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:25 smithi094 ceph-mon[86597]: Reconfiguring osd.6 (monmap changed)... 2024-09-14T03:52:26.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-14T03:52:26.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:26.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:25 smithi094 ceph-mon[86597]: Reconfiguring daemon osd.6 on smithi094 2024-09-14T03:52:26.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:25 smithi094 ceph-mon[86597]: osdmap e326: 8 total, 8 up, 8 in 2024-09-14T03:52:26.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:26.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:26.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:25 smithi070 ceph-mon[96704]: Reconfiguring osd.6 (monmap changed)... 2024-09-14T03:52:26.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-14T03:52:26.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:26.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:25 smithi070 ceph-mon[96704]: Reconfiguring daemon osd.6 on smithi094 2024-09-14T03:52:26.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:25 smithi070 ceph-mon[96704]: osdmap e326: 8 total, 8 up, 8 in 2024-09-14T03:52:26.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:26.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:26.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:25 smithi070 ceph-mon[98640]: Reconfiguring osd.6 (monmap changed)... 2024-09-14T03:52:26.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-14T03:52:26.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:26.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:25 smithi070 ceph-mon[98640]: Reconfiguring daemon osd.6 on smithi094 2024-09-14T03:52:26.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:25 smithi070 ceph-mon[98640]: osdmap e326: 8 total, 8 up, 8 in 2024-09-14T03:52:26.444 INFO:tasks.rados.rados.0.smithi070.stdout:1309: expect (ObjNum 409 snap 140 seq_num 409) 2024-09-14T03:52:26.677 INFO:tasks.rados.rados.0.smithi070.stdout:1305: done (5 left) 2024-09-14T03:52:26.677 INFO:tasks.rados.rados.0.smithi070.stdout:1306: done (4 left) 2024-09-14T03:52:26.678 INFO:tasks.rados.rados.0.smithi070.stdout:1308: done (3 left) 2024-09-14T03:52:26.678 INFO:tasks.rados.rados.0.smithi070.stdout:1309: done (2 left) 2024-09-14T03:52:26.678 INFO:tasks.rados.rados.0.smithi070.stdout:1310: done (1 left) 2024-09-14T03:52:26.678 INFO:tasks.rados.rados.0.smithi070.stdout:1311: done (0 left) 2024-09-14T03:52:26.678 INFO:tasks.rados.rados.0.smithi070.stdout:1312: rollback oid 44 current snap is 146 2024-09-14T03:52:26.678 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 44 to 120 2024-09-14T03:52:26.678 INFO:tasks.rados.rados.0.smithi070.stdout:1313: read oid 11 snap -1 2024-09-14T03:52:26.678 INFO:tasks.rados.rados.0.smithi070.stdout:1313: expect (ObjNum 293 snap 91 seq_num 293) 2024-09-14T03:52:26.678 INFO:tasks.rados.rados.0.smithi070.stdout:1314: copy_from oid 4 from oid 6 current snap is 146 2024-09-14T03:52:26.678 INFO:tasks.rados.rados.0.smithi070.stdout:1315: snap_create 2024-09-14T03:52:26.679 INFO:tasks.rados.rados.0.smithi070.stdout:1314: finishing copy_from to smithi07085154-4 2024-09-14T03:52:26.679 INFO:tasks.rados.rados.0.smithi070.stdout:1314: got expected ENOENT (src dne) 2024-09-14T03:52:26.679 INFO:tasks.rados.rados.0.smithi070.stdout:1314: finishing copy_from racing read to smithi07085154-4 2024-09-14T03:52:26.679 INFO:tasks.rados.rados.0.smithi070.stdout:1312: finishing rollback tid 0 to smithi07085154-44 2024-09-14T03:52:26.680 INFO:tasks.rados.rados.0.smithi070.stdout:1312: finishing rollback tid 1 to smithi07085154-44 2024-09-14T03:52:26.680 INFO:tasks.rados.rados.0.smithi070.stdout:1312: finishing rollback tid 2 to smithi07085154-44 2024-09-14T03:52:26.680 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 507 (ObjNum 360 snap 120 seq_num 360) dirty exists 2024-09-14T03:52:26.682 INFO:tasks.rados.rados.0.smithi070.stdout:1313: expect (ObjNum 293 snap 91 seq_num 293) 2024-09-14T03:52:27.119 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:27 smithi094 ceph-mon[86597]: osdmap e327: 8 total, 8 up, 8 in 2024-09-14T03:52:27.119 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:27 smithi094 ceph-mon[86597]: pgmap v81: 121 pgs: 121 active+clean; 362 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.8 MiB/s rd, 4.2 MiB/s wr, 19 op/s 2024-09-14T03:52:27.119 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:27 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:27.119 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:27 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:27.119 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:27 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-14T03:52:27.119 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:27 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:27.120 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:52:26 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:52:26.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:27.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:27 smithi070 ceph-mon[96704]: osdmap e327: 8 total, 8 up, 8 in 2024-09-14T03:52:27.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:27 smithi070 ceph-mon[96704]: pgmap v81: 121 pgs: 121 active+clean; 362 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.8 MiB/s rd, 4.2 MiB/s wr, 19 op/s 2024-09-14T03:52:27.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:27 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:27.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:27 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:27.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:27 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-14T03:52:27.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:27 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:27.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:27 smithi070 ceph-mon[98640]: osdmap e327: 8 total, 8 up, 8 in 2024-09-14T03:52:27.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:27 smithi070 ceph-mon[98640]: pgmap v81: 121 pgs: 121 active+clean; 362 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.8 MiB/s rd, 4.2 MiB/s wr, 19 op/s 2024-09-14T03:52:27.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:27 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:27.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:27 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:27.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:27 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-14T03:52:27.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:27 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:27.463 INFO:tasks.rados.rados.0.smithi070.stdout:1312: done (3 left) 2024-09-14T03:52:27.463 INFO:tasks.rados.rados.0.smithi070.stdout:1313: done (2 left) 2024-09-14T03:52:27.464 INFO:tasks.rados.rados.0.smithi070.stdout:1314: done (1 left) 2024-09-14T03:52:27.464 INFO:tasks.rados.rados.0.smithi070.stdout:1315: done (0 left) 2024-09-14T03:52:27.464 INFO:tasks.rados.rados.0.smithi070.stdout:1316: read oid 49 snap -1 2024-09-14T03:52:27.464 INFO:tasks.rados.rados.0.smithi070.stdout:1316: expect (ObjNum 390 snap 132 seq_num 390) 2024-09-14T03:52:27.464 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 45 current snap is 147 2024-09-14T03:52:27.464 INFO:tasks.rados.rados.0.smithi070.stdout:1317: seq_num 421 ranges {974848=655360,1630208=442368,2072576=663552,2736128=499712} 2024-09-14T03:52:27.472 INFO:tasks.rados.rados.0.smithi070.stdout:1317: writing smithi07085154-45 from 974848 to 1630208 tid 1 2024-09-14T03:52:27.475 INFO:tasks.rados.rados.0.smithi070.stdout:1317: writing smithi07085154-45 from 1630208 to 2072576 tid 2 2024-09-14T03:52:27.479 INFO:tasks.rados.rados.0.smithi070.stdout:1317: writing smithi07085154-45 from 2072576 to 2736128 tid 3 2024-09-14T03:52:27.483 INFO:tasks.rados.rados.0.smithi070.stdout:1317: writing smithi07085154-45 from 2736128 to 3235840 tid 4 2024-09-14T03:52:27.483 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 18 current snap is 147 2024-09-14T03:52:27.483 INFO:tasks.rados.rados.0.smithi070.stdout:1318: seq_num 422 ranges {0=671744,671744=729088,1400832=417792,1818624=458752} 2024-09-14T03:52:27.487 INFO:tasks.rados.rados.0.smithi070.stdout:1318: writing smithi07085154-18 from 0 to 671744 tid 1 2024-09-14T03:52:27.491 INFO:tasks.rados.rados.0.smithi070.stdout:1318: writing smithi07085154-18 from 671744 to 1400832 tid 2 2024-09-14T03:52:27.494 INFO:tasks.rados.rados.0.smithi070.stdout:1318: writing smithi07085154-18 from 1400832 to 1818624 tid 3 2024-09-14T03:52:27.497 INFO:tasks.rados.rados.0.smithi070.stdout:1318: writing smithi07085154-18 from 1818624 to 2277376 tid 4 2024-09-14T03:52:27.497 INFO:tasks.rados.rados.0.smithi070.stdout:1319: rollback oid 27 current snap is 147 2024-09-14T03:52:27.497 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 27 to 142 2024-09-14T03:52:27.497 INFO:tasks.rados.rados.0.smithi070.stdout:1316: expect (ObjNum 390 snap 132 seq_num 390) 2024-09-14T03:52:27.756 INFO:tasks.rados.rados.0.smithi070.stdout:1317: finishing write tid 1 to smithi07085154-45 2024-09-14T03:52:27.756 INFO:tasks.rados.rados.0.smithi070.stdout:1317: finishing write tid 2 to smithi07085154-45 2024-09-14T03:52:27.756 INFO:tasks.rados.rados.0.smithi070.stdout:1316: done (3 left) 2024-09-14T03:52:27.756 INFO:tasks.rados.rados.0.smithi070.stdout:1320: read oid 34 snap -1 2024-09-14T03:52:27.757 INFO:tasks.rados.rados.0.smithi070.stdout:1320: expect deleted 2024-09-14T03:52:27.757 INFO:tasks.rados.rados.0.smithi070.stdout:1321: read oid 49 snap -1 2024-09-14T03:52:27.757 INFO:tasks.rados.rados.0.smithi070.stdout:1321: expect (ObjNum 390 snap 132 seq_num 390) 2024-09-14T03:52:27.757 INFO:tasks.rados.rados.0.smithi070.stdout:1322: read oid 32 snap 79 2024-09-14T03:52:27.757 INFO:tasks.rados.rados.0.smithi070.stdout:1322: expect (ObjNum 260 snap 76 seq_num 260) 2024-09-14T03:52:27.757 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 25 current snap is 147 2024-09-14T03:52:27.757 INFO:tasks.rados.rados.0.smithi070.stdout:1317: finishing write tid 3 to smithi07085154-45 2024-09-14T03:52:27.757 INFO:tasks.rados.rados.0.smithi070.stdout:1323: seq_num 423 ranges {13385728=688128,14073856=614400,14688256=622592,15310848=368640} 2024-09-14T03:52:27.829 INFO:tasks.rados.rados.0.smithi070.stdout:1323: writing smithi07085154-25 from 13385728 to 14073856 tid 1 2024-09-14T03:52:27.833 INFO:tasks.rados.rados.0.smithi070.stdout:1323: writing smithi07085154-25 from 14073856 to 14688256 tid 2 2024-09-14T03:52:27.837 INFO:tasks.rados.rados.0.smithi070.stdout:1323: writing smithi07085154-25 from 14688256 to 15310848 tid 3 2024-09-14T03:52:27.840 INFO:tasks.rados.rados.0.smithi070.stdout:1323: writing smithi07085154-25 from 15310848 to 15679488 tid 4 2024-09-14T03:52:27.840 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 38 current snap is 147 2024-09-14T03:52:27.840 INFO:tasks.rados.rados.0.smithi070.stdout:1317: finishing write tid 4 to smithi07085154-45 2024-09-14T03:52:27.840 INFO:tasks.rados.rados.0.smithi070.stdout:1317: finishing write tid 5 to smithi07085154-45 2024-09-14T03:52:27.840 INFO:tasks.rados.rados.0.smithi070.stdout:1317: finishing write tid 6 to smithi07085154-45 2024-09-14T03:52:27.840 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 45 v 968 (ObjNum 421 snap 147 seq_num 421) dirty exists 2024-09-14T03:52:27.840 INFO:tasks.rados.rados.0.smithi070.stdout:1317: left oid 45 (ObjNum 421 snap 147 seq_num 421) 2024-09-14T03:52:27.840 INFO:tasks.rados.rados.0.smithi070.stdout:1318: finishing write tid 1 to smithi07085154-18 2024-09-14T03:52:27.841 INFO:tasks.rados.rados.0.smithi070.stdout:1318: finishing write tid 2 to smithi07085154-18 2024-09-14T03:52:27.841 INFO:tasks.rados.rados.0.smithi070.stdout:1318: finishing write tid 3 to smithi07085154-18 2024-09-14T03:52:27.841 INFO:tasks.rados.rados.0.smithi070.stdout:1318: finishing write tid 4 to smithi07085154-18 2024-09-14T03:52:27.841 INFO:tasks.rados.rados.0.smithi070.stdout:1318: finishing write tid 5 to smithi07085154-18 2024-09-14T03:52:27.841 INFO:tasks.rados.rados.0.smithi070.stdout:1318: finishing write tid 6 to smithi07085154-18 2024-09-14T03:52:27.841 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 840 (ObjNum 422 snap 147 seq_num 422) dirty exists 2024-09-14T03:52:27.841 INFO:tasks.rados.rados.0.smithi070.stdout:1318: left oid 18 (ObjNum 422 snap 147 seq_num 422) 2024-09-14T03:52:27.841 INFO:tasks.rados.rados.0.smithi070.stdout:1319: finishing rollback tid 0 to smithi07085154-27 2024-09-14T03:52:27.841 INFO:tasks.rados.rados.0.smithi070.stdout:1319: finishing rollback tid 1 to smithi07085154-27 2024-09-14T03:52:27.841 INFO:tasks.rados.rados.0.smithi070.stdout:1319: finishing rollback tid 2 to smithi07085154-27 2024-09-14T03:52:27.841 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 1197 (ObjNum 289 snap 91 seq_num 289) dirty exists 2024-09-14T03:52:27.841 INFO:tasks.rados.rados.0.smithi070.stdout:1324: seq_num 424 ranges {0=704512,704512=491520,1196032=434176,1630208=679936} 2024-09-14T03:52:27.843 INFO:tasks.rados.rados.0.smithi070.stdout:1324: writing smithi07085154-38 from 0 to 704512 tid 1 2024-09-14T03:52:27.846 INFO:tasks.rados.rados.0.smithi070.stdout:1324: writing smithi07085154-38 from 704512 to 1196032 tid 2 2024-09-14T03:52:27.849 INFO:tasks.rados.rados.0.smithi070.stdout:1324: writing smithi07085154-38 from 1196032 to 1630208 tid 3 2024-09-14T03:52:27.853 INFO:tasks.rados.rados.0.smithi070.stdout:1324: writing smithi07085154-38 from 1630208 to 2310144 tid 4 2024-09-14T03:52:27.853 INFO:tasks.rados.rados.0.smithi070.stdout:1317: done (7 left) 2024-09-14T03:52:27.853 INFO:tasks.rados.rados.0.smithi070.stdout:1318: done (6 left) 2024-09-14T03:52:27.853 INFO:tasks.rados.rados.0.smithi070.stdout:1319: done (5 left) 2024-09-14T03:52:27.853 INFO:tasks.rados.rados.0.smithi070.stdout:1320: done (4 left) 2024-09-14T03:52:27.853 INFO:tasks.rados.rados.0.smithi070.stdout:1325: read oid 48 snap -1 2024-09-14T03:52:27.854 INFO:tasks.rados.rados.0.smithi070.stdout:1325: expect (ObjNum 418 snap 146 seq_num 418) 2024-09-14T03:52:27.854 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 39 current snap is 147 2024-09-14T03:52:27.854 INFO:tasks.rados.rados.0.smithi070.stdout:1322: expect (ObjNum 260 snap 76 seq_num 260) 2024-09-14T03:52:27.904 INFO:tasks.rados.rados.0.smithi070.stdout:1321: expect (ObjNum 390 snap 132 seq_num 390) 2024-09-14T03:52:28.162 INFO:tasks.rados.rados.0.smithi070.stdout:1323: finishing write tid 1 to smithi07085154-25 2024-09-14T03:52:28.162 INFO:tasks.rados.rados.0.smithi070.stdout:1323: finishing write tid 2 to smithi07085154-25 2024-09-14T03:52:28.162 INFO:tasks.rados.rados.0.smithi070.stdout:1326: seq_num 425 ranges {4087808=720896,4808704=778240,5586944=638976,6225920=188416} 2024-09-14T03:52:28.187 INFO:tasks.rados.rados.0.smithi070.stdout:1326: writing smithi07085154-39 from 4087808 to 4808704 tid 1 2024-09-14T03:52:28.191 INFO:tasks.rados.rados.0.smithi070.stdout:1326: writing smithi07085154-39 from 4808704 to 5586944 tid 2 2024-09-14T03:52:28.195 INFO:tasks.rados.rados.0.smithi070.stdout:1326: writing smithi07085154-39 from 5586944 to 6225920 tid 3 2024-09-14T03:52:28.196 INFO:tasks.rados.rados.0.smithi070.stdout:1326: writing smithi07085154-39 from 6225920 to 6414336 tid 4 2024-09-14T03:52:28.196 INFO:tasks.rados.rados.0.smithi070.stdout:1321: done (5 left) 2024-09-14T03:52:28.196 INFO:tasks.rados.rados.0.smithi070.stdout:1322: done (4 left) 2024-09-14T03:52:28.196 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 6 current snap is 147 2024-09-14T03:52:28.196 INFO:tasks.rados.rados.0.smithi070.stdout:1323: finishing write tid 3 to smithi07085154-25 2024-09-14T03:52:28.196 INFO:tasks.rados.rados.0.smithi070.stdout:1327: seq_num 426 ranges {0=737280,737280=712704,1449984=802816,2252800=90112} 2024-09-14T03:52:28.200 INFO:tasks.rados.rados.0.smithi070.stdout:1327: writing smithi07085154-6 from 0 to 737280 tid 1 2024-09-14T03:52:28.204 INFO:tasks.rados.rados.0.smithi070.stdout:1327: writing smithi07085154-6 from 737280 to 1449984 tid 2 2024-09-14T03:52:28.208 INFO:tasks.rados.rados.0.smithi070.stdout:1327: writing smithi07085154-6 from 1449984 to 2252800 tid 3 2024-09-14T03:52:28.210 INFO:tasks.rados.rados.0.smithi070.stdout:1327: writing smithi07085154-6 from 2252800 to 2342912 tid 4 2024-09-14T03:52:28.210 INFO:tasks.rados.rados.0.smithi070.stdout:1328: rmattr oid 46 current snap is 147 2024-09-14T03:52:28.210 INFO:tasks.rados.rados.0.smithi070.stdout:1323: finishing write tid 4 to smithi07085154-25 2024-09-14T03:52:28.210 INFO:tasks.rados.rados.0.smithi070.stdout:1323: finishing write tid 5 to smithi07085154-25 2024-09-14T03:52:28.210 INFO:tasks.rados.rados.0.smithi070.stdout:1329: delete oid 9 current snap is 147 2024-09-14T03:52:28.210 INFO:tasks.rados.rados.0.smithi070.stdout:1323: finishing write tid 6 to smithi07085154-25 2024-09-14T03:52:28.210 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 608 (ObjNum 423 snap 147 seq_num 423) dirty exists 2024-09-14T03:52:28.210 INFO:tasks.rados.rados.0.smithi070.stdout:1323: left oid 25 (ObjNum 423 snap 147 seq_num 423) 2024-09-14T03:52:28.210 INFO:tasks.rados.rados.0.smithi070.stdout:1324: finishing write tid 1 to smithi07085154-38 2024-09-14T03:52:28.211 INFO:tasks.rados.rados.0.smithi070.stdout:1324: finishing write tid 2 to smithi07085154-38 2024-09-14T03:52:28.211 INFO:tasks.rados.rados.0.smithi070.stdout:1324: finishing write tid 3 to smithi07085154-38 2024-09-14T03:52:28.211 INFO:tasks.rados.rados.0.smithi070.stdout:1324: finishing write tid 4 to smithi07085154-38 2024-09-14T03:52:28.211 INFO:tasks.rados.rados.0.smithi070.stdout:1324: finishing write tid 5 to smithi07085154-38 2024-09-14T03:52:28.211 INFO:tasks.rados.rados.0.smithi070.stdout:1324: finishing write tid 6 to smithi07085154-38 2024-09-14T03:52:28.211 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 38 v 984 (ObjNum 424 snap 147 seq_num 424) dirty exists 2024-09-14T03:52:28.211 INFO:tasks.rados.rados.0.smithi070.stdout:1324: left oid 38 (ObjNum 424 snap 147 seq_num 424) 2024-09-14T03:52:28.211 INFO:tasks.rados.rados.0.smithi070.stdout:1325: expect (ObjNum 418 snap 146 seq_num 418) 2024-09-14T03:52:28.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:28 smithi094 ceph-mon[86597]: Reconfiguring osd.7 (monmap changed)... 2024-09-14T03:52:28.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:28 smithi094 ceph-mon[86597]: Reconfiguring daemon osd.7 on smithi094 2024-09-14T03:52:28.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:28 smithi094 ceph-mon[86597]: osdmap e328: 8 total, 8 up, 8 in 2024-09-14T03:52:28.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:28 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:28.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:28 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:28.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:28 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:52:28.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:28 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:28.284 INFO:tasks.rados.rados.0.smithi070.stdout:1326: finishing write tid 1 to smithi07085154-39 2024-09-14T03:52:28.284 INFO:tasks.rados.rados.0.smithi070.stdout:1326: finishing write tid 2 to smithi07085154-39 2024-09-14T03:52:28.284 INFO:tasks.rados.rados.0.smithi070.stdout:1323: done (6 left) 2024-09-14T03:52:28.284 INFO:tasks.rados.rados.0.smithi070.stdout:1324: done (5 left) 2024-09-14T03:52:28.285 INFO:tasks.rados.rados.0.smithi070.stdout:1325: done (4 left) 2024-09-14T03:52:28.285 INFO:tasks.rados.rados.0.smithi070.stdout:1329: done (3 left) 2024-09-14T03:52:28.285 INFO:tasks.rados.rados.0.smithi070.stdout:1330: snap_create 2024-09-14T03:52:28.285 INFO:tasks.rados.rados.0.smithi070.stdout:1326: finishing write tid 3 to smithi07085154-39 2024-09-14T03:52:28.285 INFO:tasks.rados.rados.0.smithi070.stdout:1326: finishing write tid 4 to smithi07085154-39 2024-09-14T03:52:28.285 INFO:tasks.rados.rados.0.smithi070.stdout:1326: finishing write tid 5 to smithi07085154-39 2024-09-14T03:52:28.285 INFO:tasks.rados.rados.0.smithi070.stdout:1326: finishing write tid 6 to smithi07085154-39 2024-09-14T03:52:28.285 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 39 v 322 (ObjNum 425 snap 147 seq_num 425) dirty exists 2024-09-14T03:52:28.285 INFO:tasks.rados.rados.0.smithi070.stdout:1326: left oid 39 (ObjNum 425 snap 147 seq_num 425) 2024-09-14T03:52:28.285 INFO:tasks.rados.rados.0.smithi070.stdout:1327: finishing write tid 1 to smithi07085154-6 2024-09-14T03:52:28.286 INFO:tasks.rados.rados.0.smithi070.stdout:1327: finishing write tid 2 to smithi07085154-6 2024-09-14T03:52:28.286 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 46 v 1199 (ObjNum 384 snap 130 seq_num 384) dirty exists 2024-09-14T03:52:28.286 INFO:tasks.rados.rados.0.smithi070.stdout:1327: finishing write tid 3 to smithi07085154-6 2024-09-14T03:52:28.286 INFO:tasks.rados.rados.0.smithi070.stdout:1327: finishing write tid 4 to smithi07085154-6 2024-09-14T03:52:28.286 INFO:tasks.rados.rados.0.smithi070.stdout:1327: finishing write tid 5 to smithi07085154-6 2024-09-14T03:52:28.286 INFO:tasks.rados.rados.0.smithi070.stdout:1327: finishing write tid 6 to smithi07085154-6 2024-09-14T03:52:28.286 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 613 (ObjNum 426 snap 147 seq_num 426) dirty exists 2024-09-14T03:52:28.286 INFO:tasks.rados.rados.0.smithi070.stdout:1327: left oid 6 (ObjNum 426 snap 147 seq_num 426) 2024-09-14T03:52:28.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:28 smithi070 ceph-mon[96704]: Reconfiguring osd.7 (monmap changed)... 2024-09-14T03:52:28.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:28 smithi070 ceph-mon[96704]: Reconfiguring daemon osd.7 on smithi094 2024-09-14T03:52:28.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:28 smithi070 ceph-mon[96704]: osdmap e328: 8 total, 8 up, 8 in 2024-09-14T03:52:28.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:28 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:28.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:28 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:28.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:28 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:52:28.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:28 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:28.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:28 smithi070 ceph-mon[98640]: Reconfiguring osd.7 (monmap changed)... 2024-09-14T03:52:28.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:28 smithi070 ceph-mon[98640]: Reconfiguring daemon osd.7 on smithi094 2024-09-14T03:52:28.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:28 smithi070 ceph-mon[98640]: osdmap e328: 8 total, 8 up, 8 in 2024-09-14T03:52:28.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:28 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:28.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:28 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:28.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:28 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:52:28.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:28 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:29.013 INFO:tasks.rados.rados.0.smithi070.stdout:1326: done (3 left) 2024-09-14T03:52:29.013 INFO:tasks.rados.rados.0.smithi070.stdout:1327: done (2 left) 2024-09-14T03:52:29.013 INFO:tasks.rados.rados.0.smithi070.stdout:1328: done (1 left) 2024-09-14T03:52:29.014 INFO:tasks.rados.rados.0.smithi070.stdout:1330: done (0 left) 2024-09-14T03:52:29.014 INFO:tasks.rados.rados.0.smithi070.stdout:1331: read oid 10 snap -1 2024-09-14T03:52:29.014 INFO:tasks.rados.rados.0.smithi070.stdout:1331: expect (ObjNum 414 snap 146 seq_num 414) 2024-09-14T03:52:29.014 INFO:tasks.rados.rados.0.smithi070.stdout:1332: delete oid 19 current snap is 148 2024-09-14T03:52:29.015 INFO:tasks.rados.rados.0.smithi070.stdout:1332: done (1 left) 2024-09-14T03:52:29.015 INFO:tasks.rados.rados.0.smithi070.stdout:1333: delete oid 5 current snap is 148 2024-09-14T03:52:29.018 INFO:tasks.rados.rados.0.smithi070.stdout:1333: done (1 left) 2024-09-14T03:52:29.018 INFO:tasks.rados.rados.0.smithi070.stdout:1334: read oid 9 snap -1 2024-09-14T03:52:29.018 INFO:tasks.rados.rados.0.smithi070.stdout:1334: expect deleted 2024-09-14T03:52:29.018 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 50 current snap is 148 2024-09-14T03:52:29.018 INFO:tasks.rados.rados.0.smithi070.stdout:1335: seq_num 430 ranges {3457024=8192} 2024-09-14T03:52:29.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:29 smithi094 ceph-mon[86597]: Reconfiguring mds.b (monmap changed)... 2024-09-14T03:52:29.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:29 smithi094 ceph-mon[86597]: Reconfiguring daemon mds.b on smithi094 2024-09-14T03:52:29.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:29 smithi094 ceph-mon[86597]: pgmap v83: 121 pgs: 121 active+clean; 362 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.9 MiB/s rd, 0 op/s 2024-09-14T03:52:29.037 INFO:tasks.rados.rados.0.smithi070.stdout:1335: writing smithi07085154-50 from 3457024 to 3465216 tid 1 2024-09-14T03:52:29.037 INFO:tasks.rados.rados.0.smithi070.stdout:1336: rmattr oid 45 current snap is 148 2024-09-14T03:52:29.037 INFO:tasks.rados.rados.0.smithi070.stdout:1336: done (3 left) 2024-09-14T03:52:29.037 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 23 current snap is 148 2024-09-14T03:52:29.037 INFO:tasks.rados.rados.0.smithi070.stdout:1337: seq_num 431 ranges {5267456=24576} 2024-09-14T03:52:29.065 INFO:tasks.rados.rados.0.smithi070.stdout:1337: writing smithi07085154-23 from 5267456 to 5292032 tid 1 2024-09-14T03:52:29.065 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 5 current snap is 148 2024-09-14T03:52:29.066 INFO:tasks.rados.rados.0.smithi070.stdout:1338: seq_num 432 ranges {0=40960} 2024-09-14T03:52:29.066 INFO:tasks.rados.rados.0.smithi070.stdout:1338: writing smithi07085154-5 from 0 to 40960 tid 1 2024-09-14T03:52:29.066 INFO:tasks.rados.rados.0.smithi070.stdout:1339: snap_remove snap 146 2024-09-14T03:52:29.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:29 smithi070 ceph-mon[96704]: Reconfiguring mds.b (monmap changed)... 2024-09-14T03:52:29.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:29 smithi070 ceph-mon[96704]: Reconfiguring daemon mds.b on smithi094 2024-09-14T03:52:29.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:29 smithi070 ceph-mon[96704]: pgmap v83: 121 pgs: 121 active+clean; 362 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.9 MiB/s rd, 0 op/s 2024-09-14T03:52:29.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:29 smithi070 ceph-mon[98640]: Reconfiguring mds.b (monmap changed)... 2024-09-14T03:52:29.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:29 smithi070 ceph-mon[98640]: Reconfiguring daemon mds.b on smithi094 2024-09-14T03:52:29.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:29 smithi070 ceph-mon[98640]: pgmap v83: 121 pgs: 121 active+clean; 362 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.9 MiB/s rd, 0 op/s 2024-09-14T03:52:30.018 INFO:tasks.rados.rados.0.smithi070.stdout:1339: done (5 left) 2024-09-14T03:52:30.019 INFO:tasks.rados.rados.0.smithi070.stdout:1340: snap_remove snap 136 2024-09-14T03:52:30.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:30 smithi070 ceph-mon[98640]: osdmap e329: 8 total, 8 up, 8 in 2024-09-14T03:52:30.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:30 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:30.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:30 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:30.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:30 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:52:30.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:30 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:52:30.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:30 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:52:30.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:30 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:30.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:30 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:52:30.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:30 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-14T03:52:30.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:30 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T03:52:30.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:30 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-14T03:52:30.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:30 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:52:30.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:30 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-14T03:52:30.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:30 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:52:30.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:30 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:30.038 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:30 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-14T03:52:30.038 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:30 smithi070 ceph-mon[96704]: osdmap e329: 8 total, 8 up, 8 in 2024-09-14T03:52:30.039 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:30 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:30.039 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:30 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:30.039 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:30 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:52:30.039 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:30 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:52:30.039 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:30 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:52:30.039 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:30 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:30.039 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:30 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:52:30.040 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:30 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-14T03:52:30.040 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:30 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T03:52:30.040 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:30 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-14T03:52:30.040 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:30 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:52:30.040 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:30 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-14T03:52:30.040 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:30 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:52:30.040 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:30 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:30.041 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:30 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-14T03:52:30.277 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:52:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:52:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:30.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:30 smithi094 ceph-mon[86597]: osdmap e329: 8 total, 8 up, 8 in 2024-09-14T03:52:30.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:30 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:30.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:30 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:30.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:30 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:52:30.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:30 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:52:30.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:30 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:52:30.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:30 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:30.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:30 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:52:30.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:30 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-14T03:52:30.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:30 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T03:52:30.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:30 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-14T03:52:30.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:30 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:52:30.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:30 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-14T03:52:30.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:30 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:52:30.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:30 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:30.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:30 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-14T03:52:31.024 INFO:tasks.rados.rados.0.smithi070.stdout:1340: done (5 left) 2024-09-14T03:52:31.024 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 18 current snap is 148 2024-09-14T03:52:31.024 INFO:tasks.rados.rados.0.smithi070.stdout:1341: seq_num 433 ranges {2277376=57344} 2024-09-14T03:52:31.037 INFO:tasks.rados.rados.0.smithi070.stdout:1341: writing smithi07085154-18 from 2277376 to 2334720 tid 1 2024-09-14T03:52:31.038 INFO:tasks.rados.rados.0.smithi070.stdout:1342: read oid 45 snap 66 2024-09-14T03:52:31.038 INFO:tasks.rados.rados.0.smithi070.stdout:1342: expect (ObjNum 49 snap 0 seq_num 49) 2024-09-14T03:52:31.038 INFO:tasks.rados.rados.0.smithi070.stdout:1343: rmattr oid 46 current snap is 148 2024-09-14T03:52:31.038 INFO:tasks.rados.rados.0.smithi070.stdout:1331: expect (ObjNum 414 snap 146 seq_num 414) 2024-09-14T03:52:31.151 INFO:tasks.rados.rados.0.smithi070.stdout:1335: finishing write tid 1 to smithi07085154-50 2024-09-14T03:52:31.151 INFO:tasks.rados.rados.0.smithi070.stdout:1335: finishing write tid 2 to smithi07085154-50 2024-09-14T03:52:31.151 INFO:tasks.rados.rados.0.smithi070.stdout:1335: finishing write tid 3 to smithi07085154-50 2024-09-14T03:52:31.152 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 719 (ObjNum 430 snap 148 seq_num 430) dirty exists 2024-09-14T03:52:31.152 INFO:tasks.rados.rados.0.smithi070.stdout:1335: left oid 50 (ObjNum 430 snap 148 seq_num 430) 2024-09-14T03:52:31.152 INFO:tasks.rados.rados.0.smithi070.stdout:1338: finishing write tid 1 to smithi07085154-5 2024-09-14T03:52:31.152 INFO:tasks.rados.rados.0.smithi070.stdout:1331: done (7 left) 2024-09-14T03:52:31.152 INFO:tasks.rados.rados.0.smithi070.stdout:1334: done (6 left) 2024-09-14T03:52:31.152 INFO:tasks.rados.rados.0.smithi070.stdout:1335: done (5 left) 2024-09-14T03:52:31.152 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 3 current snap is 148 2024-09-14T03:52:31.152 INFO:tasks.rados.rados.0.smithi070.stdout:1338: finishing write tid 2 to smithi07085154-5 2024-09-14T03:52:31.152 INFO:tasks.rados.rados.0.smithi070.stdout:1344: seq_num 434 ranges {0=73728} 2024-09-14T03:52:31.152 INFO:tasks.rados.rados.0.smithi070.stdout:1344: writing smithi07085154-3 from 0 to 73728 tid 1 2024-09-14T03:52:31.152 INFO:tasks.rados.rados.0.smithi070.stdout:1345: read oid 34 snap -1 2024-09-14T03:52:31.152 INFO:tasks.rados.rados.0.smithi070.stdout:1345: expect deleted 2024-09-14T03:52:31.153 INFO:tasks.rados.rados.0.smithi070.stdout:1346: delete oid 44 current snap is 148 2024-09-14T03:52:31.153 INFO:tasks.rados.rados.0.smithi070.stdout:1337: finishing write tid 1 to smithi07085154-23 2024-09-14T03:52:31.153 INFO:tasks.rados.rados.0.smithi070.stdout:1337: finishing write tid 2 to smithi07085154-23 2024-09-14T03:52:31.153 INFO:tasks.rados.rados.0.smithi070.stdout:1338: finishing write tid 3 to smithi07085154-5 2024-09-14T03:52:31.153 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 552 (ObjNum 432 snap 148 seq_num 432) dirty exists 2024-09-14T03:52:31.153 INFO:tasks.rados.rados.0.smithi070.stdout:1338: left oid 5 (ObjNum 432 snap 148 seq_num 432) 2024-09-14T03:52:31.153 INFO:tasks.rados.rados.0.smithi070.stdout:1337: finishing write tid 3 to smithi07085154-23 2024-09-14T03:52:31.153 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 608 (ObjNum 431 snap 148 seq_num 431) dirty exists 2024-09-14T03:52:31.153 INFO:tasks.rados.rados.0.smithi070.stdout:1337: left oid 23 (ObjNum 431 snap 148 seq_num 431) 2024-09-14T03:52:31.153 INFO:tasks.rados.rados.0.smithi070.stdout:1342: expect (ObjNum 49 snap 0 seq_num 49) 2024-09-14T03:52:31.166 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:31 smithi070 ceph-mon[96704]: Upgrade: Setting container_image for all mon 2024-09-14T03:52:31.166 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:31 smithi070 ceph-mon[96704]: Upgrade: Setting container_image for all crash 2024-09-14T03:52:31.167 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:31 smithi070 ceph-mon[96704]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-14T03:52:31.167 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:31 smithi070 ceph-mon[96704]: Upgrade: osd.0 is safe to restart 2024-09-14T03:52:31.167 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:31 smithi070 ceph-mon[96704]: osdmap e330: 8 total, 8 up, 8 in 2024-09-14T03:52:31.167 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:31 smithi070 ceph-mon[96704]: pgmap v86: 121 pgs: 121 active+clean; 388 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 25 MiB/s rd, 5.3 MiB/s wr, 26 op/s 2024-09-14T03:52:31.167 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:31 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:31.167 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:31 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-14T03:52:31.167 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:31 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:31.167 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:31 smithi070 ceph-mon[98640]: Upgrade: Setting container_image for all mon 2024-09-14T03:52:31.167 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:31 smithi070 ceph-mon[98640]: Upgrade: Setting container_image for all crash 2024-09-14T03:52:31.167 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:31 smithi070 ceph-mon[98640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-14T03:52:31.167 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:31 smithi070 ceph-mon[98640]: Upgrade: osd.0 is safe to restart 2024-09-14T03:52:31.168 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:31 smithi070 ceph-mon[98640]: osdmap e330: 8 total, 8 up, 8 in 2024-09-14T03:52:31.168 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:31 smithi070 ceph-mon[98640]: pgmap v86: 121 pgs: 121 active+clean; 388 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 25 MiB/s rd, 5.3 MiB/s wr, 26 op/s 2024-09-14T03:52:31.168 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:31 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:31.168 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:31 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-14T03:52:31.168 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:31 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:31.174 INFO:tasks.rados.rados.0.smithi070.stdout:1341: finishing write tid 1 to smithi07085154-18 2024-09-14T03:52:31.174 INFO:tasks.rados.rados.0.smithi070.stdout:1341: finishing write tid 2 to smithi07085154-18 2024-09-14T03:52:31.174 INFO:tasks.rados.rados.0.smithi070.stdout:1337: done (7 left) 2024-09-14T03:52:31.174 INFO:tasks.rados.rados.0.smithi070.stdout:1338: done (6 left) 2024-09-14T03:52:31.174 INFO:tasks.rados.rados.0.smithi070.stdout:1342: done (5 left) 2024-09-14T03:52:31.174 INFO:tasks.rados.rados.0.smithi070.stdout:1346: done (4 left) 2024-09-14T03:52:31.174 INFO:tasks.rados.rados.0.smithi070.stdout:1347: rollback oid 40 current snap is 148 2024-09-14T03:52:31.174 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 40 to 109 2024-09-14T03:52:31.174 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 46 v 1214 (ObjNum 384 snap 130 seq_num 384) dirty exists 2024-09-14T03:52:31.174 INFO:tasks.rados.rados.0.smithi070.stdout:1341: finishing write tid 3 to smithi07085154-18 2024-09-14T03:52:31.174 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 847 (ObjNum 433 snap 148 seq_num 433) dirty exists 2024-09-14T03:52:31.175 INFO:tasks.rados.rados.0.smithi070.stdout:1341: left oid 18 (ObjNum 433 snap 148 seq_num 433) 2024-09-14T03:52:31.175 INFO:tasks.rados.rados.0.smithi070.stdout:1344: finishing write tid 1 to smithi07085154-3 2024-09-14T03:52:31.175 INFO:tasks.rados.rados.0.smithi070.stdout:1344: finishing write tid 2 to smithi07085154-3 2024-09-14T03:52:31.175 INFO:tasks.rados.rados.0.smithi070.stdout:1344: finishing write tid 3 to smithi07085154-3 2024-09-14T03:52:31.175 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 3 v 986 (ObjNum 434 snap 148 seq_num 434) dirty exists 2024-09-14T03:52:31.175 INFO:tasks.rados.rados.0.smithi070.stdout:1344: left oid 3 (ObjNum 434 snap 148 seq_num 434) 2024-09-14T03:52:31.175 INFO:tasks.rados.rados.0.smithi070.stdout:1341: done (4 left) 2024-09-14T03:52:31.175 INFO:tasks.rados.rados.0.smithi070.stdout:1343: done (3 left) 2024-09-14T03:52:31.175 INFO:tasks.rados.rados.0.smithi070.stdout:1344: done (2 left) 2024-09-14T03:52:31.175 INFO:tasks.rados.rados.0.smithi070.stdout:1345: done (1 left) 2024-09-14T03:52:31.175 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 2 current snap is 148 2024-09-14T03:52:31.176 INFO:tasks.rados.rados.0.smithi070.stdout:1348: seq_num 436 ranges {0=106496} 2024-09-14T03:52:31.176 INFO:tasks.rados.rados.0.smithi070.stdout:1348: writing smithi07085154-2 from 0 to 106496 tid 1 2024-09-14T03:52:31.176 INFO:tasks.rados.rados.0.smithi070.stdout:1349: read oid 15 snap -1 2024-09-14T03:52:31.176 INFO:tasks.rados.rados.0.smithi070.stdout:1349: expect (ObjNum 385 snap 130 seq_num 385) 2024-09-14T03:52:31.176 INFO:tasks.rados.rados.0.smithi070.stdout:1350: delete oid 32 current snap is 148 2024-09-14T03:52:31.177 INFO:tasks.rados.rados.0.smithi070.stdout:1350: done (3 left) 2024-09-14T03:52:31.177 INFO:tasks.rados.rados.0.smithi070.stdout:1351: read oid 50 snap -1 2024-09-14T03:52:31.177 INFO:tasks.rados.rados.0.smithi070.stdout:1351: expect (ObjNum 430 snap 148 seq_num 430) 2024-09-14T03:52:31.177 INFO:tasks.rados.rados.0.smithi070.stdout:1352: rollback oid 24 current snap is 148 2024-09-14T03:52:31.177 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 24 to 147 2024-09-14T03:52:31.177 INFO:tasks.rados.rados.0.smithi070.stdout:1353: read oid 35 snap -1 2024-09-14T03:52:31.178 INFO:tasks.rados.rados.0.smithi070.stdout:1353: expect deleted 2024-09-14T03:52:31.178 INFO:tasks.rados.rados.0.smithi070.stdout:1354: rollback oid 3 current snap is 148 2024-09-14T03:52:31.178 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 3 to 139 2024-09-14T03:52:31.178 INFO:tasks.rados.rados.0.smithi070.stdout:1355: rollback oid 17 current snap is 148 2024-09-14T03:52:31.178 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 17 to 109 2024-09-14T03:52:31.178 INFO:tasks.rados.rados.0.smithi070.stdout:1356: copy_from oid 4 from oid 27 current snap is 148 2024-09-14T03:52:31.178 INFO:tasks.rados.rados.0.smithi070.stdout:1357: snap_create 2024-09-14T03:52:31.178 INFO:tasks.rados.rados.0.smithi070.stdout:1347: finishing rollback tid 0 to smithi07085154-40 2024-09-14T03:52:31.178 INFO:tasks.rados.rados.0.smithi070.stdout:1348: finishing write tid 1 to smithi07085154-2 2024-09-14T03:52:31.181 INFO:tasks.rados.rados.0.smithi070.stdout:1347: finishing rollback tid 1 to smithi07085154-40 2024-09-14T03:52:31.182 INFO:tasks.rados.rados.0.smithi070.stdout:1352: finishing rollback tid 0 to smithi07085154-24 2024-09-14T03:52:31.182 INFO:tasks.rados.rados.0.smithi070.stdout:1354: finishing rollback tid 0 to smithi07085154-3 2024-09-14T03:52:31.183 INFO:tasks.rados.rados.0.smithi070.stdout:1347: finishing rollback tid 2 to smithi07085154-40 2024-09-14T03:52:31.183 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 519 (ObjNum 274 snap 84 seq_num 274) dirty exists 2024-09-14T03:52:31.183 INFO:tasks.rados.rados.0.smithi070.stdout:1352: finishing rollback tid 1 to smithi07085154-24 2024-09-14T03:52:31.183 INFO:tasks.rados.rados.0.smithi070.stdout:1352: finishing rollback tid 2 to smithi07085154-24 2024-09-14T03:52:31.184 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 611 (ObjNum 409 snap 140 seq_num 409) dirty exists 2024-09-14T03:52:31.184 INFO:tasks.rados.rados.0.smithi070.stdout:1348: finishing write tid 2 to smithi07085154-2 2024-09-14T03:52:31.185 INFO:tasks.rados.rados.0.smithi070.stdout:1355: finishing rollback tid 0 to smithi07085154-17 2024-09-14T03:52:31.185 INFO:tasks.rados.rados.0.smithi070.stdout:1355: finishing rollback tid 1 to smithi07085154-17 2024-09-14T03:52:31.185 INFO:tasks.rados.rados.0.smithi070.stdout:1354: finishing rollback tid 1 to smithi07085154-3 2024-09-14T03:52:31.185 INFO:tasks.rados.rados.0.smithi070.stdout:1354: finishing rollback tid 2 to smithi07085154-3 2024-09-14T03:52:31.185 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 3 v 989 (ObjNum 81 snap 0 seq_num 1) dirty exists 2024-09-14T03:52:31.185 INFO:tasks.rados.rados.0.smithi070.stdout:1356: finishing copy_from racing read to smithi07085154-4 2024-09-14T03:52:31.186 INFO:tasks.rados.rados.0.smithi070.stdout:1355: finishing rollback tid 2 to smithi07085154-17 2024-09-14T03:52:31.186 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 623 (ObjNum 325 snap 107 seq_num 325) dirty exists 2024-09-14T03:52:31.186 INFO:tasks.rados.rados.0.smithi070.stdout:1356: finishing copy_from to smithi07085154-4 2024-09-14T03:52:31.186 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 1197 (ObjNum 289 snap 91 seq_num 289) dirty exists 2024-09-14T03:52:31.188 INFO:tasks.rados.rados.0.smithi070.stdout:1348: finishing write tid 3 to smithi07085154-2 2024-09-14T03:52:31.188 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 2 v 1222 (ObjNum 436 snap 148 seq_num 436) dirty exists 2024-09-14T03:52:31.188 INFO:tasks.rados.rados.0.smithi070.stdout:1348: left oid 2 (ObjNum 436 snap 148 seq_num 436) 2024-09-14T03:52:31.188 INFO:tasks.rados.rados.0.smithi070.stdout:1349: expect (ObjNum 385 snap 130 seq_num 385) 2024-09-14T03:52:31.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:31 smithi094 ceph-mon[86597]: Upgrade: Setting container_image for all mon 2024-09-14T03:52:31.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:31 smithi094 ceph-mon[86597]: Upgrade: Setting container_image for all crash 2024-09-14T03:52:31.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:31 smithi094 ceph-mon[86597]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-14T03:52:31.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:31 smithi094 ceph-mon[86597]: Upgrade: osd.0 is safe to restart 2024-09-14T03:52:31.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:31 smithi094 ceph-mon[86597]: osdmap e330: 8 total, 8 up, 8 in 2024-09-14T03:52:31.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:31 smithi094 ceph-mon[86597]: pgmap v86: 121 pgs: 121 active+clean; 388 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 25 MiB/s rd, 5.3 MiB/s wr, 26 op/s 2024-09-14T03:52:31.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:31 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:31.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:31 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-14T03:52:31.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:31 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:31.321 INFO:tasks.rados.rados.0.smithi070.stdout:1351: expect (ObjNum 430 snap 148 seq_num 430) 2024-09-14T03:52:32.027 INFO:tasks.rados.rados.0.smithi070.stdout:1347: done (9 left) 2024-09-14T03:52:32.027 INFO:tasks.rados.rados.0.smithi070.stdout:1348: done (8 left) 2024-09-14T03:52:32.028 INFO:tasks.rados.rados.0.smithi070.stdout:1349: done (7 left) 2024-09-14T03:52:32.028 INFO:tasks.rados.rados.0.smithi070.stdout:1351: done (6 left) 2024-09-14T03:52:32.028 INFO:tasks.rados.rados.0.smithi070.stdout:1352: done (5 left) 2024-09-14T03:52:32.028 INFO:tasks.rados.rados.0.smithi070.stdout:1353: done (4 left) 2024-09-14T03:52:32.028 INFO:tasks.rados.rados.0.smithi070.stdout:1354: done (3 left) 2024-09-14T03:52:32.028 INFO:tasks.rados.rados.0.smithi070.stdout:1355: done (2 left) 2024-09-14T03:52:32.028 INFO:tasks.rados.rados.0.smithi070.stdout:1356: done (1 left) 2024-09-14T03:52:32.028 INFO:tasks.rados.rados.0.smithi070.stdout:1357: done (0 left) 2024-09-14T03:52:32.028 INFO:tasks.rados.rados.0.smithi070.stdout:1358: read oid 31 snap 104 2024-09-14T03:52:32.028 INFO:tasks.rados.rados.0.smithi070.stdout:1358: expect (ObjNum -402636000 snap 32644 seq_num 4026574432) 2024-09-14T03:52:32.028 INFO:tasks.rados.rados.0.smithi070.stdout:1359: snap_create 2024-09-14T03:52:32.029 INFO:tasks.rados.rados.0.smithi070.stdout:1358: expect (ObjNum -402636000 snap 32644 seq_num 4026574432) 2024-09-14T03:52:32.214 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:32 smithi070 ceph-mon[96704]: Upgrade: Updating osd.0 2024-09-14T03:52:32.215 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:32 smithi070 ceph-mon[96704]: Deploying daemon osd.0 on smithi070 2024-09-14T03:52:32.215 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:32 smithi070 ceph-mon[96704]: osdmap e331: 8 total, 8 up, 8 in 2024-09-14T03:52:32.215 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:32 smithi070 ceph-mon[98640]: Upgrade: Updating osd.0 2024-09-14T03:52:32.215 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:32 smithi070 ceph-mon[98640]: Deploying daemon osd.0 on smithi070 2024-09-14T03:52:32.215 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:32 smithi070 ceph-mon[98640]: osdmap e331: 8 total, 8 up, 8 in 2024-09-14T03:52:32.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:32 smithi094 ceph-mon[86597]: Upgrade: Updating osd.0 2024-09-14T03:52:32.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:32 smithi094 ceph-mon[86597]: Deploying daemon osd.0 on smithi070 2024-09-14T03:52:32.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:32 smithi094 ceph-mon[86597]: osdmap e331: 8 total, 8 up, 8 in 2024-09-14T03:52:33.032 INFO:tasks.rados.rados.0.smithi070.stdout:1358: done (1 left) 2024-09-14T03:52:33.032 INFO:tasks.rados.rados.0.smithi070.stdout:1359: done (0 left) 2024-09-14T03:52:33.033 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 28 current snap is 150 2024-09-14T03:52:33.033 INFO:tasks.rados.rados.0.smithi070.stdout:1360: seq_num 438 ranges {7036928=139264} 2024-09-14T03:52:33.070 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:33 smithi070 ceph-mon[96704]: osdmap e332: 8 total, 8 up, 8 in 2024-09-14T03:52:33.070 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:33 smithi070 ceph-mon[96704]: pgmap v89: 121 pgs: 121 active+clean; 388 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 21 MiB/s rd, 5.3 MiB/s wr, 25 op/s 2024-09-14T03:52:33.070 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:33 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:52:33.071 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:33 smithi070 ceph-mon[98640]: osdmap e332: 8 total, 8 up, 8 in 2024-09-14T03:52:33.071 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:33 smithi070 ceph-mon[98640]: pgmap v89: 121 pgs: 121 active+clean; 388 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 21 MiB/s rd, 5.3 MiB/s wr, 25 op/s 2024-09-14T03:52:33.071 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:33 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:52:33.071 INFO:tasks.rados.rados.0.smithi070.stdout:1360: writing smithi07085154-28 from 7036928 to 7176192 tid 1 2024-09-14T03:52:33.072 INFO:tasks.rados.rados.0.smithi070.stdout:1361: rollback oid 6 current snap is 150 2024-09-14T03:52:33.072 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 6 to 114 2024-09-14T03:52:33.072 INFO:tasks.rados.rados.0.smithi070.stdout:1362: snap_remove snap 140 2024-09-14T03:52:33.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:33 smithi094 ceph-mon[86597]: osdmap e332: 8 total, 8 up, 8 in 2024-09-14T03:52:33.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:33 smithi094 ceph-mon[86597]: pgmap v89: 121 pgs: 121 active+clean; 388 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 21 MiB/s rd, 5.3 MiB/s wr, 25 op/s 2024-09-14T03:52:33.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:33 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:52:33.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:52:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:52:33] "GET /metrics HTTP/1.1" 200 36194 "" "Prometheus/2.51.0" 2024-09-14T03:52:33.702 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:33 smithi070 systemd[1]: Stopping Ceph osd.0 for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:52:34.002 INFO:tasks.rados.rados.0.smithi070.stdout:1362: done (2 left) 2024-09-14T03:52:34.002 INFO:tasks.rados.rados.0.smithi070.stdout:1363: delete oid 2 current snap is 150 2024-09-14T03:52:34.003 INFO:tasks.rados.rados.0.smithi070.stdout:1360: finishing write tid 1 to smithi07085154-28 2024-09-14T03:52:34.003 INFO:tasks.rados.rados.0.smithi070.stdout:1360: finishing write tid 2 to smithi07085154-28 2024-09-14T03:52:34.003 INFO:tasks.rados.rados.0.smithi070.stdout:1361: finishing rollback tid 0 to smithi07085154-6 2024-09-14T03:52:34.003 INFO:tasks.rados.rados.0.smithi070.stdout:1361: finishing rollback tid 1 to smithi07085154-6 2024-09-14T03:52:34.003 INFO:tasks.rados.rados.0.smithi070.stdout:1361: finishing rollback tid 2 to smithi07085154-6 2024-09-14T03:52:34.003 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 627 (ObjNum 250 snap 72 seq_num 250) dirty exists 2024-09-14T03:52:34.003 INFO:tasks.rados.rados.0.smithi070.stdout:1360: finishing write tid 3 to smithi07085154-28 2024-09-14T03:52:34.003 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 28 v 1199 (ObjNum 438 snap 150 seq_num 438) dirty exists 2024-09-14T03:52:34.003 INFO:tasks.rados.rados.0.smithi070.stdout:1360: left oid 28 (ObjNum 438 snap 150 seq_num 438) 2024-09-14T03:52:34.021 INFO:tasks.rados.rados.0.smithi070.stdout:1360: done (2 left) 2024-09-14T03:52:34.021 INFO:tasks.rados.rados.0.smithi070.stdout:1361: done (1 left) 2024-09-14T03:52:34.021 INFO:tasks.rados.rados.0.smithi070.stdout:1363: done (0 left) 2024-09-14T03:52:34.021 INFO:tasks.rados.rados.0.smithi070.stdout:1364: snap_create 2024-09-14T03:52:34.029 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0[43672]: 2024-09-14T03:52:33.700+0000 7f98ad2d1640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:52:34.030 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0[43672]: 2024-09-14T03:52:33.700+0000 7f98ad2d1640 -1 osd.0 333 *** Got signal Terminated *** 2024-09-14T03:52:34.030 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0[43672]: 2024-09-14T03:52:33.700+0000 7f98ad2d1640 -1 osd.0 333 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T03:52:34.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:34 smithi070 ceph-mon[96704]: osdmap e333: 8 total, 8 up, 8 in 2024-09-14T03:52:34.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:34 smithi070 ceph-mon[96704]: osd.0 marked itself down and dead 2024-09-14T03:52:34.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:34 smithi070 ceph-mon[96704]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:52:34.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:34 smithi070 ceph-mon[96704]: osdmap e334: 8 total, 7 up, 8 in 2024-09-14T03:52:34.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:34 smithi070 ceph-mon[98640]: osdmap e333: 8 total, 8 up, 8 in 2024-09-14T03:52:34.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:34 smithi070 ceph-mon[98640]: osd.0 marked itself down and dead 2024-09-14T03:52:34.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:34 smithi070 ceph-mon[98640]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:52:34.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:34 smithi070 ceph-mon[98640]: osdmap e334: 8 total, 7 up, 8 in 2024-09-14T03:52:34.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:34 smithi094 ceph-mon[86597]: osdmap e333: 8 total, 8 up, 8 in 2024-09-14T03:52:34.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:34 smithi094 ceph-mon[86597]: osd.0 marked itself down and dead 2024-09-14T03:52:34.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:34 smithi094 ceph-mon[86597]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:52:34.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:34 smithi094 ceph-mon[86597]: osdmap e334: 8 total, 7 up, 8 in 2024-09-14T03:52:34.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:52:34 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:52:34.149Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:34.843 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:34 smithi070 podman[103305]: 2024-09-14 03:52:34.558233885 +0000 UTC m=+0.990186480 container died 741c06b6adbf46c589e6fde616a3acb18f033fadc6b6613fcb31a356a0082bb3 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=quincy-eb16b97, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-14T03:52:35.004 INFO:tasks.rados.rados.0.smithi070.stdout:1364: done (0 left) 2024-09-14T03:52:35.004 INFO:tasks.rados.rados.0.smithi070.stdout:1365: snap_create 2024-09-14T03:52:35.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:35 smithi070 ceph-mon[96704]: pgmap v92: 121 pgs: 16 stale+active+clean, 105 active+clean; 394 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.2 MiB/s wr, 17 op/s 2024-09-14T03:52:35.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:35 smithi070 ceph-mon[96704]: osdmap e335: 8 total, 7 up, 8 in 2024-09-14T03:52:35.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:35 smithi070 ceph-mon[98640]: pgmap v92: 121 pgs: 16 stale+active+clean, 105 active+clean; 394 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.2 MiB/s wr, 17 op/s 2024-09-14T03:52:35.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:35 smithi070 ceph-mon[98640]: osdmap e335: 8 total, 7 up, 8 in 2024-09-14T03:52:35.286 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:34 smithi070 podman[103305]: 2024-09-14 03:52:34.940321995 +0000 UTC m=+1.372274597 container cleanup 741c06b6adbf46c589e6fde616a3acb18f033fadc6b6613fcb31a356a0082bb3 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-eb16b97, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-14T03:52:35.286 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:34 smithi070 bash[103305]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0 2024-09-14T03:52:35.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:35 smithi094 ceph-mon[86597]: pgmap v92: 121 pgs: 16 stale+active+clean, 105 active+clean; 394 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.2 MiB/s wr, 17 op/s 2024-09-14T03:52:35.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:35 smithi094 ceph-mon[86597]: osdmap e335: 8 total, 7 up, 8 in 2024-09-14T03:52:35.738 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:35 smithi070 podman[103325]: 2024-09-14 03:52:35.336690698 +0000 UTC m=+0.777924205 container remove 741c06b6adbf46c589e6fde616a3acb18f033fadc6b6613fcb31a356a0082bb3 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-eb16b97, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-14T03:52:36.006 INFO:tasks.rados.rados.0.smithi070.stdout:1365: done (0 left) 2024-09-14T03:52:36.006 INFO:tasks.rados.rados.0.smithi070.stdout:1366: rollback oid 2 current snap is 152 2024-09-14T03:52:36.006 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 2 to 120 2024-09-14T03:52:36.006 INFO:tasks.rados.rados.0.smithi070.stdout:1367: read oid 21 snap -1 2024-09-14T03:52:36.006 INFO:tasks.rados.rados.0.smithi070.stdout:1367: expect (ObjNum 114 snap 24 seq_num 114) 2024-09-14T03:52:36.006 INFO:tasks.rados.rados.0.smithi070.stdout:1368: rollback oid 16 current snap is 152 2024-09-14T03:52:36.006 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 16 to 133 2024-09-14T03:52:36.006 INFO:tasks.rados.rados.0.smithi070.stdout:1369: read oid 19 snap -1 2024-09-14T03:52:36.007 INFO:tasks.rados.rados.0.smithi070.stdout:1369: expect deleted 2024-09-14T03:52:36.007 INFO:tasks.rados.rados.0.smithi070.stdout:1370: delete oid 38 current snap is 152 2024-09-14T03:52:36.012 INFO:tasks.rados.rados.0.smithi070.stdout:1366: finishing rollback tid 1 to smithi07085154-2 2024-09-14T03:52:36.013 INFO:tasks.rados.rados.0.smithi070.stdout:1369: done (4 left) 2024-09-14T03:52:36.013 INFO:tasks.rados.rados.0.smithi070.stdout:1370: done (3 left) 2024-09-14T03:52:36.013 INFO:tasks.rados.rados.0.smithi070.stdout:1371: rollback oid 46 current snap is 152 2024-09-14T03:52:36.013 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 46 to 122 2024-09-14T03:52:36.013 INFO:tasks.rados.rados.0.smithi070.stdout:1372: read oid 25 snap 138 2024-09-14T03:52:36.013 INFO:tasks.rados.rados.0.smithi070.stdout:1372: expect (ObjNum 375 snap 128 seq_num 375) 2024-09-14T03:52:36.014 INFO:tasks.rados.rados.0.smithi070.stdout:1373: read oid 30 snap -1 2024-09-14T03:52:36.014 INFO:tasks.rados.rados.0.smithi070.stdout:1373: expect (ObjNum 380 snap 129 seq_num 380) 2024-09-14T03:52:36.014 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 6 current snap is 152 2024-09-14T03:52:36.014 INFO:tasks.rados.rados.0.smithi070.stdout:1374: seq_num 441 ranges {1794048=188416} 2024-09-14T03:52:36.025 INFO:tasks.rados.rados.0.smithi070.stdout:1374: writing smithi07085154-6 from 1794048 to 1982464 tid 1 2024-09-14T03:52:36.025 INFO:tasks.rados.rados.0.smithi070.stdout:1375: rmattr oid 15 current snap is 152 2024-09-14T03:52:36.025 INFO:tasks.rados.rados.0.smithi070.stdout:1366: finishing rollback tid 2 to smithi07085154-2 2024-09-14T03:52:36.025 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 2 v 1234 (ObjNum 307 snap 101 seq_num 307) dirty exists 2024-09-14T03:52:36.025 INFO:tasks.rados.rados.0.smithi070.stdout:1368: finishing rollback tid 0 to smithi07085154-16 2024-09-14T03:52:36.025 INFO:tasks.rados.rados.0.smithi070.stdout:1368: finishing rollback tid 1 to smithi07085154-16 2024-09-14T03:52:36.025 INFO:tasks.rados.rados.0.smithi070.stdout:1368: finishing rollback tid 2 to smithi07085154-16 2024-09-14T03:52:36.025 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 1238 (ObjNum 114 snap 24 seq_num 114) dirty exists 2024-09-14T03:52:36.025 INFO:tasks.rados.rados.0.smithi070.stdout:1367: expect (ObjNum 114 snap 24 seq_num 114) 2024-09-14T03:52:36.026 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:52:35 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:52:35.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:36.026 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:52:35 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:52:35.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:36.035 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:35 smithi070 podman[103420]: 2024-09-14 03:52:35.640884442 +0000 UTC m=+0.016054331 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:52:36.035 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:35 smithi070 podman[103420]: 2024-09-14 03:52:35.960191535 +0000 UTC m=+0.335361416 container create 942d3ee590acaa1d31542d4bb5ec672a6097d13c888db2bb467388d57f1f482e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, RELEASE=main-a278231, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-14T03:52:36.233 INFO:tasks.rados.rados.0.smithi070.stdout:1371: finishing rollback tid 0 to smithi07085154-46 2024-09-14T03:52:36.233 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 15 v 849 (ObjNum 385 snap 130 seq_num 385) dirty exists 2024-09-14T03:52:36.233 INFO:tasks.rados.rados.0.smithi070.stdout:1371: finishing rollback tid 1 to smithi07085154-46 2024-09-14T03:52:36.233 INFO:tasks.rados.rados.0.smithi070.stdout:1371: finishing rollback tid 2 to smithi07085154-46 2024-09-14T03:52:36.233 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 46 v 1242 (ObjNum 320 snap 107 seq_num 320) dirty exists 2024-09-14T03:52:36.233 INFO:tasks.rados.rados.0.smithi070.stdout:1373: expect (ObjNum 380 snap 129 seq_num 380) 2024-09-14T03:52:36.369 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:36 smithi070 podman[103420]: 2024-09-14 03:52:36.180380345 +0000 UTC m=+0.555550236 container init 942d3ee590acaa1d31542d4bb5ec672a6097d13c888db2bb467388d57f1f482e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-14T03:52:36.369 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:36 smithi070 podman[103420]: 2024-09-14 03:52:36.18677959 +0000 UTC m=+0.561949470 container start 942d3ee590acaa1d31542d4bb5ec672a6097d13c888db2bb467388d57f1f482e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True) 2024-09-14T03:52:36.369 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:36 smithi070 podman[103420]: 2024-09-14 03:52:36.271131409 +0000 UTC m=+0.646301315 container attach 942d3ee590acaa1d31542d4bb5ec672a6097d13c888db2bb467388d57f1f482e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-14T03:52:36.546 INFO:tasks.rados.rados.0.smithi070.stdout:1372: expect (ObjNum 375 snap 128 seq_num 375) 2024-09-14T03:52:36.734 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:36 smithi070 podman[103420]: 2024-09-14 03:52:36.368880312 +0000 UTC m=+0.744050201 container died 942d3ee590acaa1d31542d4bb5ec672a6097d13c888db2bb467388d57f1f482e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-14T03:52:37.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:37 smithi094 ceph-mon[86597]: osdmap e336: 8 total, 7 up, 8 in 2024-09-14T03:52:37.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:37 smithi094 ceph-mon[86597]: pgmap v95: 121 pgs: 16 stale+active+clean, 105 active+clean; 394 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.2 MiB/s wr, 17 op/s 2024-09-14T03:52:37.277 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:52:36 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:52:36.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:37.285 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:37 smithi070 podman[103462]: 2024-09-14 03:52:37.008688638 +0000 UTC m=+0.633972468 container remove 942d3ee590acaa1d31542d4bb5ec672a6097d13c888db2bb467388d57f1f482e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, RELEASE=main-a278231, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-14T03:52:37.286 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:37 smithi070 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.0.service: Deactivated successfully. 2024-09-14T03:52:37.286 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:37 smithi070 systemd[1]: Stopped Ceph osd.0 for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:52:37.286 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:37 smithi070 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.0.service: Consumed 10.942s CPU time. 2024-09-14T03:52:37.286 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:37 smithi070 systemd[1]: Starting Ceph osd.0 for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:52:37.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:37 smithi070 ceph-mon[96704]: osdmap e336: 8 total, 7 up, 8 in 2024-09-14T03:52:37.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:37 smithi070 ceph-mon[96704]: pgmap v95: 121 pgs: 16 stale+active+clean, 105 active+clean; 394 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.2 MiB/s wr, 17 op/s 2024-09-14T03:52:37.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:37 smithi070 ceph-mon[98640]: osdmap e336: 8 total, 7 up, 8 in 2024-09-14T03:52:37.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:37 smithi070 ceph-mon[98640]: pgmap v95: 121 pgs: 16 stale+active+clean, 105 active+clean; 394 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.2 MiB/s wr, 17 op/s 2024-09-14T03:52:37.487 INFO:tasks.rados.rados.0.smithi070.stdout:1374: finishing write tid 1 to smithi07085154-6 2024-09-14T03:52:37.488 INFO:tasks.rados.rados.0.smithi070.stdout:1374: finishing write tid 2 to smithi07085154-6 2024-09-14T03:52:37.488 INFO:tasks.rados.rados.0.smithi070.stdout:1374: finishing write tid 3 to smithi07085154-6 2024-09-14T03:52:37.488 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 630 (ObjNum 441 snap 152 seq_num 441) dirty exists 2024-09-14T03:52:37.488 INFO:tasks.rados.rados.0.smithi070.stdout:1374: left oid 6 (ObjNum 441 snap 152 seq_num 441) 2024-09-14T03:52:37.488 INFO:tasks.rados.rados.0.smithi070.stdout:1366: done (7 left) 2024-09-14T03:52:37.488 INFO:tasks.rados.rados.0.smithi070.stdout:1367: done (6 left) 2024-09-14T03:52:37.488 INFO:tasks.rados.rados.0.smithi070.stdout:1368: done (5 left) 2024-09-14T03:52:37.488 INFO:tasks.rados.rados.0.smithi070.stdout:1371: done (4 left) 2024-09-14T03:52:37.488 INFO:tasks.rados.rados.0.smithi070.stdout:1372: done (3 left) 2024-09-14T03:52:37.489 INFO:tasks.rados.rados.0.smithi070.stdout:1373: done (2 left) 2024-09-14T03:52:37.489 INFO:tasks.rados.rados.0.smithi070.stdout:1374: done (1 left) 2024-09-14T03:52:37.489 INFO:tasks.rados.rados.0.smithi070.stdout:1375: done (0 left) 2024-09-14T03:52:37.489 INFO:tasks.rados.rados.0.smithi070.stdout:1376: rmattr oid 41 current snap is 152 2024-09-14T03:52:37.489 INFO:tasks.rados.rados.0.smithi070.stdout:1377: snap_create 2024-09-14T03:52:37.491 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 851 (ObjNum 211 snap 60 seq_num 211) dirty exists 2024-09-14T03:52:37.713 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:37 smithi070 podman[103564]: 2024-09-14 03:52:37.359826779 +0000 UTC m=+0.015696973 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:52:37.713 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:37 smithi070 podman[103564]: 2024-09-14 03:52:37.484757567 +0000 UTC m=+0.140627748 container create b1d10adefbd85d001a7dbb7882f492f1ba0e1ddc780b4bc84927c64fe6d8fb01 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0-activate, maintainer=Guillaume Abrioux , RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-14T03:52:38.012 INFO:tasks.rados.rados.0.smithi070.stdout:1376: done (1 left) 2024-09-14T03:52:38.012 INFO:tasks.rados.rados.0.smithi070.stdout:1377: done (0 left) 2024-09-14T03:52:38.012 INFO:tasks.rados.rados.0.smithi070.stdout:1378: rollback oid 39 current snap is 153 2024-09-14T03:52:38.013 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 39 to 150 2024-09-14T03:52:38.013 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 44 current snap is 153 2024-09-14T03:52:38.013 INFO:tasks.rados.rados.0.smithi070.stdout:1379: seq_num 442 ranges {0=204800} 2024-09-14T03:52:38.014 INFO:tasks.rados.rados.0.smithi070.stdout:1379: writing smithi07085154-44 from 0 to 204800 tid 1 2024-09-14T03:52:38.014 INFO:tasks.rados.rados.0.smithi070.stdout:1380: read oid 16 snap 151 2024-09-14T03:52:38.014 INFO:tasks.rados.rados.0.smithi070.stdout:1380: expect (ObjNum 404 snap 140 seq_num 404) 2024-09-14T03:52:38.014 INFO:tasks.rados.rados.0.smithi070.stdout:1381: read oid 17 snap -1 2024-09-14T03:52:38.014 INFO:tasks.rados.rados.0.smithi070.stdout:1381: expect (ObjNum 325 snap 107 seq_num 325) 2024-09-14T03:52:38.014 INFO:tasks.rados.rados.0.smithi070.stdout:1382: copy_from oid 13 from oid 49 current snap is 153 2024-09-14T03:52:38.014 INFO:tasks.rados.rados.0.smithi070.stdout:1383: snap_create 2024-09-14T03:52:38.022 INFO:tasks.rados.rados.0.smithi070.stdout:1378: finishing rollback tid 0 to smithi07085154-39 2024-09-14T03:52:38.022 INFO:tasks.rados.rados.0.smithi070.stdout:1378: finishing rollback tid 1 to smithi07085154-39 2024-09-14T03:52:38.022 INFO:tasks.rados.rados.0.smithi070.stdout:1378: finishing rollback tid 2 to smithi07085154-39 2024-09-14T03:52:38.023 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 39 v 325 (ObjNum 425 snap 147 seq_num 425) dirty exists 2024-09-14T03:52:38.029 INFO:tasks.rados.rados.0.smithi070.stdout:1379: finishing write tid 1 to smithi07085154-44 2024-09-14T03:52:38.029 INFO:tasks.rados.rados.0.smithi070.stdout:1379: finishing write tid 2 to smithi07085154-44 2024-09-14T03:52:38.034 INFO:tasks.rados.rados.0.smithi070.stdout:1381: expect (ObjNum 325 snap 107 seq_num 325) 2024-09-14T03:52:38.035 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:37 smithi070 podman[103564]: 2024-09-14 03:52:37.712406344 +0000 UTC m=+0.368276526 container init b1d10adefbd85d001a7dbb7882f492f1ba0e1ddc780b4bc84927c64fe6d8fb01 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0-activate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-14T03:52:38.036 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:37 smithi070 podman[103564]: 2024-09-14 03:52:37.717310427 +0000 UTC m=+0.373180608 container start b1d10adefbd85d001a7dbb7882f492f1ba0e1ddc780b4bc84927c64fe6d8fb01 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-14T03:52:38.036 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:37 smithi070 podman[103564]: 2024-09-14 03:52:37.82066662 +0000 UTC m=+0.476536813 container attach b1d10adefbd85d001a7dbb7882f492f1ba0e1ddc780b4bc84927c64fe6d8fb01 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=main-a278231, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-14T03:52:38.036 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:37 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0-activate[103580]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:52:38.036 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:37 smithi070 bash[103564]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:52:38.036 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:37 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0-activate[103580]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:52:38.036 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:37 smithi070 bash[103564]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:52:38.138 INFO:tasks.rados.rados.0.smithi070.stdout:1379: finishing write tid 3 to smithi07085154-44 2024-09-14T03:52:38.138 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 527 (ObjNum 442 snap 153 seq_num 442) dirty exists 2024-09-14T03:52:38.138 INFO:tasks.rados.rados.0.smithi070.stdout:1379: left oid 44 (ObjNum 442 snap 153 seq_num 442) 2024-09-14T03:52:38.138 INFO:tasks.rados.rados.0.smithi070.stdout:1380: expect (ObjNum 404 snap 140 seq_num 404) 2024-09-14T03:52:38.461 INFO:tasks.rados.rados.0.smithi070.stdout:1382: finishing copy_from racing read to smithi07085154-13 2024-09-14T03:52:38.461 INFO:tasks.rados.rados.0.smithi070.stdout:1382: finishing copy_from to smithi07085154-13 2024-09-14T03:52:38.461 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 436 (ObjNum 390 snap 132 seq_num 390) dirty exists 2024-09-14T03:52:38.620 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:38 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0-activate[103580]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-14T03:52:38.620 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:38 smithi070 bash[103564]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-14T03:52:38.620 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:38 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0-activate[103580]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-14T03:52:38.620 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:38 smithi070 bash[103564]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-14T03:52:38.620 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:38 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0-activate[103580]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-14T03:52:38.620 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:38 smithi070 bash[103564]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-14T03:52:39.013 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:38 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0-activate[103580]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-14T03:52:39.013 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:38 smithi070 bash[103564]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-14T03:52:39.013 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:38 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0-activate[103580]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-14T03:52:39.013 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:38 smithi070 bash[103564]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-14T03:52:39.013 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:38 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0-activate[103580]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-14T03:52:39.013 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:38 smithi070 bash[103564]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-14T03:52:39.013 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:38 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0-activate[103580]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-14T03:52:39.013 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:38 smithi070 bash[103564]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-14T03:52:39.014 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:38 smithi070 podman[103564]: 2024-09-14 03:52:38.647709226 +0000 UTC m=+1.303579425 container died b1d10adefbd85d001a7dbb7882f492f1ba0e1ddc780b4bc84927c64fe6d8fb01 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0-activate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=main-a278231, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:52:39.016 INFO:tasks.rados.rados.0.smithi070.stdout:1378: done (5 left) 2024-09-14T03:52:39.016 INFO:tasks.rados.rados.0.smithi070.stdout:1379: done (4 left) 2024-09-14T03:52:39.016 INFO:tasks.rados.rados.0.smithi070.stdout:1380: done (3 left) 2024-09-14T03:52:39.016 INFO:tasks.rados.rados.0.smithi070.stdout:1381: done (2 left) 2024-09-14T03:52:39.016 INFO:tasks.rados.rados.0.smithi070.stdout:1382: done (1 left) 2024-09-14T03:52:39.016 INFO:tasks.rados.rados.0.smithi070.stdout:1383: done (0 left) 2024-09-14T03:52:39.017 INFO:tasks.rados.rados.0.smithi070.stdout:1384: rollback oid 1 current snap is 154 2024-09-14T03:52:39.017 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 1 to 147 2024-09-14T03:52:39.017 INFO:tasks.rados.rados.0.smithi070.stdout:1385: rollback oid 36 current snap is 154 2024-09-14T03:52:39.017 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 36 to 94 2024-09-14T03:52:39.017 INFO:tasks.rados.rados.0.smithi070.stdout:1386: snap_remove snap 133 2024-09-14T03:52:39.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:39 smithi094 ceph-mon[86597]: osdmap e337: 8 total, 7 up, 8 in 2024-09-14T03:52:39.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:39 smithi094 ceph-mon[86597]: pgmap v97: 121 pgs: 4 active+undersized, 16 stale+active+clean, 1 active+undersized+degraded, 100 active+clean; 394 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 22/942 objects degraded (2.335%) 2024-09-14T03:52:39.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:39 smithi070 ceph-mon[96704]: osdmap e337: 8 total, 7 up, 8 in 2024-09-14T03:52:39.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:39 smithi070 ceph-mon[96704]: pgmap v97: 121 pgs: 4 active+undersized, 16 stale+active+clean, 1 active+undersized+degraded, 100 active+clean; 394 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 22/942 objects degraded (2.335%) 2024-09-14T03:52:39.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:39 smithi070 ceph-mon[98640]: osdmap e337: 8 total, 7 up, 8 in 2024-09-14T03:52:39.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:39 smithi070 ceph-mon[98640]: pgmap v97: 121 pgs: 4 active+undersized, 16 stale+active+clean, 1 active+undersized+degraded, 100 active+clean; 394 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 22/942 objects degraded (2.335%) 2024-09-14T03:52:39.762 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:39 smithi070 podman[103727]: 2024-09-14 03:52:39.408283273 +0000 UTC m=+0.754869058 container remove b1d10adefbd85d001a7dbb7882f492f1ba0e1ddc780b4bc84927c64fe6d8fb01 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0-activate, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-14T03:52:39.762 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:39 smithi070 podman[103792]: 2024-09-14 03:52:39.516657704 +0000 UTC m=+0.015591497 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:52:39.762 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:39 smithi070 podman[103792]: 2024-09-14 03:52:39.625982291 +0000 UTC m=+0.124916084 container create 2345ccac208538d9c77f6cbd822cb5c03442a76571331d7a384011581cede572 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, maintainer=Guillaume Abrioux , RELEASE=main-a278231, org.label-schema.schema-version=1.0) 2024-09-14T03:52:40.017 INFO:tasks.rados.rados.0.smithi070.stdout:1386: done (2 left) 2024-09-14T03:52:40.018 INFO:tasks.rados.rados.0.smithi070.stdout:1387: snap_remove snap 130 2024-09-14T03:52:40.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:40 smithi094 ceph-mon[86597]: Health check failed: Degraded data redundancy: 22/942 objects degraded (2.335%), 1 pg degraded (PG_DEGRADED) 2024-09-14T03:52:40.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:40 smithi094 ceph-mon[86597]: osdmap e338: 8 total, 7 up, 8 in 2024-09-14T03:52:40.277 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:52:40 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:52:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:40.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:40 smithi070 ceph-mon[96704]: Health check failed: Degraded data redundancy: 22/942 objects degraded (2.335%), 1 pg degraded (PG_DEGRADED) 2024-09-14T03:52:40.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:40 smithi070 ceph-mon[96704]: osdmap e338: 8 total, 7 up, 8 in 2024-09-14T03:52:40.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:40 smithi070 ceph-mon[98640]: Health check failed: Degraded data redundancy: 22/942 objects degraded (2.335%), 1 pg degraded (PG_DEGRADED) 2024-09-14T03:52:40.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:40 smithi070 ceph-mon[98640]: osdmap e338: 8 total, 7 up, 8 in 2024-09-14T03:52:40.286 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:39 smithi070 podman[103792]: 2024-09-14 03:52:39.90352224 +0000 UTC m=+0.402456034 container init 2345ccac208538d9c77f6cbd822cb5c03442a76571331d7a384011581cede572 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2) 2024-09-14T03:52:40.286 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:39 smithi070 podman[103792]: 2024-09-14 03:52:39.908758749 +0000 UTC m=+0.407692542 container start 2345ccac208538d9c77f6cbd822cb5c03442a76571331d7a384011581cede572 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=main-a278231, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-14T03:52:40.286 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:39 smithi070 ceph-osd[103831]: -- 172.21.15.70:0/2123165512 <== mon.2 v2:172.21.15.70:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x560ea7f5fa40 con 0x560ea7f3dc00 2024-09-14T03:52:40.286 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:40 smithi070 bash[103792]: 2345ccac208538d9c77f6cbd822cb5c03442a76571331d7a384011581cede572 2024-09-14T03:52:40.286 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:40 smithi070 systemd[1]: Started Ceph osd.0 for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:52:41.018 INFO:tasks.rados.rados.0.smithi070.stdout:1387: done (2 left) 2024-09-14T03:52:41.018 INFO:tasks.rados.rados.0.smithi070.stdout:1388: snap_remove snap 125 2024-09-14T03:52:41.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:41 smithi070 ceph-mon[98640]: osdmap e339: 8 total, 7 up, 8 in 2024-09-14T03:52:41.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:41 smithi070 ceph-mon[98640]: pgmap v100: 121 pgs: 38 active+undersized, 15 active+undersized+degraded, 68 active+clean; 397 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 22 MiB/s rd, 129 KiB/s wr, 14 op/s; 125/972 objects degraded (12.860%) 2024-09-14T03:52:41.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:41 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:41.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:41 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:41.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:41 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:52:41.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:41 smithi070 ceph-mon[98640]: osdmap e340: 8 total, 7 up, 8 in 2024-09-14T03:52:41.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:41 smithi070 ceph-mon[96704]: osdmap e339: 8 total, 7 up, 8 in 2024-09-14T03:52:41.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:41 smithi070 ceph-mon[96704]: pgmap v100: 121 pgs: 38 active+undersized, 15 active+undersized+degraded, 68 active+clean; 397 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 22 MiB/s rd, 129 KiB/s wr, 14 op/s; 125/972 objects degraded (12.860%) 2024-09-14T03:52:41.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:41 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:41.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:41 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:41.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:41 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:52:41.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:41 smithi070 ceph-mon[96704]: osdmap e340: 8 total, 7 up, 8 in 2024-09-14T03:52:41.405 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:41 smithi094 ceph-mon[86597]: osdmap e339: 8 total, 7 up, 8 in 2024-09-14T03:52:41.405 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:41 smithi094 ceph-mon[86597]: pgmap v100: 121 pgs: 38 active+undersized, 15 active+undersized+degraded, 68 active+clean; 397 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 22 MiB/s rd, 129 KiB/s wr, 14 op/s; 125/972 objects degraded (12.860%) 2024-09-14T03:52:41.405 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:41 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:41.405 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:41 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:41.405 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:41 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:52:41.405 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:41 smithi094 ceph-mon[86597]: osdmap e340: 8 total, 7 up, 8 in 2024-09-14T03:52:41.594 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:41 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0[103808]: 2024-09-14T03:52:41.311+0000 7f4ae4f94640 -1 Falling back to public interface 2024-09-14T03:52:42.020 INFO:tasks.rados.rados.0.smithi070.stdout:1388: done (2 left) 2024-09-14T03:52:42.020 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 29 current snap is 154 2024-09-14T03:52:42.020 INFO:tasks.rados.rados.0.smithi070.stdout:1384: finishing rollback tid 0 to smithi07085154-1 2024-09-14T03:52:42.020 INFO:tasks.rados.rados.0.smithi070.stdout:1389: seq_num 443 ranges {5586944=221184} 2024-09-14T03:52:42.052 INFO:tasks.rados.rados.0.smithi070.stdout:1389: writing smithi07085154-29 from 5586944 to 5808128 tid 1 2024-09-14T03:52:42.052 INFO:tasks.rados.rados.0.smithi070.stdout:1390: snap_remove snap 95 2024-09-14T03:52:43.025 INFO:tasks.rados.rados.0.smithi070.stdout:1390: done (3 left) 2024-09-14T03:52:43.025 INFO:tasks.rados.rados.0.smithi070.stdout:1391: delete oid 13 current snap is 154 2024-09-14T03:52:43.025 INFO:tasks.rados.rados.0.smithi070.stdout:1384: finishing rollback tid 1 to smithi07085154-1 2024-09-14T03:52:43.025 INFO:tasks.rados.rados.0.smithi070.stdout:1384: finishing rollback tid 2 to smithi07085154-1 2024-09-14T03:52:43.025 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 1 v 531 (ObjNum 285 snap 89 seq_num 285) dirty exists 2024-09-14T03:52:43.025 INFO:tasks.rados.rados.0.smithi070.stdout:1385: finishing rollback tid 0 to smithi07085154-36 2024-09-14T03:52:43.025 INFO:tasks.rados.rados.0.smithi070.stdout:1385: finishing rollback tid 1 to smithi07085154-36 2024-09-14T03:52:43.025 INFO:tasks.rados.rados.0.smithi070.stdout:1385: finishing rollback tid 2 to smithi07085154-36 2024-09-14T03:52:43.025 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 36 v 535 (ObjNum 241 snap 69 seq_num 241) dirty exists 2024-09-14T03:52:43.026 INFO:tasks.rados.rados.0.smithi070.stdout:1389: finishing write tid 1 to smithi07085154-29 2024-09-14T03:52:43.026 INFO:tasks.rados.rados.0.smithi070.stdout:1389: finishing write tid 2 to smithi07085154-29 2024-09-14T03:52:43.026 INFO:tasks.rados.rados.0.smithi070.stdout:1389: finishing write tid 3 to smithi07085154-29 2024-09-14T03:52:43.026 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 758 (ObjNum 443 snap 154 seq_num 443) dirty exists 2024-09-14T03:52:43.026 INFO:tasks.rados.rados.0.smithi070.stdout:1389: left oid 29 (ObjNum 443 snap 154 seq_num 443) 2024-09-14T03:52:43.027 INFO:tasks.rados.rados.0.smithi070.stdout:1384: done (3 left) 2024-09-14T03:52:43.027 INFO:tasks.rados.rados.0.smithi070.stdout:1385: done (2 left) 2024-09-14T03:52:43.027 INFO:tasks.rados.rados.0.smithi070.stdout:1389: done (1 left) 2024-09-14T03:52:43.027 INFO:tasks.rados.rados.0.smithi070.stdout:1391: done (0 left) 2024-09-14T03:52:43.027 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 15 current snap is 154 2024-09-14T03:52:43.027 INFO:tasks.rados.rados.0.smithi070.stdout:1392: seq_num 445 ranges {3424256=253952} 2024-09-14T03:52:43.048 INFO:tasks.rados.rados.0.smithi070.stdout:1392: writing smithi07085154-15 from 3424256 to 3678208 tid 1 2024-09-14T03:52:43.048 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 22 current snap is 154 2024-09-14T03:52:43.048 INFO:tasks.rados.rados.0.smithi070.stdout:1393: seq_num 446 ranges {0=278528} 2024-09-14T03:52:43.050 INFO:tasks.rados.rados.0.smithi070.stdout:1393: writing smithi07085154-22 from 0 to 278528 tid 1 2024-09-14T03:52:43.050 INFO:tasks.rados.rados.0.smithi070.stdout:1394: read oid 30 snap -1 2024-09-14T03:52:43.050 INFO:tasks.rados.rados.0.smithi070.stdout:1394: expect (ObjNum 380 snap 129 seq_num 380) 2024-09-14T03:52:43.050 INFO:tasks.rados.rados.0.smithi070.stdout:1395: read oid 50 snap -1 2024-09-14T03:52:43.050 INFO:tasks.rados.rados.0.smithi070.stdout:1395: expect (ObjNum 430 snap 148 seq_num 430) 2024-09-14T03:52:43.050 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 29 current snap is 154 2024-09-14T03:52:43.050 INFO:tasks.rados.rados.0.smithi070.stdout:1396: seq_num 447 ranges {5808128=294912} 2024-09-14T03:52:43.083 INFO:tasks.rados.rados.0.smithi070.stdout:1396: writing smithi07085154-29 from 5808128 to 6103040 tid 1 2024-09-14T03:52:43.084 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 34 current snap is 154 2024-09-14T03:52:43.084 INFO:tasks.rados.rados.0.smithi070.stdout:1392: finishing write tid 1 to smithi07085154-15 2024-09-14T03:52:43.084 INFO:tasks.rados.rados.0.smithi070.stdout:1397: seq_num 448 ranges {0=311296} 2024-09-14T03:52:43.085 INFO:tasks.rados.rados.0.smithi070.stdout:1397: writing smithi07085154-34 from 0 to 311296 tid 1 2024-09-14T03:52:43.086 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 33 current snap is 154 2024-09-14T03:52:43.086 INFO:tasks.rados.rados.0.smithi070.stdout:1392: finishing write tid 2 to smithi07085154-15 2024-09-14T03:52:43.086 INFO:tasks.rados.rados.0.smithi070.stdout:1392: finishing write tid 3 to smithi07085154-15 2024-09-14T03:52:43.086 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 15 v 890 (ObjNum 445 snap 154 seq_num 445) dirty exists 2024-09-14T03:52:43.086 INFO:tasks.rados.rados.0.smithi070.stdout:1392: left oid 15 (ObjNum 445 snap 154 seq_num 445) 2024-09-14T03:52:43.086 INFO:tasks.rados.rados.0.smithi070.stdout:1393: finishing write tid 1 to smithi07085154-22 2024-09-14T03:52:43.086 INFO:tasks.rados.rados.0.smithi070.stdout:1393: finishing write tid 2 to smithi07085154-22 2024-09-14T03:52:43.086 INFO:tasks.rados.rados.0.smithi070.stdout:1398: seq_num 449 ranges {548864=327680} 2024-09-14T03:52:43.090 INFO:tasks.rados.rados.0.smithi070.stdout:1398: writing smithi07085154-33 from 548864 to 876544 tid 1 2024-09-14T03:52:43.091 INFO:tasks.rados.rados.0.smithi070.stdout:1392: done (6 left) 2024-09-14T03:52:43.091 INFO:tasks.rados.rados.0.smithi070.stdout:1399: rmattr oid 27 current snap is 154 2024-09-14T03:52:43.091 INFO:tasks.rados.rados.0.smithi070.stdout:1399: done (6 left) 2024-09-14T03:52:43.091 INFO:tasks.rados.rados.0.smithi070.stdout:1400: delete oid 20 current snap is 154 2024-09-14T03:52:43.091 INFO:tasks.rados.rados.0.smithi070.stdout:1393: finishing write tid 3 to smithi07085154-22 2024-09-14T03:52:43.091 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 762 (ObjNum 446 snap 154 seq_num 446) dirty exists 2024-09-14T03:52:43.091 INFO:tasks.rados.rados.0.smithi070.stdout:1393: left oid 22 (ObjNum 446 snap 154 seq_num 446) 2024-09-14T03:52:43.091 INFO:tasks.rados.rados.0.smithi070.stdout:1395: expect (ObjNum 430 snap 148 seq_num 430) 2024-09-14T03:52:43.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:43 smithi094 ceph-mon[86597]: osdmap e341: 8 total, 7 up, 8 in 2024-09-14T03:52:43.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:43 smithi094 ceph-mon[86597]: pgmap v103: 121 pgs: 38 active+undersized, 15 active+undersized+degraded, 68 active+clean; 397 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 22 MiB/s rd, 130 KiB/s wr, 14 op/s; 125/972 objects degraded (12.860%) 2024-09-14T03:52:43.283 INFO:tasks.rados.rados.0.smithi070.stdout:1394: expect (ObjNum 380 snap 129 seq_num 380) 2024-09-14T03:52:43.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:43 smithi070 ceph-mon[96704]: osdmap e341: 8 total, 7 up, 8 in 2024-09-14T03:52:43.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:43 smithi070 ceph-mon[96704]: pgmap v103: 121 pgs: 38 active+undersized, 15 active+undersized+degraded, 68 active+clean; 397 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 22 MiB/s rd, 130 KiB/s wr, 14 op/s; 125/972 objects degraded (12.860%) 2024-09-14T03:52:43.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:43 smithi070 ceph-mon[98640]: osdmap e341: 8 total, 7 up, 8 in 2024-09-14T03:52:43.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:43 smithi070 ceph-mon[98640]: pgmap v103: 121 pgs: 38 active+undersized, 15 active+undersized+degraded, 68 active+clean; 397 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 22 MiB/s rd, 130 KiB/s wr, 14 op/s; 125/972 objects degraded (12.860%) 2024-09-14T03:52:43.606 INFO:tasks.rados.rados.0.smithi070.stdout:1396: finishing write tid 1 to smithi07085154-29 2024-09-14T03:52:43.606 INFO:tasks.rados.rados.0.smithi070.stdout:1396: finishing write tid 2 to smithi07085154-29 2024-09-14T03:52:43.606 INFO:tasks.rados.rados.0.smithi070.stdout:1397: finishing write tid 1 to smithi07085154-34 2024-09-14T03:52:43.606 INFO:tasks.rados.rados.0.smithi070.stdout:1397: finishing write tid 2 to smithi07085154-34 2024-09-14T03:52:43.606 INFO:tasks.rados.rados.0.smithi070.stdout:1396: finishing write tid 3 to smithi07085154-29 2024-09-14T03:52:43.606 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 764 (ObjNum 447 snap 154 seq_num 447) dirty exists 2024-09-14T03:52:43.606 INFO:tasks.rados.rados.0.smithi070.stdout:1396: left oid 29 (ObjNum 447 snap 154 seq_num 447) 2024-09-14T03:52:43.606 INFO:tasks.rados.rados.0.smithi070.stdout:1397: finishing write tid 3 to smithi07085154-34 2024-09-14T03:52:43.606 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 766 (ObjNum 448 snap 154 seq_num 448) dirty exists 2024-09-14T03:52:43.607 INFO:tasks.rados.rados.0.smithi070.stdout:1397: left oid 34 (ObjNum 448 snap 154 seq_num 448) 2024-09-14T03:52:43.607 INFO:tasks.rados.rados.0.smithi070.stdout:1398: finishing write tid 1 to smithi07085154-33 2024-09-14T03:52:43.607 INFO:tasks.rados.rados.0.smithi070.stdout:1398: finishing write tid 2 to smithi07085154-33 2024-09-14T03:52:43.607 INFO:tasks.rados.rados.0.smithi070.stdout:1398: finishing write tid 3 to smithi07085154-33 2024-09-14T03:52:43.607 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 33 v 970 (ObjNum 449 snap 154 seq_num 449) dirty exists 2024-09-14T03:52:43.607 INFO:tasks.rados.rados.0.smithi070.stdout:1398: left oid 33 (ObjNum 449 snap 154 seq_num 449) 2024-09-14T03:52:43.607 INFO:tasks.rados.rados.0.smithi070.stdout:1393: done (6 left) 2024-09-14T03:52:43.607 INFO:tasks.rados.rados.0.smithi070.stdout:1394: done (5 left) 2024-09-14T03:52:43.607 INFO:tasks.rados.rados.0.smithi070.stdout:1395: done (4 left) 2024-09-14T03:52:43.607 INFO:tasks.rados.rados.0.smithi070.stdout:1396: done (3 left) 2024-09-14T03:52:43.607 INFO:tasks.rados.rados.0.smithi070.stdout:1397: done (2 left) 2024-09-14T03:52:43.607 INFO:tasks.rados.rados.0.smithi070.stdout:1398: done (1 left) 2024-09-14T03:52:43.607 INFO:tasks.rados.rados.0.smithi070.stdout:1400: done (0 left) 2024-09-14T03:52:43.607 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 48 current snap is 154 2024-09-14T03:52:43.608 INFO:tasks.rados.rados.0.smithi070.stdout:1401: seq_num 451 ranges {2662400=360448} 2024-09-14T03:52:43.621 INFO:tasks.rados.rados.0.smithi070.stdout:1401: writing smithi07085154-48 from 2662400 to 3022848 tid 1 2024-09-14T03:52:43.621 INFO:tasks.rados.rados.0.smithi070.stdout:1402: delete oid 20 current snap is 154 2024-09-14T03:52:43.622 INFO:tasks.rados.rados.0.smithi070.stdout:1402: done (1 left) 2024-09-14T03:52:43.622 INFO:tasks.rados.rados.0.smithi070.stdout:1403: setattr oid 17 current snap is 154 2024-09-14T03:52:43.624 INFO:tasks.rados.rados.0.smithi070.stdout:1404: read oid 21 snap -1 2024-09-14T03:52:43.624 INFO:tasks.rados.rados.0.smithi070.stdout:1404: expect (ObjNum 114 snap 24 seq_num 114) 2024-09-14T03:52:43.624 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 26 current snap is 154 2024-09-14T03:52:43.624 INFO:tasks.rados.rados.0.smithi070.stdout:1405: seq_num 453 ranges {0=393216} 2024-09-14T03:52:43.626 INFO:tasks.rados.rados.0.smithi070.stdout:1405: writing smithi07085154-26 from 0 to 393216 tid 1 2024-09-14T03:52:43.626 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 7 current snap is 154 2024-09-14T03:52:43.626 INFO:tasks.rados.rados.0.smithi070.stdout:1401: finishing write tid 1 to smithi07085154-48 2024-09-14T03:52:43.627 INFO:tasks.rados.rados.0.smithi070.stdout:1401: finishing write tid 2 to smithi07085154-48 2024-09-14T03:52:43.627 INFO:tasks.rados.rados.0.smithi070.stdout:1406: seq_num 454 ranges {5103616=409600} 2024-09-14T03:52:43.656 INFO:tasks.rados.rados.0.smithi070.stdout:1406: writing smithi07085154-7 from 5103616 to 5513216 tid 1 2024-09-14T03:52:43.657 INFO:tasks.rados.rados.0.smithi070.stdout:1407: rollback oid 34 current snap is 154 2024-09-14T03:52:43.657 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 34 to 147 2024-09-14T03:52:43.657 INFO:tasks.rados.rados.0.smithi070.stdout:1401: finishing write tid 3 to smithi07085154-48 2024-09-14T03:52:43.657 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 549 (ObjNum 451 snap 154 seq_num 451) dirty exists 2024-09-14T03:52:43.657 INFO:tasks.rados.rados.0.smithi070.stdout:1401: left oid 48 (ObjNum 451 snap 154 seq_num 451) 2024-09-14T03:52:43.657 INFO:tasks.rados.rados.0.smithi070.stdout:1401: done (5 left) 2024-09-14T03:52:43.657 INFO:tasks.rados.rados.0.smithi070.stdout:1408: rollback oid 13 current snap is 154 2024-09-14T03:52:43.657 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 13 to 83 2024-09-14T03:52:43.657 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 632 (ObjNum 325 snap 107 seq_num 325) dirty exists 2024-09-14T03:52:43.657 INFO:tasks.rados.rados.0.smithi070.stdout:1405: finishing write tid 1 to smithi07085154-26 2024-09-14T03:52:43.657 INFO:tasks.rados.rados.0.smithi070.stdout:1405: finishing write tid 2 to smithi07085154-26 2024-09-14T03:52:43.657 INFO:tasks.rados.rados.0.smithi070.stdout:1405: finishing write tid 3 to smithi07085154-26 2024-09-14T03:52:43.658 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 1302 (ObjNum 453 snap 154 seq_num 453) dirty exists 2024-09-14T03:52:43.658 INFO:tasks.rados.rados.0.smithi070.stdout:1405: left oid 26 (ObjNum 453 snap 154 seq_num 453) 2024-09-14T03:52:43.658 INFO:tasks.rados.rados.0.smithi070.stdout:1404: expect (ObjNum 114 snap 24 seq_num 114) 2024-09-14T03:52:43.661 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:52:43 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:52:43] "GET /metrics HTTP/1.1" 200 36194 "" "Prometheus/2.51.0" 2024-09-14T03:52:43.858 INFO:tasks.rados.rados.0.smithi070.stdout:1407: finishing rollback tid 0 to smithi07085154-34 2024-09-14T03:52:43.858 INFO:tasks.rados.rados.0.smithi070.stdout:1408: finishing rollback tid 1 to smithi07085154-13 2024-09-14T03:52:43.858 INFO:tasks.rados.rados.0.smithi070.stdout:1407: finishing rollback tid 1 to smithi07085154-34 2024-09-14T03:52:43.858 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 768 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-14T03:52:43.858 INFO:tasks.rados.rados.0.smithi070.stdout:1408: finishing rollback tid 2 to smithi07085154-13 2024-09-14T03:52:43.858 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 494 (ObjNum 138 snap 39 seq_num 138) dirty exists 2024-09-14T03:52:43.858 INFO:tasks.rados.rados.0.smithi070.stdout:1406: finishing write tid 1 to smithi07085154-7 2024-09-14T03:52:43.858 INFO:tasks.rados.rados.0.smithi070.stdout:1406: finishing write tid 2 to smithi07085154-7 2024-09-14T03:52:43.859 INFO:tasks.rados.rados.0.smithi070.stdout:1406: finishing write tid 3 to smithi07085154-7 2024-09-14T03:52:43.859 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 1202 (ObjNum 454 snap 154 seq_num 454) dirty exists 2024-09-14T03:52:43.859 INFO:tasks.rados.rados.0.smithi070.stdout:1406: left oid 7 (ObjNum 454 snap 154 seq_num 454) 2024-09-14T03:52:43.859 INFO:tasks.rados.rados.0.smithi070.stdout:1403: done (5 left) 2024-09-14T03:52:43.859 INFO:tasks.rados.rados.0.smithi070.stdout:1404: done (4 left) 2024-09-14T03:52:43.859 INFO:tasks.rados.rados.0.smithi070.stdout:1405: done (3 left) 2024-09-14T03:52:43.859 INFO:tasks.rados.rados.0.smithi070.stdout:1406: done (2 left) 2024-09-14T03:52:43.859 INFO:tasks.rados.rados.0.smithi070.stdout:1407: done (1 left) 2024-09-14T03:52:43.859 INFO:tasks.rados.rados.0.smithi070.stdout:1408: done (0 left) 2024-09-14T03:52:43.859 INFO:tasks.rados.rados.0.smithi070.stdout:1409: snap_create 2024-09-14T03:52:44.004 INFO:tasks.rados.rados.0.smithi070.stdout:1409: done (0 left) 2024-09-14T03:52:44.005 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 5 current snap is 155 2024-09-14T03:52:44.005 INFO:tasks.rados.rados.0.smithi070.stdout:1410: seq_num 455 ranges {40960=425984} 2024-09-14T03:52:44.007 INFO:tasks.rados.rados.0.smithi070.stdout:1410: writing smithi07085154-5 from 40960 to 466944 tid 1 2024-09-14T03:52:44.007 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 10 current snap is 155 2024-09-14T03:52:44.007 INFO:tasks.rados.rados.0.smithi070.stdout:1411: seq_num 456 ranges {3203072=442368} 2024-09-14T03:52:44.028 INFO:tasks.rados.rados.0.smithi070.stdout:1411: writing smithi07085154-10 from 3203072 to 3645440 tid 1 2024-09-14T03:52:44.028 INFO:tasks.rados.rados.0.smithi070.stdout:1412: rmattr oid 30 current snap is 155 2024-09-14T03:52:44.028 INFO:tasks.rados.rados.0.smithi070.stdout:1412: done (2 left) 2024-09-14T03:52:44.028 INFO:tasks.rados.rados.0.smithi070.stdout:1413: copy_from oid 33 from oid 50 current snap is 155 2024-09-14T03:52:44.028 INFO:tasks.rados.rados.0.smithi070.stdout:1410: finishing write tid 1 to smithi07085154-5 2024-09-14T03:52:44.028 INFO:tasks.rados.rados.0.smithi070.stdout:1410: finishing write tid 2 to smithi07085154-5 2024-09-14T03:52:44.028 INFO:tasks.rados.rados.0.smithi070.stdout:1410: finishing write tid 3 to smithi07085154-5 2024-09-14T03:52:44.029 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 554 (ObjNum 455 snap 155 seq_num 455) dirty exists 2024-09-14T03:52:44.029 INFO:tasks.rados.rados.0.smithi070.stdout:1410: left oid 5 (ObjNum 455 snap 155 seq_num 455) 2024-09-14T03:52:44.029 INFO:tasks.rados.rados.0.smithi070.stdout:1414: read oid 28 snap -1 2024-09-14T03:52:44.029 INFO:tasks.rados.rados.0.smithi070.stdout:1414: expect (ObjNum 438 snap 150 seq_num 438) 2024-09-14T03:52:44.029 INFO:tasks.rados.rados.0.smithi070.stdout:1410: done (3 left) 2024-09-14T03:52:44.029 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 20 current snap is 155 2024-09-14T03:52:44.029 INFO:tasks.rados.rados.0.smithi070.stdout:1415: seq_num 457 ranges {0=458752} 2024-09-14T03:52:44.030 INFO:tasks.rados.rados.0.smithi070.stdout:1415: writing smithi07085154-20 from 0 to 458752 tid 1 2024-09-14T03:52:44.030 INFO:tasks.rados.rados.0.smithi070.stdout:1416: copy_from oid 36 from oid 38 current snap is 155 2024-09-14T03:52:44.030 INFO:tasks.rados.rados.0.smithi070.stdout:1417: snap_remove snap 154 2024-09-14T03:52:44.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:44 smithi094 ceph-mon[86597]: osdmap e342: 8 total, 7 up, 8 in 2024-09-14T03:52:44.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:44 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:44.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:44 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:44.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:44 smithi094 ceph-mon[86597]: osdmap e343: 8 total, 7 up, 8 in 2024-09-14T03:52:44.278 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:52:44 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:52:44.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:44.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:44 smithi070 ceph-mon[96704]: osdmap e342: 8 total, 7 up, 8 in 2024-09-14T03:52:44.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:44 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:44.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:44 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:44.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:44 smithi070 ceph-mon[96704]: osdmap e343: 8 total, 7 up, 8 in 2024-09-14T03:52:44.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:44 smithi070 ceph-mon[98640]: osdmap e342: 8 total, 7 up, 8 in 2024-09-14T03:52:44.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:44 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:44.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:44 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:44.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:44 smithi070 ceph-mon[98640]: osdmap e343: 8 total, 7 up, 8 in 2024-09-14T03:52:45.030 INFO:tasks.rados.rados.0.smithi070.stdout:1417: done (5 left) 2024-09-14T03:52:45.030 INFO:tasks.rados.rados.0.smithi070.stdout:1418: snap_remove snap 93 2024-09-14T03:52:45.091 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:45 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0[103808]: 2024-09-14T03:52:45.017+0000 7f4ae4f94640 -1 osd.0 333 log_to_monitors true 2024-09-14T03:52:45.091 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:45 smithi070 ceph-mon[98640]: pgmap v106: 121 pgs: 2 active+clean+snaptrim, 38 active+undersized, 15 active+undersized+degraded, 66 active+clean; 401 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 1.3 MiB/s wr, 2 op/s; 127/981 objects degraded (12.946%) 2024-09-14T03:52:45.091 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:45 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:45.091 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:45 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:45.091 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:45 smithi070 ceph-mon[96704]: pgmap v106: 121 pgs: 2 active+clean+snaptrim, 38 active+undersized, 15 active+undersized+degraded, 66 active+clean; 401 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 1.3 MiB/s wr, 2 op/s; 127/981 objects degraded (12.946%) 2024-09-14T03:52:45.092 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:45 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:45.092 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:45 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:45.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:45 smithi094 ceph-mon[86597]: pgmap v106: 121 pgs: 2 active+clean+snaptrim, 38 active+undersized, 15 active+undersized+degraded, 66 active+clean; 401 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 1.3 MiB/s wr, 2 op/s; 127/981 objects degraded (12.946%) 2024-09-14T03:52:45.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:45 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:45.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:45 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:46.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:52:45 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:52:45.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:46.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:52:45 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:52:45.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:46.032 INFO:tasks.rados.rados.0.smithi070.stdout:1418: done (5 left) 2024-09-14T03:52:46.032 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 29 current snap is 155 2024-09-14T03:52:46.032 INFO:tasks.rados.rados.0.smithi070.stdout:1419: seq_num 458 ranges {6103040=475136} 2024-09-14T03:52:46.070 INFO:tasks.rados.rados.0.smithi070.stdout:1419: writing smithi07085154-29 from 6103040 to 6578176 tid 1 2024-09-14T03:52:46.070 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 25 current snap is 155 2024-09-14T03:52:46.071 INFO:tasks.rados.rados.0.smithi070.stdout:1411: finishing write tid 1 to smithi07085154-10 2024-09-14T03:52:46.071 INFO:tasks.rados.rados.0.smithi070.stdout:1420: seq_num 459 ranges {15679488=491520} 2024-09-14T03:52:46.135 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:52:46 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0[103808]: 2024-09-14T03:52:46.070+0000 7f4adcd3e640 -1 osd.0 333 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:52:46.136 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:46 smithi070 ceph-mon[98640]: Health check update: Degraded data redundancy: 127/981 objects degraded (12.946%), 15 pgs degraded (PG_DEGRADED) 2024-09-14T03:52:46.136 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:46 smithi070 ceph-mon[98640]: from='osd.0 [v2:172.21.15.70:6802/1152327285,v1:172.21.15.70:6803/1152327285]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-14T03:52:46.136 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:46 smithi070 ceph-mon[98640]: osdmap e344: 8 total, 7 up, 8 in 2024-09-14T03:52:46.136 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:46 smithi070 ceph-mon[96704]: Health check update: Degraded data redundancy: 127/981 objects degraded (12.946%), 15 pgs degraded (PG_DEGRADED) 2024-09-14T03:52:46.136 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:46 smithi070 ceph-mon[96704]: from='osd.0 [v2:172.21.15.70:6802/1152327285,v1:172.21.15.70:6803/1152327285]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-14T03:52:46.136 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:46 smithi070 ceph-mon[96704]: osdmap e344: 8 total, 7 up, 8 in 2024-09-14T03:52:46.157 INFO:tasks.rados.rados.0.smithi070.stdout:1420: writing smithi07085154-25 from 15679488 to 16171008 tid 1 2024-09-14T03:52:46.158 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 39 current snap is 155 2024-09-14T03:52:46.158 INFO:tasks.rados.rados.0.smithi070.stdout:1421: seq_num 460 ranges {6414336=507904} 2024-09-14T03:52:46.196 INFO:tasks.rados.rados.0.smithi070.stdout:1421: writing smithi07085154-39 from 6414336 to 6922240 tid 1 2024-09-14T03:52:46.196 INFO:tasks.rados.rados.0.smithi070.stdout:1422: copy_from oid 32 from oid 42 current snap is 155 2024-09-14T03:52:46.196 INFO:tasks.rados.rados.0.smithi070.stdout:1423: copy_from oid 11 from oid 9 current snap is 155 2024-09-14T03:52:46.196 INFO:tasks.rados.rados.0.smithi070.stdout:1411: finishing write tid 2 to smithi07085154-10 2024-09-14T03:52:46.196 INFO:tasks.rados.rados.0.smithi070.stdout:1424: rollback oid 45 current snap is 155 2024-09-14T03:52:46.196 INFO:tasks.rados.rados.0.smithi070.stdout:1411: finishing write tid 3 to smithi07085154-10 2024-09-14T03:52:46.196 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 1305 (ObjNum 456 snap 155 seq_num 456) dirty exists 2024-09-14T03:52:46.197 INFO:tasks.rados.rados.0.smithi070.stdout:1411: left oid 10 (ObjNum 456 snap 155 seq_num 456) 2024-09-14T03:52:46.197 INFO:tasks.rados.rados.0.smithi070.stdout:1416: finishing copy_from to smithi07085154-36 2024-09-14T03:52:46.197 INFO:tasks.rados.rados.0.smithi070.stdout:1416: got expected ENOENT (src dne) 2024-09-14T03:52:46.197 INFO:tasks.rados.rados.0.smithi070.stdout:1416: finishing copy_from racing read to smithi07085154-36 2024-09-14T03:52:46.197 INFO:tasks.rados.rados.0.smithi070.stdout:1413: finishing copy_from racing read to smithi07085154-33 2024-09-14T03:52:46.197 INFO:tasks.rados.rados.0.smithi070.stdout:1413: finishing copy_from to smithi07085154-33 2024-09-14T03:52:46.197 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 33 v 719 (ObjNum 430 snap 148 seq_num 430) dirty exists 2024-09-14T03:52:46.197 INFO:tasks.rados.rados.0.smithi070.stdout:1414: expect (ObjNum 438 snap 150 seq_num 438) 2024-09-14T03:52:46.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:46 smithi094 ceph-mon[86597]: Health check update: Degraded data redundancy: 127/981 objects degraded (12.946%), 15 pgs degraded (PG_DEGRADED) 2024-09-14T03:52:46.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:46 smithi094 ceph-mon[86597]: from='osd.0 [v2:172.21.15.70:6802/1152327285,v1:172.21.15.70:6803/1152327285]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-14T03:52:46.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:46 smithi094 ceph-mon[86597]: osdmap e344: 8 total, 7 up, 8 in 2024-09-14T03:52:46.685 INFO:tasks.rados.rados.0.smithi070.stdout:1415: finishing write tid 1 to smithi07085154-20 2024-09-14T03:52:46.685 INFO:tasks.rados.rados.0.smithi070.stdout:1415: finishing write tid 2 to smithi07085154-20 2024-09-14T03:52:46.685 INFO:tasks.rados.rados.0.smithi070.stdout:1415: finishing write tid 3 to smithi07085154-20 2024-09-14T03:52:46.685 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 20 v 1204 (ObjNum 457 snap 155 seq_num 457) dirty exists 2024-09-14T03:52:46.685 INFO:tasks.rados.rados.0.smithi070.stdout:1415: left oid 20 (ObjNum 457 snap 155 seq_num 457) 2024-09-14T03:52:46.685 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 45 to 122 2024-09-14T03:52:46.685 INFO:tasks.rados.rados.0.smithi070.stdout:1419: finishing write tid 1 to smithi07085154-29 2024-09-14T03:52:46.685 INFO:tasks.rados.rados.0.smithi070.stdout:1419: finishing write tid 2 to smithi07085154-29 2024-09-14T03:52:46.685 INFO:tasks.rados.rados.0.smithi070.stdout:1419: finishing write tid 3 to smithi07085154-29 2024-09-14T03:52:46.685 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 772 (ObjNum 458 snap 155 seq_num 458) dirty exists 2024-09-14T03:52:46.685 INFO:tasks.rados.rados.0.smithi070.stdout:1419: left oid 29 (ObjNum 458 snap 155 seq_num 458) 2024-09-14T03:52:46.685 INFO:tasks.rados.rados.0.smithi070.stdout:1420: finishing write tid 1 to smithi07085154-25 2024-09-14T03:52:46.686 INFO:tasks.rados.rados.0.smithi070.stdout:1420: finishing write tid 2 to smithi07085154-25 2024-09-14T03:52:46.686 INFO:tasks.rados.rados.0.smithi070.stdout:1420: finishing write tid 3 to smithi07085154-25 2024-09-14T03:52:46.686 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 635 (ObjNum 459 snap 155 seq_num 459) dirty exists 2024-09-14T03:52:46.686 INFO:tasks.rados.rados.0.smithi070.stdout:1420: left oid 25 (ObjNum 459 snap 155 seq_num 459) 2024-09-14T03:52:46.686 INFO:tasks.rados.rados.0.smithi070.stdout:1423: finishing copy_from to smithi07085154-11 2024-09-14T03:52:46.686 INFO:tasks.rados.rados.0.smithi070.stdout:1423: got expected ENOENT (src dne) 2024-09-14T03:52:46.686 INFO:tasks.rados.rados.0.smithi070.stdout:1423: finishing copy_from racing read to smithi07085154-11 2024-09-14T03:52:46.686 INFO:tasks.rados.rados.0.smithi070.stdout:1421: finishing write tid 1 to smithi07085154-39 2024-09-14T03:52:46.686 INFO:tasks.rados.rados.0.smithi070.stdout:1421: finishing write tid 2 to smithi07085154-39 2024-09-14T03:52:46.686 INFO:tasks.rados.rados.0.smithi070.stdout:1421: finishing write tid 3 to smithi07085154-39 2024-09-14T03:52:46.686 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 39 v 327 (ObjNum 460 snap 155 seq_num 460) dirty exists 2024-09-14T03:52:46.686 INFO:tasks.rados.rados.0.smithi070.stdout:1421: left oid 39 (ObjNum 460 snap 155 seq_num 460) 2024-09-14T03:52:46.686 INFO:tasks.rados.rados.0.smithi070.stdout:1422: finishing copy_from racing read to smithi07085154-32 2024-09-14T03:52:46.686 INFO:tasks.rados.rados.0.smithi070.stdout:1422: finishing copy_from to smithi07085154-32 2024-09-14T03:52:46.686 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 1165 (ObjNum 417 snap 146 seq_num 417) dirty exists 2024-09-14T03:52:46.687 INFO:tasks.rados.rados.0.smithi070.stdout:1411: done (10 left) 2024-09-14T03:52:46.687 INFO:tasks.rados.rados.0.smithi070.stdout:1413: done (9 left) 2024-09-14T03:52:46.687 INFO:tasks.rados.rados.0.smithi070.stdout:1414: done (8 left) 2024-09-14T03:52:46.687 INFO:tasks.rados.rados.0.smithi070.stdout:1415: done (7 left) 2024-09-14T03:52:46.687 INFO:tasks.rados.rados.0.smithi070.stdout:1416: done (6 left) 2024-09-14T03:52:46.687 INFO:tasks.rados.rados.0.smithi070.stdout:1419: done (5 left) 2024-09-14T03:52:46.687 INFO:tasks.rados.rados.0.smithi070.stdout:1420: done (4 left) 2024-09-14T03:52:46.687 INFO:tasks.rados.rados.0.smithi070.stdout:1421: done (3 left) 2024-09-14T03:52:46.687 INFO:tasks.rados.rados.0.smithi070.stdout:1422: done (2 left) 2024-09-14T03:52:46.687 INFO:tasks.rados.rados.0.smithi070.stdout:1423: done (1 left) 2024-09-14T03:52:46.687 INFO:tasks.rados.rados.0.smithi070.stdout:1425: read oid 18 snap -1 2024-09-14T03:52:46.687 INFO:tasks.rados.rados.0.smithi070.stdout:1425: expect (ObjNum 433 snap 148 seq_num 433) 2024-09-14T03:52:46.687 INFO:tasks.rados.rados.0.smithi070.stdout:1426: snap_remove snap 149 2024-09-14T03:52:47.041 INFO:tasks.rados.rados.0.smithi070.stdout:1424: finishing rollback tid 0 to smithi07085154-45 2024-09-14T03:52:47.041 INFO:tasks.rados.rados.0.smithi070.stdout:1424: finishing rollback tid 1 to smithi07085154-45 2024-09-14T03:52:47.041 INFO:tasks.rados.rados.0.smithi070.stdout:1424: finishing rollback tid 2 to smithi07085154-45 2024-09-14T03:52:47.041 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 45 v 973 (ObjNum 295 snap 92 seq_num 295) dirty exists 2024-09-14T03:52:47.041 INFO:tasks.rados.rados.0.smithi070.stdout:1425: expect (ObjNum 433 snap 148 seq_num 433) 2024-09-14T03:52:47.152 INFO:tasks.rados.rados.0.smithi070.stdout:1424: done (2 left) 2024-09-14T03:52:47.152 INFO:tasks.rados.rados.0.smithi070.stdout:1425: done (1 left) 2024-09-14T03:52:47.152 INFO:tasks.rados.rados.0.smithi070.stdout:1426: done (0 left) 2024-09-14T03:52:47.152 INFO:tasks.rados.rados.0.smithi070.stdout:1427: copy_from oid 27 from oid 25 current snap is 155 2024-09-14T03:52:47.153 INFO:tasks.rados.rados.0.smithi070.stdout:1428: read oid 24 snap 152 2024-09-14T03:52:47.153 INFO:tasks.rados.rados.0.smithi070.stdout:1428: expect (ObjNum 409 snap 140 seq_num 409) 2024-09-14T03:52:47.153 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 28 current snap is 155 2024-09-14T03:52:47.153 INFO:tasks.rados.rados.0.smithi070.stdout:1429: seq_num 461 ranges {7176192=524288} 2024-09-14T03:52:47.177 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:47 smithi070 ceph-mon[98640]: from='osd.0 [v2:172.21.15.70:6802/1152327285,v1:172.21.15.70:6803/1152327285]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-14T03:52:47.177 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:47 smithi070 ceph-mon[98640]: osdmap e345: 8 total, 7 up, 8 in 2024-09-14T03:52:47.178 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:47 smithi070 ceph-mon[98640]: from='osd.0 [v2:172.21.15.70:6802/1152327285,v1:172.21.15.70:6803/1152327285]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi070", "root=default"]} : dispatch 2024-09-14T03:52:47.178 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:47 smithi070 ceph-mon[98640]: pgmap v109: 121 pgs: 2 active+clean+snaptrim, 38 active+undersized, 15 active+undersized+degraded, 66 active+clean; 401 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 1.3 MiB/s wr, 2 op/s; 127/981 objects degraded (12.946%) 2024-09-14T03:52:47.178 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:47 smithi070 ceph-mon[96704]: from='osd.0 [v2:172.21.15.70:6802/1152327285,v1:172.21.15.70:6803/1152327285]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-14T03:52:47.178 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:47 smithi070 ceph-mon[96704]: osdmap e345: 8 total, 7 up, 8 in 2024-09-14T03:52:47.178 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:47 smithi070 ceph-mon[96704]: from='osd.0 [v2:172.21.15.70:6802/1152327285,v1:172.21.15.70:6803/1152327285]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi070", "root=default"]} : dispatch 2024-09-14T03:52:47.178 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:47 smithi070 ceph-mon[96704]: pgmap v109: 121 pgs: 2 active+clean+snaptrim, 38 active+undersized, 15 active+undersized+degraded, 66 active+clean; 401 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 1.3 MiB/s wr, 2 op/s; 127/981 objects degraded (12.946%) 2024-09-14T03:52:47.194 INFO:tasks.rados.rados.0.smithi070.stdout:1429: writing smithi07085154-28 from 7176192 to 7700480 tid 1 2024-09-14T03:52:47.194 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 19 current snap is 155 2024-09-14T03:52:47.194 INFO:tasks.rados.rados.0.smithi070.stdout:1430: seq_num 462 ranges {0=540672} 2024-09-14T03:52:47.197 INFO:tasks.rados.rados.0.smithi070.stdout:1430: writing smithi07085154-19 from 0 to 540672 tid 1 2024-09-14T03:52:47.197 INFO:tasks.rados.rados.0.smithi070.stdout:1431: setattr oid 39 current snap is 155 2024-09-14T03:52:47.198 INFO:tasks.rados.rados.0.smithi070.stdout:1432: rollback oid 20 current snap is 155 2024-09-14T03:52:47.199 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 20 to 99 2024-09-14T03:52:47.199 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 45 current snap is 155 2024-09-14T03:52:47.199 INFO:tasks.rados.rados.0.smithi070.stdout:1433: seq_num 463 ranges {974848=557056} 2024-09-14T03:52:47.207 INFO:tasks.rados.rados.0.smithi070.stdout:1433: writing smithi07085154-45 from 974848 to 1531904 tid 1 2024-09-14T03:52:47.207 INFO:tasks.rados.rados.0.smithi070.stdout:1434: rmattr oid 3 current snap is 155 2024-09-14T03:52:47.207 INFO:tasks.rados.rados.0.smithi070.stdout:1430: finishing write tid 1 to smithi07085154-19 2024-09-14T03:52:47.207 INFO:tasks.rados.rados.0.smithi070.stdout:1430: finishing write tid 2 to smithi07085154-19 2024-09-14T03:52:47.207 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 39 v 328 (ObjNum 460 snap 155 seq_num 460) dirty exists 2024-09-14T03:52:47.207 INFO:tasks.rados.rados.0.smithi070.stdout:1431: done (7 left) 2024-09-14T03:52:47.207 INFO:tasks.rados.rados.0.smithi070.stdout:1435: read oid 43 snap -1 2024-09-14T03:52:47.207 INFO:tasks.rados.rados.0.smithi070.stdout:1435: expect (ObjNum 390 snap 132 seq_num 390) 2024-09-14T03:52:47.207 INFO:tasks.rados.rados.0.smithi070.stdout:1436: snap_create 2024-09-14T03:52:47.207 INFO:tasks.rados.rados.0.smithi070.stdout:1430: finishing write tid 3 to smithi07085154-19 2024-09-14T03:52:47.208 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 19 v 776 (ObjNum 462 snap 155 seq_num 462) dirty exists 2024-09-14T03:52:47.208 INFO:tasks.rados.rados.0.smithi070.stdout:1430: left oid 19 (ObjNum 462 snap 155 seq_num 462) 2024-09-14T03:52:47.210 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 3 v 991 (ObjNum 225 snap 0 seq_num 4214144556) dirty exists 2024-09-14T03:52:47.221 INFO:tasks.rados.rados.0.smithi070.stdout:1435: expect (ObjNum 390 snap 132 seq_num 390) 2024-09-14T03:52:47.277 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:52:46 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:52:46.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:47.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:47 smithi094 ceph-mon[86597]: from='osd.0 [v2:172.21.15.70:6802/1152327285,v1:172.21.15.70:6803/1152327285]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-14T03:52:47.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:47 smithi094 ceph-mon[86597]: osdmap e345: 8 total, 7 up, 8 in 2024-09-14T03:52:47.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:47 smithi094 ceph-mon[86597]: from='osd.0 [v2:172.21.15.70:6802/1152327285,v1:172.21.15.70:6803/1152327285]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi070", "root=default"]} : dispatch 2024-09-14T03:52:47.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:47 smithi094 ceph-mon[86597]: pgmap v109: 121 pgs: 2 active+clean+snaptrim, 38 active+undersized, 15 active+undersized+degraded, 66 active+clean; 401 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 1.3 MiB/s wr, 2 op/s; 127/981 objects degraded (12.946%) 2024-09-14T03:52:48.035 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:52:47 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:52:47.713+0000 7fa4af5ba640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-14T03:52:48.041 INFO:tasks.rados.rados.0.smithi070.stdout:1430: done (8 left) 2024-09-14T03:52:48.042 INFO:tasks.rados.rados.0.smithi070.stdout:1434: done (7 left) 2024-09-14T03:52:48.042 INFO:tasks.rados.rados.0.smithi070.stdout:1435: done (6 left) 2024-09-14T03:52:48.042 INFO:tasks.rados.rados.0.smithi070.stdout:1436: done (5 left) 2024-09-14T03:52:48.042 INFO:tasks.rados.rados.0.smithi070.stdout:1437: read oid 26 snap -1 2024-09-14T03:52:48.042 INFO:tasks.rados.rados.0.smithi070.stdout:1437: expect (ObjNum 453 snap 154 seq_num 453) 2024-09-14T03:52:48.042 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 31 current snap is 156 2024-09-14T03:52:48.042 INFO:tasks.rados.rados.0.smithi070.stdout:1438: seq_num 464 ranges {1810432=573440} 2024-09-14T03:52:48.056 INFO:tasks.rados.rados.0.smithi070.stdout:1438: writing smithi07085154-31 from 1810432 to 2383872 tid 1 2024-09-14T03:52:48.056 INFO:tasks.rados.rados.0.smithi070.stdout:1439: snap_create 2024-09-14T03:52:48.058 INFO:tasks.rados.rados.0.smithi070.stdout:1437: expect (ObjNum 453 snap 154 seq_num 453) 2024-09-14T03:52:48.076 INFO:tasks.rados.rados.0.smithi070.stdout:1429: finishing write tid 1 to smithi07085154-28 2024-09-14T03:52:48.076 INFO:tasks.rados.rados.0.smithi070.stdout:1429: finishing write tid 2 to smithi07085154-28 2024-09-14T03:52:48.077 INFO:tasks.rados.rados.0.smithi070.stdout:1438: finishing write tid 1 to smithi07085154-31 2024-09-14T03:52:48.077 INFO:tasks.rados.rados.0.smithi070.stdout:1438: finishing write tid 2 to smithi07085154-31 2024-09-14T03:52:48.080 INFO:tasks.rados.rados.0.smithi070.stdout:1438: finishing write tid 3 to smithi07085154-31 2024-09-14T03:52:48.080 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 638 (ObjNum 464 snap 156 seq_num 464) dirty exists 2024-09-14T03:52:48.080 INFO:tasks.rados.rados.0.smithi070.stdout:1438: left oid 31 (ObjNum 464 snap 156 seq_num 464) 2024-09-14T03:52:48.080 INFO:tasks.rados.rados.0.smithi070.stdout:1433: finishing write tid 1 to smithi07085154-45 2024-09-14T03:52:48.080 INFO:tasks.rados.rados.0.smithi070.stdout:1433: finishing write tid 2 to smithi07085154-45 2024-09-14T03:52:48.082 INFO:tasks.rados.rados.0.smithi070.stdout:1429: finishing write tid 3 to smithi07085154-28 2024-09-14T03:52:48.082 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 28 v 1206 (ObjNum 461 snap 155 seq_num 461) dirty exists 2024-09-14T03:52:48.082 INFO:tasks.rados.rados.0.smithi070.stdout:1429: left oid 28 (ObjNum 461 snap 155 seq_num 461) 2024-09-14T03:52:48.083 INFO:tasks.rados.rados.0.smithi070.stdout:1432: finishing rollback tid 0 to smithi07085154-20 2024-09-14T03:52:48.083 INFO:tasks.rados.rados.0.smithi070.stdout:1433: finishing write tid 3 to smithi07085154-45 2024-09-14T03:52:48.083 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 45 v 975 (ObjNum 463 snap 155 seq_num 463) dirty exists 2024-09-14T03:52:48.083 INFO:tasks.rados.rados.0.smithi070.stdout:1433: left oid 45 (ObjNum 463 snap 155 seq_num 463) 2024-09-14T03:52:48.083 INFO:tasks.rados.rados.0.smithi070.stdout:1432: finishing rollback tid 1 to smithi07085154-20 2024-09-14T03:52:48.083 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 20 v 1208 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-14T03:52:48.097 INFO:tasks.rados.rados.0.smithi070.stdout:1428: expect (ObjNum 409 snap 140 seq_num 409) 2024-09-14T03:52:48.362 INFO:tasks.rados.rados.0.smithi070.stdout:1427: finishing copy_from racing read to smithi07085154-27 2024-09-14T03:52:48.362 INFO:tasks.rados.rados.0.smithi070.stdout:1427: finishing copy_from to smithi07085154-27 2024-09-14T03:52:48.362 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 635 (ObjNum 459 snap 155 seq_num 459) dirty exists 2024-09-14T03:52:48.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:48 smithi094 ceph-mon[86597]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:52:48.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:48 smithi094 ceph-mon[86597]: osd.0 [v2:172.21.15.70:6802/1152327285,v1:172.21.15.70:6803/1152327285] boot 2024-09-14T03:52:48.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:48 smithi094 ceph-mon[86597]: osdmap e346: 8 total, 8 up, 8 in 2024-09-14T03:52:48.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:48 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-14T03:52:48.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:48 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:48.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:48 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:52:48.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:48 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:48.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:48 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:48.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:48 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:48.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:48 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:52:48.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:48 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:48.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:48 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:52:48.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:48 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:52:48.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:48 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:52:48.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:48 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:52:48.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:48 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-14T03:52:48.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:48 smithi094 ceph-mon[86597]: osdmap e347: 8 total, 8 up, 8 in 2024-09-14T03:52:48.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:48 smithi070 ceph-mon[96704]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:52:48.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:48 smithi070 ceph-mon[96704]: osd.0 [v2:172.21.15.70:6802/1152327285,v1:172.21.15.70:6803/1152327285] boot 2024-09-14T03:52:48.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:48 smithi070 ceph-mon[96704]: osdmap e346: 8 total, 8 up, 8 in 2024-09-14T03:52:48.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:48 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-14T03:52:48.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:48 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:48.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:48 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:52:48.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:48 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:48.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:48 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:48.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:48 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:48.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:48 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:52:48.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:48 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:48.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:48 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:52:48.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:48 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:52:48.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:48 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:52:48.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:48 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:52:48.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:48 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-14T03:52:48.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:48 smithi070 ceph-mon[96704]: osdmap e347: 8 total, 8 up, 8 in 2024-09-14T03:52:48.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:48 smithi070 ceph-mon[98640]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:52:48.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:48 smithi070 ceph-mon[98640]: osd.0 [v2:172.21.15.70:6802/1152327285,v1:172.21.15.70:6803/1152327285] boot 2024-09-14T03:52:48.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:48 smithi070 ceph-mon[98640]: osdmap e346: 8 total, 8 up, 8 in 2024-09-14T03:52:48.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:48 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-14T03:52:48.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:48 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:48.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:48 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:52:48.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:48 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:48.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:48 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:48.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:48 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:52:48.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:48 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:52:48.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:48 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:52:48.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:48 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:52:48.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:48 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:52:48.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:48 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:52:48.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:48 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:52:48.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:48 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-14T03:52:48.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:48 smithi070 ceph-mon[98640]: osdmap e347: 8 total, 8 up, 8 in 2024-09-14T03:52:49.036 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-09-14T03:52:49.044 INFO:tasks.rados.rados.0.smithi070.stdout:1427: done (7 left) 2024-09-14T03:52:49.044 INFO:tasks.rados.rados.0.smithi070.stdout:1428: done (6 left) 2024-09-14T03:52:49.044 INFO:tasks.rados.rados.0.smithi070.stdout:1429: done (5 left) 2024-09-14T03:52:49.044 INFO:tasks.rados.rados.0.smithi070.stdout:1432: done (4 left) 2024-09-14T03:52:49.044 INFO:tasks.rados.rados.0.smithi070.stdout:1433: done (3 left) 2024-09-14T03:52:49.044 INFO:tasks.rados.rados.0.smithi070.stdout:1437: done (2 left) 2024-09-14T03:52:49.044 INFO:tasks.rados.rados.0.smithi070.stdout:1438: done (1 left) 2024-09-14T03:52:49.044 INFO:tasks.rados.rados.0.smithi070.stdout:1439: done (0 left) 2024-09-14T03:52:49.045 INFO:tasks.rados.rados.0.smithi070.stdout:1440: snap_create 2024-09-14T03:52:49.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:49 smithi094 ceph-mon[86597]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-14T03:52:49.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:49 smithi094 ceph-mon[86597]: Upgrade: unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-14T03:52:49.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:49 smithi094 ceph-mon[86597]: pgmap v111: 121 pgs: 9 peering, 2 active+clean+snaptrim, 31 active+undersized, 13 active+undersized+degraded, 66 active+clean; 402 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 507 B/s rd, 210 KiB/s wr, 6 op/s; 90/987 objects degraded (9.119%) 2024-09-14T03:52:49.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:49 smithi094 ceph-mon[86597]: osdmap e348: 8 total, 8 up, 8 in 2024-09-14T03:52:49.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:49 smithi070 ceph-mon[98640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-14T03:52:49.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:49 smithi070 ceph-mon[98640]: Upgrade: unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-14T03:52:49.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:49 smithi070 ceph-mon[98640]: pgmap v111: 121 pgs: 9 peering, 2 active+clean+snaptrim, 31 active+undersized, 13 active+undersized+degraded, 66 active+clean; 402 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 507 B/s rd, 210 KiB/s wr, 6 op/s; 90/987 objects degraded (9.119%) 2024-09-14T03:52:49.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:49 smithi070 ceph-mon[98640]: osdmap e348: 8 total, 8 up, 8 in 2024-09-14T03:52:49.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:49 smithi070 ceph-mon[96704]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-14T03:52:49.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:49 smithi070 ceph-mon[96704]: Upgrade: unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-14T03:52:49.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:49 smithi070 ceph-mon[96704]: pgmap v111: 121 pgs: 9 peering, 2 active+clean+snaptrim, 31 active+undersized, 13 active+undersized+degraded, 66 active+clean; 402 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 507 B/s rd, 210 KiB/s wr, 6 op/s; 90/987 objects degraded (9.119%) 2024-09-14T03:52:49.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:49 smithi070 ceph-mon[96704]: osdmap e348: 8 total, 8 up, 8 in 2024-09-14T03:52:49.586 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:52:49.586 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (2m) 5s ago 7m 18.6M - 0.27.0 11f11916f8cd 45e79e51074c 2024-09-14T03:52:49.586 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi094 *:3000 running (2m) 45s ago 7m 79.5M - 9.4.7 954c08fa6188 2240810d0368 2024-09-14T03:52:49.587 INFO:teuthology.orchestra.run.smithi070.stdout:mds.a smithi070 running (9m) 5s ago 9m 20.5M - 17.2.7-1629-geb16b97b ab51d1bebd12 52e0dbb7be50 2024-09-14T03:52:49.587 INFO:teuthology.orchestra.run.smithi070.stdout:mds.b smithi094 running (9m) 45s ago 9m 15.4M - 17.2.7-1629-geb16b97b ab51d1bebd12 e3b5e3e00189 2024-09-14T03:52:49.587 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi094 *:8443,9283,8765 running (99s) 45s ago 12m 452M - 19.3.0-4907-ga2782315 8801d2529e43 569d9c7e0f06 2024-09-14T03:52:49.587 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (2m) 5s ago 14m 517M - 19.3.0-4907-ga2782315 8801d2529e43 0ef4d8e0cdbe 2024-09-14T03:52:49.587 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (86s) 5s ago 14m 48.4M 2048M 19.3.0-4907-ga2782315 8801d2529e43 96542de31cac 2024-09-14T03:52:49.587 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi094 running (51s) 45s ago 12m 21.3M 2048M 19.3.0-4907-ga2782315 8801d2529e43 86827be98659 2024-09-14T03:52:49.587 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (63s) 5s ago 12m 43.1M 2048M 19.3.0-4907-ga2782315 8801d2529e43 5741e4fcf4b6 2024-09-14T03:52:49.587 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (2m) 5s ago 7m 10.0M - 1.7.0 72c9c2088986 e316760a30b3 2024-09-14T03:52:49.587 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi094 *:9100 running (2m) 45s ago 6m 7750k - 1.7.0 72c9c2088986 767f8a093175 2024-09-14T03:52:49.587 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (10s) 5s ago 12m 12.2M 2480M 19.3.0-4907-ga2782315 8801d2529e43 2345ccac2085 2024-09-14T03:52:49.587 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (11m) 5s ago 11m 222M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 af459ea7ef68 2024-09-14T03:52:49.588 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (11m) 5s ago 11m 170M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 109b66e4e60f 2024-09-14T03:52:49.588 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (10m) 5s ago 10m 138M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 2e4199b58392 2024-09-14T03:52:49.588 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi094 running (10m) 45s ago 10m 180M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 52b64527e24c 2024-09-14T03:52:49.588 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi094 running (10m) 45s ago 10m 168M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 b543f152ba87 2024-09-14T03:52:49.588 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi094 running (9m) 45s ago 9m 114M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 e66b592ac940 2024-09-14T03:52:49.588 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi094 running (9m) 45s ago 9m 152M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 037c6208e9ce 2024-09-14T03:52:49.588 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi094 *:9095 running (104s) 45s ago 7m 46.3M - 2.51.0 1d3b7f56885b ce1b353828dc 2024-09-14T03:52:49.916 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:52:49.917 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-09-14T03:52:49.917 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:52:49.917 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:52:49.917 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-09-14T03:52:49.917 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:52:49.917 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:52:49.917 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-09-14T03:52:49.917 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 7, 2024-09-14T03:52:49.917 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 1 2024-09-14T03:52:49.917 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:52:49.917 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": { 2024-09-14T03:52:49.918 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:52:49.918 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:52:49.918 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-09-14T03:52:49.918 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 9, 2024-09-14T03:52:49.918 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 6 2024-09-14T03:52:49.918 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-09-14T03:52:49.918 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:52:50.046 INFO:tasks.rados.rados.0.smithi070.stdout:1440: done (0 left) 2024-09-14T03:52:50.046 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 44 current snap is 158 2024-09-14T03:52:50.046 INFO:tasks.rados.rados.0.smithi070.stdout:1441: seq_num 465 ranges {204800=598016} 2024-09-14T03:52:50.050 INFO:tasks.rados.rados.0.smithi070.stdout:1441: writing smithi07085154-44 from 204800 to 802816 tid 1 2024-09-14T03:52:50.050 INFO:tasks.rados.rados.0.smithi070.stdout:1442: read oid 10 snap -1 2024-09-14T03:52:50.050 INFO:tasks.rados.rados.0.smithi070.stdout:1442: expect (ObjNum 456 snap 155 seq_num 456) 2024-09-14T03:52:50.050 INFO:tasks.rados.rados.0.smithi070.stdout:1443: read oid 1 snap -1 2024-09-14T03:52:50.051 INFO:tasks.rados.rados.0.smithi070.stdout:1443: expect (ObjNum 285 snap 89 seq_num 285) 2024-09-14T03:52:50.051 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 19 current snap is 158 2024-09-14T03:52:50.051 INFO:tasks.rados.rados.0.smithi070.stdout:1444: seq_num 466 ranges {540672=614400} 2024-09-14T03:52:50.056 INFO:tasks.rados.rados.0.smithi070.stdout:1444: writing smithi07085154-19 from 540672 to 1155072 tid 1 2024-09-14T03:52:50.056 INFO:tasks.rados.rados.0.smithi070.stdout:1445: copy_from oid 17 from oid 18 current snap is 158 2024-09-14T03:52:50.056 INFO:tasks.rados.rados.0.smithi070.stdout:1446: snap_create 2024-09-14T03:52:50.063 INFO:tasks.rados.rados.0.smithi070.stdout:1444: finishing write tid 1 to smithi07085154-19 2024-09-14T03:52:50.064 INFO:tasks.rados.rados.0.smithi070.stdout:1444: finishing write tid 2 to smithi07085154-19 2024-09-14T03:52:50.068 INFO:tasks.rados.rados.0.smithi070.stdout:1444: finishing write tid 3 to smithi07085154-19 2024-09-14T03:52:50.068 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 19 v 779 (ObjNum 466 snap 158 seq_num 466) dirty exists 2024-09-14T03:52:50.069 INFO:tasks.rados.rados.0.smithi070.stdout:1444: left oid 19 (ObjNum 466 snap 158 seq_num 466) 2024-09-14T03:52:50.069 INFO:tasks.rados.rados.0.smithi070.stdout:1442: expect (ObjNum 456 snap 155 seq_num 456) 2024-09-14T03:52:50.217 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:52:50.217 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:52:50.217 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-09-14T03:52:50.217 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:52:50.217 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [ 2024-09-14T03:52:50.217 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr", 2024-09-14T03:52:50.217 INFO:teuthology.orchestra.run.smithi070.stdout: "mon" 2024-09-14T03:52:50.217 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2024-09-14T03:52:50.217 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "6/20 daemons upgraded", 2024-09-14T03:52:50.217 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "Currently upgrading osd daemons", 2024-09-14T03:52:50.217 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-09-14T03:52:50.218 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:52:50.218 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:50 smithi070 ceph-mon[96704]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:52:50.218 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:50 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/4095751807' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:52:50.218 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:50 smithi070 ceph-mon[96704]: osdmap e349: 8 total, 8 up, 8 in 2024-09-14T03:52:50.218 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:50 smithi070 ceph-mon[98640]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:52:50.218 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:50 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/4095751807' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:52:50.218 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:50 smithi070 ceph-mon[98640]: osdmap e349: 8 total, 8 up, 8 in 2024-09-14T03:52:50.244 INFO:tasks.rados.rados.0.smithi070.stdout:1441: finishing write tid 1 to smithi07085154-44 2024-09-14T03:52:50.244 INFO:tasks.rados.rados.0.smithi070.stdout:1441: finishing write tid 2 to smithi07085154-44 2024-09-14T03:52:50.244 INFO:tasks.rados.rados.0.smithi070.stdout:1443: expect (ObjNum 285 snap 89 seq_num 285) 2024-09-14T03:52:50.304 INFO:tasks.rados.rados.0.smithi070.stdout:1441: finishing write tid 3 to smithi07085154-44 2024-09-14T03:52:50.304 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 578 (ObjNum 465 snap 158 seq_num 465) dirty exists 2024-09-14T03:52:50.305 INFO:tasks.rados.rados.0.smithi070.stdout:1441: left oid 44 (ObjNum 465 snap 158 seq_num 465) 2024-09-14T03:52:50.305 INFO:tasks.rados.rados.0.smithi070.stdout:1445: finishing copy_from racing read to smithi07085154-17 2024-09-14T03:52:50.305 INFO:tasks.rados.rados.0.smithi070.stdout:1445: finishing copy_from to smithi07085154-17 2024-09-14T03:52:50.305 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 847 (ObjNum 433 snap 148 seq_num 433) dirty exists 2024-09-14T03:52:50.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:50 smithi094 ceph-mon[86597]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:52:50.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:50 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/4095751807' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:52:50.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:50 smithi094 ceph-mon[86597]: osdmap e349: 8 total, 8 up, 8 in 2024-09-14T03:52:50.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:52:50 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:52:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:51.046 INFO:tasks.rados.rados.0.smithi070.stdout:1441: done (5 left) 2024-09-14T03:52:51.047 INFO:tasks.rados.rados.0.smithi070.stdout:1442: done (4 left) 2024-09-14T03:52:51.047 INFO:tasks.rados.rados.0.smithi070.stdout:1443: done (3 left) 2024-09-14T03:52:51.047 INFO:tasks.rados.rados.0.smithi070.stdout:1444: done (2 left) 2024-09-14T03:52:51.047 INFO:tasks.rados.rados.0.smithi070.stdout:1445: done (1 left) 2024-09-14T03:52:51.047 INFO:tasks.rados.rados.0.smithi070.stdout:1446: done (0 left) 2024-09-14T03:52:51.047 INFO:tasks.rados.rados.0.smithi070.stdout:1447: snap_create 2024-09-14T03:52:51.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:51 smithi094 ceph-mon[86597]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:52:51.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:51 smithi094 ceph-mon[86597]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:52:51.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:51 smithi094 ceph-mon[86597]: pgmap v114: 121 pgs: 1 activating+degraded, 12 activating, 24 peering, 12 active+undersized, 4 active+undersized+degraded, 68 active+clean; 410 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 2.2 MiB/s wr, 30 op/s; 37/1020 objects degraded (3.627%) 2024-09-14T03:52:51.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:51 smithi094 ceph-mon[86597]: Health check update: Degraded data redundancy: 37/1020 objects degraded (3.627%), 5 pgs degraded (PG_DEGRADED) 2024-09-14T03:52:51.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:51 smithi094 ceph-mon[86597]: osdmap e350: 8 total, 8 up, 8 in 2024-09-14T03:52:51.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:51 smithi070 ceph-mon[96704]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:52:51.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:51 smithi070 ceph-mon[96704]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:52:51.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:51 smithi070 ceph-mon[96704]: pgmap v114: 121 pgs: 1 activating+degraded, 12 activating, 24 peering, 12 active+undersized, 4 active+undersized+degraded, 68 active+clean; 410 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 2.2 MiB/s wr, 30 op/s; 37/1020 objects degraded (3.627%) 2024-09-14T03:52:51.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:51 smithi070 ceph-mon[96704]: Health check update: Degraded data redundancy: 37/1020 objects degraded (3.627%), 5 pgs degraded (PG_DEGRADED) 2024-09-14T03:52:51.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:51 smithi070 ceph-mon[96704]: osdmap e350: 8 total, 8 up, 8 in 2024-09-14T03:52:51.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:51 smithi070 ceph-mon[98640]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:52:51.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:51 smithi070 ceph-mon[98640]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:52:51.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:51 smithi070 ceph-mon[98640]: pgmap v114: 121 pgs: 1 activating+degraded, 12 activating, 24 peering, 12 active+undersized, 4 active+undersized+degraded, 68 active+clean; 410 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 2.2 MiB/s wr, 30 op/s; 37/1020 objects degraded (3.627%) 2024-09-14T03:52:51.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:51 smithi070 ceph-mon[98640]: Health check update: Degraded data redundancy: 37/1020 objects degraded (3.627%), 5 pgs degraded (PG_DEGRADED) 2024-09-14T03:52:51.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:51 smithi070 ceph-mon[98640]: osdmap e350: 8 total, 8 up, 8 in 2024-09-14T03:52:52.048 INFO:tasks.rados.rados.0.smithi070.stdout:1447: done (0 left) 2024-09-14T03:52:52.048 INFO:tasks.rados.rados.0.smithi070.stdout:1448: snap_remove snap 143 2024-09-14T03:52:52.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:52 smithi094 ceph-mon[86597]: from='client.34248 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:52:52.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:52 smithi094 ceph-mon[86597]: osdmap e351: 8 total, 8 up, 8 in 2024-09-14T03:52:52.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:52 smithi070 ceph-mon[96704]: from='client.34248 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:52:52.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:52 smithi070 ceph-mon[96704]: osdmap e351: 8 total, 8 up, 8 in 2024-09-14T03:52:52.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:52 smithi070 ceph-mon[98640]: from='client.34248 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:52:52.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:52 smithi070 ceph-mon[98640]: osdmap e351: 8 total, 8 up, 8 in 2024-09-14T03:52:53.050 INFO:tasks.rados.rados.0.smithi070.stdout:1448: done (0 left) 2024-09-14T03:52:53.050 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 33 current snap is 160 2024-09-14T03:52:53.050 INFO:tasks.rados.rados.0.smithi070.stdout:1449: seq_num 467 ranges {3465216=630784} 2024-09-14T03:52:53.074 INFO:tasks.rados.rados.0.smithi070.stdout:1449: writing smithi07085154-33 from 3465216 to 4096000 tid 1 2024-09-14T03:52:53.074 INFO:tasks.rados.rados.0.smithi070.stdout:1450: copy_from oid 22 from oid 50 current snap is 160 2024-09-14T03:52:53.074 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 38 current snap is 160 2024-09-14T03:52:53.074 INFO:tasks.rados.rados.0.smithi070.stdout:1451: seq_num 468 ranges {0=647168} 2024-09-14T03:52:53.078 INFO:tasks.rados.rados.0.smithi070.stdout:1451: writing smithi07085154-38 from 0 to 647168 tid 1 2024-09-14T03:52:53.078 INFO:tasks.rados.rados.0.smithi070.stdout:1452: read oid 17 snap -1 2024-09-14T03:52:53.078 INFO:tasks.rados.rados.0.smithi070.stdout:1452: expect (ObjNum 433 snap 148 seq_num 433) 2024-09-14T03:52:53.078 INFO:tasks.rados.rados.0.smithi070.stdout:1453: read oid 43 snap -1 2024-09-14T03:52:53.078 INFO:tasks.rados.rados.0.smithi070.stdout:1453: expect (ObjNum 390 snap 132 seq_num 390) 2024-09-14T03:52:53.079 INFO:tasks.rados.rados.0.smithi070.stdout:1454: read oid 10 snap -1 2024-09-14T03:52:53.079 INFO:tasks.rados.rados.0.smithi070.stdout:1454: expect (ObjNum 456 snap 155 seq_num 456) 2024-09-14T03:52:53.079 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 15 current snap is 160 2024-09-14T03:52:53.079 INFO:tasks.rados.rados.0.smithi070.stdout:1455: seq_num 469 ranges {3678208=663552} 2024-09-14T03:52:53.103 INFO:tasks.rados.rados.0.smithi070.stdout:1455: writing smithi07085154-15 from 3678208 to 4341760 tid 1 2024-09-14T03:52:53.104 INFO:tasks.rados.rados.0.smithi070.stdout:1456: rollback oid 39 current snap is 160 2024-09-14T03:52:53.104 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 39 to 157 2024-09-14T03:52:53.104 INFO:tasks.rados.rados.0.smithi070.stdout:1449: finishing write tid 1 to smithi07085154-33 2024-09-14T03:52:53.104 INFO:tasks.rados.rados.0.smithi070.stdout:1449: finishing write tid 2 to smithi07085154-33 2024-09-14T03:52:53.104 INFO:tasks.rados.rados.0.smithi070.stdout:1449: finishing write tid 3 to smithi07085154-33 2024-09-14T03:52:53.104 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 33 v 977 (ObjNum 467 snap 160 seq_num 467) dirty exists 2024-09-14T03:52:53.104 INFO:tasks.rados.rados.0.smithi070.stdout:1449: left oid 33 (ObjNum 467 snap 160 seq_num 467) 2024-09-14T03:52:53.104 INFO:tasks.rados.rados.0.smithi070.stdout:1451: finishing write tid 1 to smithi07085154-38 2024-09-14T03:52:53.104 INFO:tasks.rados.rados.0.smithi070.stdout:1449: done (7 left) 2024-09-14T03:52:53.104 INFO:tasks.rados.rados.0.smithi070.stdout:1457: read oid 6 snap -1 2024-09-14T03:52:53.105 INFO:tasks.rados.rados.0.smithi070.stdout:1457: expect (ObjNum 441 snap 152 seq_num 441) 2024-09-14T03:52:53.105 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 1 current snap is 160 2024-09-14T03:52:53.105 INFO:tasks.rados.rados.0.smithi070.stdout:1450: finishing copy_from racing read to smithi07085154-22 2024-09-14T03:52:53.105 INFO:tasks.rados.rados.0.smithi070.stdout:1458: seq_num 470 ranges {2383872=679936} 2024-09-14T03:52:53.120 INFO:tasks.rados.rados.0.smithi070.stdout:1458: writing smithi07085154-1 from 2383872 to 3063808 tid 1 2024-09-14T03:52:53.121 INFO:tasks.rados.rados.0.smithi070.stdout:1459: snap_remove snap 83 2024-09-14T03:52:53.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:53 smithi070 ceph-mon[98640]: pgmap v117: 121 pgs: 1 activating+degraded, 12 activating, 24 peering, 84 active+clean; 421 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 3.8 MiB/s wr, 24 op/s; 17/1020 objects degraded (1.667%); 6.4 MiB/s, 0 objects/s recovering 2024-09-14T03:52:53.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:53 smithi070 ceph-mon[98640]: osdmap e352: 8 total, 8 up, 8 in 2024-09-14T03:52:53.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:53 smithi070 ceph-mon[96704]: pgmap v117: 121 pgs: 1 activating+degraded, 12 activating, 24 peering, 84 active+clean; 421 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 3.8 MiB/s wr, 24 op/s; 17/1020 objects degraded (1.667%); 6.4 MiB/s, 0 objects/s recovering 2024-09-14T03:52:53.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:53 smithi070 ceph-mon[96704]: osdmap e352: 8 total, 8 up, 8 in 2024-09-14T03:52:53.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:53 smithi094 ceph-mon[86597]: pgmap v117: 121 pgs: 1 activating+degraded, 12 activating, 24 peering, 84 active+clean; 421 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 3.8 MiB/s wr, 24 op/s; 17/1020 objects degraded (1.667%); 6.4 MiB/s, 0 objects/s recovering 2024-09-14T03:52:53.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:53 smithi094 ceph-mon[86597]: osdmap e352: 8 total, 8 up, 8 in 2024-09-14T03:52:53.679 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:52:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:52:53] "GET /metrics HTTP/1.1" 200 36257 "" "Prometheus/2.51.0" 2024-09-14T03:52:54.085 INFO:tasks.rados.rados.0.smithi070.stdout:1459: done (9 left) 2024-09-14T03:52:54.086 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 16 current snap is 160 2024-09-14T03:52:54.086 INFO:tasks.rados.rados.0.smithi070.stdout:1460: seq_num 471 ranges {4784128=696320} 2024-09-14T03:52:54.117 INFO:tasks.rados.rados.0.smithi070.stdout:1460: writing smithi07085154-16 from 4784128 to 5480448 tid 1 2024-09-14T03:52:54.117 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 28 current snap is 160 2024-09-14T03:52:54.117 INFO:tasks.rados.rados.0.smithi070.stdout:1461: seq_num 472 ranges {7700480=712704} 2024-09-14T03:52:54.162 INFO:tasks.rados.rados.0.smithi070.stdout:1461: writing smithi07085154-28 from 7700480 to 8413184 tid 1 2024-09-14T03:52:54.162 INFO:tasks.rados.rados.0.smithi070.stdout:1462: copy_from oid 20 from oid 40 current snap is 160 2024-09-14T03:52:54.162 INFO:tasks.rados.rados.0.smithi070.stdout:1451: finishing write tid 2 to smithi07085154-38 2024-09-14T03:52:54.163 INFO:tasks.rados.rados.0.smithi070.stdout:1451: finishing write tid 3 to smithi07085154-38 2024-09-14T03:52:54.163 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 38 v 993 (ObjNum 468 snap 160 seq_num 468) dirty exists 2024-09-14T03:52:54.163 INFO:tasks.rados.rados.0.smithi070.stdout:1451: left oid 38 (ObjNum 468 snap 160 seq_num 468) 2024-09-14T03:52:54.163 INFO:tasks.rados.rados.0.smithi070.stdout:1450: finishing copy_from to smithi07085154-22 2024-09-14T03:52:54.163 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 719 (ObjNum 430 snap 148 seq_num 430) dirty exists 2024-09-14T03:52:54.163 INFO:tasks.rados.rados.0.smithi070.stdout:1452: expect (ObjNum 433 snap 148 seq_num 433) 2024-09-14T03:52:54.263 INFO:tasks.rados.rados.0.smithi070.stdout:1453: expect (ObjNum 390 snap 132 seq_num 390) 2024-09-14T03:52:54.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:54 smithi094 ceph-mon[86597]: pgmap v120: 121 pgs: 121 active+clean; 441 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.5 MiB/s rd, 6.9 MiB/s wr, 10 op/s; 22 MiB/s, 5 objects/s recovering 2024-09-14T03:52:54.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:54 smithi094 ceph-mon[86597]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17/1020 objects degraded (1.667%), 1 pg degraded) 2024-09-14T03:52:54.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:54 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T03:52:54.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:54 smithi094 ceph-mon[86597]: osdmap e353: 8 total, 8 up, 8 in 2024-09-14T03:52:54.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:52:54 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:52:54.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.1\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:54.530 INFO:tasks.rados.rados.0.smithi070.stdout:1454: expect (ObjNum 456 snap 155 seq_num 456) 2024-09-14T03:52:54.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:54 smithi070 ceph-mon[98640]: pgmap v120: 121 pgs: 121 active+clean; 441 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.5 MiB/s rd, 6.9 MiB/s wr, 10 op/s; 22 MiB/s, 5 objects/s recovering 2024-09-14T03:52:54.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:54 smithi070 ceph-mon[98640]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17/1020 objects degraded (1.667%), 1 pg degraded) 2024-09-14T03:52:54.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:54 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T03:52:54.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:54 smithi070 ceph-mon[98640]: osdmap e353: 8 total, 8 up, 8 in 2024-09-14T03:52:54.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:54 smithi070 ceph-mon[96704]: pgmap v120: 121 pgs: 121 active+clean; 441 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.5 MiB/s rd, 6.9 MiB/s wr, 10 op/s; 22 MiB/s, 5 objects/s recovering 2024-09-14T03:52:54.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:54 smithi070 ceph-mon[96704]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17/1020 objects degraded (1.667%), 1 pg degraded) 2024-09-14T03:52:54.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:54 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T03:52:54.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:54 smithi070 ceph-mon[96704]: osdmap e353: 8 total, 8 up, 8 in 2024-09-14T03:52:54.707 INFO:tasks.rados.rados.0.smithi070.stdout:1456: finishing rollback tid 0 to smithi07085154-39 2024-09-14T03:52:54.707 INFO:tasks.rados.rados.0.smithi070.stdout:1450: done (11 left) 2024-09-14T03:52:54.707 INFO:tasks.rados.rados.0.smithi070.stdout:1451: done (10 left) 2024-09-14T03:52:54.707 INFO:tasks.rados.rados.0.smithi070.stdout:1452: done (9 left) 2024-09-14T03:52:54.707 INFO:tasks.rados.rados.0.smithi070.stdout:1453: done (8 left) 2024-09-14T03:52:54.707 INFO:tasks.rados.rados.0.smithi070.stdout:1454: done (7 left) 2024-09-14T03:52:54.707 INFO:tasks.rados.rados.0.smithi070.stdout:1463: read oid 50 snap 102 2024-09-14T03:52:54.707 INFO:tasks.rados.rados.0.smithi070.stdout:1463: expect deleted 2024-09-14T03:52:54.707 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 13 current snap is 160 2024-09-14T03:52:54.707 INFO:tasks.rados.rados.0.smithi070.stdout:1464: seq_num 473 ranges {4734976=729088} 2024-09-14T03:52:54.735 INFO:tasks.rados.rados.0.smithi070.stdout:1464: writing smithi07085154-13 from 4734976 to 5464064 tid 1 2024-09-14T03:52:54.735 INFO:tasks.rados.rados.0.smithi070.stdout:1465: setattr oid 25 current snap is 160 2024-09-14T03:52:54.735 INFO:tasks.rados.rados.0.smithi070.stdout:1456: finishing rollback tid 1 to smithi07085154-39 2024-09-14T03:52:54.735 INFO:tasks.rados.rados.0.smithi070.stdout:1456: finishing rollback tid 2 to smithi07085154-39 2024-09-14T03:52:54.735 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 39 v 331 (ObjNum 460 snap 155 seq_num 460) dirty exists 2024-09-14T03:52:54.735 INFO:tasks.rados.rados.0.smithi070.stdout:1455: finishing write tid 1 to smithi07085154-15 2024-09-14T03:52:54.736 INFO:tasks.rados.rados.0.smithi070.stdout:1455: finishing write tid 2 to smithi07085154-15 2024-09-14T03:52:54.736 INFO:tasks.rados.rados.0.smithi070.stdout:1455: finishing write tid 3 to smithi07085154-15 2024-09-14T03:52:54.736 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 15 v 911 (ObjNum 469 snap 160 seq_num 469) dirty exists 2024-09-14T03:52:54.736 INFO:tasks.rados.rados.0.smithi070.stdout:1455: left oid 15 (ObjNum 469 snap 160 seq_num 469) 2024-09-14T03:52:54.736 INFO:tasks.rados.rados.0.smithi070.stdout:1457: expect (ObjNum 441 snap 152 seq_num 441) 2024-09-14T03:52:54.816 INFO:tasks.rados.rados.0.smithi070.stdout:1458: finishing write tid 1 to smithi07085154-1 2024-09-14T03:52:54.816 INFO:tasks.rados.rados.0.smithi070.stdout:1458: finishing write tid 2 to smithi07085154-1 2024-09-14T03:52:54.816 INFO:tasks.rados.rados.0.smithi070.stdout:1458: finishing write tid 3 to smithi07085154-1 2024-09-14T03:52:54.816 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 1 v 589 (ObjNum 470 snap 160 seq_num 470) dirty exists 2024-09-14T03:52:54.817 INFO:tasks.rados.rados.0.smithi070.stdout:1458: left oid 1 (ObjNum 470 snap 160 seq_num 470) 2024-09-14T03:52:54.817 INFO:tasks.rados.rados.0.smithi070.stdout:1460: finishing write tid 1 to smithi07085154-16 2024-09-14T03:52:54.817 INFO:tasks.rados.rados.0.smithi070.stdout:1460: finishing write tid 2 to smithi07085154-16 2024-09-14T03:52:54.817 INFO:tasks.rados.rados.0.smithi070.stdout:1460: finishing write tid 3 to smithi07085154-16 2024-09-14T03:52:54.817 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 1352 (ObjNum 471 snap 160 seq_num 471) dirty exists 2024-09-14T03:52:54.817 INFO:tasks.rados.rados.0.smithi070.stdout:1460: left oid 16 (ObjNum 471 snap 160 seq_num 471) 2024-09-14T03:52:54.817 INFO:tasks.rados.rados.0.smithi070.stdout:1461: finishing write tid 1 to smithi07085154-28 2024-09-14T03:52:54.817 INFO:tasks.rados.rados.0.smithi070.stdout:1461: finishing write tid 2 to smithi07085154-28 2024-09-14T03:52:54.817 INFO:tasks.rados.rados.0.smithi070.stdout:1461: finishing write tid 3 to smithi07085154-28 2024-09-14T03:52:54.817 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 28 v 1210 (ObjNum 472 snap 160 seq_num 472) dirty exists 2024-09-14T03:52:54.817 INFO:tasks.rados.rados.0.smithi070.stdout:1461: left oid 28 (ObjNum 472 snap 160 seq_num 472) 2024-09-14T03:52:54.817 INFO:tasks.rados.rados.0.smithi070.stdout:1462: finishing copy_from racing read to smithi07085154-20 2024-09-14T03:52:54.817 INFO:tasks.rados.rados.0.smithi070.stdout:1462: finishing copy_from to smithi07085154-20 2024-09-14T03:52:54.817 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 20 v 519 (ObjNum 274 snap 84 seq_num 274) dirty exists 2024-09-14T03:52:54.818 INFO:tasks.rados.rados.0.smithi070.stdout:1464: finishing write tid 1 to smithi07085154-13 2024-09-14T03:52:54.818 INFO:tasks.rados.rados.0.smithi070.stdout:1464: finishing write tid 2 to smithi07085154-13 2024-09-14T03:52:54.818 INFO:tasks.rados.rados.0.smithi070.stdout:1464: finishing write tid 3 to smithi07085154-13 2024-09-14T03:52:54.818 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 1167 (ObjNum 473 snap 160 seq_num 473) dirty exists 2024-09-14T03:52:54.818 INFO:tasks.rados.rados.0.smithi070.stdout:1464: left oid 13 (ObjNum 473 snap 160 seq_num 473) 2024-09-14T03:52:54.818 INFO:tasks.rados.rados.0.smithi070.stdout:1455: done (9 left) 2024-09-14T03:52:54.818 INFO:tasks.rados.rados.0.smithi070.stdout:1456: done (8 left) 2024-09-14T03:52:54.818 INFO:tasks.rados.rados.0.smithi070.stdout:1457: done (7 left) 2024-09-14T03:52:54.818 INFO:tasks.rados.rados.0.smithi070.stdout:1458: done (6 left) 2024-09-14T03:52:54.818 INFO:tasks.rados.rados.0.smithi070.stdout:1460: done (5 left) 2024-09-14T03:52:54.818 INFO:tasks.rados.rados.0.smithi070.stdout:1461: done (4 left) 2024-09-14T03:52:54.818 INFO:tasks.rados.rados.0.smithi070.stdout:1462: done (3 left) 2024-09-14T03:52:54.818 INFO:tasks.rados.rados.0.smithi070.stdout:1463: done (2 left) 2024-09-14T03:52:54.818 INFO:tasks.rados.rados.0.smithi070.stdout:1464: done (1 left) 2024-09-14T03:52:54.819 INFO:tasks.rados.rados.0.smithi070.stdout:1466: snap_create 2024-09-14T03:52:54.819 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 848 (ObjNum 459 snap 155 seq_num 459) dirty exists 2024-09-14T03:52:55.134 INFO:tasks.rados.rados.0.smithi070.stdout:1465: done (1 left) 2024-09-14T03:52:55.134 INFO:tasks.rados.rados.0.smithi070.stdout:1466: done (0 left) 2024-09-14T03:52:55.134 INFO:tasks.rados.rados.0.smithi070.stdout:1467: rmattr oid 41 current snap is 161 2024-09-14T03:52:55.134 INFO:tasks.rados.rados.0.smithi070.stdout:1468: snap_remove snap 109 2024-09-14T03:52:56.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:52:55 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:52:55.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:56.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:52:55 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:52:55.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:56.141 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 923 (ObjNum 211 snap 60 seq_num 211) dirty exists 2024-09-14T03:52:56.141 INFO:tasks.rados.rados.0.smithi070.stdout:1467: done (1 left) 2024-09-14T03:52:56.141 INFO:tasks.rados.rados.0.smithi070.stdout:1468: done (0 left) 2024-09-14T03:52:56.141 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 45 current snap is 161 2024-09-14T03:52:56.141 INFO:tasks.rados.rados.0.smithi070.stdout:1469: seq_num 474 ranges {1531904=745472} 2024-09-14T03:52:56.155 INFO:tasks.rados.rados.0.smithi070.stdout:1469: writing smithi07085154-45 from 1531904 to 2277376 tid 1 2024-09-14T03:52:56.155 INFO:tasks.rados.rados.0.smithi070.stdout:1470: delete oid 26 current snap is 161 2024-09-14T03:52:56.159 INFO:tasks.rados.rados.0.smithi070.stdout:1470: done (1 left) 2024-09-14T03:52:56.159 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 18 current snap is 161 2024-09-14T03:52:56.159 INFO:tasks.rados.rados.0.smithi070.stdout:1471: seq_num 476 ranges {2334720=778240} 2024-09-14T03:52:56.177 INFO:tasks.rados.rados.0.smithi070.stdout:1471: writing smithi07085154-18 from 2334720 to 3112960 tid 1 2024-09-14T03:52:56.177 INFO:tasks.rados.rados.0.smithi070.stdout:1472: delete oid 1 current snap is 161 2024-09-14T03:52:56.177 INFO:tasks.rados.rados.0.smithi070.stdout:1469: finishing write tid 1 to smithi07085154-45 2024-09-14T03:52:56.177 INFO:tasks.rados.rados.0.smithi070.stdout:1469: finishing write tid 2 to smithi07085154-45 2024-09-14T03:52:56.177 INFO:tasks.rados.rados.0.smithi070.stdout:1469: finishing write tid 3 to smithi07085154-45 2024-09-14T03:52:56.177 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 45 v 979 (ObjNum 474 snap 161 seq_num 474) dirty exists 2024-09-14T03:52:56.177 INFO:tasks.rados.rados.0.smithi070.stdout:1469: left oid 45 (ObjNum 474 snap 161 seq_num 474) 2024-09-14T03:52:56.179 INFO:tasks.rados.rados.0.smithi070.stdout:1469: done (2 left) 2024-09-14T03:52:56.179 INFO:tasks.rados.rados.0.smithi070.stdout:1472: done (1 left) 2024-09-14T03:52:56.179 INFO:tasks.rados.rados.0.smithi070.stdout:1473: copy_from oid 4 from oid 1 current snap is 161 2024-09-14T03:52:56.180 INFO:tasks.rados.rados.0.smithi070.stdout:1474: snap_create 2024-09-14T03:52:56.180 INFO:tasks.rados.rados.0.smithi070.stdout:1473: finishing copy_from to smithi07085154-4 2024-09-14T03:52:56.180 INFO:tasks.rados.rados.0.smithi070.stdout:1473: got expected ENOENT (src dne) 2024-09-14T03:52:56.181 INFO:tasks.rados.rados.0.smithi070.stdout:1473: finishing copy_from racing read to smithi07085154-4 2024-09-14T03:52:56.181 INFO:tasks.rados.rados.0.smithi070.stdout:1471: finishing write tid 1 to smithi07085154-18 2024-09-14T03:52:56.181 INFO:tasks.rados.rados.0.smithi070.stdout:1471: finishing write tid 2 to smithi07085154-18 2024-09-14T03:52:56.181 INFO:tasks.rados.rados.0.smithi070.stdout:1471: finishing write tid 3 to smithi07085154-18 2024-09-14T03:52:56.181 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 934 (ObjNum 476 snap 161 seq_num 476) dirty exists 2024-09-14T03:52:56.182 INFO:tasks.rados.rados.0.smithi070.stdout:1471: left oid 18 (ObjNum 476 snap 161 seq_num 476) 2024-09-14T03:52:56.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:56 smithi094 ceph-mon[86597]: osdmap e354: 8 total, 8 up, 8 in 2024-09-14T03:52:56.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:56 smithi094 ceph-mon[86597]: pgmap v123: 121 pgs: 121 active+clean; 441 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.5 MiB/s rd, 5.1 MiB/s wr, 9 op/s; 16 MiB/s, 4 objects/s recovering 2024-09-14T03:52:56.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:56 smithi070 ceph-mon[96704]: osdmap e354: 8 total, 8 up, 8 in 2024-09-14T03:52:56.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:56 smithi070 ceph-mon[96704]: pgmap v123: 121 pgs: 121 active+clean; 441 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.5 MiB/s rd, 5.1 MiB/s wr, 9 op/s; 16 MiB/s, 4 objects/s recovering 2024-09-14T03:52:56.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:56 smithi070 ceph-mon[98640]: osdmap e354: 8 total, 8 up, 8 in 2024-09-14T03:52:56.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:56 smithi070 ceph-mon[98640]: pgmap v123: 121 pgs: 121 active+clean; 441 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.5 MiB/s rd, 5.1 MiB/s wr, 9 op/s; 16 MiB/s, 4 objects/s recovering 2024-09-14T03:52:57.144 INFO:tasks.rados.rados.0.smithi070.stdout:1471: done (2 left) 2024-09-14T03:52:57.144 INFO:tasks.rados.rados.0.smithi070.stdout:1473: done (1 left) 2024-09-14T03:52:57.144 INFO:tasks.rados.rados.0.smithi070.stdout:1474: done (0 left) 2024-09-14T03:52:57.144 INFO:tasks.rados.rados.0.smithi070.stdout:1475: setattr oid 25 current snap is 162 2024-09-14T03:52:57.145 INFO:tasks.rados.rados.0.smithi070.stdout:1476: setattr oid 47 current snap is 162 2024-09-14T03:52:57.145 INFO:tasks.rados.rados.0.smithi070.stdout:1477: snap_create 2024-09-14T03:52:57.149 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 47 v 994 (ObjNum 382 snap 130 seq_num 382) dirty exists 2024-09-14T03:52:57.151 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 849 (ObjNum 459 snap 155 seq_num 459) dirty exists 2024-09-14T03:52:57.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:57 smithi094 ceph-mon[86597]: osdmap e355: 8 total, 8 up, 8 in 2024-09-14T03:52:57.277 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:52:56 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:52:56.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:52:57.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:57 smithi070 ceph-mon[96704]: osdmap e355: 8 total, 8 up, 8 in 2024-09-14T03:52:57.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:57 smithi070 ceph-mon[98640]: osdmap e355: 8 total, 8 up, 8 in 2024-09-14T03:52:58.151 INFO:tasks.rados.rados.0.smithi070.stdout:1475: done (2 left) 2024-09-14T03:52:58.151 INFO:tasks.rados.rados.0.smithi070.stdout:1476: done (1 left) 2024-09-14T03:52:58.151 INFO:tasks.rados.rados.0.smithi070.stdout:1477: done (0 left) 2024-09-14T03:52:58.151 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 7 current snap is 163 2024-09-14T03:52:58.151 INFO:tasks.rados.rados.0.smithi070.stdout:1478: seq_num 478 ranges {5513216=409600,5922816=401408} 2024-09-14T03:52:58.182 INFO:tasks.rados.rados.0.smithi070.stdout:1478: writing smithi07085154-7 from 5513216 to 5922816 tid 1 2024-09-14T03:52:58.185 INFO:tasks.rados.rados.0.smithi070.stdout:1478: writing smithi07085154-7 from 5922816 to 6324224 tid 2 2024-09-14T03:52:58.185 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 32 current snap is 163 2024-09-14T03:52:58.185 INFO:tasks.rados.rados.0.smithi070.stdout:1479: seq_num 479 ranges {7528448=425984,7954432=401408} 2024-09-14T03:52:58.228 INFO:tasks.rados.rados.0.smithi070.stdout:1479: writing smithi07085154-32 from 7528448 to 7954432 tid 1 2024-09-14T03:52:58.230 INFO:tasks.rados.rados.0.smithi070.stdout:1479: writing smithi07085154-32 from 7954432 to 8355840 tid 2 2024-09-14T03:52:58.230 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 34 current snap is 163 2024-09-14T03:52:58.231 INFO:tasks.rados.rados.0.smithi070.stdout:1478: finishing write tid 1 to smithi07085154-7 2024-09-14T03:52:58.231 INFO:tasks.rados.rados.0.smithi070.stdout:1480: seq_num 480 ranges {0=442368,442368=401408} 2024-09-14T03:52:58.233 INFO:tasks.rados.rados.0.smithi070.stdout:1480: writing smithi07085154-34 from 0 to 442368 tid 1 2024-09-14T03:52:58.235 INFO:tasks.rados.rados.0.smithi070.stdout:1480: writing smithi07085154-34 from 442368 to 843776 tid 2 2024-09-14T03:52:58.235 INFO:tasks.rados.rados.0.smithi070.stdout:1481: snap_create 2024-09-14T03:52:58.236 INFO:tasks.rados.rados.0.smithi070.stdout:1478: finishing write tid 2 to smithi07085154-7 2024-09-14T03:52:58.236 INFO:tasks.rados.rados.0.smithi070.stdout:1478: finishing write tid 3 to smithi07085154-7 2024-09-14T03:52:58.236 INFO:tasks.rados.rados.0.smithi070.stdout:1478: finishing write tid 4 to smithi07085154-7 2024-09-14T03:52:58.236 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 1213 (ObjNum 478 snap 163 seq_num 478) dirty exists 2024-09-14T03:52:58.236 INFO:tasks.rados.rados.0.smithi070.stdout:1478: left oid 7 (ObjNum 478 snap 163 seq_num 478) 2024-09-14T03:52:58.236 INFO:tasks.rados.rados.0.smithi070.stdout:1479: finishing write tid 1 to smithi07085154-32 2024-09-14T03:52:58.236 INFO:tasks.rados.rados.0.smithi070.stdout:1479: finishing write tid 2 to smithi07085154-32 2024-09-14T03:52:58.236 INFO:tasks.rados.rados.0.smithi070.stdout:1479: finishing write tid 3 to smithi07085154-32 2024-09-14T03:52:58.236 INFO:tasks.rados.rados.0.smithi070.stdout:1479: finishing write tid 4 to smithi07085154-32 2024-09-14T03:52:58.236 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 1170 (ObjNum 479 snap 163 seq_num 479) dirty exists 2024-09-14T03:52:58.236 INFO:tasks.rados.rados.0.smithi070.stdout:1479: left oid 32 (ObjNum 479 snap 163 seq_num 479) 2024-09-14T03:52:58.241 INFO:tasks.rados.rados.0.smithi070.stdout:1480: finishing write tid 1 to smithi07085154-34 2024-09-14T03:52:58.241 INFO:tasks.rados.rados.0.smithi070.stdout:1480: finishing write tid 2 to smithi07085154-34 2024-09-14T03:52:58.241 INFO:tasks.rados.rados.0.smithi070.stdout:1480: finishing write tid 3 to smithi07085154-34 2024-09-14T03:52:58.241 INFO:tasks.rados.rados.0.smithi070.stdout:1480: finishing write tid 4 to smithi07085154-34 2024-09-14T03:52:58.242 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 808 (ObjNum 480 snap 163 seq_num 480) dirty exists 2024-09-14T03:52:58.242 INFO:tasks.rados.rados.0.smithi070.stdout:1480: left oid 34 (ObjNum 480 snap 163 seq_num 480) 2024-09-14T03:52:58.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:58 smithi094 ceph-mon[86597]: osdmap e356: 8 total, 8 up, 8 in 2024-09-14T03:52:58.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:58 smithi094 ceph-mon[86597]: pgmap v126: 121 pgs: 121 active+clean; 438 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 178 KiB/s wr, 1 op/s 2024-09-14T03:52:58.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:58 smithi070 ceph-mon[98640]: osdmap e356: 8 total, 8 up, 8 in 2024-09-14T03:52:58.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:58 smithi070 ceph-mon[98640]: pgmap v126: 121 pgs: 121 active+clean; 438 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 178 KiB/s wr, 1 op/s 2024-09-14T03:52:58.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:58 smithi070 ceph-mon[96704]: osdmap e356: 8 total, 8 up, 8 in 2024-09-14T03:52:58.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:58 smithi070 ceph-mon[96704]: pgmap v126: 121 pgs: 121 active+clean; 438 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 178 KiB/s wr, 1 op/s 2024-09-14T03:52:59.008 INFO:tasks.rados.rados.0.smithi070.stdout:1478: done (3 left) 2024-09-14T03:52:59.008 INFO:tasks.rados.rados.0.smithi070.stdout:1479: done (2 left) 2024-09-14T03:52:59.008 INFO:tasks.rados.rados.0.smithi070.stdout:1480: done (1 left) 2024-09-14T03:52:59.008 INFO:tasks.rados.rados.0.smithi070.stdout:1481: done (0 left) 2024-09-14T03:52:59.008 INFO:tasks.rados.rados.0.smithi070.stdout:1482: delete oid 40 current snap is 164 2024-09-14T03:52:59.011 INFO:tasks.rados.rados.0.smithi070.stdout:1482: done (0 left) 2024-09-14T03:52:59.011 INFO:tasks.rados.rados.0.smithi070.stdout:1483: snap_remove snap 138 2024-09-14T03:52:59.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:59 smithi094 ceph-mon[86597]: osdmap e357: 8 total, 8 up, 8 in 2024-09-14T03:52:59.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:52:59 smithi094 ceph-mon[86597]: osdmap e358: 8 total, 8 up, 8 in 2024-09-14T03:52:59.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:59 smithi070 ceph-mon[96704]: osdmap e357: 8 total, 8 up, 8 in 2024-09-14T03:52:59.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:52:59 smithi070 ceph-mon[96704]: osdmap e358: 8 total, 8 up, 8 in 2024-09-14T03:52:59.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:59 smithi070 ceph-mon[98640]: osdmap e357: 8 total, 8 up, 8 in 2024-09-14T03:52:59.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:52:59 smithi070 ceph-mon[98640]: osdmap e358: 8 total, 8 up, 8 in 2024-09-14T03:53:00.011 INFO:tasks.rados.rados.0.smithi070.stdout:1483: done (0 left) 2024-09-14T03:53:00.011 INFO:tasks.rados.rados.0.smithi070.stdout:1484: snap_remove snap 98 2024-09-14T03:53:00.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:53:00 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:53:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:01.016 INFO:tasks.rados.rados.0.smithi070.stdout:1484: done (0 left) 2024-09-14T03:53:01.016 INFO:tasks.rados.rados.0.smithi070.stdout:1485: snap_remove snap 135 2024-09-14T03:53:01.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:01 smithi094 ceph-mon[86597]: osdmap e359: 8 total, 8 up, 8 in 2024-09-14T03:53:01.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:01 smithi094 ceph-mon[86597]: pgmap v130: 121 pgs: 121 active+clean; 450 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 9.8 MiB/s rd, 3.9 MiB/s wr, 29 op/s 2024-09-14T03:53:01.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:01 smithi070 ceph-mon[98640]: osdmap e359: 8 total, 8 up, 8 in 2024-09-14T03:53:01.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:01 smithi070 ceph-mon[98640]: pgmap v130: 121 pgs: 121 active+clean; 450 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 9.8 MiB/s rd, 3.9 MiB/s wr, 29 op/s 2024-09-14T03:53:01.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:01 smithi070 ceph-mon[96704]: osdmap e359: 8 total, 8 up, 8 in 2024-09-14T03:53:01.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:01 smithi070 ceph-mon[96704]: pgmap v130: 121 pgs: 121 active+clean; 450 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 9.8 MiB/s rd, 3.9 MiB/s wr, 29 op/s 2024-09-14T03:53:02.017 INFO:tasks.rados.rados.0.smithi070.stdout:1485: done (0 left) 2024-09-14T03:53:02.017 INFO:tasks.rados.rados.0.smithi070.stdout:1486: delete oid 34 current snap is 164 2024-09-14T03:53:02.027 INFO:tasks.rados.rados.0.smithi070.stdout:1486: done (0 left) 2024-09-14T03:53:02.027 INFO:tasks.rados.rados.0.smithi070.stdout:1487: read oid 33 snap -1 2024-09-14T03:53:02.027 INFO:tasks.rados.rados.0.smithi070.stdout:1487: expect (ObjNum 467 snap 160 seq_num 467) 2024-09-14T03:53:02.027 INFO:tasks.rados.rados.0.smithi070.stdout:1488: delete oid 36 current snap is 164 2024-09-14T03:53:02.033 INFO:tasks.rados.rados.0.smithi070.stdout:1488: done (1 left) 2024-09-14T03:53:02.033 INFO:tasks.rados.rados.0.smithi070.stdout:1489: read oid 3 snap -1 2024-09-14T03:53:02.033 INFO:tasks.rados.rados.0.smithi070.stdout:1489: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:53:02.033 INFO:tasks.rados.rados.0.smithi070.stdout:1490: read oid 35 snap -1 2024-09-14T03:53:02.033 INFO:tasks.rados.rados.0.smithi070.stdout:1490: expect deleted 2024-09-14T03:53:02.033 INFO:tasks.rados.rados.0.smithi070.stdout:1491: setattr oid 31 current snap is 164 2024-09-14T03:53:02.034 INFO:tasks.rados.rados.0.smithi070.stdout:1489: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:53:02.037 INFO:tasks.rados.rados.0.smithi070.stdout:1489: done (3 left) 2024-09-14T03:53:02.037 INFO:tasks.rados.rados.0.smithi070.stdout:1490: done (2 left) 2024-09-14T03:53:02.037 INFO:tasks.rados.rados.0.smithi070.stdout:1492: snap_remove snap 114 2024-09-14T03:53:02.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:02 smithi094 ceph-mon[86597]: osdmap e360: 8 total, 8 up, 8 in 2024-09-14T03:53:02.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:02 smithi070 ceph-mon[98640]: osdmap e360: 8 total, 8 up, 8 in 2024-09-14T03:53:02.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:02 smithi070 ceph-mon[96704]: osdmap e360: 8 total, 8 up, 8 in 2024-09-14T03:53:03.030 INFO:tasks.rados.rados.0.smithi070.stdout:1492: done (2 left) 2024-09-14T03:53:03.031 INFO:tasks.rados.rados.0.smithi070.stdout:1493: snap_remove snap 147 2024-09-14T03:53:03.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:03 smithi094 ceph-mon[86597]: osdmap e361: 8 total, 8 up, 8 in 2024-09-14T03:53:03.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:03 smithi094 ceph-mon[86597]: pgmap v133: 121 pgs: 121 active+clean; 451 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 263 B/s rd, 208 KiB/s wr, 1 op/s 2024-09-14T03:53:03.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:03 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:03.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:03 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:03.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:03 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:03.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:03 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-14T03:53:03.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:03 smithi070 ceph-mon[96704]: osdmap e361: 8 total, 8 up, 8 in 2024-09-14T03:53:03.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:03 smithi070 ceph-mon[96704]: pgmap v133: 121 pgs: 121 active+clean; 451 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 263 B/s rd, 208 KiB/s wr, 1 op/s 2024-09-14T03:53:03.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:03 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:03.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:03 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:03.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:03 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:03.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:03 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-14T03:53:03.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:03 smithi070 ceph-mon[98640]: osdmap e361: 8 total, 8 up, 8 in 2024-09-14T03:53:03.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:03 smithi070 ceph-mon[98640]: pgmap v133: 121 pgs: 121 active+clean; 451 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 263 B/s rd, 208 KiB/s wr, 1 op/s 2024-09-14T03:53:03.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:03 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:03.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:03 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:03.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:03 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:03.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:03 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-14T03:53:03.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:53:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:53:03] "GET /metrics HTTP/1.1" 200 36253 "" "Prometheus/2.51.0" 2024-09-14T03:53:04.009 INFO:tasks.rados.rados.0.smithi070.stdout:1493: done (2 left) 2024-09-14T03:53:04.009 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 16 current snap is 164 2024-09-14T03:53:04.009 INFO:tasks.rados.rados.0.smithi070.stdout:1494: seq_num 484 ranges {5480448=507904,5988352=401408} 2024-09-14T03:53:04.043 INFO:tasks.rados.rados.0.smithi070.stdout:1494: writing smithi07085154-16 from 5480448 to 5988352 tid 1 2024-09-14T03:53:04.046 INFO:tasks.rados.rados.0.smithi070.stdout:1494: writing smithi07085154-16 from 5988352 to 6389760 tid 2 2024-09-14T03:53:04.046 INFO:tasks.rados.rados.0.smithi070.stdout:1495: rmattr oid 46 current snap is 164 2024-09-14T03:53:04.046 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 850 (ObjNum 464 snap 156 seq_num 464) dirty exists 2024-09-14T03:53:04.046 INFO:tasks.rados.rados.0.smithi070.stdout:1487: expect (ObjNum 467 snap 160 seq_num 467) 2024-09-14T03:53:04.154 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:04 smithi070 ceph-mon[96704]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-14T03:53:04.154 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:04 smithi070 ceph-mon[96704]: Upgrade: osd.1 is safe to restart 2024-09-14T03:53:04.154 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:04 smithi070 ceph-mon[96704]: osdmap e362: 8 total, 8 up, 8 in 2024-09-14T03:53:04.154 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:04 smithi070 ceph-mon[96704]: osdmap e363: 8 total, 8 up, 8 in 2024-09-14T03:53:04.154 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:04 smithi070 ceph-mon[98640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-14T03:53:04.154 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:04 smithi070 ceph-mon[98640]: Upgrade: osd.1 is safe to restart 2024-09-14T03:53:04.154 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:04 smithi070 ceph-mon[98640]: osdmap e362: 8 total, 8 up, 8 in 2024-09-14T03:53:04.155 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:04 smithi070 ceph-mon[98640]: osdmap e363: 8 total, 8 up, 8 in 2024-09-14T03:53:04.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:04 smithi094 ceph-mon[86597]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-14T03:53:04.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:04 smithi094 ceph-mon[86597]: Upgrade: osd.1 is safe to restart 2024-09-14T03:53:04.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:04 smithi094 ceph-mon[86597]: osdmap e362: 8 total, 8 up, 8 in 2024-09-14T03:53:04.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:04 smithi094 ceph-mon[86597]: osdmap e363: 8 total, 8 up, 8 in 2024-09-14T03:53:04.277 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:53:04 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:53:04.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.1\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:04.313 INFO:tasks.rados.rados.0.smithi070.stdout:1494: finishing write tid 1 to smithi07085154-16 2024-09-14T03:53:04.313 INFO:tasks.rados.rados.0.smithi070.stdout:1494: finishing write tid 2 to smithi07085154-16 2024-09-14T03:53:04.314 INFO:tasks.rados.rados.0.smithi070.stdout:1494: finishing write tid 3 to smithi07085154-16 2024-09-14T03:53:04.314 INFO:tasks.rados.rados.0.smithi070.stdout:1494: finishing write tid 4 to smithi07085154-16 2024-09-14T03:53:04.314 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 1434 (ObjNum 484 snap 164 seq_num 484) dirty exists 2024-09-14T03:53:04.314 INFO:tasks.rados.rados.0.smithi070.stdout:1494: left oid 16 (ObjNum 484 snap 164 seq_num 484) 2024-09-14T03:53:04.314 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 46 v 1436 (ObjNum 320 snap 107 seq_num 320) dirty exists 2024-09-14T03:53:04.314 INFO:tasks.rados.rados.0.smithi070.stdout:1487: done (3 left) 2024-09-14T03:53:04.314 INFO:tasks.rados.rados.0.smithi070.stdout:1491: done (2 left) 2024-09-14T03:53:04.314 INFO:tasks.rados.rados.0.smithi070.stdout:1494: done (1 left) 2024-09-14T03:53:04.314 INFO:tasks.rados.rados.0.smithi070.stdout:1495: done (0 left) 2024-09-14T03:53:04.314 INFO:tasks.rados.rados.0.smithi070.stdout:1496: read oid 16 snap 151 2024-09-14T03:53:04.314 INFO:tasks.rados.rados.0.smithi070.stdout:1496: expect (ObjNum 404 snap 140 seq_num 404) 2024-09-14T03:53:04.314 INFO:tasks.rados.rados.0.smithi070.stdout:1497: snap_remove snap 79 2024-09-14T03:53:05.243 INFO:tasks.rados.rados.0.smithi070.stdout:1497: done (1 left) 2024-09-14T03:53:05.243 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 43 current snap is 164 2024-09-14T03:53:05.243 INFO:tasks.rados.rados.0.smithi070.stdout:1498: seq_num 485 ranges {5365760=532480,5898240=401408} 2024-09-14T03:53:05.276 INFO:tasks.rados.rados.0.smithi070.stdout:1498: writing smithi07085154-43 from 5365760 to 5898240 tid 1 2024-09-14T03:53:05.279 INFO:tasks.rados.rados.0.smithi070.stdout:1498: writing smithi07085154-43 from 5898240 to 6299648 tid 2 2024-09-14T03:53:05.279 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 37 current snap is 164 2024-09-14T03:53:05.279 INFO:tasks.rados.rados.0.smithi070.stdout:1499: seq_num 486 ranges {3457024=548864,4005888=401408} 2024-09-14T03:53:05.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:05 smithi070 ceph-mon[96704]: pgmap v136: 121 pgs: 7 active+clean+snaptrim, 114 active+clean; 452 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.0 MiB/s rd, 607 KiB/s wr, 11 op/s 2024-09-14T03:53:05.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:05 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:05.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:05 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-14T03:53:05.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:05 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:05.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:05 smithi070 ceph-mon[96704]: mgrmap e44: y(active, since 93s), standbys: x 2024-09-14T03:53:05.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:05 smithi070 ceph-mon[98640]: pgmap v136: 121 pgs: 7 active+clean+snaptrim, 114 active+clean; 452 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.0 MiB/s rd, 607 KiB/s wr, 11 op/s 2024-09-14T03:53:05.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:05 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:05.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:05 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-14T03:53:05.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:05 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:05.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:05 smithi070 ceph-mon[98640]: mgrmap e44: y(active, since 93s), standbys: x 2024-09-14T03:53:05.300 INFO:tasks.rados.rados.0.smithi070.stdout:1499: writing smithi07085154-37 from 3457024 to 4005888 tid 1 2024-09-14T03:53:05.304 INFO:tasks.rados.rados.0.smithi070.stdout:1499: writing smithi07085154-37 from 4005888 to 4407296 tid 2 2024-09-14T03:53:05.304 INFO:tasks.rados.rados.0.smithi070.stdout:1500: read oid 39 snap 151 2024-09-14T03:53:05.304 INFO:tasks.rados.rados.0.smithi070.stdout:1500: expect (ObjNum 425 snap 147 seq_num 425) 2024-09-14T03:53:05.304 INFO:tasks.rados.rados.0.smithi070.stdout:1501: rollback oid 2 current snap is 164 2024-09-14T03:53:05.304 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 2 to 77 2024-09-14T03:53:05.304 INFO:tasks.rados.rados.0.smithi070.stdout:1496: expect (ObjNum 404 snap 140 seq_num 404) 2024-09-14T03:53:05.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:05 smithi094 ceph-mon[86597]: pgmap v136: 121 pgs: 7 active+clean+snaptrim, 114 active+clean; 452 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.0 MiB/s rd, 607 KiB/s wr, 11 op/s 2024-09-14T03:53:05.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:05 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:05.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:05 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-14T03:53:05.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:05 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:05.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:05 smithi094 ceph-mon[86597]: mgrmap e44: y(active, since 93s), standbys: x 2024-09-14T03:53:05.631 INFO:tasks.rados.rados.0.smithi070.stdout:1498: finishing write tid 1 to smithi07085154-43 2024-09-14T03:53:05.631 INFO:tasks.rados.rados.0.smithi070.stdout:1498: finishing write tid 2 to smithi07085154-43 2024-09-14T03:53:05.631 INFO:tasks.rados.rados.0.smithi070.stdout:1496: done (4 left) 2024-09-14T03:53:05.631 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 32 current snap is 164 2024-09-14T03:53:05.631 INFO:tasks.rados.rados.0.smithi070.stdout:1498: finishing write tid 3 to smithi07085154-43 2024-09-14T03:53:05.631 INFO:tasks.rados.rados.0.smithi070.stdout:1502: seq_num 487 ranges {8355840=565248,8921088=401408} 2024-09-14T03:53:05.677 INFO:tasks.rados.rados.0.smithi070.stdout:1502: writing smithi07085154-32 from 8355840 to 8921088 tid 1 2024-09-14T03:53:05.680 INFO:tasks.rados.rados.0.smithi070.stdout:1502: writing smithi07085154-32 from 8921088 to 9322496 tid 2 2024-09-14T03:53:05.680 INFO:tasks.rados.rados.0.smithi070.stdout:1503: snap_remove snap 153 2024-09-14T03:53:06.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:53:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:53:05.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:06.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:53:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:53:05.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:06.245 INFO:tasks.rados.rados.0.smithi070.stdout:1503: done (5 left) 2024-09-14T03:53:06.245 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 8 current snap is 164 2024-09-14T03:53:06.245 INFO:tasks.rados.rados.0.smithi070.stdout:1504: seq_num 488 ranges {3710976=581632,4292608=401408} 2024-09-14T03:53:06.262 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:06 smithi070 ceph-mon[96704]: Upgrade: Updating osd.1 2024-09-14T03:53:06.262 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:06 smithi070 ceph-mon[96704]: Deploying daemon osd.1 on smithi070 2024-09-14T03:53:06.262 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:06 smithi070 ceph-mon[96704]: osdmap e364: 8 total, 8 up, 8 in 2024-09-14T03:53:06.263 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:06 smithi070 ceph-mon[98640]: Upgrade: Updating osd.1 2024-09-14T03:53:06.263 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:06 smithi070 ceph-mon[98640]: Deploying daemon osd.1 on smithi070 2024-09-14T03:53:06.263 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:06 smithi070 ceph-mon[98640]: osdmap e364: 8 total, 8 up, 8 in 2024-09-14T03:53:06.269 INFO:tasks.rados.rados.0.smithi070.stdout:1504: writing smithi07085154-8 from 3710976 to 4292608 tid 1 2024-09-14T03:53:06.272 INFO:tasks.rados.rados.0.smithi070.stdout:1504: writing smithi07085154-8 from 4292608 to 4694016 tid 2 2024-09-14T03:53:06.272 INFO:tasks.rados.rados.0.smithi070.stdout:1505: read oid 30 snap -1 2024-09-14T03:53:06.272 INFO:tasks.rados.rados.0.smithi070.stdout:1505: expect (ObjNum 380 snap 129 seq_num 380) 2024-09-14T03:53:06.272 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 48 current snap is 164 2024-09-14T03:53:06.273 INFO:tasks.rados.rados.0.smithi070.stdout:1498: finishing write tid 4 to smithi07085154-43 2024-09-14T03:53:06.273 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 990 (ObjNum 485 snap 164 seq_num 485) dirty exists 2024-09-14T03:53:06.273 INFO:tasks.rados.rados.0.smithi070.stdout:1498: left oid 43 (ObjNum 485 snap 164 seq_num 485) 2024-09-14T03:53:06.273 INFO:tasks.rados.rados.0.smithi070.stdout:1499: finishing write tid 1 to smithi07085154-37 2024-09-14T03:53:06.273 INFO:tasks.rados.rados.0.smithi070.stdout:1501: finishing rollback tid 0 to smithi07085154-2 2024-09-14T03:53:06.273 INFO:tasks.rados.rados.0.smithi070.stdout:1499: finishing write tid 2 to smithi07085154-37 2024-09-14T03:53:06.273 INFO:tasks.rados.rados.0.smithi070.stdout:1501: finishing rollback tid 1 to smithi07085154-2 2024-09-14T03:53:06.273 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 2 v 1449 (ObjNum 65 snap 0 seq_num 1449548622) dirty dne 2024-09-14T03:53:06.273 INFO:tasks.rados.rados.0.smithi070.stdout:1499: finishing write tid 3 to smithi07085154-37 2024-09-14T03:53:06.273 INFO:tasks.rados.rados.0.smithi070.stdout:1499: finishing write tid 4 to smithi07085154-37 2024-09-14T03:53:06.273 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 37 v 994 (ObjNum 486 snap 164 seq_num 486) dirty exists 2024-09-14T03:53:06.273 INFO:tasks.rados.rados.0.smithi070.stdout:1499: left oid 37 (ObjNum 486 snap 164 seq_num 486) 2024-09-14T03:53:06.274 INFO:tasks.rados.rados.0.smithi070.stdout:1506: seq_num 489 ranges {3022848=598016,3620864=401408} 2024-09-14T03:53:06.291 INFO:tasks.rados.rados.0.smithi070.stdout:1506: writing smithi07085154-48 from 3022848 to 3620864 tid 1 2024-09-14T03:53:06.294 INFO:tasks.rados.rados.0.smithi070.stdout:1506: writing smithi07085154-48 from 3620864 to 4022272 tid 2 2024-09-14T03:53:06.294 INFO:tasks.rados.rados.0.smithi070.stdout:1498: done (7 left) 2024-09-14T03:53:06.294 INFO:tasks.rados.rados.0.smithi070.stdout:1499: done (6 left) 2024-09-14T03:53:06.294 INFO:tasks.rados.rados.0.smithi070.stdout:1501: done (5 left) 2024-09-14T03:53:06.294 INFO:tasks.rados.rados.0.smithi070.stdout:1507: read oid 3 snap -1 2024-09-14T03:53:06.294 INFO:tasks.rados.rados.0.smithi070.stdout:1507: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:53:06.294 INFO:tasks.rados.rados.0.smithi070.stdout:1508: snap_remove snap 159 2024-09-14T03:53:06.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:06 smithi094 ceph-mon[86597]: Upgrade: Updating osd.1 2024-09-14T03:53:06.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:06 smithi094 ceph-mon[86597]: Deploying daemon osd.1 on smithi070 2024-09-14T03:53:06.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:06 smithi094 ceph-mon[86597]: osdmap e364: 8 total, 8 up, 8 in 2024-09-14T03:53:07.251 INFO:tasks.rados.rados.0.smithi070.stdout:1508: done (6 left) 2024-09-14T03:53:07.251 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 40 current snap is 164 2024-09-14T03:53:07.252 INFO:tasks.rados.rados.0.smithi070.stdout:1509: seq_num 490 ranges {0=614400,614400=401408} 2024-09-14T03:53:07.255 INFO:tasks.rados.rados.0.smithi070.stdout:1509: writing smithi07085154-40 from 0 to 614400 tid 1 2024-09-14T03:53:07.259 INFO:tasks.rados.rados.0.smithi070.stdout:1509: writing smithi07085154-40 from 614400 to 1015808 tid 2 2024-09-14T03:53:07.259 INFO:tasks.rados.rados.0.smithi070.stdout:1510: setattr oid 26 current snap is 164 2024-09-14T03:53:07.260 INFO:tasks.rados.rados.0.smithi070.stdout:1500: expect (ObjNum 425 snap 147 seq_num 425) 2024-09-14T03:53:07.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:07 smithi094 ceph-mon[86597]: pgmap v138: 121 pgs: 7 active+clean+snaptrim, 114 active+clean; 452 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.9 MiB/s rd, 402 KiB/s wr, 10 op/s 2024-09-14T03:53:07.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:07 smithi094 ceph-mon[86597]: osdmap e365: 8 total, 8 up, 8 in 2024-09-14T03:53:07.277 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:53:06 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:53:06.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:07.399 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:07 smithi070 ceph-mon[96704]: pgmap v138: 121 pgs: 7 active+clean+snaptrim, 114 active+clean; 452 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.9 MiB/s rd, 402 KiB/s wr, 10 op/s 2024-09-14T03:53:07.400 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:07 smithi070 ceph-mon[96704]: osdmap e365: 8 total, 8 up, 8 in 2024-09-14T03:53:07.400 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:07 smithi070 ceph-mon[98640]: pgmap v138: 121 pgs: 7 active+clean+snaptrim, 114 active+clean; 452 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 2.9 MiB/s rd, 402 KiB/s wr, 10 op/s 2024-09-14T03:53:07.400 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:07 smithi070 ceph-mon[98640]: osdmap e365: 8 total, 8 up, 8 in 2024-09-14T03:53:07.568 INFO:tasks.rados.rados.0.smithi070.stdout:1502: finishing write tid 1 to smithi07085154-32 2024-09-14T03:53:07.569 INFO:tasks.rados.rados.0.smithi070.stdout:1502: finishing write tid 2 to smithi07085154-32 2024-09-14T03:53:07.569 INFO:tasks.rados.rados.0.smithi070.stdout:1502: finishing write tid 3 to smithi07085154-32 2024-09-14T03:53:07.569 INFO:tasks.rados.rados.0.smithi070.stdout:1502: finishing write tid 4 to smithi07085154-32 2024-09-14T03:53:07.569 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 1173 (ObjNum 487 snap 164 seq_num 487) dirty exists 2024-09-14T03:53:07.569 INFO:tasks.rados.rados.0.smithi070.stdout:1502: left oid 32 (ObjNum 487 snap 164 seq_num 487) 2024-09-14T03:53:07.569 INFO:tasks.rados.rados.0.smithi070.stdout:1504: finishing write tid 1 to smithi07085154-8 2024-09-14T03:53:07.569 INFO:tasks.rados.rados.0.smithi070.stdout:1504: finishing write tid 2 to smithi07085154-8 2024-09-14T03:53:07.569 INFO:tasks.rados.rados.0.smithi070.stdout:1504: finishing write tid 3 to smithi07085154-8 2024-09-14T03:53:07.569 INFO:tasks.rados.rados.0.smithi070.stdout:1504: finishing write tid 4 to smithi07085154-8 2024-09-14T03:53:07.569 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 8 v 1008 (ObjNum 488 snap 164 seq_num 488) dirty exists 2024-09-14T03:53:07.569 INFO:tasks.rados.rados.0.smithi070.stdout:1504: left oid 8 (ObjNum 488 snap 164 seq_num 488) 2024-09-14T03:53:07.569 INFO:tasks.rados.rados.0.smithi070.stdout:1506: finishing write tid 1 to smithi07085154-48 2024-09-14T03:53:07.570 INFO:tasks.rados.rados.0.smithi070.stdout:1506: finishing write tid 2 to smithi07085154-48 2024-09-14T03:53:07.570 INFO:tasks.rados.rados.0.smithi070.stdout:1506: finishing write tid 3 to smithi07085154-48 2024-09-14T03:53:07.570 INFO:tasks.rados.rados.0.smithi070.stdout:1505: expect (ObjNum 380 snap 129 seq_num 380) 2024-09-14T03:53:07.664 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:07 smithi070 systemd[1]: Stopping Ceph osd.1 for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:53:07.879 INFO:tasks.rados.rados.0.smithi070.stdout:1507: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:53:07.880 INFO:tasks.rados.rados.0.smithi070.stdout:1506: finishing write tid 4 to smithi07085154-48 2024-09-14T03:53:07.881 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 552 (ObjNum 489 snap 164 seq_num 489) dirty exists 2024-09-14T03:53:07.881 INFO:tasks.rados.rados.0.smithi070.stdout:1506: left oid 48 (ObjNum 489 snap 164 seq_num 489) 2024-09-14T03:53:07.881 INFO:tasks.rados.rados.0.smithi070.stdout:1509: finishing write tid 1 to smithi07085154-40 2024-09-14T03:53:07.881 INFO:tasks.rados.rados.0.smithi070.stdout:1509: finishing write tid 2 to smithi07085154-40 2024-09-14T03:53:07.881 INFO:tasks.rados.rados.0.smithi070.stdout:1509: finishing write tid 3 to smithi07085154-40 2024-09-14T03:53:07.881 INFO:tasks.rados.rados.0.smithi070.stdout:1509: finishing write tid 4 to smithi07085154-40 2024-09-14T03:53:07.881 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 668 (ObjNum 490 snap 164 seq_num 490) dirty exists 2024-09-14T03:53:07.882 INFO:tasks.rados.rados.0.smithi070.stdout:1509: left oid 40 (ObjNum 490 snap 164 seq_num 490) 2024-09-14T03:53:07.882 INFO:tasks.rados.rados.0.smithi070.stdout:1500: done (7 left) 2024-09-14T03:53:07.882 INFO:tasks.rados.rados.0.smithi070.stdout:1502: done (6 left) 2024-09-14T03:53:07.882 INFO:tasks.rados.rados.0.smithi070.stdout:1504: done (5 left) 2024-09-14T03:53:07.882 INFO:tasks.rados.rados.0.smithi070.stdout:1505: done (4 left) 2024-09-14T03:53:07.882 INFO:tasks.rados.rados.0.smithi070.stdout:1506: done (3 left) 2024-09-14T03:53:07.883 INFO:tasks.rados.rados.0.smithi070.stdout:1507: done (2 left) 2024-09-14T03:53:07.883 INFO:tasks.rados.rados.0.smithi070.stdout:1509: done (1 left) 2024-09-14T03:53:07.883 INFO:tasks.rados.rados.0.smithi070.stdout:1511: snap_remove snap 131 2024-09-14T03:53:08.036 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:07 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1[49601]: 2024-09-14T03:53:07.662+0000 7f961dc92640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:53:08.036 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:07 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1[49601]: 2024-09-14T03:53:07.662+0000 7f961dc92640 -1 osd.1 366 *** Got signal Terminated *** 2024-09-14T03:53:08.036 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:07 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1[49601]: 2024-09-14T03:53:07.662+0000 7f961dc92640 -1 osd.1 366 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T03:53:08.258 INFO:tasks.rados.rados.0.smithi070.stdout:1511: done (1 left) 2024-09-14T03:53:08.258 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 37 current snap is 164 2024-09-14T03:53:08.258 INFO:tasks.rados.rados.0.smithi070.stdout:1512: seq_num 491 ranges {4407296=630784,5038080=401408} 2024-09-14T03:53:08.286 INFO:tasks.rados.rados.0.smithi070.stdout:1512: writing smithi07085154-37 from 4407296 to 5038080 tid 1 2024-09-14T03:53:08.289 INFO:tasks.rados.rados.0.smithi070.stdout:1512: writing smithi07085154-37 from 5038080 to 5439488 tid 2 2024-09-14T03:53:08.289 INFO:tasks.rados.rados.0.smithi070.stdout:1513: copy_from oid 43 from oid 15 current snap is 164 2024-09-14T03:53:08.289 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 1468 (ObjNum 177 snap 0 seq_num 1) dirty exists 2024-09-14T03:53:08.289 INFO:tasks.rados.rados.0.smithi070.stdout:1510: done (2 left) 2024-09-14T03:53:08.289 INFO:tasks.rados.rados.0.smithi070.stdout:1514: setattr oid 25 current snap is 164 2024-09-14T03:53:08.290 INFO:tasks.rados.rados.0.smithi070.stdout:1515: rollback oid 46 current snap is 164 2024-09-14T03:53:08.290 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 46 to 137 2024-09-14T03:53:08.290 INFO:tasks.rados.rados.0.smithi070.stdout:1516: rollback oid 6 current snap is 164 2024-09-14T03:53:08.290 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 6 to 104 2024-09-14T03:53:08.290 INFO:tasks.rados.rados.0.smithi070.stdout:1517: delete oid 7 current snap is 164 2024-09-14T03:53:08.292 INFO:tasks.rados.rados.0.smithi070.stdout:1515: finishing rollback tid 0 to smithi07085154-46 2024-09-14T03:53:08.293 INFO:tasks.rados.rados.0.smithi070.stdout:1515: finishing rollback tid 1 to smithi07085154-46 2024-09-14T03:53:08.294 INFO:tasks.rados.rados.0.smithi070.stdout:1515: finishing rollback tid 2 to smithi07085154-46 2024-09-14T03:53:08.294 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 46 v 1485 (ObjNum 384 snap 130 seq_num 384) dirty exists 2024-09-14T03:53:08.295 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 851 (ObjNum 459 snap 155 seq_num 459) dirty exists 2024-09-14T03:53:08.296 INFO:tasks.rados.rados.0.smithi070.stdout:1516: finishing rollback tid 0 to smithi07085154-6 2024-09-14T03:53:08.296 INFO:tasks.rados.rados.0.smithi070.stdout:1516: finishing rollback tid 1 to smithi07085154-6 2024-09-14T03:53:08.297 INFO:tasks.rados.rados.0.smithi070.stdout:1516: finishing rollback tid 2 to smithi07085154-6 2024-09-14T03:53:08.297 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 854 (ObjNum 250 snap 72 seq_num 250) dirty exists 2024-09-14T03:53:08.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:08 smithi094 ceph-mon[86597]: osdmap e366: 8 total, 8 up, 8 in 2024-09-14T03:53:08.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:08 smithi094 ceph-mon[86597]: osd.1 marked itself down and dead 2024-09-14T03:53:08.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:08 smithi094 ceph-mon[86597]: pgmap v141: 121 pgs: 7 active+clean+snaptrim, 114 active+clean; 451 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 253 B/s rd, 234 KiB/s wr, 0 op/s 2024-09-14T03:53:08.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:08 smithi070 ceph-mon[96704]: osdmap e366: 8 total, 8 up, 8 in 2024-09-14T03:53:08.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:08 smithi070 ceph-mon[96704]: osd.1 marked itself down and dead 2024-09-14T03:53:08.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:08 smithi070 ceph-mon[96704]: pgmap v141: 121 pgs: 7 active+clean+snaptrim, 114 active+clean; 451 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 253 B/s rd, 234 KiB/s wr, 0 op/s 2024-09-14T03:53:08.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:08 smithi070 ceph-mon[98640]: osdmap e366: 8 total, 8 up, 8 in 2024-09-14T03:53:08.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:08 smithi070 ceph-mon[98640]: osd.1 marked itself down and dead 2024-09-14T03:53:08.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:08 smithi070 ceph-mon[98640]: pgmap v141: 121 pgs: 7 active+clean+snaptrim, 114 active+clean; 451 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 253 B/s rd, 234 KiB/s wr, 0 op/s 2024-09-14T03:53:09.021 INFO:tasks.rados.rados.0.smithi070.stdout:1514: done (5 left) 2024-09-14T03:53:09.022 INFO:tasks.rados.rados.0.smithi070.stdout:1515: done (4 left) 2024-09-14T03:53:09.022 INFO:tasks.rados.rados.0.smithi070.stdout:1516: done (3 left) 2024-09-14T03:53:09.022 INFO:tasks.rados.rados.0.smithi070.stdout:1517: done (2 left) 2024-09-14T03:53:09.022 INFO:tasks.rados.rados.0.smithi070.stdout:1518: read oid 7 snap -1 2024-09-14T03:53:09.022 INFO:tasks.rados.rados.0.smithi070.stdout:1518: expect deleted 2024-09-14T03:53:09.022 INFO:tasks.rados.rados.0.smithi070.stdout:1519: read oid 28 snap 120 2024-09-14T03:53:09.022 INFO:tasks.rados.rados.0.smithi070.stdout:1519: expect (ObjNum 160 snap 47 seq_num 160) 2024-09-14T03:53:09.022 INFO:tasks.rados.rados.0.smithi070.stdout:1520: read oid 39 snap 157 2024-09-14T03:53:09.022 INFO:tasks.rados.rados.0.smithi070.stdout:1520: expect (ObjNum 460 snap 155 seq_num 460) 2024-09-14T03:53:09.022 INFO:tasks.rados.rados.0.smithi070.stdout:1521: snap_create 2024-09-14T03:53:09.032 INFO:tasks.rados.rados.0.smithi070.stdout:1519: expect (ObjNum 160 snap 47 seq_num 160) 2024-09-14T03:53:09.123 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:08 smithi070 podman[107909]: 2024-09-14 03:53:08.815689161 +0000 UTC m=+1.257163348 container died af459ea7ef6869959f69f4b548207ba4b5b07996fedff00c2fad32b6d3a6b051 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=quincy-eb16b97, CEPH_POINT_RELEASE=) 2024-09-14T03:53:09.168 INFO:tasks.rados.rados.0.smithi070.stdout:1512: finishing write tid 1 to smithi07085154-37 2024-09-14T03:53:09.168 INFO:tasks.rados.rados.0.smithi070.stdout:1512: finishing write tid 2 to smithi07085154-37 2024-09-14T03:53:09.168 INFO:tasks.rados.rados.0.smithi070.stdout:1512: finishing write tid 3 to smithi07085154-37 2024-09-14T03:53:09.168 INFO:tasks.rados.rados.0.smithi070.stdout:1512: finishing write tid 4 to smithi07085154-37 2024-09-14T03:53:09.168 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 37 v 1023 (ObjNum 491 snap 164 seq_num 491) dirty exists 2024-09-14T03:53:09.168 INFO:tasks.rados.rados.0.smithi070.stdout:1512: left oid 37 (ObjNum 491 snap 164 seq_num 491) 2024-09-14T03:53:09.168 INFO:tasks.rados.rados.0.smithi070.stdout:1520: expect (ObjNum 460 snap 155 seq_num 460) 2024-09-14T03:53:09.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:09 smithi094 ceph-mon[86597]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:53:09.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:09 smithi094 ceph-mon[86597]: osdmap e367: 8 total, 7 up, 8 in 2024-09-14T03:53:09.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:09 smithi094 ceph-mon[86597]: osdmap e368: 8 total, 7 up, 8 in 2024-09-14T03:53:09.535 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:09 smithi070 podman[107909]: 2024-09-14 03:53:09.295181292 +0000 UTC m=+1.736655480 container cleanup af459ea7ef6869959f69f4b548207ba4b5b07996fedff00c2fad32b6d3a6b051 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=quincy-eb16b97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-14T03:53:09.535 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:09 smithi070 bash[107909]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1 2024-09-14T03:53:09.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:09 smithi070 ceph-mon[96704]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:53:09.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:09 smithi070 ceph-mon[96704]: osdmap e367: 8 total, 7 up, 8 in 2024-09-14T03:53:09.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:09 smithi070 ceph-mon[96704]: osdmap e368: 8 total, 7 up, 8 in 2024-09-14T03:53:09.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:09 smithi070 ceph-mon[98640]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:53:09.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:09 smithi070 ceph-mon[98640]: osdmap e367: 8 total, 7 up, 8 in 2024-09-14T03:53:09.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:09 smithi070 ceph-mon[98640]: osdmap e368: 8 total, 7 up, 8 in 2024-09-14T03:53:09.571 INFO:tasks.rados.rados.0.smithi070.stdout:1513: finishing copy_from racing read to smithi07085154-43 2024-09-14T03:53:09.571 INFO:tasks.rados.rados.0.smithi070.stdout:1513: finishing copy_from to smithi07085154-43 2024-09-14T03:53:09.571 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 911 (ObjNum 469 snap 160 seq_num 469) dirty exists 2024-09-14T03:53:09.905 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:09 smithi070 podman[107929]: 2024-09-14 03:53:09.573148241 +0000 UTC m=+0.752244906 container remove af459ea7ef6869959f69f4b548207ba4b5b07996fedff00c2fad32b6d3a6b051 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-eb16b97, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True) 2024-09-14T03:53:10.012 INFO:tasks.rados.rados.0.smithi070.stdout:1512: done (5 left) 2024-09-14T03:53:10.012 INFO:tasks.rados.rados.0.smithi070.stdout:1513: done (4 left) 2024-09-14T03:53:10.012 INFO:tasks.rados.rados.0.smithi070.stdout:1518: done (3 left) 2024-09-14T03:53:10.012 INFO:tasks.rados.rados.0.smithi070.stdout:1519: done (2 left) 2024-09-14T03:53:10.012 INFO:tasks.rados.rados.0.smithi070.stdout:1520: done (1 left) 2024-09-14T03:53:10.012 INFO:tasks.rados.rados.0.smithi070.stdout:1521: done (0 left) 2024-09-14T03:53:10.012 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 48 current snap is 165 2024-09-14T03:53:10.012 INFO:tasks.rados.rados.0.smithi070.stdout:1522: seq_num 493 ranges {4022272=663552,4685824=401408} 2024-09-14T03:53:10.037 INFO:tasks.rados.rados.0.smithi070.stdout:1522: writing smithi07085154-48 from 4022272 to 4685824 tid 1 2024-09-14T03:53:10.039 INFO:tasks.rados.rados.0.smithi070.stdout:1522: writing smithi07085154-48 from 4685824 to 5087232 tid 2 2024-09-14T03:53:10.040 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 42 current snap is 165 2024-09-14T03:53:10.040 INFO:tasks.rados.rados.0.smithi070.stdout:1523: seq_num 494 ranges {7528448=679936,8208384=401408} 2024-09-14T03:53:10.084 INFO:tasks.rados.rados.0.smithi070.stdout:1523: writing smithi07085154-42 from 7528448 to 8208384 tid 1 2024-09-14T03:53:10.087 INFO:tasks.rados.rados.0.smithi070.stdout:1523: writing smithi07085154-42 from 8208384 to 8609792 tid 2 2024-09-14T03:53:10.087 INFO:tasks.rados.rados.0.smithi070.stdout:1524: delete oid 39 current snap is 165 2024-09-14T03:53:10.087 INFO:tasks.rados.rados.0.smithi070.stdout:1522: finishing write tid 1 to smithi07085154-48 2024-09-14T03:53:10.087 INFO:tasks.rados.rados.0.smithi070.stdout:1522: finishing write tid 2 to smithi07085154-48 2024-09-14T03:53:10.087 INFO:tasks.rados.rados.0.smithi070.stdout:1522: finishing write tid 3 to smithi07085154-48 2024-09-14T03:53:10.087 INFO:tasks.rados.rados.0.smithi070.stdout:1522: finishing write tid 4 to smithi07085154-48 2024-09-14T03:53:10.087 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 555 (ObjNum 493 snap 165 seq_num 493) dirty exists 2024-09-14T03:53:10.087 INFO:tasks.rados.rados.0.smithi070.stdout:1522: left oid 48 (ObjNum 493 snap 165 seq_num 493) 2024-09-14T03:53:10.090 INFO:tasks.rados.rados.0.smithi070.stdout:1522: done (2 left) 2024-09-14T03:53:10.090 INFO:tasks.rados.rados.0.smithi070.stdout:1524: done (1 left) 2024-09-14T03:53:10.090 INFO:tasks.rados.rados.0.smithi070.stdout:1525: snap_remove snap 155 2024-09-14T03:53:10.201 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:09 smithi070 podman[108023]: 2024-09-14 03:53:09.807086725 +0000 UTC m=+0.015934682 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:53:10.201 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:09 smithi070 podman[108023]: 2024-09-14 03:53:09.947928362 +0000 UTC m=+0.156776323 container create 08e544974d2f91d6ec76182487d4c5e54aa39071941b24ed50ec48337e942dca (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1-deactivate, RELEASE=main-a278231, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, ceph=True) 2024-09-14T03:53:10.201 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:10 smithi070 podman[108023]: 2024-09-14 03:53:10.200634961 +0000 UTC m=+0.409482914 container init 08e544974d2f91d6ec76182487d4c5e54aa39071941b24ed50ec48337e942dca (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-14T03:53:10.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:53:10 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:53:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:10.536 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:10 smithi070 podman[108023]: 2024-09-14 03:53:10.205508474 +0000 UTC m=+0.414356434 container start 08e544974d2f91d6ec76182487d4c5e54aa39071941b24ed50ec48337e942dca (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-14T03:53:10.536 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:10 smithi070 podman[108023]: 2024-09-14 03:53:10.300760247 +0000 UTC m=+0.509608206 container attach 08e544974d2f91d6ec76182487d4c5e54aa39071941b24ed50ec48337e942dca (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=main-a278231, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:53:10.536 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:10 smithi070 podman[108023]: 2024-09-14 03:53:10.392072291 +0000 UTC m=+0.600920246 container died 08e544974d2f91d6ec76182487d4c5e54aa39071941b24ed50ec48337e942dca (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, ceph=True, RELEASE=main-a278231, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-09-14T03:53:11.019 INFO:tasks.rados.rados.0.smithi070.stdout:1525: done (1 left) 2024-09-14T03:53:11.019 INFO:tasks.rados.rados.0.smithi070.stdout:1526: delete oid 23 current snap is 165 2024-09-14T03:53:11.019 INFO:tasks.rados.rados.0.smithi070.stdout:1523: finishing write tid 1 to smithi07085154-42 2024-09-14T03:53:11.019 INFO:tasks.rados.rados.0.smithi070.stdout:1523: finishing write tid 2 to smithi07085154-42 2024-09-14T03:53:11.019 INFO:tasks.rados.rados.0.smithi070.stdout:1523: finishing write tid 3 to smithi07085154-42 2024-09-14T03:53:11.019 INFO:tasks.rados.rados.0.smithi070.stdout:1523: finishing write tid 4 to smithi07085154-42 2024-09-14T03:53:11.019 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 1489 (ObjNum 494 snap 165 seq_num 494) dirty exists 2024-09-14T03:53:11.019 INFO:tasks.rados.rados.0.smithi070.stdout:1523: left oid 42 (ObjNum 494 snap 165 seq_num 494) 2024-09-14T03:53:11.030 INFO:tasks.rados.rados.0.smithi070.stdout:1523: done (1 left) 2024-09-14T03:53:11.030 INFO:tasks.rados.rados.0.smithi070.stdout:1526: done (0 left) 2024-09-14T03:53:11.030 INFO:tasks.rados.rados.0.smithi070.stdout:1527: read oid 30 snap -1 2024-09-14T03:53:11.030 INFO:tasks.rados.rados.0.smithi070.stdout:1527: expect (ObjNum 380 snap 129 seq_num 380) 2024-09-14T03:53:11.030 INFO:tasks.rados.rados.0.smithi070.stdout:1528: read oid 43 snap -1 2024-09-14T03:53:11.030 INFO:tasks.rados.rados.0.smithi070.stdout:1528: expect (ObjNum 469 snap 160 seq_num 469) 2024-09-14T03:53:11.030 INFO:tasks.rados.rados.0.smithi070.stdout:1529: rollback oid 37 current snap is 165 2024-09-14T03:53:11.030 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 37 to 116 2024-09-14T03:53:11.030 INFO:tasks.rados.rados.0.smithi070.stdout:1530: snap_remove snap 127 2024-09-14T03:53:11.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:11 smithi094 ceph-mon[86597]: osdmap e369: 8 total, 7 up, 8 in 2024-09-14T03:53:11.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:11 smithi094 ceph-mon[86597]: pgmap v145: 121 pgs: 16 stale+active+clean, 105 active+clean; 447 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 1.5 MiB/s wr, 17 op/s 2024-09-14T03:53:11.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:11 smithi070 ceph-mon[96704]: osdmap e369: 8 total, 7 up, 8 in 2024-09-14T03:53:11.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:11 smithi070 ceph-mon[96704]: pgmap v145: 121 pgs: 16 stale+active+clean, 105 active+clean; 447 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 1.5 MiB/s wr, 17 op/s 2024-09-14T03:53:11.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:11 smithi070 ceph-mon[98640]: osdmap e369: 8 total, 7 up, 8 in 2024-09-14T03:53:11.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:11 smithi070 ceph-mon[98640]: pgmap v145: 121 pgs: 16 stale+active+clean, 105 active+clean; 447 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 1.5 MiB/s wr, 17 op/s 2024-09-14T03:53:11.286 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:10 smithi070 podman[108066]: 2024-09-14 03:53:10.971399741 +0000 UTC m=+0.570891361 container remove 08e544974d2f91d6ec76182487d4c5e54aa39071941b24ed50ec48337e942dca (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=main-a278231, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True) 2024-09-14T03:53:11.286 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:10 smithi070 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.1.service: Deactivated successfully. 2024-09-14T03:53:11.286 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:10 smithi070 systemd[1]: Stopped Ceph osd.1 for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:53:11.286 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:10 smithi070 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.1.service: Consumed 12.159s CPU time. 2024-09-14T03:53:11.286 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:11 smithi070 systemd[1]: Starting Ceph osd.1 for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:53:11.725 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:11 smithi070 podman[108171]: 2024-09-14 03:53:11.315183606 +0000 UTC m=+0.016297198 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:53:11.725 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:11 smithi070 podman[108171]: 2024-09-14 03:53:11.41413195 +0000 UTC m=+0.115245588 container create b52f7494a1dea208c24f912958c751b215e8d997bbff053fb447156b3b2385c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS) 2024-09-14T03:53:11.725 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:11 smithi070 podman[108171]: 2024-09-14 03:53:11.64164941 +0000 UTC m=+0.342763002 container init b52f7494a1dea208c24f912958c751b215e8d997bbff053fb447156b3b2385c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=main-a278231, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T03:53:11.726 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:11 smithi070 podman[108171]: 2024-09-14 03:53:11.646633853 +0000 UTC m=+0.347747445 container start b52f7494a1dea208c24f912958c751b215e8d997bbff053fb447156b3b2385c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=main-a278231, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-14T03:53:11.726 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:11 smithi070 podman[108171]: 2024-09-14 03:53:11.724803825 +0000 UTC m=+0.425917434 container attach b52f7494a1dea208c24f912958c751b215e8d997bbff053fb447156b3b2385c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=main-a278231, org.label-schema.schema-version=1.0) 2024-09-14T03:53:12.014 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:11 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1-activate[108186]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:53:12.014 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:11 smithi070 bash[108171]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:53:12.015 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:11 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1-activate[108186]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:53:12.015 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:11 smithi070 bash[108171]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:53:12.020 INFO:tasks.rados.rados.0.smithi070.stdout:1530: done (3 left) 2024-09-14T03:53:12.020 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 4 current snap is 165 2024-09-14T03:53:12.020 INFO:tasks.rados.rados.0.smithi070.stdout:1531: seq_num 497 ranges {40960=729088,770048=401408} 2024-09-14T03:53:12.025 INFO:tasks.rados.rados.0.smithi070.stdout:1531: writing smithi07085154-4 from 40960 to 770048 tid 1 2024-09-14T03:53:12.028 INFO:tasks.rados.rados.0.smithi070.stdout:1531: writing smithi07085154-4 from 770048 to 1171456 tid 2 2024-09-14T03:53:12.028 INFO:tasks.rados.rados.0.smithi070.stdout:1532: delete oid 23 current snap is 165 2024-09-14T03:53:12.028 INFO:tasks.rados.rados.0.smithi070.stdout:1529: finishing rollback tid 0 to smithi07085154-37 2024-09-14T03:53:12.028 INFO:tasks.rados.rados.0.smithi070.stdout:1529: finishing rollback tid 1 to smithi07085154-37 2024-09-14T03:53:12.028 INFO:tasks.rados.rados.0.smithi070.stdout:1528: expect (ObjNum 469 snap 160 seq_num 469) 2024-09-14T03:53:12.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:12 smithi094 ceph-mon[86597]: osdmap e370: 8 total, 7 up, 8 in 2024-09-14T03:53:12.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:12 smithi070 ceph-mon[96704]: osdmap e370: 8 total, 7 up, 8 in 2024-09-14T03:53:12.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:12 smithi070 ceph-mon[98640]: osdmap e370: 8 total, 7 up, 8 in 2024-09-14T03:53:12.286 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:12 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1-activate[108186]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-14T03:53:12.286 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:12 smithi070 bash[108171]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-14T03:53:12.286 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:12 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1-activate[108186]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-14T03:53:12.286 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:12 smithi070 bash[108171]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-14T03:53:12.302 INFO:tasks.rados.rados.0.smithi070.stdout:1529: finishing rollback tid 2 to smithi07085154-37 2024-09-14T03:53:12.302 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 37 v 1028 (ObjNum 312 snap 105 seq_num 312) dirty exists 2024-09-14T03:53:12.302 INFO:tasks.rados.rados.0.smithi070.stdout:1527: expect (ObjNum 380 snap 129 seq_num 380) 2024-09-14T03:53:12.614 INFO:tasks.rados.rados.0.smithi070.stdout:1531: finishing write tid 1 to smithi07085154-4 2024-09-14T03:53:12.614 INFO:tasks.rados.rados.0.smithi070.stdout:1531: finishing write tid 2 to smithi07085154-4 2024-09-14T03:53:12.614 INFO:tasks.rados.rados.0.smithi070.stdout:1531: finishing write tid 3 to smithi07085154-4 2024-09-14T03:53:12.614 INFO:tasks.rados.rados.0.smithi070.stdout:1531: finishing write tid 4 to smithi07085154-4 2024-09-14T03:53:12.614 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 1217 (ObjNum 497 snap 165 seq_num 497) dirty exists 2024-09-14T03:53:12.615 INFO:tasks.rados.rados.0.smithi070.stdout:1531: left oid 4 (ObjNum 497 snap 165 seq_num 497) 2024-09-14T03:53:12.615 INFO:tasks.rados.rados.0.smithi070.stdout:1527: done (4 left) 2024-09-14T03:53:12.615 INFO:tasks.rados.rados.0.smithi070.stdout:1528: done (3 left) 2024-09-14T03:53:12.615 INFO:tasks.rados.rados.0.smithi070.stdout:1529: done (2 left) 2024-09-14T03:53:12.615 INFO:tasks.rados.rados.0.smithi070.stdout:1531: done (1 left) 2024-09-14T03:53:12.615 INFO:tasks.rados.rados.0.smithi070.stdout:1532: done (0 left) 2024-09-14T03:53:12.615 INFO:tasks.rados.rados.0.smithi070.stdout:1533: snap_remove snap 156 2024-09-14T03:53:12.786 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:12 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1-activate[108186]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-14T03:53:12.786 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:12 smithi070 bash[108171]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-14T03:53:12.786 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:12 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1-activate[108186]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-14T03:53:12.786 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:12 smithi070 bash[108171]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-14T03:53:12.786 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:12 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1-activate[108186]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-14T03:53:12.786 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:12 smithi070 bash[108171]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-14T03:53:12.786 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:12 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1-activate[108186]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-14T03:53:12.786 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:12 smithi070 bash[108171]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-14T03:53:12.786 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:12 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1-activate[108186]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-14T03:53:12.786 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:12 smithi070 bash[108171]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-14T03:53:12.787 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:12 smithi070 podman[108171]: 2024-09-14 03:53:12.556050739 +0000 UTC m=+1.257164334 container died b52f7494a1dea208c24f912958c751b215e8d997bbff053fb447156b3b2385c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , RELEASE=main-a278231, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-14T03:53:13.027 INFO:tasks.rados.rados.0.smithi070.stdout:1533: done (0 left) 2024-09-14T03:53:13.027 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 32 current snap is 165 2024-09-14T03:53:13.027 INFO:tasks.rados.rados.0.smithi070.stdout:1534: seq_num 499 ranges {9322496=761856,10084352=401408} 2024-09-14T03:53:13.082 INFO:tasks.rados.rados.0.smithi070.stdout:1534: writing smithi07085154-32 from 9322496 to 10084352 tid 1 2024-09-14T03:53:13.084 INFO:tasks.rados.rados.0.smithi070.stdout:1534: writing smithi07085154-32 from 10084352 to 10485760 tid 2 2024-09-14T03:53:13.084 INFO:tasks.rados.rados.0.smithi070.stdout:1535: read oid 23 snap -1 2024-09-14T03:53:13.084 INFO:tasks.rados.rados.0.smithi070.stdout:1535: expect deleted 2024-09-14T03:53:13.084 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 41 current snap is 165 2024-09-14T03:53:13.084 INFO:tasks.rados.rados.0.smithi070.stdout:1536: seq_num 500 ranges {2195456=778240,2973696=401408} 2024-09-14T03:53:13.100 INFO:tasks.rados.rados.0.smithi070.stdout:1536: writing smithi07085154-41 from 2195456 to 2973696 tid 1 2024-09-14T03:53:13.103 INFO:tasks.rados.rados.0.smithi070.stdout:1536: writing smithi07085154-41 from 2973696 to 3375104 tid 2 2024-09-14T03:53:13.104 INFO:tasks.rados.rados.0.smithi070.stdout:1537: read oid 36 snap -1 2024-09-14T03:53:13.104 INFO:tasks.rados.rados.0.smithi070.stdout:1537: expect deleted 2024-09-14T03:53:13.104 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 21 current snap is 165 2024-09-14T03:53:13.104 INFO:tasks.rados.rados.0.smithi070.stdout:1538: seq_num 501 ranges {4784128=794624,5578752=401408} 2024-09-14T03:53:13.133 INFO:tasks.rados.rados.0.smithi070.stdout:1538: writing smithi07085154-21 from 4784128 to 5578752 tid 1 2024-09-14T03:53:13.136 INFO:tasks.rados.rados.0.smithi070.stdout:1538: writing smithi07085154-21 from 5578752 to 5980160 tid 2 2024-09-14T03:53:13.136 INFO:tasks.rados.rados.0.smithi070.stdout:1539: setattr oid 20 current snap is 165 2024-09-14T03:53:13.136 INFO:tasks.rados.rados.0.smithi070.stdout:1534: finishing write tid 1 to smithi07085154-32 2024-09-14T03:53:13.136 INFO:tasks.rados.rados.0.smithi070.stdout:1534: finishing write tid 2 to smithi07085154-32 2024-09-14T03:53:13.136 INFO:tasks.rados.rados.0.smithi070.stdout:1534: finishing write tid 3 to smithi07085154-32 2024-09-14T03:53:13.136 INFO:tasks.rados.rados.0.smithi070.stdout:1534: finishing write tid 4 to smithi07085154-32 2024-09-14T03:53:13.136 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 1176 (ObjNum 499 snap 165 seq_num 499) dirty exists 2024-09-14T03:53:13.136 INFO:tasks.rados.rados.0.smithi070.stdout:1534: left oid 32 (ObjNum 499 snap 165 seq_num 499) 2024-09-14T03:53:13.136 INFO:tasks.rados.rados.0.smithi070.stdout:1536: finishing write tid 1 to smithi07085154-41 2024-09-14T03:53:13.136 INFO:tasks.rados.rados.0.smithi070.stdout:1536: finishing write tid 2 to smithi07085154-41 2024-09-14T03:53:13.136 INFO:tasks.rados.rados.0.smithi070.stdout:1536: finishing write tid 3 to smithi07085154-41 2024-09-14T03:53:13.137 INFO:tasks.rados.rados.0.smithi070.stdout:1536: finishing write tid 4 to smithi07085154-41 2024-09-14T03:53:13.137 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 1032 (ObjNum 500 snap 165 seq_num 500) dirty exists 2024-09-14T03:53:13.137 INFO:tasks.rados.rados.0.smithi070.stdout:1536: left oid 41 (ObjNum 500 snap 165 seq_num 500) 2024-09-14T03:53:13.137 INFO:tasks.rados.rados.0.smithi070.stdout:1534: done (5 left) 2024-09-14T03:53:13.137 INFO:tasks.rados.rados.0.smithi070.stdout:1535: done (4 left) 2024-09-14T03:53:13.138 INFO:tasks.rados.rados.0.smithi070.stdout:1536: done (3 left) 2024-09-14T03:53:13.138 INFO:tasks.rados.rados.0.smithi070.stdout:1537: done (2 left) 2024-09-14T03:53:13.138 INFO:tasks.rados.rados.0.smithi070.stdout:1540: delete oid 18 current snap is 165 2024-09-14T03:53:13.138 INFO:tasks.rados.rados.0.smithi070.stdout:1538: finishing write tid 1 to smithi07085154-21 2024-09-14T03:53:13.138 INFO:tasks.rados.rados.0.smithi070.stdout:1538: finishing write tid 2 to smithi07085154-21 2024-09-14T03:53:13.139 INFO:tasks.rados.rados.0.smithi070.stdout:1538: finishing write tid 3 to smithi07085154-21 2024-09-14T03:53:13.139 INFO:tasks.rados.rados.0.smithi070.stdout:1538: finishing write tid 4 to smithi07085154-21 2024-09-14T03:53:13.140 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 549 (ObjNum 501 snap 165 seq_num 501) dirty exists 2024-09-14T03:53:13.140 INFO:tasks.rados.rados.0.smithi070.stdout:1538: left oid 21 (ObjNum 501 snap 165 seq_num 501) 2024-09-14T03:53:13.140 INFO:tasks.rados.rados.0.smithi070.stdout:1538: done (2 left) 2024-09-14T03:53:13.140 INFO:tasks.rados.rados.0.smithi070.stdout:1540: done (1 left) 2024-09-14T03:53:13.140 INFO:tasks.rados.rados.0.smithi070.stdout:1541: rollback oid 41 current snap is 165 2024-09-14T03:53:13.140 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 41 to 142 2024-09-14T03:53:13.140 INFO:tasks.rados.rados.0.smithi070.stdout:1542: delete oid 13 current snap is 165 2024-09-14T03:53:13.141 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 20 v 1218 (ObjNum 274 snap 84 seq_num 274) dirty exists 2024-09-14T03:53:13.142 INFO:tasks.rados.rados.0.smithi070.stdout:1539: done (2 left) 2024-09-14T03:53:13.142 INFO:tasks.rados.rados.0.smithi070.stdout:1542: done (1 left) 2024-09-14T03:53:13.142 INFO:tasks.rados.rados.0.smithi070.stdout:1543: delete oid 22 current snap is 165 2024-09-14T03:53:13.142 INFO:tasks.rados.rados.0.smithi070.stdout:1541: finishing rollback tid 0 to smithi07085154-41 2024-09-14T03:53:13.143 INFO:tasks.rados.rados.0.smithi070.stdout:1541: finishing rollback tid 1 to smithi07085154-41 2024-09-14T03:53:13.143 INFO:tasks.rados.rados.0.smithi070.stdout:1541: finishing rollback tid 2 to smithi07085154-41 2024-09-14T03:53:13.144 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 1037 (ObjNum 211 snap 60 seq_num 211) dirty exists 2024-09-14T03:53:13.145 INFO:tasks.rados.rados.0.smithi070.stdout:1541: done (1 left) 2024-09-14T03:53:13.145 INFO:tasks.rados.rados.0.smithi070.stdout:1543: done (0 left) 2024-09-14T03:53:13.146 INFO:tasks.rados.rados.0.smithi070.stdout:1544: read oid 32 snap -1 2024-09-14T03:53:13.146 INFO:tasks.rados.rados.0.smithi070.stdout:1544: expect (ObjNum 499 snap 165 seq_num 499) 2024-09-14T03:53:13.146 INFO:tasks.rados.rados.0.smithi070.stdout:1545: setattr oid 19 current snap is 165 2024-09-14T03:53:13.146 INFO:tasks.rados.rados.0.smithi070.stdout:1546: snap_remove snap 122 2024-09-14T03:53:13.163 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:13 smithi070 ceph-mon[98640]: osdmap e371: 8 total, 7 up, 8 in 2024-09-14T03:53:13.163 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:13 smithi070 ceph-mon[98640]: pgmap v148: 121 pgs: 7 active+undersized, 14 stale+active+clean, 1 active+undersized+degraded, 99 active+clean; 442 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 15/1068 objects degraded (1.404%) 2024-09-14T03:53:13.163 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:13 smithi070 ceph-mon[96704]: osdmap e371: 8 total, 7 up, 8 in 2024-09-14T03:53:13.163 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:13 smithi070 ceph-mon[96704]: pgmap v148: 121 pgs: 7 active+undersized, 14 stale+active+clean, 1 active+undersized+degraded, 99 active+clean; 442 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 15/1068 objects degraded (1.404%) 2024-09-14T03:53:13.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:13 smithi094 ceph-mon[86597]: osdmap e371: 8 total, 7 up, 8 in 2024-09-14T03:53:13.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:13 smithi094 ceph-mon[86597]: pgmap v148: 121 pgs: 7 active+undersized, 14 stale+active+clean, 1 active+undersized+degraded, 99 active+clean; 442 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 15/1068 objects degraded (1.404%) 2024-09-14T03:53:13.489 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:53:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:53:13] "GET /metrics HTTP/1.1" 200 36253 "" "Prometheus/2.51.0" 2024-09-14T03:53:13.489 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:13 smithi070 podman[108331]: 2024-09-14 03:53:13.162293024 +0000 UTC m=+0.600145349 container remove b52f7494a1dea208c24f912958c751b215e8d997bbff053fb447156b3b2385c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1-activate, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, RELEASE=main-a278231, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-14T03:53:13.490 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:13 smithi070 podman[108398]: 2024-09-14 03:53:13.27195555 +0000 UTC m=+0.015847352 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:53:13.490 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:13 smithi070 podman[108398]: 2024-09-14 03:53:13.371530264 +0000 UTC m=+0.115422057 container create 53868856512a30fe44c93e5115ca093444ae01f9821f3d11b1376d762ed75798 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=main-a278231, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-14T03:53:14.017 INFO:tasks.rados.rados.0.smithi070.stdout:1546: done (2 left) 2024-09-14T03:53:14.018 INFO:tasks.rados.rados.0.smithi070.stdout:1547: read oid 10 snap -1 2024-09-14T03:53:14.018 INFO:tasks.rados.rados.0.smithi070.stdout:1547: expect (ObjNum 456 snap 155 seq_num 456) 2024-09-14T03:53:14.018 INFO:tasks.rados.rados.0.smithi070.stdout:1548: rmattr oid 50 current snap is 165 2024-09-14T03:53:14.018 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 38 current snap is 165 2024-09-14T03:53:14.018 INFO:tasks.rados.rados.0.smithi070.stdout:1549: seq_num 505 ranges {647168=466944,1114112=475136,1589248=327680} 2024-09-14T03:53:14.021 INFO:tasks.rados.rados.0.smithi070.stdout:1549: writing smithi07085154-38 from 647168 to 1114112 tid 1 2024-09-14T03:53:14.025 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:13 smithi070 podman[108398]: 2024-09-14 03:53:13.599317513 +0000 UTC m=+0.343209305 container init 53868856512a30fe44c93e5115ca093444ae01f9821f3d11b1376d762ed75798 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2) 2024-09-14T03:53:14.025 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:13 smithi070 podman[108398]: 2024-09-14 03:53:13.604079473 +0000 UTC m=+0.347971271 container start 53868856512a30fe44c93e5115ca093444ae01f9821f3d11b1376d762ed75798 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1, RELEASE=main-a278231, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-14T03:53:14.025 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:13 smithi070 ceph-osd[108436]: -- 172.21.15.70:0/3185226146 <== mon.2 v2:172.21.15.70:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55e963b6fa40 con 0x55e963b4dc00 2024-09-14T03:53:14.026 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:13 smithi070 bash[108398]: 53868856512a30fe44c93e5115ca093444ae01f9821f3d11b1376d762ed75798 2024-09-14T03:53:14.026 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:13 smithi070 systemd[1]: Started Ceph osd.1 for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:53:14.026 INFO:tasks.rados.rados.0.smithi070.stdout:1549: writing smithi07085154-38 from 1114112 to 1589248 tid 2 2024-09-14T03:53:14.026 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:14 smithi070 ceph-mon[98640]: Health check failed: Degraded data redundancy: 15/1068 objects degraded (1.404%), 1 pg degraded (PG_DEGRADED) 2024-09-14T03:53:14.026 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:14 smithi070 ceph-mon[98640]: osdmap e372: 8 total, 7 up, 8 in 2024-09-14T03:53:14.026 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:14 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:14.026 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:14 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:14.027 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:14 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:53:14.027 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:14 smithi070 ceph-mon[98640]: osdmap e373: 8 total, 7 up, 8 in 2024-09-14T03:53:14.028 INFO:tasks.rados.rados.0.smithi070.stdout:1549: writing smithi07085154-38 from 1589248 to 1916928 tid 3 2024-09-14T03:53:14.028 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 11 current snap is 165 2024-09-14T03:53:14.028 INFO:tasks.rados.rados.0.smithi070.stdout:1550: seq_num 506 ranges {2310144=483328,2793472=761856,3555328=40960} 2024-09-14T03:53:14.044 INFO:tasks.rados.rados.0.smithi070.stdout:1550: writing smithi07085154-11 from 2310144 to 2793472 tid 1 2024-09-14T03:53:14.050 INFO:tasks.rados.rados.0.smithi070.stdout:1550: writing smithi07085154-11 from 2793472 to 3555328 tid 2 2024-09-14T03:53:14.050 INFO:tasks.rados.rados.0.smithi070.stdout:1550: writing smithi07085154-11 from 3555328 to 3596288 tid 3 2024-09-14T03:53:14.050 INFO:tasks.rados.rados.0.smithi070.stdout:1551: snap_create 2024-09-14T03:53:14.050 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 19 v 856 (ObjNum 466 snap 158 seq_num 466) dirty exists 2024-09-14T03:53:14.050 INFO:tasks.rados.rados.0.smithi070.stdout:1544: expect (ObjNum 499 snap 165 seq_num 499) 2024-09-14T03:53:14.277 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:53:14 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:53:14.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.1\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:14.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:14 smithi094 ceph-mon[86597]: Health check failed: Degraded data redundancy: 15/1068 objects degraded (1.404%), 1 pg degraded (PG_DEGRADED) 2024-09-14T03:53:14.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:14 smithi094 ceph-mon[86597]: osdmap e372: 8 total, 7 up, 8 in 2024-09-14T03:53:14.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:14 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:14.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:14 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:14.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:14 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:53:14.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:14 smithi094 ceph-mon[86597]: osdmap e373: 8 total, 7 up, 8 in 2024-09-14T03:53:14.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:14 smithi070 ceph-mon[96704]: Health check failed: Degraded data redundancy: 15/1068 objects degraded (1.404%), 1 pg degraded (PG_DEGRADED) 2024-09-14T03:53:14.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:14 smithi070 ceph-mon[96704]: osdmap e372: 8 total, 7 up, 8 in 2024-09-14T03:53:14.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:14 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:14.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:14 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:14.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:14 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:53:14.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:14 smithi070 ceph-mon[96704]: osdmap e373: 8 total, 7 up, 8 in 2024-09-14T03:53:14.924 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 858 (ObjNum 430 snap 148 seq_num 430) dirty exists 2024-09-14T03:53:14.924 INFO:tasks.rados.rados.0.smithi070.stdout:1549: finishing write tid 1 to smithi07085154-38 2024-09-14T03:53:14.924 INFO:tasks.rados.rados.0.smithi070.stdout:1549: finishing write tid 2 to smithi07085154-38 2024-09-14T03:53:14.924 INFO:tasks.rados.rados.0.smithi070.stdout:1549: finishing write tid 3 to smithi07085154-38 2024-09-14T03:53:14.924 INFO:tasks.rados.rados.0.smithi070.stdout:1549: finishing write tid 4 to smithi07085154-38 2024-09-14T03:53:14.924 INFO:tasks.rados.rados.0.smithi070.stdout:1549: finishing write tid 5 to smithi07085154-38 2024-09-14T03:53:14.925 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 38 v 998 (ObjNum 505 snap 165 seq_num 505) dirty exists 2024-09-14T03:53:14.925 INFO:tasks.rados.rados.0.smithi070.stdout:1549: left oid 38 (ObjNum 505 snap 165 seq_num 505) 2024-09-14T03:53:14.925 INFO:tasks.rados.rados.0.smithi070.stdout:1547: expect (ObjNum 456 snap 155 seq_num 456) 2024-09-14T03:53:15.108 INFO:tasks.rados.rados.0.smithi070.stdout:1550: finishing write tid 1 to smithi07085154-11 2024-09-14T03:53:15.108 INFO:tasks.rados.rados.0.smithi070.stdout:1550: finishing write tid 2 to smithi07085154-11 2024-09-14T03:53:15.108 INFO:tasks.rados.rados.0.smithi070.stdout:1550: finishing write tid 3 to smithi07085154-11 2024-09-14T03:53:15.108 INFO:tasks.rados.rados.0.smithi070.stdout:1550: finishing write tid 4 to smithi07085154-11 2024-09-14T03:53:15.108 INFO:tasks.rados.rados.0.smithi070.stdout:1550: finishing write tid 5 to smithi07085154-11 2024-09-14T03:53:15.108 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 11 v 1181 (ObjNum 506 snap 165 seq_num 506) dirty exists 2024-09-14T03:53:15.108 INFO:tasks.rados.rados.0.smithi070.stdout:1550: left oid 11 (ObjNum 506 snap 165 seq_num 506) 2024-09-14T03:53:15.108 INFO:tasks.rados.rados.0.smithi070.stdout:1544: done (6 left) 2024-09-14T03:53:15.108 INFO:tasks.rados.rados.0.smithi070.stdout:1545: done (5 left) 2024-09-14T03:53:15.109 INFO:tasks.rados.rados.0.smithi070.stdout:1547: done (4 left) 2024-09-14T03:53:15.109 INFO:tasks.rados.rados.0.smithi070.stdout:1548: done (3 left) 2024-09-14T03:53:15.109 INFO:tasks.rados.rados.0.smithi070.stdout:1549: done (2 left) 2024-09-14T03:53:15.109 INFO:tasks.rados.rados.0.smithi070.stdout:1550: done (1 left) 2024-09-14T03:53:15.109 INFO:tasks.rados.rados.0.smithi070.stdout:1551: done (0 left) 2024-09-14T03:53:15.109 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 9 current snap is 166 2024-09-14T03:53:15.109 INFO:tasks.rados.rados.0.smithi070.stdout:1552: seq_num 507 ranges {0=499712,499712=638976,1138688=163840} 2024-09-14T03:53:15.110 INFO:tasks.rados.rados.0.smithi070.stdout:1552: writing smithi07085154-9 from 0 to 499712 tid 1 2024-09-14T03:53:15.114 INFO:tasks.rados.rados.0.smithi070.stdout:1552: writing smithi07085154-9 from 499712 to 1138688 tid 2 2024-09-14T03:53:15.115 INFO:tasks.rados.rados.0.smithi070.stdout:1552: writing smithi07085154-9 from 1138688 to 1302528 tid 3 2024-09-14T03:53:15.115 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 10 current snap is 166 2024-09-14T03:53:15.115 INFO:tasks.rados.rados.0.smithi070.stdout:1553: seq_num 508 ranges {3645440=516096,4161536=524288,4685824=278528} 2024-09-14T03:53:15.137 INFO:tasks.rados.rados.0.smithi070.stdout:1553: writing smithi07085154-10 from 3645440 to 4161536 tid 1 2024-09-14T03:53:15.140 INFO:tasks.rados.rados.0.smithi070.stdout:1553: writing smithi07085154-10 from 4161536 to 4685824 tid 2 2024-09-14T03:53:15.142 INFO:tasks.rados.rados.0.smithi070.stdout:1553: writing smithi07085154-10 from 4685824 to 4964352 tid 3 2024-09-14T03:53:15.142 INFO:tasks.rados.rados.0.smithi070.stdout:1554: copy_from oid 6 from oid 7 current snap is 166 2024-09-14T03:53:15.142 INFO:tasks.rados.rados.0.smithi070.stdout:1552: finishing write tid 1 to smithi07085154-9 2024-09-14T03:53:15.142 INFO:tasks.rados.rados.0.smithi070.stdout:1552: finishing write tid 2 to smithi07085154-9 2024-09-14T03:53:15.143 INFO:tasks.rados.rados.0.smithi070.stdout:1552: finishing write tid 3 to smithi07085154-9 2024-09-14T03:53:15.143 INFO:tasks.rados.rados.0.smithi070.stdout:1552: finishing write tid 4 to smithi07085154-9 2024-09-14T03:53:15.143 INFO:tasks.rados.rados.0.smithi070.stdout:1552: finishing write tid 5 to smithi07085154-9 2024-09-14T03:53:15.143 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 1545 (ObjNum 507 snap 166 seq_num 507) dirty exists 2024-09-14T03:53:15.143 INFO:tasks.rados.rados.0.smithi070.stdout:1552: left oid 9 (ObjNum 507 snap 166 seq_num 507) 2024-09-14T03:53:15.143 INFO:tasks.rados.rados.0.smithi070.stdout:1553: finishing write tid 1 to smithi07085154-10 2024-09-14T03:53:15.143 INFO:tasks.rados.rados.0.smithi070.stdout:1552: done (2 left) 2024-09-14T03:53:15.143 INFO:tasks.rados.rados.0.smithi070.stdout:1555: rmattr oid 50 current snap is 166 2024-09-14T03:53:15.143 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 4 current snap is 166 2024-09-14T03:53:15.143 INFO:tasks.rados.rados.0.smithi070.stdout:1556: seq_num 509 ranges {1171456=532480,1703936=409600,2113536=393216} 2024-09-14T03:53:15.151 INFO:tasks.rados.rados.0.smithi070.stdout:1556: writing smithi07085154-4 from 1171456 to 1703936 tid 1 2024-09-14T03:53:15.154 INFO:tasks.rados.rados.0.smithi070.stdout:1556: writing smithi07085154-4 from 1703936 to 2113536 tid 2 2024-09-14T03:53:15.156 INFO:tasks.rados.rados.0.smithi070.stdout:1556: writing smithi07085154-4 from 2113536 to 2506752 tid 3 2024-09-14T03:53:15.156 INFO:tasks.rados.rados.0.smithi070.stdout:1557: delete oid 15 current snap is 166 2024-09-14T03:53:15.157 INFO:tasks.rados.rados.0.smithi070.stdout:1553: finishing write tid 2 to smithi07085154-10 2024-09-14T03:53:15.157 INFO:tasks.rados.rados.0.smithi070.stdout:1553: finishing write tid 3 to smithi07085154-10 2024-09-14T03:53:15.157 INFO:tasks.rados.rados.0.smithi070.stdout:1553: finishing write tid 4 to smithi07085154-10 2024-09-14T03:53:15.157 INFO:tasks.rados.rados.0.smithi070.stdout:1554: finishing copy_from to smithi07085154-6 2024-09-14T03:53:15.157 INFO:tasks.rados.rados.0.smithi070.stdout:1554: got expected ENOENT (src dne) 2024-09-14T03:53:15.157 INFO:tasks.rados.rados.0.smithi070.stdout:1554: finishing copy_from racing read to smithi07085154-6 2024-09-14T03:53:15.157 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 860 (ObjNum 430 snap 148 seq_num 430) dirty exists 2024-09-14T03:53:15.157 INFO:tasks.rados.rados.0.smithi070.stdout:1553: finishing write tid 5 to smithi07085154-10 2024-09-14T03:53:15.157 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 1550 (ObjNum 508 snap 166 seq_num 508) dirty exists 2024-09-14T03:53:15.158 INFO:tasks.rados.rados.0.smithi070.stdout:1553: left oid 10 (ObjNum 508 snap 166 seq_num 508) 2024-09-14T03:53:15.158 INFO:tasks.rados.rados.0.smithi070.stdout:1556: finishing write tid 1 to smithi07085154-4 2024-09-14T03:53:15.158 INFO:tasks.rados.rados.0.smithi070.stdout:1556: finishing write tid 2 to smithi07085154-4 2024-09-14T03:53:15.159 INFO:tasks.rados.rados.0.smithi070.stdout:1556: finishing write tid 3 to smithi07085154-4 2024-09-14T03:53:15.159 INFO:tasks.rados.rados.0.smithi070.stdout:1556: finishing write tid 4 to smithi07085154-4 2024-09-14T03:53:15.161 INFO:tasks.rados.rados.0.smithi070.stdout:1556: finishing write tid 5 to smithi07085154-4 2024-09-14T03:53:15.161 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 1222 (ObjNum 509 snap 166 seq_num 509) dirty exists 2024-09-14T03:53:15.161 INFO:tasks.rados.rados.0.smithi070.stdout:1556: left oid 4 (ObjNum 509 snap 166 seq_num 509) 2024-09-14T03:53:15.162 INFO:tasks.rados.rados.0.smithi070.stdout:1553: done (4 left) 2024-09-14T03:53:15.162 INFO:tasks.rados.rados.0.smithi070.stdout:1554: done (3 left) 2024-09-14T03:53:15.162 INFO:tasks.rados.rados.0.smithi070.stdout:1555: done (2 left) 2024-09-14T03:53:15.162 INFO:tasks.rados.rados.0.smithi070.stdout:1556: done (1 left) 2024-09-14T03:53:15.162 INFO:tasks.rados.rados.0.smithi070.stdout:1557: done (0 left) 2024-09-14T03:53:15.162 INFO:tasks.rados.rados.0.smithi070.stdout:1558: read oid 38 snap -1 2024-09-14T03:53:15.162 INFO:tasks.rados.rados.0.smithi070.stdout:1558: expect (ObjNum 505 snap 165 seq_num 505) 2024-09-14T03:53:15.163 INFO:tasks.rados.rados.0.smithi070.stdout:1559: snap_create 2024-09-14T03:53:15.170 INFO:tasks.rados.rados.0.smithi070.stdout:1558: expect (ObjNum 505 snap 165 seq_num 505) 2024-09-14T03:53:15.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:15 smithi070 ceph-mon[96704]: pgmap v151: 121 pgs: 33 active+undersized, 12 active+undersized+degraded, 76 active+clean; 454 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 2.1 MiB/s wr, 15 op/s; 156/1095 objects degraded (14.247%) 2024-09-14T03:53:15.286 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:15 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1[108413]: 2024-09-14T03:53:15.243+0000 7fde53e18640 -1 Falling back to public interface 2024-09-14T03:53:15.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:15 smithi070 ceph-mon[98640]: pgmap v151: 121 pgs: 33 active+undersized, 12 active+undersized+degraded, 76 active+clean; 454 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 2.1 MiB/s wr, 15 op/s; 156/1095 objects degraded (14.247%) 2024-09-14T03:53:15.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:15 smithi094 ceph-mon[86597]: pgmap v151: 121 pgs: 33 active+undersized, 12 active+undersized+degraded, 76 active+clean; 454 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 2.1 MiB/s wr, 15 op/s; 156/1095 objects degraded (14.247%) 2024-09-14T03:53:16.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:53:15 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:53:15.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:16.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:53:15 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:53:15.742Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:16.037 INFO:tasks.rados.rados.0.smithi070.stdout:1558: done (1 left) 2024-09-14T03:53:16.038 INFO:tasks.rados.rados.0.smithi070.stdout:1559: done (0 left) 2024-09-14T03:53:16.038 INFO:tasks.rados.rados.0.smithi070.stdout:1560: delete oid 24 current snap is 167 2024-09-14T03:53:16.042 INFO:tasks.rados.rados.0.smithi070.stdout:1560: done (0 left) 2024-09-14T03:53:16.042 INFO:tasks.rados.rados.0.smithi070.stdout:1561: snap_create 2024-09-14T03:53:16.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:16 smithi070 ceph-mon[96704]: osdmap e374: 8 total, 7 up, 8 in 2024-09-14T03:53:16.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:16 smithi070 ceph-mon[98640]: osdmap e374: 8 total, 7 up, 8 in 2024-09-14T03:53:16.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:16 smithi094 ceph-mon[86597]: osdmap e374: 8 total, 7 up, 8 in 2024-09-14T03:53:17.044 INFO:tasks.rados.rados.0.smithi070.stdout:1561: done (0 left) 2024-09-14T03:53:17.044 INFO:tasks.rados.rados.0.smithi070.stdout:1562: snap_remove snap 66 2024-09-14T03:53:17.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:17 smithi094 ceph-mon[86597]: osdmap e375: 8 total, 7 up, 8 in 2024-09-14T03:53:17.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:17 smithi094 ceph-mon[86597]: pgmap v154: 121 pgs: 33 active+undersized, 12 active+undersized+degraded, 76 active+clean; 454 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 2.1 MiB/s wr, 15 op/s; 156/1095 objects degraded (14.247%) 2024-09-14T03:53:17.277 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:53:16 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:53:16.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:17.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:17 smithi070 ceph-mon[96704]: osdmap e375: 8 total, 7 up, 8 in 2024-09-14T03:53:17.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:17 smithi070 ceph-mon[96704]: pgmap v154: 121 pgs: 33 active+undersized, 12 active+undersized+degraded, 76 active+clean; 454 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 2.1 MiB/s wr, 15 op/s; 156/1095 objects degraded (14.247%) 2024-09-14T03:53:17.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:17 smithi070 ceph-mon[98640]: osdmap e375: 8 total, 7 up, 8 in 2024-09-14T03:53:17.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:17 smithi070 ceph-mon[98640]: pgmap v154: 121 pgs: 33 active+undersized, 12 active+undersized+degraded, 76 active+clean; 454 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 2.1 MiB/s wr, 15 op/s; 156/1095 objects degraded (14.247%) 2024-09-14T03:53:18.045 INFO:tasks.rados.rados.0.smithi070.stdout:1562: done (0 left) 2024-09-14T03:53:18.045 INFO:tasks.rados.rados.0.smithi070.stdout:1563: snap_create 2024-09-14T03:53:18.314 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:18 smithi070 ceph-mon[96704]: osdmap e376: 8 total, 7 up, 8 in 2024-09-14T03:53:18.314 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:18 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:18.314 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:18 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:18.314 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:18 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:18.314 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:18 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:18.314 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:18 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:18.315 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:18 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:18.315 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:18 smithi070 ceph-mon[98640]: osdmap e376: 8 total, 7 up, 8 in 2024-09-14T03:53:18.315 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:18 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:18.315 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:18 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:18.315 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:18 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:18.315 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:18 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:18.315 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:18 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:18.315 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:18 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:18.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:18 smithi094 ceph-mon[86597]: osdmap e376: 8 total, 7 up, 8 in 2024-09-14T03:53:18.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:18 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:18.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:18 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:18.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:18 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:18.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:18 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:18.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:18 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:18.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:18 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:19.053 INFO:tasks.rados.rados.0.smithi070.stdout:1563: done (0 left) 2024-09-14T03:53:19.054 INFO:tasks.rados.rados.0.smithi070.stdout:1564: read oid 11 snap -1 2024-09-14T03:53:19.054 INFO:tasks.rados.rados.0.smithi070.stdout:1564: expect (ObjNum 506 snap 165 seq_num 506) 2024-09-14T03:53:19.054 INFO:tasks.rados.rados.0.smithi070.stdout:1565: snap_create 2024-09-14T03:53:19.068 INFO:tasks.rados.rados.0.smithi070.stdout:1564: expect (ObjNum 506 snap 165 seq_num 506) 2024-09-14T03:53:19.320 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:19 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1[108413]: 2024-09-14T03:53:19.034+0000 7fde53e18640 -1 osd.1 366 log_to_monitors true 2024-09-14T03:53:19.321 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:19 smithi070 ceph-mon[96704]: pgmap v157: 121 pgs: 33 active+undersized, 12 active+undersized+degraded, 76 active+clean; 456 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.5 MiB/s rd, 598 KiB/s wr, 4 op/s; 159/1107 objects degraded (14.363%) 2024-09-14T03:53:19.321 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:19 smithi070 ceph-mon[96704]: osdmap e377: 8 total, 7 up, 8 in 2024-09-14T03:53:19.321 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:19 smithi070 ceph-mon[96704]: Health check update: Degraded data redundancy: 156/1095 objects degraded (14.247%), 12 pgs degraded (PG_DEGRADED) 2024-09-14T03:53:19.321 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:19 smithi070 ceph-mon[96704]: from='osd.1 [v2:172.21.15.70:6810/478220551,v1:172.21.15.70:6811/478220551]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-14T03:53:19.321 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:19 smithi070 ceph-mon[98640]: pgmap v157: 121 pgs: 33 active+undersized, 12 active+undersized+degraded, 76 active+clean; 456 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.5 MiB/s rd, 598 KiB/s wr, 4 op/s; 159/1107 objects degraded (14.363%) 2024-09-14T03:53:19.321 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:19 smithi070 ceph-mon[98640]: osdmap e377: 8 total, 7 up, 8 in 2024-09-14T03:53:19.322 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:19 smithi070 ceph-mon[98640]: Health check update: Degraded data redundancy: 156/1095 objects degraded (14.247%), 12 pgs degraded (PG_DEGRADED) 2024-09-14T03:53:19.322 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:19 smithi070 ceph-mon[98640]: from='osd.1 [v2:172.21.15.70:6810/478220551,v1:172.21.15.70:6811/478220551]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-14T03:53:19.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:19 smithi094 ceph-mon[86597]: pgmap v157: 121 pgs: 33 active+undersized, 12 active+undersized+degraded, 76 active+clean; 456 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.5 MiB/s rd, 598 KiB/s wr, 4 op/s; 159/1107 objects degraded (14.363%) 2024-09-14T03:53:19.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:19 smithi094 ceph-mon[86597]: osdmap e377: 8 total, 7 up, 8 in 2024-09-14T03:53:19.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:19 smithi094 ceph-mon[86597]: Health check update: Degraded data redundancy: 156/1095 objects degraded (14.247%), 12 pgs degraded (PG_DEGRADED) 2024-09-14T03:53:19.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:19 smithi094 ceph-mon[86597]: from='osd.1 [v2:172.21.15.70:6810/478220551,v1:172.21.15.70:6811/478220551]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-14T03:53:19.575 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 03:53:19 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1[108413]: 2024-09-14T03:53:19.319+0000 7fde47bba640 -1 osd.1 366 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:53:20.055 INFO:tasks.rados.rados.0.smithi070.stdout:1564: done (1 left) 2024-09-14T03:53:20.056 INFO:tasks.rados.rados.0.smithi070.stdout:1565: done (0 left) 2024-09-14T03:53:20.056 INFO:tasks.rados.rados.0.smithi070.stdout:1566: snap_remove snap 76 2024-09-14T03:53:20.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:20 smithi070 ceph-mon[98640]: from='osd.1 [v2:172.21.15.70:6810/478220551,v1:172.21.15.70:6811/478220551]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-14T03:53:20.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:20 smithi070 ceph-mon[98640]: osdmap e378: 8 total, 7 up, 8 in 2024-09-14T03:53:20.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:20 smithi070 ceph-mon[98640]: from='osd.1 [v2:172.21.15.70:6810/478220551,v1:172.21.15.70:6811/478220551]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi070", "root=default"]} : dispatch 2024-09-14T03:53:20.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:20 smithi070 ceph-mon[98640]: from='osd.1 [v2:172.21.15.70:6810/478220551,v1:172.21.15.70:6811/478220551]' entity='osd.1' 2024-09-14T03:53:20.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:20 smithi070 ceph-mon[96704]: from='osd.1 [v2:172.21.15.70:6810/478220551,v1:172.21.15.70:6811/478220551]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-14T03:53:20.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:20 smithi070 ceph-mon[96704]: osdmap e378: 8 total, 7 up, 8 in 2024-09-14T03:53:20.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:20 smithi070 ceph-mon[96704]: from='osd.1 [v2:172.21.15.70:6810/478220551,v1:172.21.15.70:6811/478220551]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi070", "root=default"]} : dispatch 2024-09-14T03:53:20.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:20 smithi070 ceph-mon[96704]: from='osd.1 [v2:172.21.15.70:6810/478220551,v1:172.21.15.70:6811/478220551]' entity='osd.1' 2024-09-14T03:53:20.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:53:20 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:53:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:20.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:20 smithi094 ceph-mon[86597]: from='osd.1 [v2:172.21.15.70:6810/478220551,v1:172.21.15.70:6811/478220551]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-14T03:53:20.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:20 smithi094 ceph-mon[86597]: osdmap e378: 8 total, 7 up, 8 in 2024-09-14T03:53:20.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:20 smithi094 ceph-mon[86597]: from='osd.1 [v2:172.21.15.70:6810/478220551,v1:172.21.15.70:6811/478220551]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi070", "root=default"]} : dispatch 2024-09-14T03:53:20.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:20 smithi094 ceph-mon[86597]: from='osd.1 [v2:172.21.15.70:6810/478220551,v1:172.21.15.70:6811/478220551]' entity='osd.1' 2024-09-14T03:53:20.559 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-09-14T03:53:21.061 INFO:tasks.rados.rados.0.smithi070.stdout:1566: done (0 left) 2024-09-14T03:53:21.061 INFO:tasks.rados.rados.0.smithi070.stdout:1567: snap_create 2024-09-14T03:53:21.166 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:53:21.166 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (3m) 3s ago 7m 18.6M - 0.27.0 11f11916f8cd 45e79e51074c 2024-09-14T03:53:21.166 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi094 *:3000 running (2m) 77s ago 7m 79.5M - 9.4.7 954c08fa6188 2240810d0368 2024-09-14T03:53:21.166 INFO:teuthology.orchestra.run.smithi070.stdout:mds.a smithi070 running (9m) 3s ago 9m 20.9M - 17.2.7-1629-geb16b97b ab51d1bebd12 52e0dbb7be50 2024-09-14T03:53:21.166 INFO:teuthology.orchestra.run.smithi070.stdout:mds.b smithi094 running (9m) 77s ago 9m 15.4M - 17.2.7-1629-geb16b97b ab51d1bebd12 e3b5e3e00189 2024-09-14T03:53:21.166 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi094 *:8443,9283,8765 running (2m) 77s ago 12m 452M - 19.3.0-4907-ga2782315 8801d2529e43 569d9c7e0f06 2024-09-14T03:53:21.166 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (2m) 3s ago 14m 518M - 19.3.0-4907-ga2782315 8801d2529e43 0ef4d8e0cdbe 2024-09-14T03:53:21.167 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (117s) 3s ago 14m 52.5M 2048M 19.3.0-4907-ga2782315 8801d2529e43 96542de31cac 2024-09-14T03:53:21.167 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi094 running (82s) 77s ago 13m 21.3M 2048M 19.3.0-4907-ga2782315 8801d2529e43 86827be98659 2024-09-14T03:53:21.167 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (95s) 3s ago 13m 42.3M 2048M 19.3.0-4907-ga2782315 8801d2529e43 5741e4fcf4b6 2024-09-14T03:53:21.167 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (3m) 3s ago 7m 9801k - 1.7.0 72c9c2088986 e316760a30b3 2024-09-14T03:53:21.167 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi094 *:9100 running (2m) 77s ago 7m 7750k - 1.7.0 72c9c2088986 767f8a093175 2024-09-14T03:53:21.167 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (41s) 3s ago 12m 136M 2480M 19.3.0-4907-ga2782315 8801d2529e43 2345ccac2085 2024-09-14T03:53:21.167 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (7s) 3s ago 12m 12.2M 2480M 19.3.0-4907-ga2782315 8801d2529e43 53868856512a 2024-09-14T03:53:21.167 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (11m) 3s ago 11m 189M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 109b66e4e60f 2024-09-14T03:53:21.167 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (11m) 3s ago 11m 152M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 2e4199b58392 2024-09-14T03:53:21.167 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi094 running (11m) 77s ago 11m 180M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 52b64527e24c 2024-09-14T03:53:21.167 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi094 running (10m) 77s ago 10m 168M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 b543f152ba87 2024-09-14T03:53:21.167 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi094 running (10m) 77s ago 10m 114M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 e66b592ac940 2024-09-14T03:53:21.167 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi094 running (9m) 77s ago 9m 152M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 037c6208e9ce 2024-09-14T03:53:21.168 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi094 *:9095 running (2m) 77s ago 8m 46.3M - 2.51.0 1d3b7f56885b ce1b353828dc 2024-09-14T03:53:21.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:21 smithi094 ceph-mon[86597]: pgmap v159: 121 pgs: 2 active+clean+snaptrim, 33 active+undersized, 12 active+undersized+degraded, 74 active+clean; 462 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.4 MiB/s wr, 30 op/s; 168/1143 objects degraded (14.698%) 2024-09-14T03:53:21.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:21 smithi094 ceph-mon[86597]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:53:21.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:21 smithi094 ceph-mon[86597]: osd.1 [v2:172.21.15.70:6810/478220551,v1:172.21.15.70:6811/478220551] boot 2024-09-14T03:53:21.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:21 smithi094 ceph-mon[86597]: osdmap e379: 8 total, 8 up, 8 in 2024-09-14T03:53:21.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:21 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-14T03:53:21.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:21 smithi094 ceph-mon[86597]: osdmap e380: 8 total, 8 up, 8 in 2024-09-14T03:53:21.535 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:53:21 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:53:21.214+0000 7fa4af5ba640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-14T03:53:21.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:21 smithi070 ceph-mon[96704]: pgmap v159: 121 pgs: 2 active+clean+snaptrim, 33 active+undersized, 12 active+undersized+degraded, 74 active+clean; 462 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.4 MiB/s wr, 30 op/s; 168/1143 objects degraded (14.698%) 2024-09-14T03:53:21.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:21 smithi070 ceph-mon[96704]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:53:21.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:21 smithi070 ceph-mon[96704]: osd.1 [v2:172.21.15.70:6810/478220551,v1:172.21.15.70:6811/478220551] boot 2024-09-14T03:53:21.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:21 smithi070 ceph-mon[96704]: osdmap e379: 8 total, 8 up, 8 in 2024-09-14T03:53:21.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:21 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-14T03:53:21.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:21 smithi070 ceph-mon[96704]: osdmap e380: 8 total, 8 up, 8 in 2024-09-14T03:53:21.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:21 smithi070 ceph-mon[98640]: pgmap v159: 121 pgs: 2 active+clean+snaptrim, 33 active+undersized, 12 active+undersized+degraded, 74 active+clean; 462 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.4 MiB/s wr, 30 op/s; 168/1143 objects degraded (14.698%) 2024-09-14T03:53:21.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:21 smithi070 ceph-mon[98640]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:53:21.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:21 smithi070 ceph-mon[98640]: osd.1 [v2:172.21.15.70:6810/478220551,v1:172.21.15.70:6811/478220551] boot 2024-09-14T03:53:21.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:21 smithi070 ceph-mon[98640]: osdmap e379: 8 total, 8 up, 8 in 2024-09-14T03:53:21.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:21 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-14T03:53:21.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:21 smithi070 ceph-mon[98640]: osdmap e380: 8 total, 8 up, 8 in 2024-09-14T03:53:21.564 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:53:21.564 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-09-14T03:53:21.564 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:53:21.564 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:53:21.565 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-09-14T03:53:21.565 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:53:21.565 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:53:21.565 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-09-14T03:53:21.565 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 6, 2024-09-14T03:53:21.565 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:53:21.565 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:53:21.565 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": { 2024-09-14T03:53:21.565 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:53:21.565 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:53:21.565 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-09-14T03:53:21.566 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 8, 2024-09-14T03:53:21.566 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 7 2024-09-14T03:53:21.566 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-09-14T03:53:21.566 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:53:21.845 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:53:21.845 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:53:21.845 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-09-14T03:53:21.845 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:53:21.845 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [ 2024-09-14T03:53:21.845 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr", 2024-09-14T03:53:21.845 INFO:teuthology.orchestra.run.smithi070.stdout: "mon" 2024-09-14T03:53:21.846 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2024-09-14T03:53:21.846 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "7/20 daemons upgraded", 2024-09-14T03:53:21.846 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "Currently upgrading osd daemons", 2024-09-14T03:53:21.846 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-09-14T03:53:21.846 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:53:22.057 INFO:tasks.rados.rados.0.smithi070.stdout:1567: done (0 left) 2024-09-14T03:53:22.057 INFO:tasks.rados.rados.0.smithi070.stdout:1568: read oid 28 snap -1 2024-09-14T03:53:22.058 INFO:tasks.rados.rados.0.smithi070.stdout:1568: expect (ObjNum 472 snap 160 seq_num 472) 2024-09-14T03:53:22.058 INFO:tasks.rados.rados.0.smithi070.stdout:1569: read oid 11 snap -1 2024-09-14T03:53:22.058 INFO:tasks.rados.rados.0.smithi070.stdout:1569: expect (ObjNum 506 snap 165 seq_num 506) 2024-09-14T03:53:22.058 INFO:tasks.rados.rados.0.smithi070.stdout:1570: rmattr oid 36 current snap is 171 2024-09-14T03:53:22.058 INFO:tasks.rados.rados.0.smithi070.stdout:1570: done (2 left) 2024-09-14T03:53:22.058 INFO:tasks.rados.rados.0.smithi070.stdout:1571: delete oid 19 current snap is 171 2024-09-14T03:53:22.063 INFO:tasks.rados.rados.0.smithi070.stdout:1571: done (2 left) 2024-09-14T03:53:22.063 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 24 current snap is 171 2024-09-14T03:53:22.064 INFO:tasks.rados.rados.0.smithi070.stdout:1572: seq_num 513 ranges {0=598016,598016=794624,1392640=8192} 2024-09-14T03:53:22.067 INFO:tasks.rados.rados.0.smithi070.stdout:1572: writing smithi07085154-24 from 0 to 598016 tid 1 2024-09-14T03:53:22.073 INFO:tasks.rados.rados.0.smithi070.stdout:1572: writing smithi07085154-24 from 598016 to 1392640 tid 2 2024-09-14T03:53:22.073 INFO:tasks.rados.rados.0.smithi070.stdout:1572: writing smithi07085154-24 from 1392640 to 1400832 tid 3 2024-09-14T03:53:22.073 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 41 current snap is 171 2024-09-14T03:53:22.073 INFO:tasks.rados.rados.0.smithi070.stdout:1573: seq_num 514 ranges {2195456=614400,2809856=679936,3489792=122880} 2024-09-14T03:53:22.089 INFO:tasks.rados.rados.0.smithi070.stdout:1573: writing smithi07085154-41 from 2195456 to 2809856 tid 1 2024-09-14T03:53:22.093 INFO:tasks.rados.rados.0.smithi070.stdout:1573: writing smithi07085154-41 from 2809856 to 3489792 tid 2 2024-09-14T03:53:22.094 INFO:tasks.rados.rados.0.smithi070.stdout:1573: writing smithi07085154-41 from 3489792 to 3612672 tid 3 2024-09-14T03:53:22.094 INFO:tasks.rados.rados.0.smithi070.stdout:1574: snap_remove snap 148 2024-09-14T03:53:22.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:22 smithi070 ceph-mon[96704]: from='client.44174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:22.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:22 smithi070 ceph-mon[96704]: from='client.34266 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:22.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:22 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:22.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:22 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:22.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:22 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:22.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:22 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:53:22.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:22 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:22.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:22 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:53:22.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:22 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:22.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:22 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:22.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:22 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:22.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:22 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:53:22.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:22 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/2500795296' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:22.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:22 smithi070 ceph-mon[96704]: osdmap e381: 8 total, 8 up, 8 in 2024-09-14T03:53:22.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:22 smithi070 ceph-mon[98640]: from='client.44174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:22.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:22 smithi070 ceph-mon[98640]: from='client.34266 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:22.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:22 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:22.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:22 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:22.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:22 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:22.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:22 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:53:22.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:22 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:22.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:22 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:53:22.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:22 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:22.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:22 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:22.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:22 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:22.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:22 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:53:22.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:22 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/2500795296' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:22.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:22 smithi070 ceph-mon[98640]: osdmap e381: 8 total, 8 up, 8 in 2024-09-14T03:53:22.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:22 smithi094 ceph-mon[86597]: from='client.44174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:22.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:22 smithi094 ceph-mon[86597]: from='client.34266 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:22.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:22 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:22.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:22 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:22.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:22 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:22.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:22 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:53:22.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:22 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:22.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:22 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:53:22.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:22 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:22.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:22 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:22.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:22 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:22.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:22 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:53:22.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:22 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/2500795296' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:22.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:22 smithi094 ceph-mon[86597]: osdmap e381: 8 total, 8 up, 8 in 2024-09-14T03:53:23.099 INFO:tasks.rados.rados.0.smithi070.stdout:1572: finishing write tid 1 to smithi07085154-24 2024-09-14T03:53:23.099 INFO:tasks.rados.rados.0.smithi070.stdout:1572: finishing write tid 2 to smithi07085154-24 2024-09-14T03:53:23.099 INFO:tasks.rados.rados.0.smithi070.stdout:1572: finishing write tid 3 to smithi07085154-24 2024-09-14T03:53:23.099 INFO:tasks.rados.rados.0.smithi070.stdout:1572: finishing write tid 4 to smithi07085154-24 2024-09-14T03:53:23.099 INFO:tasks.rados.rados.0.smithi070.stdout:1572: finishing write tid 5 to smithi07085154-24 2024-09-14T03:53:23.099 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 617 (ObjNum 513 snap 171 seq_num 513) dirty exists 2024-09-14T03:53:23.099 INFO:tasks.rados.rados.0.smithi070.stdout:1572: left oid 24 (ObjNum 513 snap 171 seq_num 513) 2024-09-14T03:53:23.099 INFO:tasks.rados.rados.0.smithi070.stdout:1569: expect (ObjNum 506 snap 165 seq_num 506) 2024-09-14T03:53:23.292 INFO:tasks.rados.rados.0.smithi070.stdout:1568: expect (ObjNum 472 snap 160 seq_num 472) 2024-09-14T03:53:23.406 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:23 smithi070 ceph-mon[96704]: from='client.34272 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:23.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:23 smithi070 ceph-mon[96704]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:53:23.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:23 smithi070 ceph-mon[96704]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-14T03:53:23.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:23 smithi070 ceph-mon[96704]: from='client.44198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:23.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:23 smithi070 ceph-mon[96704]: pgmap v162: 121 pgs: 6 activating, 2 active+clean+snaptrim, 28 active+undersized, 11 active+undersized+degraded, 74 active+clean; 462 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.8 MiB/s wr, 26 op/s; 156/1143 objects degraded (13.648%) 2024-09-14T03:53:23.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:23 smithi070 ceph-mon[98640]: from='client.34272 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:23.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:23 smithi070 ceph-mon[98640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:53:23.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:23 smithi070 ceph-mon[98640]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-14T03:53:23.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:23 smithi070 ceph-mon[98640]: from='client.44198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:23.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:23 smithi070 ceph-mon[98640]: pgmap v162: 121 pgs: 6 activating, 2 active+clean+snaptrim, 28 active+undersized, 11 active+undersized+degraded, 74 active+clean; 462 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.8 MiB/s wr, 26 op/s; 156/1143 objects degraded (13.648%) 2024-09-14T03:53:23.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:23 smithi094 ceph-mon[86597]: from='client.34272 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:23.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:23 smithi094 ceph-mon[86597]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:53:23.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:23 smithi094 ceph-mon[86597]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-14T03:53:23.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:23 smithi094 ceph-mon[86597]: from='client.44198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:23.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:23 smithi094 ceph-mon[86597]: pgmap v162: 121 pgs: 6 activating, 2 active+clean+snaptrim, 28 active+undersized, 11 active+undersized+degraded, 74 active+clean; 462 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.8 MiB/s wr, 26 op/s; 156/1143 objects degraded (13.648%) 2024-09-14T03:53:23.678 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:53:23 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:53:23] "GET /metrics HTTP/1.1" 200 36319 "" "Prometheus/2.51.0" 2024-09-14T03:53:24.053 INFO:tasks.rados.rados.0.smithi070.stdout:1573: finishing write tid 1 to smithi07085154-41 2024-09-14T03:53:24.053 INFO:tasks.rados.rados.0.smithi070.stdout:1573: finishing write tid 2 to smithi07085154-41 2024-09-14T03:53:24.053 INFO:tasks.rados.rados.0.smithi070.stdout:1573: finishing write tid 3 to smithi07085154-41 2024-09-14T03:53:24.053 INFO:tasks.rados.rados.0.smithi070.stdout:1573: finishing write tid 4 to smithi07085154-41 2024-09-14T03:53:24.053 INFO:tasks.rados.rados.0.smithi070.stdout:1573: finishing write tid 5 to smithi07085154-41 2024-09-14T03:53:24.053 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 1086 (ObjNum 514 snap 171 seq_num 514) dirty exists 2024-09-14T03:53:24.053 INFO:tasks.rados.rados.0.smithi070.stdout:1573: left oid 41 (ObjNum 514 snap 171 seq_num 514) 2024-09-14T03:53:24.053 INFO:tasks.rados.rados.0.smithi070.stdout:1568: done (4 left) 2024-09-14T03:53:24.053 INFO:tasks.rados.rados.0.smithi070.stdout:1569: done (3 left) 2024-09-14T03:53:24.054 INFO:tasks.rados.rados.0.smithi070.stdout:1572: done (2 left) 2024-09-14T03:53:24.054 INFO:tasks.rados.rados.0.smithi070.stdout:1573: done (1 left) 2024-09-14T03:53:24.054 INFO:tasks.rados.rados.0.smithi070.stdout:1574: done (0 left) 2024-09-14T03:53:24.054 INFO:tasks.rados.rados.0.smithi070.stdout:1575: snap_remove snap 158 2024-09-14T03:53:24.104 INFO:tasks.rados.rados.0.smithi070.stdout:1575: done (0 left) 2024-09-14T03:53:24.105 INFO:tasks.rados.rados.0.smithi070.stdout:1576: snap_create 2024-09-14T03:53:24.444 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:24 smithi070 ceph-mon[96704]: osdmap e382: 8 total, 8 up, 8 in 2024-09-14T03:53:24.444 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:24 smithi070 ceph-mon[96704]: Health check update: Degraded data redundancy: 156/1143 objects degraded (13.648%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T03:53:24.445 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:24 smithi070 ceph-mon[96704]: pgmap v165: 121 pgs: 6 activating, 11 active+undersized, 5 active+undersized+degraded, 99 active+clean; 444 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 688 KiB/s wr, 3 op/s; 26/1083 objects degraded (2.401%); 17 MiB/s, 6 objects/s recovering 2024-09-14T03:53:24.445 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:24 smithi070 ceph-mon[98640]: osdmap e382: 8 total, 8 up, 8 in 2024-09-14T03:53:24.445 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:24 smithi070 ceph-mon[98640]: Health check update: Degraded data redundancy: 156/1143 objects degraded (13.648%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T03:53:24.445 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:24 smithi070 ceph-mon[98640]: pgmap v165: 121 pgs: 6 activating, 11 active+undersized, 5 active+undersized+degraded, 99 active+clean; 444 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 688 KiB/s wr, 3 op/s; 26/1083 objects degraded (2.401%); 17 MiB/s, 6 objects/s recovering 2024-09-14T03:53:24.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:24 smithi094 ceph-mon[86597]: osdmap e382: 8 total, 8 up, 8 in 2024-09-14T03:53:24.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:24 smithi094 ceph-mon[86597]: Health check update: Degraded data redundancy: 156/1143 objects degraded (13.648%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T03:53:24.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:24 smithi094 ceph-mon[86597]: pgmap v165: 121 pgs: 6 activating, 11 active+undersized, 5 active+undersized+degraded, 99 active+clean; 444 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 688 KiB/s wr, 3 op/s; 26/1083 objects degraded (2.401%); 17 MiB/s, 6 objects/s recovering 2024-09-14T03:53:24.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:53:24 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:53:24.147Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.1\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-14T03:53:24.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:53:24 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:53:24.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.1\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:25.106 INFO:tasks.rados.rados.0.smithi070.stdout:1576: done (0 left) 2024-09-14T03:53:25.106 INFO:tasks.rados.rados.0.smithi070.stdout:1577: read oid 50 snap -1 2024-09-14T03:53:25.107 INFO:tasks.rados.rados.0.smithi070.stdout:1577: expect (ObjNum 430 snap 148 seq_num 430) 2024-09-14T03:53:25.107 INFO:tasks.rados.rados.0.smithi070.stdout:1578: read oid 17 snap 107 2024-09-14T03:53:25.107 INFO:tasks.rados.rados.0.smithi070.stdout:1578: expect (ObjNum 325 snap 107 seq_num 325) 2024-09-14T03:53:25.107 INFO:tasks.rados.rados.0.smithi070.stdout:1579: read oid 21 snap -1 2024-09-14T03:53:25.107 INFO:tasks.rados.rados.0.smithi070.stdout:1579: expect (ObjNum 501 snap 165 seq_num 501) 2024-09-14T03:53:25.107 INFO:tasks.rados.rados.0.smithi070.stdout:1580: read oid 5 snap 120 2024-09-14T03:53:25.107 INFO:tasks.rados.rados.0.smithi070.stdout:1580: expect (ObjNum 114 snap 24 seq_num 114) 2024-09-14T03:53:25.107 INFO:tasks.rados.rados.0.smithi070.stdout:1581: delete oid 6 current snap is 172 2024-09-14T03:53:25.122 INFO:tasks.rados.rados.0.smithi070.stdout:1578: expect (ObjNum 325 snap 107 seq_num 325) 2024-09-14T03:53:25.225 INFO:tasks.rados.rados.0.smithi070.stdout:1580: expect (ObjNum 114 snap 24 seq_num 114) 2024-09-14T03:53:25.418 INFO:tasks.rados.rados.0.smithi070.stdout:1577: expect (ObjNum 430 snap 148 seq_num 430) 2024-09-14T03:53:25.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:25 smithi094 ceph-mon[86597]: osdmap e383: 8 total, 8 up, 8 in 2024-09-14T03:53:25.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:25 smithi070 ceph-mon[96704]: osdmap e383: 8 total, 8 up, 8 in 2024-09-14T03:53:25.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:25 smithi070 ceph-mon[98640]: osdmap e383: 8 total, 8 up, 8 in 2024-09-14T03:53:25.600 INFO:tasks.rados.rados.0.smithi070.stdout:1579: expect (ObjNum 501 snap 165 seq_num 501) 2024-09-14T03:53:25.899 INFO:tasks.rados.rados.0.smithi070.stdout:1577: done (4 left) 2024-09-14T03:53:25.899 INFO:tasks.rados.rados.0.smithi070.stdout:1578: done (3 left) 2024-09-14T03:53:25.899 INFO:tasks.rados.rados.0.smithi070.stdout:1579: done (2 left) 2024-09-14T03:53:25.899 INFO:tasks.rados.rados.0.smithi070.stdout:1580: done (1 left) 2024-09-14T03:53:25.899 INFO:tasks.rados.rados.0.smithi070.stdout:1581: done (0 left) 2024-09-14T03:53:25.899 INFO:tasks.rados.rados.0.smithi070.stdout:1582: snap_create 2024-09-14T03:53:26.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:53:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:53:25.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:26.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:53:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:53:25.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:26.114 INFO:tasks.rados.rados.0.smithi070.stdout:1582: done (0 left) 2024-09-14T03:53:26.114 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 5 current snap is 173 2024-09-14T03:53:26.115 INFO:tasks.rados.rados.0.smithi070.stdout:1583: seq_num 516 ranges {466944=647168,1114112=450560,1564672=352256} 2024-09-14T03:53:26.120 INFO:tasks.rados.rados.0.smithi070.stdout:1583: writing smithi07085154-5 from 466944 to 1114112 tid 1 2024-09-14T03:53:26.123 INFO:tasks.rados.rados.0.smithi070.stdout:1583: writing smithi07085154-5 from 1114112 to 1564672 tid 2 2024-09-14T03:53:26.126 INFO:tasks.rados.rados.0.smithi070.stdout:1583: writing smithi07085154-5 from 1564672 to 1916928 tid 3 2024-09-14T03:53:26.126 INFO:tasks.rados.rados.0.smithi070.stdout:1583: finishing write tid 1 to smithi07085154-5 2024-09-14T03:53:26.126 INFO:tasks.rados.rados.0.smithi070.stdout:1584: rollback oid 11 current snap is 173 2024-09-14T03:53:26.126 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 11 to 145 2024-09-14T03:53:26.126 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 4 current snap is 173 2024-09-14T03:53:26.126 INFO:tasks.rados.rados.0.smithi070.stdout:1585: seq_num 517 ranges {2506752=663552,3170304=786432,3956736=16384} 2024-09-14T03:53:26.143 INFO:tasks.rados.rados.0.smithi070.stdout:1585: writing smithi07085154-4 from 2506752 to 3170304 tid 1 2024-09-14T03:53:26.148 INFO:tasks.rados.rados.0.smithi070.stdout:1585: writing smithi07085154-4 from 3170304 to 3956736 tid 2 2024-09-14T03:53:26.148 INFO:tasks.rados.rados.0.smithi070.stdout:1585: writing smithi07085154-4 from 3956736 to 3973120 tid 3 2024-09-14T03:53:26.149 INFO:tasks.rados.rados.0.smithi070.stdout:1586: delete oid 30 current snap is 173 2024-09-14T03:53:26.149 INFO:tasks.rados.rados.0.smithi070.stdout:1583: finishing write tid 2 to smithi07085154-5 2024-09-14T03:53:26.149 INFO:tasks.rados.rados.0.smithi070.stdout:1583: finishing write tid 3 to smithi07085154-5 2024-09-14T03:53:26.149 INFO:tasks.rados.rados.0.smithi070.stdout:1584: finishing rollback tid 0 to smithi07085154-11 2024-09-14T03:53:26.149 INFO:tasks.rados.rados.0.smithi070.stdout:1583: finishing write tid 4 to smithi07085154-5 2024-09-14T03:53:26.149 INFO:tasks.rados.rados.0.smithi070.stdout:1584: finishing rollback tid 1 to smithi07085154-11 2024-09-14T03:53:26.149 INFO:tasks.rados.rados.0.smithi070.stdout:1584: finishing rollback tid 2 to smithi07085154-11 2024-09-14T03:53:26.149 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 11 v 1184 (ObjNum 293 snap 91 seq_num 293) dirty exists 2024-09-14T03:53:26.149 INFO:tasks.rados.rados.0.smithi070.stdout:1583: finishing write tid 5 to smithi07085154-5 2024-09-14T03:53:26.149 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 558 (ObjNum 516 snap 173 seq_num 516) dirty exists 2024-09-14T03:53:26.149 INFO:tasks.rados.rados.0.smithi070.stdout:1583: left oid 5 (ObjNum 516 snap 173 seq_num 516) 2024-09-14T03:53:26.149 INFO:tasks.rados.rados.0.smithi070.stdout:1585: finishing write tid 1 to smithi07085154-4 2024-09-14T03:53:26.152 INFO:tasks.rados.rados.0.smithi070.stdout:1585: finishing write tid 2 to smithi07085154-4 2024-09-14T03:53:26.152 INFO:tasks.rados.rados.0.smithi070.stdout:1585: finishing write tid 3 to smithi07085154-4 2024-09-14T03:53:26.152 INFO:tasks.rados.rados.0.smithi070.stdout:1585: finishing write tid 4 to smithi07085154-4 2024-09-14T03:53:26.153 INFO:tasks.rados.rados.0.smithi070.stdout:1583: done (3 left) 2024-09-14T03:53:26.153 INFO:tasks.rados.rados.0.smithi070.stdout:1584: done (2 left) 2024-09-14T03:53:26.154 INFO:tasks.rados.rados.0.smithi070.stdout:1586: done (1 left) 2024-09-14T03:53:26.154 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 15 current snap is 173 2024-09-14T03:53:26.154 INFO:tasks.rados.rados.0.smithi070.stdout:1587: seq_num 519 ranges {0=696320,696320=548864,1245184=253952} 2024-09-14T03:53:26.157 INFO:tasks.rados.rados.0.smithi070.stdout:1587: writing smithi07085154-15 from 0 to 696320 tid 1 2024-09-14T03:53:26.160 INFO:tasks.rados.rados.0.smithi070.stdout:1587: writing smithi07085154-15 from 696320 to 1245184 tid 2 2024-09-14T03:53:26.162 INFO:tasks.rados.rados.0.smithi070.stdout:1587: writing smithi07085154-15 from 1245184 to 1499136 tid 3 2024-09-14T03:53:26.162 INFO:tasks.rados.rados.0.smithi070.stdout:1588: delete oid 34 current snap is 173 2024-09-14T03:53:26.163 INFO:tasks.rados.rados.0.smithi070.stdout:1585: finishing write tid 5 to smithi07085154-4 2024-09-14T03:53:26.163 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 1226 (ObjNum 517 snap 173 seq_num 517) dirty exists 2024-09-14T03:53:26.163 INFO:tasks.rados.rados.0.smithi070.stdout:1585: left oid 4 (ObjNum 517 snap 173 seq_num 517) 2024-09-14T03:53:26.163 INFO:tasks.rados.rados.0.smithi070.stdout:1587: finishing write tid 1 to smithi07085154-15 2024-09-14T03:53:26.165 INFO:tasks.rados.rados.0.smithi070.stdout:1585: done (2 left) 2024-09-14T03:53:26.165 INFO:tasks.rados.rados.0.smithi070.stdout:1588: done (1 left) 2024-09-14T03:53:26.165 INFO:tasks.rados.rados.0.smithi070.stdout:1589: rollback oid 13 current snap is 173 2024-09-14T03:53:26.165 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 13 to 150 2024-09-14T03:53:26.165 INFO:tasks.rados.rados.0.smithi070.stdout:1587: finishing write tid 2 to smithi07085154-15 2024-09-14T03:53:26.165 INFO:tasks.rados.rados.0.smithi070.stdout:1590: snap_create 2024-09-14T03:53:26.165 INFO:tasks.rados.rados.0.smithi070.stdout:1587: finishing write tid 3 to smithi07085154-15 2024-09-14T03:53:26.165 INFO:tasks.rados.rados.0.smithi070.stdout:1587: finishing write tid 4 to smithi07085154-15 2024-09-14T03:53:26.166 INFO:tasks.rados.rados.0.smithi070.stdout:1587: finishing write tid 5 to smithi07085154-15 2024-09-14T03:53:26.166 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 15 v 1114 (ObjNum 519 snap 173 seq_num 519) dirty exists 2024-09-14T03:53:26.166 INFO:tasks.rados.rados.0.smithi070.stdout:1587: left oid 15 (ObjNum 519 snap 173 seq_num 519) 2024-09-14T03:53:26.167 INFO:tasks.rados.rados.0.smithi070.stdout:1589: finishing rollback tid 1 to smithi07085154-13 2024-09-14T03:53:26.168 INFO:tasks.rados.rados.0.smithi070.stdout:1589: finishing rollback tid 2 to smithi07085154-13 2024-09-14T03:53:26.168 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 1186 (ObjNum 403 snap 140 seq_num 403) dirty exists 2024-09-14T03:53:26.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:26 smithi094 ceph-mon[86597]: osdmap e384: 8 total, 8 up, 8 in 2024-09-14T03:53:26.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:26 smithi094 ceph-mon[86597]: pgmap v168: 121 pgs: 6 activating, 115 active+clean; 442 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 688 KiB/s wr, 3 op/s; 6/1077 objects degraded (0.557%); 17 MiB/s, 6 objects/s recovering 2024-09-14T03:53:26.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:26 smithi070 ceph-mon[96704]: osdmap e384: 8 total, 8 up, 8 in 2024-09-14T03:53:26.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:26 smithi070 ceph-mon[96704]: pgmap v168: 121 pgs: 6 activating, 115 active+clean; 442 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 688 KiB/s wr, 3 op/s; 6/1077 objects degraded (0.557%); 17 MiB/s, 6 objects/s recovering 2024-09-14T03:53:26.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:26 smithi070 ceph-mon[98640]: osdmap e384: 8 total, 8 up, 8 in 2024-09-14T03:53:26.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:26 smithi070 ceph-mon[98640]: pgmap v168: 121 pgs: 6 activating, 115 active+clean; 442 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 688 KiB/s wr, 3 op/s; 6/1077 objects degraded (0.557%); 17 MiB/s, 6 objects/s recovering 2024-09-14T03:53:27.116 INFO:tasks.rados.rados.0.smithi070.stdout:1587: done (2 left) 2024-09-14T03:53:27.116 INFO:tasks.rados.rados.0.smithi070.stdout:1589: done (1 left) 2024-09-14T03:53:27.116 INFO:tasks.rados.rados.0.smithi070.stdout:1590: done (0 left) 2024-09-14T03:53:27.116 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 38 current snap is 174 2024-09-14T03:53:27.116 INFO:tasks.rados.rados.0.smithi070.stdout:1591: seq_num 521 ranges {1916928=729088,2646016=720896,3366912=81920} 2024-09-14T03:53:27.130 INFO:tasks.rados.rados.0.smithi070.stdout:1591: writing smithi07085154-38 from 1916928 to 2646016 tid 1 2024-09-14T03:53:27.134 INFO:tasks.rados.rados.0.smithi070.stdout:1591: writing smithi07085154-38 from 2646016 to 3366912 tid 2 2024-09-14T03:53:27.135 INFO:tasks.rados.rados.0.smithi070.stdout:1591: writing smithi07085154-38 from 3366912 to 3448832 tid 3 2024-09-14T03:53:27.135 INFO:tasks.rados.rados.0.smithi070.stdout:1592: rollback oid 50 current snap is 174 2024-09-14T03:53:27.135 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 50 to 102 2024-09-14T03:53:27.135 INFO:tasks.rados.rados.0.smithi070.stdout:1593: setattr oid 41 current snap is 174 2024-09-14T03:53:27.138 INFO:tasks.rados.rados.0.smithi070.stdout:1594: rollback oid 7 current snap is 174 2024-09-14T03:53:27.138 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 7 to 126 2024-09-14T03:53:27.138 INFO:tasks.rados.rados.0.smithi070.stdout:1595: snap_remove snap 166 2024-09-14T03:53:27.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:27 smithi094 ceph-mon[86597]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 26/1083 objects degraded (2.401%), 5 pgs degraded) 2024-09-14T03:53:27.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:27 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T03:53:27.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:27 smithi094 ceph-mon[86597]: osdmap e385: 8 total, 8 up, 8 in 2024-09-14T03:53:27.277 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:53:26 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:53:26.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:27.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:27 smithi070 ceph-mon[96704]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 26/1083 objects degraded (2.401%), 5 pgs degraded) 2024-09-14T03:53:27.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:27 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T03:53:27.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:27 smithi070 ceph-mon[96704]: osdmap e385: 8 total, 8 up, 8 in 2024-09-14T03:53:27.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:27 smithi070 ceph-mon[98640]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 26/1083 objects degraded (2.401%), 5 pgs degraded) 2024-09-14T03:53:27.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:27 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T03:53:27.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:27 smithi070 ceph-mon[98640]: osdmap e385: 8 total, 8 up, 8 in 2024-09-14T03:53:28.126 INFO:tasks.rados.rados.0.smithi070.stdout:1595: done (4 left) 2024-09-14T03:53:28.126 INFO:tasks.rados.rados.0.smithi070.stdout:1596: rollback oid 14 current snap is 174 2024-09-14T03:53:28.126 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 14 to 169 2024-09-14T03:53:28.126 INFO:tasks.rados.rados.0.smithi070.stdout:1591: finishing write tid 1 to smithi07085154-38 2024-09-14T03:53:28.127 INFO:tasks.rados.rados.0.smithi070.stdout:1591: finishing write tid 2 to smithi07085154-38 2024-09-14T03:53:28.127 INFO:tasks.rados.rados.0.smithi070.stdout:1591: finishing write tid 3 to smithi07085154-38 2024-09-14T03:53:28.127 INFO:tasks.rados.rados.0.smithi070.stdout:1591: finishing write tid 4 to smithi07085154-38 2024-09-14T03:53:28.127 INFO:tasks.rados.rados.0.smithi070.stdout:1592: finishing rollback tid 0 to smithi07085154-50 2024-09-14T03:53:28.127 INFO:tasks.rados.rados.0.smithi070.stdout:1592: finishing rollback tid 1 to smithi07085154-50 2024-09-14T03:53:28.127 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 908 (ObjNum 33 snap 0 seq_num 2930099152) dirty dne 2024-09-14T03:53:28.127 INFO:tasks.rados.rados.0.smithi070.stdout:1591: finishing write tid 5 to smithi07085154-38 2024-09-14T03:53:28.127 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 38 v 1003 (ObjNum 521 snap 174 seq_num 521) dirty exists 2024-09-14T03:53:28.127 INFO:tasks.rados.rados.0.smithi070.stdout:1591: left oid 38 (ObjNum 521 snap 174 seq_num 521) 2024-09-14T03:53:28.127 INFO:tasks.rados.rados.0.smithi070.stdout:1594: finishing rollback tid 1 to smithi07085154-7 2024-09-14T03:53:28.127 INFO:tasks.rados.rados.0.smithi070.stdout:1594: finishing rollback tid 2 to smithi07085154-7 2024-09-14T03:53:28.127 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 1228 (ObjNum 325 snap 107 seq_num 325) dirty exists 2024-09-14T03:53:28.128 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 1116 (ObjNum 514 snap 171 seq_num 514) dirty exists 2024-09-14T03:53:28.128 INFO:tasks.rados.rados.0.smithi070.stdout:1591: done (4 left) 2024-09-14T03:53:28.128 INFO:tasks.rados.rados.0.smithi070.stdout:1592: done (3 left) 2024-09-14T03:53:28.128 INFO:tasks.rados.rados.0.smithi070.stdout:1593: done (2 left) 2024-09-14T03:53:28.128 INFO:tasks.rados.rados.0.smithi070.stdout:1594: done (1 left) 2024-09-14T03:53:28.128 INFO:tasks.rados.rados.0.smithi070.stdout:1597: delete oid 31 current snap is 174 2024-09-14T03:53:28.130 INFO:tasks.rados.rados.0.smithi070.stdout:1597: done (1 left) 2024-09-14T03:53:28.130 INFO:tasks.rados.rados.0.smithi070.stdout:1598: snap_remove snap 170 2024-09-14T03:53:28.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:28 smithi094 ceph-mon[86597]: osdmap e386: 8 total, 8 up, 8 in 2024-09-14T03:53:28.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:28 smithi094 ceph-mon[86597]: pgmap v171: 121 pgs: 121 active+clean; 440 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 0 op/s; 7.4 MiB/s, 1 objects/s recovering 2024-09-14T03:53:28.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:28 smithi070 ceph-mon[96704]: osdmap e386: 8 total, 8 up, 8 in 2024-09-14T03:53:28.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:28 smithi070 ceph-mon[96704]: pgmap v171: 121 pgs: 121 active+clean; 440 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 0 op/s; 7.4 MiB/s, 1 objects/s recovering 2024-09-14T03:53:28.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:28 smithi070 ceph-mon[98640]: osdmap e386: 8 total, 8 up, 8 in 2024-09-14T03:53:28.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:28 smithi070 ceph-mon[98640]: pgmap v171: 121 pgs: 121 active+clean; 440 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 0 op/s; 7.4 MiB/s, 1 objects/s recovering 2024-09-14T03:53:29.017 INFO:tasks.rados.rados.0.smithi070.stdout:1598: done (1 left) 2024-09-14T03:53:29.017 INFO:tasks.rados.rados.0.smithi070.stdout:1599: read oid 23 snap 150 2024-09-14T03:53:29.017 INFO:tasks.rados.rados.0.smithi070.stdout:1599: expect (ObjNum 431 snap 148 seq_num 431) 2024-09-14T03:53:29.017 INFO:tasks.rados.rados.0.smithi070.stdout:1600: copy_from oid 3 from oid 39 current snap is 174 2024-09-14T03:53:29.017 INFO:tasks.rados.rados.0.smithi070.stdout:1596: finishing rollback tid 0 to smithi07085154-14 2024-09-14T03:53:29.017 INFO:tasks.rados.rados.0.smithi070.stdout:1596: finishing rollback tid 1 to smithi07085154-14 2024-09-14T03:53:29.017 INFO:tasks.rados.rados.0.smithi070.stdout:1596: finishing rollback tid 2 to smithi07085154-14 2024-09-14T03:53:29.017 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 620 (ObjNum 398 snap 138 seq_num 398) dirty exists 2024-09-14T03:53:29.017 INFO:tasks.rados.rados.0.smithi070.stdout:1596: done (2 left) 2024-09-14T03:53:29.017 INFO:tasks.rados.rados.0.smithi070.stdout:1601: delete oid 16 current snap is 174 2024-09-14T03:53:29.027 INFO:tasks.rados.rados.0.smithi070.stdout:1601: done (2 left) 2024-09-14T03:53:29.027 INFO:tasks.rados.rados.0.smithi070.stdout:1602: read oid 36 snap 73 2024-09-14T03:53:29.028 INFO:tasks.rados.rados.0.smithi070.stdout:1602: expect (ObjNum 241 snap 69 seq_num 241) 2024-09-14T03:53:29.028 INFO:tasks.rados.rados.0.smithi070.stdout:1603: read oid 1 snap -1 2024-09-14T03:53:29.028 INFO:tasks.rados.rados.0.smithi070.stdout:1603: expect deleted 2024-09-14T03:53:29.028 INFO:tasks.rados.rados.0.smithi070.stdout:1604: read oid 5 snap -1 2024-09-14T03:53:29.028 INFO:tasks.rados.rados.0.smithi070.stdout:1604: expect (ObjNum 516 snap 173 seq_num 516) 2024-09-14T03:53:29.028 INFO:tasks.rados.rados.0.smithi070.stdout:1605: snap_remove snap 169 2024-09-14T03:53:29.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:29 smithi094 ceph-mon[86597]: osdmap e387: 8 total, 8 up, 8 in 2024-09-14T03:53:29.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:29 smithi094 ceph-mon[86597]: osdmap e388: 8 total, 8 up, 8 in 2024-09-14T03:53:29.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:29 smithi070 ceph-mon[96704]: osdmap e387: 8 total, 8 up, 8 in 2024-09-14T03:53:29.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:29 smithi070 ceph-mon[96704]: osdmap e388: 8 total, 8 up, 8 in 2024-09-14T03:53:29.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:29 smithi070 ceph-mon[98640]: osdmap e387: 8 total, 8 up, 8 in 2024-09-14T03:53:29.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:29 smithi070 ceph-mon[98640]: osdmap e388: 8 total, 8 up, 8 in 2024-09-14T03:53:30.017 INFO:tasks.rados.rados.0.smithi070.stdout:1605: done (5 left) 2024-09-14T03:53:30.017 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 20 current snap is 174 2024-09-14T03:53:30.017 INFO:tasks.rados.rados.0.smithi070.stdout:1606: seq_num 524 ranges {6856704=786432,7643136=778240,8421376=24576} 2024-09-14T03:53:30.059 INFO:tasks.rados.rados.0.smithi070.stdout:1606: writing smithi07085154-20 from 6856704 to 7643136 tid 1 2024-09-14T03:53:30.064 INFO:tasks.rados.rados.0.smithi070.stdout:1606: writing smithi07085154-20 from 7643136 to 8421376 tid 2 2024-09-14T03:53:30.064 INFO:tasks.rados.rados.0.smithi070.stdout:1606: writing smithi07085154-20 from 8421376 to 8445952 tid 3 2024-09-14T03:53:30.064 INFO:tasks.rados.rados.0.smithi070.stdout:1607: rollback oid 17 current snap is 174 2024-09-14T03:53:30.064 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 17 to 121 2024-09-14T03:53:30.064 INFO:tasks.rados.rados.0.smithi070.stdout:1600: finishing copy_from to smithi07085154-3 2024-09-14T03:53:30.064 INFO:tasks.rados.rados.0.smithi070.stdout:1600: got expected ENOENT (src dne) 2024-09-14T03:53:30.064 INFO:tasks.rados.rados.0.smithi070.stdout:1600: finishing copy_from racing read to smithi07085154-3 2024-09-14T03:53:30.064 INFO:tasks.rados.rados.0.smithi070.stdout:1604: expect (ObjNum 516 snap 173 seq_num 516) 2024-09-14T03:53:30.122 INFO:tasks.rados.rados.0.smithi070.stdout:1602: expect (ObjNum 241 snap 69 seq_num 241) 2024-09-14T03:53:30.163 INFO:tasks.rados.rados.0.smithi070.stdout:1599: expect (ObjNum 431 snap 148 seq_num 431) 2024-09-14T03:53:30.475 INFO:tasks.rados.rados.0.smithi070.stdout:1606: finishing write tid 1 to smithi07085154-20 2024-09-14T03:53:30.476 INFO:tasks.rados.rados.0.smithi070.stdout:1607: finishing rollback tid 0 to smithi07085154-17 2024-09-14T03:53:30.476 INFO:tasks.rados.rados.0.smithi070.stdout:1607: finishing rollback tid 1 to smithi07085154-17 2024-09-14T03:53:30.476 INFO:tasks.rados.rados.0.smithi070.stdout:1607: finishing rollback tid 2 to smithi07085154-17 2024-09-14T03:53:30.476 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 859 (ObjNum 325 snap 107 seq_num 325) dirty exists 2024-09-14T03:53:30.476 INFO:tasks.rados.rados.0.smithi070.stdout:1599: done (6 left) 2024-09-14T03:53:30.476 INFO:tasks.rados.rados.0.smithi070.stdout:1600: done (5 left) 2024-09-14T03:53:30.476 INFO:tasks.rados.rados.0.smithi070.stdout:1602: done (4 left) 2024-09-14T03:53:30.476 INFO:tasks.rados.rados.0.smithi070.stdout:1603: done (3 left) 2024-09-14T03:53:30.476 INFO:tasks.rados.rados.0.smithi070.stdout:1604: done (2 left) 2024-09-14T03:53:30.476 INFO:tasks.rados.rados.0.smithi070.stdout:1607: done (1 left) 2024-09-14T03:53:30.476 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 7 current snap is 174 2024-09-14T03:53:30.476 INFO:tasks.rados.rados.0.smithi070.stdout:1606: finishing write tid 2 to smithi07085154-20 2024-09-14T03:53:30.477 INFO:tasks.rados.rados.0.smithi070.stdout:1606: finishing write tid 3 to smithi07085154-20 2024-09-14T03:53:30.477 INFO:tasks.rados.rados.0.smithi070.stdout:1608: seq_num 525 ranges {2154496=802816,2957312=712704,3670016=90112} 2024-09-14T03:53:30.490 INFO:tasks.rados.rados.0.smithi070.stdout:1608: writing smithi07085154-7 from 2154496 to 2957312 tid 1 2024-09-14T03:53:30.495 INFO:tasks.rados.rados.0.smithi070.stdout:1608: writing smithi07085154-7 from 2957312 to 3670016 tid 2 2024-09-14T03:53:30.496 INFO:tasks.rados.rados.0.smithi070.stdout:1608: writing smithi07085154-7 from 3670016 to 3760128 tid 3 2024-09-14T03:53:30.496 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 27 current snap is 174 2024-09-14T03:53:30.496 INFO:tasks.rados.rados.0.smithi070.stdout:1606: finishing write tid 4 to smithi07085154-20 2024-09-14T03:53:30.496 INFO:tasks.rados.rados.0.smithi070.stdout:1606: finishing write tid 5 to smithi07085154-20 2024-09-14T03:53:30.496 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 20 v 1232 (ObjNum 524 snap 174 seq_num 524) dirty exists 2024-09-14T03:53:30.496 INFO:tasks.rados.rados.0.smithi070.stdout:1606: left oid 20 (ObjNum 524 snap 174 seq_num 524) 2024-09-14T03:53:30.496 INFO:tasks.rados.rados.0.smithi070.stdout:1608: finishing write tid 1 to smithi07085154-7 2024-09-14T03:53:30.496 INFO:tasks.rados.rados.0.smithi070.stdout:1609: seq_num 526 ranges {16171008=417792,16588800=589824,17178624=614400} 2024-09-14T03:53:30.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:53:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:53:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:30.582 INFO:tasks.rados.rados.0.smithi070.stdout:1609: writing smithi07085154-27 from 16171008 to 16588800 tid 1 2024-09-14T03:53:30.586 INFO:tasks.rados.rados.0.smithi070.stdout:1609: writing smithi07085154-27 from 16588800 to 17178624 tid 2 2024-09-14T03:53:30.590 INFO:tasks.rados.rados.0.smithi070.stdout:1609: writing smithi07085154-27 from 17178624 to 17793024 tid 3 2024-09-14T03:53:30.590 INFO:tasks.rados.rados.0.smithi070.stdout:1606: done (2 left) 2024-09-14T03:53:30.590 INFO:tasks.rados.rados.0.smithi070.stdout:1610: copy_from oid 50 from oid 9 current snap is 174 2024-09-14T03:53:30.591 INFO:tasks.rados.rados.0.smithi070.stdout:1608: finishing write tid 2 to smithi07085154-7 2024-09-14T03:53:30.591 INFO:tasks.rados.rados.0.smithi070.stdout:1608: finishing write tid 3 to smithi07085154-7 2024-09-14T03:53:30.591 INFO:tasks.rados.rados.0.smithi070.stdout:1608: finishing write tid 4 to smithi07085154-7 2024-09-14T03:53:30.591 INFO:tasks.rados.rados.0.smithi070.stdout:1608: finishing write tid 5 to smithi07085154-7 2024-09-14T03:53:30.591 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 1236 (ObjNum 525 snap 174 seq_num 525) dirty exists 2024-09-14T03:53:30.591 INFO:tasks.rados.rados.0.smithi070.stdout:1608: left oid 7 (ObjNum 525 snap 174 seq_num 525) 2024-09-14T03:53:30.591 INFO:tasks.rados.rados.0.smithi070.stdout:1609: finishing write tid 1 to smithi07085154-27 2024-09-14T03:53:30.591 INFO:tasks.rados.rados.0.smithi070.stdout:1609: finishing write tid 2 to smithi07085154-27 2024-09-14T03:53:30.591 INFO:tasks.rados.rados.0.smithi070.stdout:1608: done (2 left) 2024-09-14T03:53:30.591 INFO:tasks.rados.rados.0.smithi070.stdout:1611: snap_remove snap 145 2024-09-14T03:53:31.024 INFO:tasks.rados.rados.0.smithi070.stdout:1611: done (2 left) 2024-09-14T03:53:31.024 INFO:tasks.rados.rados.0.smithi070.stdout:1612: snap_create 2024-09-14T03:53:31.024 INFO:tasks.rados.rados.0.smithi070.stdout:1609: finishing write tid 3 to smithi07085154-27 2024-09-14T03:53:31.025 INFO:tasks.rados.rados.0.smithi070.stdout:1609: finishing write tid 4 to smithi07085154-27 2024-09-14T03:53:31.025 INFO:tasks.rados.rados.0.smithi070.stdout:1609: finishing write tid 5 to smithi07085154-27 2024-09-14T03:53:31.025 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 1603 (ObjNum 526 snap 174 seq_num 526) dirty exists 2024-09-14T03:53:31.025 INFO:tasks.rados.rados.0.smithi070.stdout:1609: left oid 27 (ObjNum 526 snap 174 seq_num 526) 2024-09-14T03:53:31.025 INFO:tasks.rados.rados.0.smithi070.stdout:1610: finishing copy_from racing read to smithi07085154-50 2024-09-14T03:53:31.025 INFO:tasks.rados.rados.0.smithi070.stdout:1610: finishing copy_from to smithi07085154-50 2024-09-14T03:53:31.025 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 1545 (ObjNum 507 snap 166 seq_num 507) dirty exists 2024-09-14T03:53:31.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:31 smithi094 ceph-mon[86597]: osdmap e389: 8 total, 8 up, 8 in 2024-09-14T03:53:31.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:31 smithi094 ceph-mon[86597]: pgmap v175: 121 pgs: 121 active+clean; 445 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.4 MiB/s wr, 15 op/s 2024-09-14T03:53:31.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:31 smithi070 ceph-mon[98640]: osdmap e389: 8 total, 8 up, 8 in 2024-09-14T03:53:31.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:31 smithi070 ceph-mon[98640]: pgmap v175: 121 pgs: 121 active+clean; 445 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.4 MiB/s wr, 15 op/s 2024-09-14T03:53:31.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:31 smithi070 ceph-mon[96704]: osdmap e389: 8 total, 8 up, 8 in 2024-09-14T03:53:31.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:31 smithi070 ceph-mon[96704]: pgmap v175: 121 pgs: 121 active+clean; 445 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.4 MiB/s wr, 15 op/s 2024-09-14T03:53:32.025 INFO:tasks.rados.rados.0.smithi070.stdout:1609: done (2 left) 2024-09-14T03:53:32.025 INFO:tasks.rados.rados.0.smithi070.stdout:1610: done (1 left) 2024-09-14T03:53:32.025 INFO:tasks.rados.rados.0.smithi070.stdout:1612: done (0 left) 2024-09-14T03:53:32.025 INFO:tasks.rados.rados.0.smithi070.stdout:1613: delete oid 39 current snap is 175 2024-09-14T03:53:32.029 INFO:tasks.rados.rados.0.smithi070.stdout:1613: done (0 left) 2024-09-14T03:53:32.029 INFO:tasks.rados.rados.0.smithi070.stdout:1614: snap_remove snap 129 2024-09-14T03:53:32.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:32 smithi094 ceph-mon[86597]: osdmap e390: 8 total, 8 up, 8 in 2024-09-14T03:53:32.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:32 smithi070 ceph-mon[96704]: osdmap e390: 8 total, 8 up, 8 in 2024-09-14T03:53:32.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:32 smithi070 ceph-mon[98640]: osdmap e390: 8 total, 8 up, 8 in 2024-09-14T03:53:33.038 INFO:tasks.rados.rados.0.smithi070.stdout:1614: done (0 left) 2024-09-14T03:53:33.038 INFO:tasks.rados.rados.0.smithi070.stdout:1615: snap_remove snap 104 2024-09-14T03:53:33.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:33 smithi070 ceph-mon[96704]: osdmap e391: 8 total, 8 up, 8 in 2024-09-14T03:53:33.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:33 smithi070 ceph-mon[96704]: pgmap v178: 121 pgs: 121 active+clean; 454 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-14T03:53:33.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:33 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:33.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:33 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:33.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:33 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:33.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:33 smithi070 ceph-mon[98640]: osdmap e391: 8 total, 8 up, 8 in 2024-09-14T03:53:33.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:33 smithi070 ceph-mon[98640]: pgmap v178: 121 pgs: 121 active+clean; 454 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-14T03:53:33.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:33 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:33.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:33 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:33.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:33 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:33.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:33 smithi094 ceph-mon[86597]: osdmap e391: 8 total, 8 up, 8 in 2024-09-14T03:53:33.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:33 smithi094 ceph-mon[86597]: pgmap v178: 121 pgs: 121 active+clean; 454 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-14T03:53:33.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:33 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:33.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:33 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:33.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:33 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:33.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:53:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:53:33] "GET /metrics HTTP/1.1" 200 36312 "" "Prometheus/2.51.0" 2024-09-14T03:53:34.018 INFO:tasks.rados.rados.0.smithi070.stdout:1615: done (0 left) 2024-09-14T03:53:34.018 INFO:tasks.rados.rados.0.smithi070.stdout:1616: rollback oid 19 current snap is 175 2024-09-14T03:53:34.018 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 19 to 94 2024-09-14T03:53:34.018 INFO:tasks.rados.rados.0.smithi070.stdout:1617: rmattr oid 33 current snap is 175 2024-09-14T03:53:34.018 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 46 current snap is 175 2024-09-14T03:53:34.018 INFO:tasks.rados.rados.0.smithi070.stdout:1618: seq_num 528 ranges {3244032=450560,3694592=761856,4456448=442368} 2024-09-14T03:53:34.040 INFO:tasks.rados.rados.0.smithi070.stdout:1618: writing smithi07085154-46 from 3244032 to 3694592 tid 1 2024-09-14T03:53:34.045 INFO:tasks.rados.rados.0.smithi070.stdout:1618: writing smithi07085154-46 from 3694592 to 4456448 tid 2 2024-09-14T03:53:34.048 INFO:tasks.rados.rados.0.smithi070.stdout:1618: writing smithi07085154-46 from 4456448 to 4898816 tid 3 2024-09-14T03:53:34.049 INFO:tasks.rados.rados.0.smithi070.stdout:1619: rmattr oid 40 current snap is 175 2024-09-14T03:53:34.049 INFO:tasks.rados.rados.0.smithi070.stdout:1619: done (3 left) 2024-09-14T03:53:34.049 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 3 current snap is 175 2024-09-14T03:53:34.049 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 33 v 980 (ObjNum 467 snap 160 seq_num 467) dirty exists 2024-09-14T03:53:34.049 INFO:tasks.rados.rados.0.smithi070.stdout:1620: seq_num 529 ranges {0=466944,466944=647168,1114112=532480,1646592=24576} 2024-09-14T03:53:34.051 INFO:tasks.rados.rados.0.smithi070.stdout:1620: writing smithi07085154-3 from 0 to 466944 tid 1 2024-09-14T03:53:34.055 INFO:tasks.rados.rados.0.smithi070.stdout:1620: writing smithi07085154-3 from 466944 to 1114112 tid 2 2024-09-14T03:53:34.058 INFO:tasks.rados.rados.0.smithi070.stdout:1620: writing smithi07085154-3 from 1114112 to 1646592 tid 3 2024-09-14T03:53:34.059 INFO:tasks.rados.rados.0.smithi070.stdout:1620: writing smithi07085154-3 from 1646592 to 1671168 tid 4 2024-09-14T03:53:34.060 INFO:tasks.rados.rados.0.smithi070.stdout:1617: done (3 left) 2024-09-14T03:53:34.060 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 18 current snap is 175 2024-09-14T03:53:34.060 INFO:tasks.rados.rados.0.smithi070.stdout:1616: finishing rollback tid 1 to smithi07085154-19 2024-09-14T03:53:34.060 INFO:tasks.rados.rados.0.smithi070.stdout:1616: finishing rollback tid 2 to smithi07085154-19 2024-09-14T03:53:34.060 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 19 v 1547 (ObjNum 262 snap 76 seq_num 262) dirty exists 2024-09-14T03:53:34.060 INFO:tasks.rados.rados.0.smithi070.stdout:1618: finishing write tid 1 to smithi07085154-46 2024-09-14T03:53:34.060 INFO:tasks.rados.rados.0.smithi070.stdout:1618: finishing write tid 2 to smithi07085154-46 2024-09-14T03:53:34.060 INFO:tasks.rados.rados.0.smithi070.stdout:1618: finishing write tid 3 to smithi07085154-46 2024-09-14T03:53:34.060 INFO:tasks.rados.rados.0.smithi070.stdout:1621: seq_num 530 ranges {0=483328,483328=532480,1015808=671744} 2024-09-14T03:53:34.061 INFO:tasks.rados.rados.0.smithi070.stdout:1621: writing smithi07085154-18 from 0 to 483328 tid 1 2024-09-14T03:53:34.065 INFO:tasks.rados.rados.0.smithi070.stdout:1621: writing smithi07085154-18 from 483328 to 1015808 tid 2 2024-09-14T03:53:34.070 INFO:tasks.rados.rados.0.smithi070.stdout:1621: writing smithi07085154-18 from 1015808 to 1687552 tid 3 2024-09-14T03:53:34.070 INFO:tasks.rados.rados.0.smithi070.stdout:1616: done (3 left) 2024-09-14T03:53:34.070 INFO:tasks.rados.rados.0.smithi070.stdout:1622: read oid 33 snap -1 2024-09-14T03:53:34.070 INFO:tasks.rados.rados.0.smithi070.stdout:1622: expect (ObjNum 467 snap 160 seq_num 467) 2024-09-14T03:53:34.070 INFO:tasks.rados.rados.0.smithi070.stdout:1623: copy_from oid 42 from oid 9 current snap is 175 2024-09-14T03:53:34.070 INFO:tasks.rados.rados.0.smithi070.stdout:1618: finishing write tid 4 to smithi07085154-46 2024-09-14T03:53:34.070 INFO:tasks.rados.rados.0.smithi070.stdout:1618: finishing write tid 5 to smithi07085154-46 2024-09-14T03:53:34.070 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 46 v 1646 (ObjNum 528 snap 175 seq_num 528) dirty exists 2024-09-14T03:53:34.070 INFO:tasks.rados.rados.0.smithi070.stdout:1618: left oid 46 (ObjNum 528 snap 175 seq_num 528) 2024-09-14T03:53:34.070 INFO:tasks.rados.rados.0.smithi070.stdout:1620: finishing write tid 1 to smithi07085154-3 2024-09-14T03:53:34.071 INFO:tasks.rados.rados.0.smithi070.stdout:1620: finishing write tid 2 to smithi07085154-3 2024-09-14T03:53:34.071 INFO:tasks.rados.rados.0.smithi070.stdout:1620: finishing write tid 3 to smithi07085154-3 2024-09-14T03:53:34.071 INFO:tasks.rados.rados.0.smithi070.stdout:1618: done (4 left) 2024-09-14T03:53:34.071 INFO:tasks.rados.rados.0.smithi070.stdout:1624: read oid 26 snap 174 2024-09-14T03:53:34.071 INFO:tasks.rados.rados.0.smithi070.stdout:1624: expect (ObjNum -1364505072 snap 21849 seq_num 2930739344) 2024-09-14T03:53:34.071 INFO:tasks.rados.rados.0.smithi070.stdout:1625: snap_create 2024-09-14T03:53:34.071 INFO:tasks.rados.rados.0.smithi070.stdout:1620: finishing write tid 4 to smithi07085154-3 2024-09-14T03:53:34.071 INFO:tasks.rados.rados.0.smithi070.stdout:1620: finishing write tid 5 to smithi07085154-3 2024-09-14T03:53:34.071 INFO:tasks.rados.rados.0.smithi070.stdout:1620: finishing write tid 6 to smithi07085154-3 2024-09-14T03:53:34.071 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 3 v 1008 (ObjNum 529 snap 175 seq_num 529) dirty exists 2024-09-14T03:53:34.071 INFO:tasks.rados.rados.0.smithi070.stdout:1620: left oid 3 (ObjNum 529 snap 175 seq_num 529) 2024-09-14T03:53:34.071 INFO:tasks.rados.rados.0.smithi070.stdout:1621: finishing write tid 1 to smithi07085154-18 2024-09-14T03:53:34.071 INFO:tasks.rados.rados.0.smithi070.stdout:1621: finishing write tid 2 to smithi07085154-18 2024-09-14T03:53:34.072 INFO:tasks.rados.rados.0.smithi070.stdout:1624: expect (ObjNum -1364505072 snap 21849 seq_num 2930739344) 2024-09-14T03:53:34.072 INFO:tasks.rados.rados.0.smithi070.stdout:1621: finishing write tid 3 to smithi07085154-18 2024-09-14T03:53:34.073 INFO:tasks.rados.rados.0.smithi070.stdout:1621: finishing write tid 4 to smithi07085154-18 2024-09-14T03:53:34.074 INFO:tasks.rados.rados.0.smithi070.stdout:1621: finishing write tid 5 to smithi07085154-18 2024-09-14T03:53:34.074 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 1156 (ObjNum 530 snap 175 seq_num 530) dirty exists 2024-09-14T03:53:34.075 INFO:tasks.rados.rados.0.smithi070.stdout:1621: left oid 18 (ObjNum 530 snap 175 seq_num 530) 2024-09-14T03:53:34.077 INFO:tasks.rados.rados.0.smithi070.stdout:1623: finishing copy_from racing read to smithi07085154-42 2024-09-14T03:53:34.078 INFO:tasks.rados.rados.0.smithi070.stdout:1623: finishing copy_from to smithi07085154-42 2024-09-14T03:53:34.078 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 1545 (ObjNum 507 snap 166 seq_num 507) dirty exists 2024-09-14T03:53:34.084 INFO:tasks.rados.rados.0.smithi070.stdout:1622: expect (ObjNum 467 snap 160 seq_num 467) 2024-09-14T03:53:34.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:34 smithi070 ceph-mon[96704]: osdmap e392: 8 total, 8 up, 8 in 2024-09-14T03:53:34.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:34 smithi070 ceph-mon[96704]: osdmap e393: 8 total, 8 up, 8 in 2024-09-14T03:53:34.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:34 smithi070 ceph-mon[98640]: osdmap e392: 8 total, 8 up, 8 in 2024-09-14T03:53:34.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:34 smithi070 ceph-mon[98640]: osdmap e393: 8 total, 8 up, 8 in 2024-09-14T03:53:34.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:34 smithi094 ceph-mon[86597]: osdmap e392: 8 total, 8 up, 8 in 2024-09-14T03:53:34.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:34 smithi094 ceph-mon[86597]: osdmap e393: 8 total, 8 up, 8 in 2024-09-14T03:53:34.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:53:34 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:53:34.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.2\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:35.041 INFO:tasks.rados.rados.0.smithi070.stdout:1620: done (5 left) 2024-09-14T03:53:35.041 INFO:tasks.rados.rados.0.smithi070.stdout:1621: done (4 left) 2024-09-14T03:53:35.041 INFO:tasks.rados.rados.0.smithi070.stdout:1622: done (3 left) 2024-09-14T03:53:35.042 INFO:tasks.rados.rados.0.smithi070.stdout:1623: done (2 left) 2024-09-14T03:53:35.042 INFO:tasks.rados.rados.0.smithi070.stdout:1624: done (1 left) 2024-09-14T03:53:35.042 INFO:tasks.rados.rados.0.smithi070.stdout:1625: done (0 left) 2024-09-14T03:53:35.042 INFO:tasks.rados.rados.0.smithi070.stdout:1626: snap_remove snap 78 2024-09-14T03:53:35.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:35 smithi094 ceph-mon[86597]: pgmap v181: 121 pgs: 8 active+clean+snaptrim, 113 active+clean; 458 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.5 MiB/s wr, 9 op/s 2024-09-14T03:53:35.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:35 smithi070 ceph-mon[96704]: pgmap v181: 121 pgs: 8 active+clean+snaptrim, 113 active+clean; 458 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.5 MiB/s wr, 9 op/s 2024-09-14T03:53:35.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:35 smithi070 ceph-mon[98640]: pgmap v181: 121 pgs: 8 active+clean+snaptrim, 113 active+clean; 458 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.5 MiB/s wr, 9 op/s 2024-09-14T03:53:36.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:53:35 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:53:35.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:36.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:53:35 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:53:35.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:36.044 INFO:tasks.rados.rados.0.smithi070.stdout:1626: done (0 left) 2024-09-14T03:53:36.044 INFO:tasks.rados.rados.0.smithi070.stdout:1627: read oid 42 snap -1 2024-09-14T03:53:36.044 INFO:tasks.rados.rados.0.smithi070.stdout:1627: expect (ObjNum 507 snap 166 seq_num 507) 2024-09-14T03:53:36.044 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 11 current snap is 176 2024-09-14T03:53:36.044 INFO:tasks.rados.rados.0.smithi070.stdout:1628: seq_num 531 ranges {2310144=499712,2809856=417792,3227648=548864,3776512=237568} 2024-09-14T03:53:36.061 INFO:tasks.rados.rados.0.smithi070.stdout:1628: writing smithi07085154-11 from 2310144 to 2809856 tid 1 2024-09-14T03:53:36.064 INFO:tasks.rados.rados.0.smithi070.stdout:1628: writing smithi07085154-11 from 2809856 to 3227648 tid 2 2024-09-14T03:53:36.067 INFO:tasks.rados.rados.0.smithi070.stdout:1628: writing smithi07085154-11 from 3227648 to 3776512 tid 3 2024-09-14T03:53:36.069 INFO:tasks.rados.rados.0.smithi070.stdout:1628: writing smithi07085154-11 from 3776512 to 4014080 tid 4 2024-09-14T03:53:36.069 INFO:tasks.rados.rados.0.smithi070.stdout:1627: expect (ObjNum 507 snap 166 seq_num 507) 2024-09-14T03:53:36.109 INFO:tasks.rados.rados.0.smithi070.stdout:1628: finishing write tid 1 to smithi07085154-11 2024-09-14T03:53:36.109 INFO:tasks.rados.rados.0.smithi070.stdout:1628: finishing write tid 2 to smithi07085154-11 2024-09-14T03:53:36.109 INFO:tasks.rados.rados.0.smithi070.stdout:1628: finishing write tid 3 to smithi07085154-11 2024-09-14T03:53:36.109 INFO:tasks.rados.rados.0.smithi070.stdout:1628: finishing write tid 4 to smithi07085154-11 2024-09-14T03:53:36.109 INFO:tasks.rados.rados.0.smithi070.stdout:1628: finishing write tid 5 to smithi07085154-11 2024-09-14T03:53:36.109 INFO:tasks.rados.rados.0.smithi070.stdout:1628: finishing write tid 6 to smithi07085154-11 2024-09-14T03:53:36.109 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 11 v 1191 (ObjNum 531 snap 176 seq_num 531) dirty exists 2024-09-14T03:53:36.109 INFO:tasks.rados.rados.0.smithi070.stdout:1628: left oid 11 (ObjNum 531 snap 176 seq_num 531) 2024-09-14T03:53:36.109 INFO:tasks.rados.rados.0.smithi070.stdout:1627: done (1 left) 2024-09-14T03:53:36.110 INFO:tasks.rados.rados.0.smithi070.stdout:1628: done (0 left) 2024-09-14T03:53:36.110 INFO:tasks.rados.rados.0.smithi070.stdout:1629: delete oid 1 current snap is 176 2024-09-14T03:53:36.110 INFO:tasks.rados.rados.0.smithi070.stdout:1629: done (0 left) 2024-09-14T03:53:36.110 INFO:tasks.rados.rados.0.smithi070.stdout:1630: rollback oid 16 current snap is 176 2024-09-14T03:53:36.110 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 16 to 73 2024-09-14T03:53:36.110 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 24 current snap is 176 2024-09-14T03:53:36.111 INFO:tasks.rados.rados.0.smithi070.stdout:1631: seq_num 533 ranges {1400832=532480,1933312=630784,2564096=516096,3080192=57344} 2024-09-14T03:53:36.121 INFO:tasks.rados.rados.0.smithi070.stdout:1631: writing smithi07085154-24 from 1400832 to 1933312 tid 1 2024-09-14T03:53:36.124 INFO:tasks.rados.rados.0.smithi070.stdout:1631: writing smithi07085154-24 from 1933312 to 2564096 tid 2 2024-09-14T03:53:36.127 INFO:tasks.rados.rados.0.smithi070.stdout:1631: writing smithi07085154-24 from 2564096 to 3080192 tid 3 2024-09-14T03:53:36.128 INFO:tasks.rados.rados.0.smithi070.stdout:1631: writing smithi07085154-24 from 3080192 to 3137536 tid 4 2024-09-14T03:53:36.128 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 12 current snap is 176 2024-09-14T03:53:36.128 INFO:tasks.rados.rados.0.smithi070.stdout:1630: finishing rollback tid 1 to smithi07085154-16 2024-09-14T03:53:36.128 INFO:tasks.rados.rados.0.smithi070.stdout:1630: finishing rollback tid 2 to smithi07085154-16 2024-09-14T03:53:36.129 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 1660 (ObjNum 114 snap 24 seq_num 114) dirty exists 2024-09-14T03:53:36.129 INFO:tasks.rados.rados.0.smithi070.stdout:1631: finishing write tid 1 to smithi07085154-24 2024-09-14T03:53:36.129 INFO:tasks.rados.rados.0.smithi070.stdout:1631: finishing write tid 2 to smithi07085154-24 2024-09-14T03:53:36.129 INFO:tasks.rados.rados.0.smithi070.stdout:1632: seq_num 534 ranges {3964928=548864,4513792=516096,5029888=688128} 2024-09-14T03:53:36.151 INFO:tasks.rados.rados.0.smithi070.stdout:1632: writing smithi07085154-12 from 3964928 to 4513792 tid 1 2024-09-14T03:53:36.154 INFO:tasks.rados.rados.0.smithi070.stdout:1632: writing smithi07085154-12 from 4513792 to 5029888 tid 2 2024-09-14T03:53:36.159 INFO:tasks.rados.rados.0.smithi070.stdout:1632: writing smithi07085154-12 from 5029888 to 5718016 tid 3 2024-09-14T03:53:36.159 INFO:tasks.rados.rados.0.smithi070.stdout:1630: done (2 left) 2024-09-14T03:53:36.159 INFO:tasks.rados.rados.0.smithi070.stdout:1633: read oid 9 snap -1 2024-09-14T03:53:36.159 INFO:tasks.rados.rados.0.smithi070.stdout:1633: expect (ObjNum 507 snap 166 seq_num 507) 2024-09-14T03:53:36.159 INFO:tasks.rados.rados.0.smithi070.stdout:1634: delete oid 3 current snap is 176 2024-09-14T03:53:36.160 INFO:tasks.rados.rados.0.smithi070.stdout:1631: finishing write tid 3 to smithi07085154-24 2024-09-14T03:53:36.160 INFO:tasks.rados.rados.0.smithi070.stdout:1631: finishing write tid 4 to smithi07085154-24 2024-09-14T03:53:36.160 INFO:tasks.rados.rados.0.smithi070.stdout:1631: finishing write tid 5 to smithi07085154-24 2024-09-14T03:53:36.160 INFO:tasks.rados.rados.0.smithi070.stdout:1631: finishing write tid 6 to smithi07085154-24 2024-09-14T03:53:36.160 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 625 (ObjNum 533 snap 176 seq_num 533) dirty exists 2024-09-14T03:53:36.160 INFO:tasks.rados.rados.0.smithi070.stdout:1631: left oid 24 (ObjNum 533 snap 176 seq_num 533) 2024-09-14T03:53:36.160 INFO:tasks.rados.rados.0.smithi070.stdout:1632: finishing write tid 1 to smithi07085154-12 2024-09-14T03:53:36.160 INFO:tasks.rados.rados.0.smithi070.stdout:1632: finishing write tid 2 to smithi07085154-12 2024-09-14T03:53:36.163 INFO:tasks.rados.rados.0.smithi070.stdout:1632: finishing write tid 3 to smithi07085154-12 2024-09-14T03:53:36.163 INFO:tasks.rados.rados.0.smithi070.stdout:1631: done (3 left) 2024-09-14T03:53:36.164 INFO:tasks.rados.rados.0.smithi070.stdout:1634: done (2 left) 2024-09-14T03:53:36.164 INFO:tasks.rados.rados.0.smithi070.stdout:1635: copy_from oid 34 from oid 4 current snap is 176 2024-09-14T03:53:36.164 INFO:tasks.rados.rados.0.smithi070.stdout:1632: finishing write tid 4 to smithi07085154-12 2024-09-14T03:53:36.164 INFO:tasks.rados.rados.0.smithi070.stdout:1636: rmattr oid 43 current snap is 176 2024-09-14T03:53:36.164 INFO:tasks.rados.rados.0.smithi070.stdout:1633: expect (ObjNum 507 snap 166 seq_num 507) 2024-09-14T03:53:36.198 INFO:tasks.rados.rados.0.smithi070.stdout:1632: finishing write tid 5 to smithi07085154-12 2024-09-14T03:53:36.198 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 559 (ObjNum 534 snap 176 seq_num 534) dirty exists 2024-09-14T03:53:36.199 INFO:tasks.rados.rados.0.smithi070.stdout:1632: left oid 12 (ObjNum 534 snap 176 seq_num 534) 2024-09-14T03:53:36.199 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 1168 (ObjNum 469 snap 160 seq_num 469) dirty exists 2024-09-14T03:53:36.199 INFO:tasks.rados.rados.0.smithi070.stdout:1635: finishing copy_from racing read to smithi07085154-34 2024-09-14T03:53:36.199 INFO:tasks.rados.rados.0.smithi070.stdout:1635: finishing copy_from to smithi07085154-34 2024-09-14T03:53:36.199 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 1226 (ObjNum 517 snap 173 seq_num 517) dirty exists 2024-09-14T03:53:36.199 INFO:tasks.rados.rados.0.smithi070.stdout:1632: done (3 left) 2024-09-14T03:53:36.199 INFO:tasks.rados.rados.0.smithi070.stdout:1633: done (2 left) 2024-09-14T03:53:36.199 INFO:tasks.rados.rados.0.smithi070.stdout:1635: done (1 left) 2024-09-14T03:53:36.199 INFO:tasks.rados.rados.0.smithi070.stdout:1636: done (0 left) 2024-09-14T03:53:36.199 INFO:tasks.rados.rados.0.smithi070.stdout:1637: snap_remove snap 157 2024-09-14T03:53:36.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:36 smithi094 ceph-mon[86597]: osdmap e394: 8 total, 8 up, 8 in 2024-09-14T03:53:36.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:36 smithi070 ceph-mon[98640]: osdmap e394: 8 total, 8 up, 8 in 2024-09-14T03:53:36.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:36 smithi070 ceph-mon[96704]: osdmap e394: 8 total, 8 up, 8 in 2024-09-14T03:53:37.054 INFO:tasks.rados.rados.0.smithi070.stdout:1637: done (0 left) 2024-09-14T03:53:37.055 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 44 current snap is 176 2024-09-14T03:53:37.055 INFO:tasks.rados.rados.0.smithi070.stdout:1638: seq_num 536 ranges {802816=581632,1384448=688128,2072576=516096} 2024-09-14T03:53:37.063 INFO:tasks.rados.rados.0.smithi070.stdout:1638: writing smithi07085154-44 from 802816 to 1384448 tid 1 2024-09-14T03:53:37.068 INFO:tasks.rados.rados.0.smithi070.stdout:1638: writing smithi07085154-44 from 1384448 to 2072576 tid 2 2024-09-14T03:53:37.072 INFO:tasks.rados.rados.0.smithi070.stdout:1638: writing smithi07085154-44 from 2072576 to 2588672 tid 3 2024-09-14T03:53:37.072 INFO:tasks.rados.rados.0.smithi070.stdout:1639: copy_from oid 45 from oid 37 current snap is 176 2024-09-14T03:53:37.072 INFO:tasks.rados.rados.0.smithi070.stdout:1640: rollback oid 30 current snap is 176 2024-09-14T03:53:37.072 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 30 to 160 2024-09-14T03:53:37.072 INFO:tasks.rados.rados.0.smithi070.stdout:1641: rollback oid 16 current snap is 176 2024-09-14T03:53:37.072 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 16 to 94 2024-09-14T03:53:37.072 INFO:tasks.rados.rados.0.smithi070.stdout:1642: snap_remove snap 77 2024-09-14T03:53:37.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:37 smithi094 ceph-mon[86597]: osdmap e395: 8 total, 8 up, 8 in 2024-09-14T03:53:37.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:37 smithi094 ceph-mon[86597]: pgmap v184: 121 pgs: 8 active+clean+snaptrim, 113 active+clean; 458 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.3 MiB/s rd, 1.5 MiB/s wr, 12 op/s 2024-09-14T03:53:37.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:37 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:53:37.277 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:53:36 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:53:36.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:37.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:37 smithi070 ceph-mon[96704]: osdmap e395: 8 total, 8 up, 8 in 2024-09-14T03:53:37.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:37 smithi070 ceph-mon[96704]: pgmap v184: 121 pgs: 8 active+clean+snaptrim, 113 active+clean; 458 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.3 MiB/s rd, 1.5 MiB/s wr, 12 op/s 2024-09-14T03:53:37.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:37 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:53:37.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:37 smithi070 ceph-mon[98640]: osdmap e395: 8 total, 8 up, 8 in 2024-09-14T03:53:37.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:37 smithi070 ceph-mon[98640]: pgmap v184: 121 pgs: 8 active+clean+snaptrim, 113 active+clean; 458 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.3 MiB/s rd, 1.5 MiB/s wr, 12 op/s 2024-09-14T03:53:37.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:37 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T03:53:38.059 INFO:tasks.rados.rados.0.smithi070.stdout:1642: done (4 left) 2024-09-14T03:53:38.059 INFO:tasks.rados.rados.0.smithi070.stdout:1643: snap_remove snap 150 2024-09-14T03:53:38.368 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:38 smithi070 ceph-mon[96704]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:53:38.369 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:38 smithi070 ceph-mon[96704]: Upgrade: osd.2 is safe to restart 2024-09-14T03:53:38.369 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:38 smithi070 ceph-mon[96704]: osdmap e396: 8 total, 8 up, 8 in 2024-09-14T03:53:38.369 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:38 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:38.369 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:38 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-14T03:53:38.369 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:38 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:38.369 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:38 smithi070 ceph-mon[98640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:53:38.369 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:38 smithi070 ceph-mon[98640]: Upgrade: osd.2 is safe to restart 2024-09-14T03:53:38.369 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:38 smithi070 ceph-mon[98640]: osdmap e396: 8 total, 8 up, 8 in 2024-09-14T03:53:38.369 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:38 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:38.370 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:38 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-14T03:53:38.370 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:38 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:38.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:38 smithi094 ceph-mon[86597]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T03:53:38.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:38 smithi094 ceph-mon[86597]: Upgrade: osd.2 is safe to restart 2024-09-14T03:53:38.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:38 smithi094 ceph-mon[86597]: osdmap e396: 8 total, 8 up, 8 in 2024-09-14T03:53:38.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:38 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:38.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:38 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-14T03:53:38.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:38 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:39.018 INFO:tasks.rados.rados.0.smithi070.stdout:1643: done (4 left) 2024-09-14T03:53:39.018 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 40 current snap is 176 2024-09-14T03:53:39.018 INFO:tasks.rados.rados.0.smithi070.stdout:1640: finishing rollback tid 1 to smithi07085154-30 2024-09-14T03:53:39.018 INFO:tasks.rados.rados.0.smithi070.stdout:1644: seq_num 537 ranges {1015808=598016,1613824=573440,2187264=491520,2678784=139264} 2024-09-14T03:53:39.028 INFO:tasks.rados.rados.0.smithi070.stdout:1644: writing smithi07085154-40 from 1015808 to 1613824 tid 1 2024-09-14T03:53:39.032 INFO:tasks.rados.rados.0.smithi070.stdout:1644: writing smithi07085154-40 from 1613824 to 2187264 tid 2 2024-09-14T03:53:39.035 INFO:tasks.rados.rados.0.smithi070.stdout:1644: writing smithi07085154-40 from 2187264 to 2678784 tid 3 2024-09-14T03:53:39.037 INFO:tasks.rados.rados.0.smithi070.stdout:1644: writing smithi07085154-40 from 2678784 to 2818048 tid 4 2024-09-14T03:53:39.037 INFO:tasks.rados.rados.0.smithi070.stdout:1645: read oid 46 snap -1 2024-09-14T03:53:39.037 INFO:tasks.rados.rados.0.smithi070.stdout:1645: expect (ObjNum 528 snap 175 seq_num 528) 2024-09-14T03:53:39.037 INFO:tasks.rados.rados.0.smithi070.stdout:1646: delete oid 2 current snap is 176 2024-09-14T03:53:39.037 INFO:tasks.rados.rados.0.smithi070.stdout:1640: finishing rollback tid 2 to smithi07085154-30 2024-09-14T03:53:39.037 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 30 v 1011 (ObjNum 380 snap 129 seq_num 380) dirty exists 2024-09-14T03:53:39.037 INFO:tasks.rados.rados.0.smithi070.stdout:1638: finishing write tid 1 to smithi07085154-44 2024-09-14T03:53:39.037 INFO:tasks.rados.rados.0.smithi070.stdout:1641: finishing rollback tid 0 to smithi07085154-16 2024-09-14T03:53:39.037 INFO:tasks.rados.rados.0.smithi070.stdout:1641: finishing rollback tid 1 to smithi07085154-16 2024-09-14T03:53:39.037 INFO:tasks.rados.rados.0.smithi070.stdout:1641: finishing rollback tid 2 to smithi07085154-16 2024-09-14T03:53:39.037 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 1667 (ObjNum 114 snap 24 seq_num 114) dirty exists 2024-09-14T03:53:39.038 INFO:tasks.rados.rados.0.smithi070.stdout:1638: finishing write tid 2 to smithi07085154-44 2024-09-14T03:53:39.038 INFO:tasks.rados.rados.0.smithi070.stdout:1638: finishing write tid 3 to smithi07085154-44 2024-09-14T03:53:39.038 INFO:tasks.rados.rados.0.smithi070.stdout:1638: finishing write tid 4 to smithi07085154-44 2024-09-14T03:53:39.038 INFO:tasks.rados.rados.0.smithi070.stdout:1639: finishing copy_from racing read to smithi07085154-45 2024-09-14T03:53:39.038 INFO:tasks.rados.rados.0.smithi070.stdout:1638: finishing write tid 5 to smithi07085154-44 2024-09-14T03:53:39.038 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 774 (ObjNum 536 snap 176 seq_num 536) dirty exists 2024-09-14T03:53:39.038 INFO:tasks.rados.rados.0.smithi070.stdout:1638: left oid 44 (ObjNum 536 snap 176 seq_num 536) 2024-09-14T03:53:39.038 INFO:tasks.rados.rados.0.smithi070.stdout:1639: finishing copy_from to smithi07085154-45 2024-09-14T03:53:39.038 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 45 v 1028 (ObjNum 312 snap 105 seq_num 312) dirty exists 2024-09-14T03:53:39.038 INFO:tasks.rados.rados.0.smithi070.stdout:1644: finishing write tid 1 to smithi07085154-40 2024-09-14T03:53:39.039 INFO:tasks.rados.rados.0.smithi070.stdout:1644: finishing write tid 2 to smithi07085154-40 2024-09-14T03:53:39.041 INFO:tasks.rados.rados.0.smithi070.stdout:1644: finishing write tid 3 to smithi07085154-40 2024-09-14T03:53:39.042 INFO:tasks.rados.rados.0.smithi070.stdout:1644: finishing write tid 4 to smithi07085154-40 2024-09-14T03:53:39.043 INFO:tasks.rados.rados.0.smithi070.stdout:1644: finishing write tid 5 to smithi07085154-40 2024-09-14T03:53:39.044 INFO:tasks.rados.rados.0.smithi070.stdout:1644: finishing write tid 6 to smithi07085154-40 2024-09-14T03:53:39.044 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 792 (ObjNum 537 snap 176 seq_num 537) dirty exists 2024-09-14T03:53:39.044 INFO:tasks.rados.rados.0.smithi070.stdout:1644: left oid 40 (ObjNum 537 snap 176 seq_num 537) 2024-09-14T03:53:39.051 INFO:tasks.rados.rados.0.smithi070.stdout:1638: done (6 left) 2024-09-14T03:53:39.051 INFO:tasks.rados.rados.0.smithi070.stdout:1639: done (5 left) 2024-09-14T03:53:39.051 INFO:tasks.rados.rados.0.smithi070.stdout:1640: done (4 left) 2024-09-14T03:53:39.051 INFO:tasks.rados.rados.0.smithi070.stdout:1641: done (3 left) 2024-09-14T03:53:39.052 INFO:tasks.rados.rados.0.smithi070.stdout:1644: done (2 left) 2024-09-14T03:53:39.052 INFO:tasks.rados.rados.0.smithi070.stdout:1646: done (1 left) 2024-09-14T03:53:39.052 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 1 current snap is 176 2024-09-14T03:53:39.052 INFO:tasks.rados.rados.0.smithi070.stdout:1647: seq_num 539 ranges {0=630784,630784=737280,1368064=466944} 2024-09-14T03:53:39.055 INFO:tasks.rados.rados.0.smithi070.stdout:1647: writing smithi07085154-1 from 0 to 630784 tid 1 2024-09-14T03:53:39.060 INFO:tasks.rados.rados.0.smithi070.stdout:1647: writing smithi07085154-1 from 630784 to 1368064 tid 2 2024-09-14T03:53:39.063 INFO:tasks.rados.rados.0.smithi070.stdout:1647: writing smithi07085154-1 from 1368064 to 1835008 tid 3 2024-09-14T03:53:39.063 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 31 current snap is 176 2024-09-14T03:53:39.063 INFO:tasks.rados.rados.0.smithi070.stdout:1648: seq_num 540 ranges {0=647168,647168=671744,1318912=532480} 2024-09-14T03:53:39.067 INFO:tasks.rados.rados.0.smithi070.stdout:1648: writing smithi07085154-31 from 0 to 647168 tid 1 2024-09-14T03:53:39.070 INFO:tasks.rados.rados.0.smithi070.stdout:1648: writing smithi07085154-31 from 647168 to 1318912 tid 2 2024-09-14T03:53:39.074 INFO:tasks.rados.rados.0.smithi070.stdout:1648: writing smithi07085154-31 from 1318912 to 1851392 tid 3 2024-09-14T03:53:39.074 INFO:tasks.rados.rados.0.smithi070.stdout:1649: rollback oid 40 current snap is 176 2024-09-14T03:53:39.074 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 40 to 139 2024-09-14T03:53:39.074 INFO:tasks.rados.rados.0.smithi070.stdout:1647: finishing write tid 1 to smithi07085154-1 2024-09-14T03:53:39.074 INFO:tasks.rados.rados.0.smithi070.stdout:1650: snap_remove snap 151 2024-09-14T03:53:39.123 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:39 smithi070 ceph-mon[96704]: Upgrade: Updating osd.2 2024-09-14T03:53:39.123 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:39 smithi070 ceph-mon[96704]: Deploying daemon osd.2 on smithi070 2024-09-14T03:53:39.123 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:39 smithi070 ceph-mon[96704]: pgmap v186: 121 pgs: 5 active+clean+snaptrim, 116 active+clean; 464 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-14T03:53:39.123 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:39 smithi070 ceph-mon[96704]: osdmap e397: 8 total, 8 up, 8 in 2024-09-14T03:53:39.123 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:39 smithi070 ceph-mon[96704]: osdmap e398: 8 total, 8 up, 8 in 2024-09-14T03:53:39.123 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:39 smithi070 ceph-mon[98640]: Upgrade: Updating osd.2 2024-09-14T03:53:39.124 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:39 smithi070 ceph-mon[98640]: Deploying daemon osd.2 on smithi070 2024-09-14T03:53:39.124 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:39 smithi070 ceph-mon[98640]: pgmap v186: 121 pgs: 5 active+clean+snaptrim, 116 active+clean; 464 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-14T03:53:39.124 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:39 smithi070 ceph-mon[98640]: osdmap e397: 8 total, 8 up, 8 in 2024-09-14T03:53:39.124 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:39 smithi070 ceph-mon[98640]: osdmap e398: 8 total, 8 up, 8 in 2024-09-14T03:53:39.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:39 smithi094 ceph-mon[86597]: Upgrade: Updating osd.2 2024-09-14T03:53:39.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:39 smithi094 ceph-mon[86597]: Deploying daemon osd.2 on smithi070 2024-09-14T03:53:39.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:39 smithi094 ceph-mon[86597]: pgmap v186: 121 pgs: 5 active+clean+snaptrim, 116 active+clean; 464 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-14T03:53:39.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:39 smithi094 ceph-mon[86597]: osdmap e397: 8 total, 8 up, 8 in 2024-09-14T03:53:39.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:39 smithi094 ceph-mon[86597]: osdmap e398: 8 total, 8 up, 8 in 2024-09-14T03:53:40.061 INFO:tasks.rados.rados.0.smithi070.stdout:1650: done (4 left) 2024-09-14T03:53:40.062 INFO:tasks.rados.rados.0.smithi070.stdout:1651: snap_remove snap 141 2024-09-14T03:53:40.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:53:40 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:53:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:41.013 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:40 smithi070 systemd[1]: Stopping Ceph osd.2 for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:53:41.069 INFO:tasks.rados.rados.0.smithi070.stdout:1651: done (4 left) 2024-09-14T03:53:41.069 INFO:tasks.rados.rados.0.smithi070.stdout:1652: rmattr oid 22 current snap is 176 2024-09-14T03:53:41.070 INFO:tasks.rados.rados.0.smithi070.stdout:1652: done (4 left) 2024-09-14T03:53:41.070 INFO:tasks.rados.rados.0.smithi070.stdout:1653: rollback oid 35 current snap is 176 2024-09-14T03:53:41.070 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 35 to 119 2024-09-14T03:53:41.070 INFO:tasks.rados.rados.0.smithi070.stdout:1654: read oid 33 snap 162 2024-09-14T03:53:41.070 INFO:tasks.rados.rados.0.smithi070.stdout:1654: expect (ObjNum 467 snap 160 seq_num 467) 2024-09-14T03:53:41.070 INFO:tasks.rados.rados.0.smithi070.stdout:1655: snap_remove snap 94 2024-09-14T03:53:41.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:41 smithi070 ceph-mon[96704]: pgmap v189: 121 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 118 active+clean; 487 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 4.3 MiB/s wr, 16 op/s 2024-09-14T03:53:41.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:41 smithi070 ceph-mon[96704]: osdmap e399: 8 total, 8 up, 8 in 2024-09-14T03:53:41.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:41 smithi070 ceph-mon[96704]: osd.2 marked itself down and dead 2024-09-14T03:53:41.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:41 smithi070 ceph-mon[98640]: pgmap v189: 121 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 118 active+clean; 487 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 4.3 MiB/s wr, 16 op/s 2024-09-14T03:53:41.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:41 smithi070 ceph-mon[98640]: osdmap e399: 8 total, 8 up, 8 in 2024-09-14T03:53:41.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:41 smithi070 ceph-mon[98640]: osd.2 marked itself down and dead 2024-09-14T03:53:41.286 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:41 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2[55574]: 2024-09-14T03:53:41.010+0000 7fbf80186640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:53:41.286 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:41 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2[55574]: 2024-09-14T03:53:41.010+0000 7fbf80186640 -1 osd.2 399 *** Got signal Terminated *** 2024-09-14T03:53:41.286 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:41 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2[55574]: 2024-09-14T03:53:41.010+0000 7fbf80186640 -1 osd.2 399 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T03:53:41.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:41 smithi094 ceph-mon[86597]: pgmap v189: 121 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 118 active+clean; 487 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 4.3 MiB/s wr, 16 op/s 2024-09-14T03:53:41.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:41 smithi094 ceph-mon[86597]: osdmap e399: 8 total, 8 up, 8 in 2024-09-14T03:53:41.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:41 smithi094 ceph-mon[86597]: osd.2 marked itself down and dead 2024-09-14T03:53:41.719 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:41 smithi070 podman[112518]: 2024-09-14 03:53:41.420406362 +0000 UTC m=+0.532762823 container died 109b66e4e60fb2ded7002092727e75a0a43595887c12a5275f924cc9430a8cdc (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-eb16b97, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-14T03:53:42.028 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:41 smithi070 podman[112518]: 2024-09-14 03:53:41.834191356 +0000 UTC m=+0.946547817 container cleanup 109b66e4e60fb2ded7002092727e75a0a43595887c12a5275f924cc9430a8cdc (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=quincy-eb16b97) 2024-09-14T03:53:42.028 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:41 smithi070 bash[112518]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2 2024-09-14T03:53:42.073 INFO:tasks.rados.rados.0.smithi070.stdout:1655: done (6 left) 2024-09-14T03:53:42.073 INFO:tasks.rados.rados.0.smithi070.stdout:1656: rmattr oid 48 current snap is 176 2024-09-14T03:53:42.073 INFO:tasks.rados.rados.0.smithi070.stdout:1656: done (6 left) 2024-09-14T03:53:42.074 INFO:tasks.rados.rados.0.smithi070.stdout:1657: rollback oid 48 current snap is 176 2024-09-14T03:53:42.074 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 48 to 137 2024-09-14T03:53:42.074 INFO:tasks.rados.rados.0.smithi070.stdout:1658: rollback oid 8 current snap is 176 2024-09-14T03:53:42.074 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 8 to 163 2024-09-14T03:53:42.074 INFO:tasks.rados.rados.0.smithi070.stdout:1659: read oid 39 snap -1 2024-09-14T03:53:42.074 INFO:tasks.rados.rados.0.smithi070.stdout:1659: expect deleted 2024-09-14T03:53:42.074 INFO:tasks.rados.rados.0.smithi070.stdout:1660: read oid 18 snap -1 2024-09-14T03:53:42.074 INFO:tasks.rados.rados.0.smithi070.stdout:1660: expect (ObjNum 530 snap 175 seq_num 530) 2024-09-14T03:53:42.074 INFO:tasks.rados.rados.0.smithi070.stdout:1661: setattr oid 14 current snap is 176 2024-09-14T03:53:42.074 INFO:tasks.rados.rados.0.smithi070.stdout:1662: snap_remove snap 164 2024-09-14T03:53:42.285 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:42 smithi070 podman[112540]: 2024-09-14 03:53:42.027258036 +0000 UTC m=+0.599839136 container remove 109b66e4e60fb2ded7002092727e75a0a43595887c12a5275f924cc9430a8cdc (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=quincy-eb16b97, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:53:42.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:42 smithi070 ceph-mon[96704]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:53:42.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:42 smithi070 ceph-mon[96704]: osdmap e400: 8 total, 7 up, 8 in 2024-09-14T03:53:42.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:42 smithi070 ceph-mon[98640]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:53:42.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:42 smithi070 ceph-mon[98640]: osdmap e400: 8 total, 7 up, 8 in 2024-09-14T03:53:42.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:42 smithi094 ceph-mon[86597]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:53:42.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:42 smithi094 ceph-mon[86597]: osdmap e400: 8 total, 7 up, 8 in 2024-09-14T03:53:42.773 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:42 smithi070 podman[112633]: 2024-09-14 03:53:42.453251717 +0000 UTC m=+0.110548382 container create bdb2908507a2cdb500a325c59f22e6fd92e6490029796c9563f0b6d713de06c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231) 2024-09-14T03:53:42.773 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:42 smithi070 podman[112633]: 2024-09-14 03:53:42.358429376 +0000 UTC m=+0.015726048 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:53:42.773 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:42 smithi070 podman[112633]: 2024-09-14 03:53:42.680892183 +0000 UTC m=+0.338188847 container init bdb2908507a2cdb500a325c59f22e6fd92e6490029796c9563f0b6d713de06c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2-deactivate, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=main-a278231, ceph=True, CEPH_POINT_RELEASE=) 2024-09-14T03:53:42.773 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:42 smithi070 podman[112633]: 2024-09-14 03:53:42.686047628 +0000 UTC m=+0.343344284 container start bdb2908507a2cdb500a325c59f22e6fd92e6490029796c9563f0b6d713de06c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=main-a278231, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-14T03:53:42.773 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:42 smithi070 podman[112633]: 2024-09-14 03:53:42.772363825 +0000 UTC m=+0.429660498 container attach bdb2908507a2cdb500a325c59f22e6fd92e6490029796c9563f0b6d713de06c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T03:53:43.035 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:42 smithi070 podman[112633]: 2024-09-14 03:53:42.872037895 +0000 UTC m=+0.529334553 container died bdb2908507a2cdb500a325c59f22e6fd92e6490029796c9563f0b6d713de06c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=main-a278231, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2) 2024-09-14T03:53:43.079 INFO:tasks.rados.rados.0.smithi070.stdout:1662: done (11 left) 2024-09-14T03:53:43.079 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 15 current snap is 176 2024-09-14T03:53:43.079 INFO:tasks.rados.rados.0.smithi070.stdout:1663: seq_num 541 ranges {1499136=663552,2162688=557056,2719744=475136,3194880=172032} 2024-09-14T03:53:43.091 INFO:tasks.rados.rados.0.smithi070.stdout:1663: writing smithi07085154-15 from 1499136 to 2162688 tid 1 2024-09-14T03:53:43.095 INFO:tasks.rados.rados.0.smithi070.stdout:1663: writing smithi07085154-15 from 2162688 to 2719744 tid 2 2024-09-14T03:53:43.097 INFO:tasks.rados.rados.0.smithi070.stdout:1663: writing smithi07085154-15 from 2719744 to 3194880 tid 3 2024-09-14T03:53:43.099 INFO:tasks.rados.rados.0.smithi070.stdout:1663: writing smithi07085154-15 from 3194880 to 3366912 tid 4 2024-09-14T03:53:43.099 INFO:tasks.rados.rados.0.smithi070.stdout:1664: snap_create 2024-09-14T03:53:43.099 INFO:tasks.rados.rados.0.smithi070.stdout:1645: expect (ObjNum 528 snap 175 seq_num 528) 2024-09-14T03:53:43.326 INFO:tasks.rados.rados.0.smithi070.stdout:1647: finishing write tid 2 to smithi07085154-1 2024-09-14T03:53:43.327 INFO:tasks.rados.rados.0.smithi070.stdout:1647: finishing write tid 3 to smithi07085154-1 2024-09-14T03:53:43.327 INFO:tasks.rados.rados.0.smithi070.stdout:1647: finishing write tid 4 to smithi07085154-1 2024-09-14T03:53:43.327 INFO:tasks.rados.rados.0.smithi070.stdout:1647: finishing write tid 5 to smithi07085154-1 2024-09-14T03:53:43.327 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 1 v 796 (ObjNum 539 snap 176 seq_num 539) dirty exists 2024-09-14T03:53:43.327 INFO:tasks.rados.rados.0.smithi070.stdout:1647: left oid 1 (ObjNum 539 snap 176 seq_num 539) 2024-09-14T03:53:43.327 INFO:tasks.rados.rados.0.smithi070.stdout:1648: finishing write tid 1 to smithi07085154-31 2024-09-14T03:53:43.327 INFO:tasks.rados.rados.0.smithi070.stdout:1648: finishing write tid 2 to smithi07085154-31 2024-09-14T03:53:43.327 INFO:tasks.rados.rados.0.smithi070.stdout:1648: finishing write tid 3 to smithi07085154-31 2024-09-14T03:53:43.328 INFO:tasks.rados.rados.0.smithi070.stdout:1648: finishing write tid 4 to smithi07085154-31 2024-09-14T03:53:43.328 INFO:tasks.rados.rados.0.smithi070.stdout:1649: finishing rollback tid 0 to smithi07085154-40 2024-09-14T03:53:43.328 INFO:tasks.rados.rados.0.smithi070.stdout:1649: finishing rollback tid 1 to smithi07085154-40 2024-09-14T03:53:43.328 INFO:tasks.rados.rados.0.smithi070.stdout:1649: finishing rollback tid 2 to smithi07085154-40 2024-09-14T03:53:43.328 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 799 (ObjNum 353 snap 116 seq_num 353) dirty exists 2024-09-14T03:53:43.328 INFO:tasks.rados.rados.0.smithi070.stdout:1648: finishing write tid 5 to smithi07085154-31 2024-09-14T03:53:43.328 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 884 (ObjNum 540 snap 176 seq_num 540) dirty exists 2024-09-14T03:53:43.328 INFO:tasks.rados.rados.0.smithi070.stdout:1648: left oid 31 (ObjNum 540 snap 176 seq_num 540) 2024-09-14T03:53:43.328 INFO:tasks.rados.rados.0.smithi070.stdout:1654: expect (ObjNum 467 snap 160 seq_num 467) 2024-09-14T03:53:43.406 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:43 smithi070 ceph-mon[96704]: pgmap v192: 121 pgs: 1 active+clean+snaptrim, 5 peering, 1 stale+active+clean+snaptrim, 1 stale+active+clean+snaptrim_wait, 8 stale+active+clean, 105 active+clean; 485 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.7 MiB/s wr, 10 op/s 2024-09-14T03:53:43.406 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:43 smithi070 ceph-mon[96704]: osdmap e401: 8 total, 7 up, 8 in 2024-09-14T03:53:43.406 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:43 smithi070 ceph-mon[98640]: pgmap v192: 121 pgs: 1 active+clean+snaptrim, 5 peering, 1 stale+active+clean+snaptrim, 1 stale+active+clean+snaptrim_wait, 8 stale+active+clean, 105 active+clean; 485 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.7 MiB/s wr, 10 op/s 2024-09-14T03:53:43.406 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:43 smithi070 ceph-mon[98640]: osdmap e401: 8 total, 7 up, 8 in 2024-09-14T03:53:43.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:43 smithi094 ceph-mon[86597]: pgmap v192: 121 pgs: 1 active+clean+snaptrim, 5 peering, 1 stale+active+clean+snaptrim, 1 stale+active+clean+snaptrim_wait, 8 stale+active+clean, 105 active+clean; 485 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.7 MiB/s wr, 10 op/s 2024-09-14T03:53:43.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:43 smithi094 ceph-mon[86597]: osdmap e401: 8 total, 7 up, 8 in 2024-09-14T03:53:43.594 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 641 (ObjNum 398 snap 138 seq_num 398) dirty exists 2024-09-14T03:53:43.595 INFO:tasks.rados.rados.0.smithi070.stdout:1658: finishing rollback tid 0 to smithi07085154-8 2024-09-14T03:53:43.595 INFO:tasks.rados.rados.0.smithi070.stdout:1657: finishing rollback tid 0 to smithi07085154-48 2024-09-14T03:53:43.595 INFO:tasks.rados.rados.0.smithi070.stdout:1657: finishing rollback tid 1 to smithi07085154-48 2024-09-14T03:53:43.595 INFO:tasks.rados.rados.0.smithi070.stdout:1653: finishing rollback tid 1 to smithi07085154-35 2024-09-14T03:53:43.595 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 35 v 24 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-14T03:53:43.595 INFO:tasks.rados.rados.0.smithi070.stdout:1657: finishing rollback tid 2 to smithi07085154-48 2024-09-14T03:53:43.595 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 562 (ObjNum 313 snap 105 seq_num 313) dirty exists 2024-09-14T03:53:43.595 INFO:tasks.rados.rados.0.smithi070.stdout:1658: finishing rollback tid 1 to smithi07085154-8 2024-09-14T03:53:43.595 INFO:tasks.rados.rados.0.smithi070.stdout:1658: finishing rollback tid 2 to smithi07085154-8 2024-09-14T03:53:43.595 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 8 v 1230 (ObjNum 402 snap 140 seq_num 402) dirty exists 2024-09-14T03:53:43.595 INFO:tasks.rados.rados.0.smithi070.stdout:1660: expect (ObjNum 530 snap 175 seq_num 530) 2024-09-14T03:53:43.638 INFO:tasks.rados.rados.0.smithi070.stdout:1663: finishing write tid 1 to smithi07085154-15 2024-09-14T03:53:43.639 INFO:tasks.rados.rados.0.smithi070.stdout:1663: finishing write tid 2 to smithi07085154-15 2024-09-14T03:53:43.639 INFO:tasks.rados.rados.0.smithi070.stdout:1663: finishing write tid 3 to smithi07085154-15 2024-09-14T03:53:43.639 INFO:tasks.rados.rados.0.smithi070.stdout:1663: finishing write tid 4 to smithi07085154-15 2024-09-14T03:53:43.639 INFO:tasks.rados.rados.0.smithi070.stdout:1663: finishing write tid 5 to smithi07085154-15 2024-09-14T03:53:43.639 INFO:tasks.rados.rados.0.smithi070.stdout:1663: finishing write tid 6 to smithi07085154-15 2024-09-14T03:53:43.639 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 15 v 1254 (ObjNum 541 snap 176 seq_num 541) dirty exists 2024-09-14T03:53:43.639 INFO:tasks.rados.rados.0.smithi070.stdout:1663: left oid 15 (ObjNum 541 snap 176 seq_num 541) 2024-09-14T03:53:43.685 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:53:43 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:53:43] "GET /metrics HTTP/1.1" 200 36312 "" "Prometheus/2.51.0" 2024-09-14T03:53:43.686 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:43 smithi070 podman[112676]: 2024-09-14 03:53:43.451336328 +0000 UTC m=+0.573606774 container remove bdb2908507a2cdb500a325c59f22e6fd92e6490029796c9563f0b6d713de06c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=main-a278231, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-14T03:53:43.686 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:43 smithi070 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.2.service: Deactivated successfully. 2024-09-14T03:53:43.686 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:43 smithi070 systemd[1]: Stopped Ceph osd.2 for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:53:43.686 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:43 smithi070 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.2.service: Consumed 14.489s CPU time. 2024-09-14T03:53:44.020 INFO:tasks.rados.rados.0.smithi070.stdout:1645: done (12 left) 2024-09-14T03:53:44.021 INFO:tasks.rados.rados.0.smithi070.stdout:1647: done (11 left) 2024-09-14T03:53:44.021 INFO:tasks.rados.rados.0.smithi070.stdout:1648: done (10 left) 2024-09-14T03:53:44.021 INFO:tasks.rados.rados.0.smithi070.stdout:1649: done (9 left) 2024-09-14T03:53:44.021 INFO:tasks.rados.rados.0.smithi070.stdout:1653: done (8 left) 2024-09-14T03:53:44.021 INFO:tasks.rados.rados.0.smithi070.stdout:1654: done (7 left) 2024-09-14T03:53:44.021 INFO:tasks.rados.rados.0.smithi070.stdout:1657: done (6 left) 2024-09-14T03:53:44.021 INFO:tasks.rados.rados.0.smithi070.stdout:1658: done (5 left) 2024-09-14T03:53:44.021 INFO:tasks.rados.rados.0.smithi070.stdout:1659: done (4 left) 2024-09-14T03:53:44.021 INFO:tasks.rados.rados.0.smithi070.stdout:1660: done (3 left) 2024-09-14T03:53:44.021 INFO:tasks.rados.rados.0.smithi070.stdout:1661: done (2 left) 2024-09-14T03:53:44.021 INFO:tasks.rados.rados.0.smithi070.stdout:1663: done (1 left) 2024-09-14T03:53:44.022 INFO:tasks.rados.rados.0.smithi070.stdout:1664: done (0 left) 2024-09-14T03:53:44.022 INFO:tasks.rados.rados.0.smithi070.stdout:1665: read oid 27 snap -1 2024-09-14T03:53:44.022 INFO:tasks.rados.rados.0.smithi070.stdout:1665: expect (ObjNum 526 snap 174 seq_num 526) 2024-09-14T03:53:44.022 INFO:tasks.rados.rados.0.smithi070.stdout:1666: rmattr oid 22 current snap is 177 2024-09-14T03:53:44.022 INFO:tasks.rados.rados.0.smithi070.stdout:1666: done (1 left) 2024-09-14T03:53:44.022 INFO:tasks.rados.rados.0.smithi070.stdout:1667: read oid 21 snap 142 2024-09-14T03:53:44.022 INFO:tasks.rados.rados.0.smithi070.stdout:1667: expect (ObjNum 114 snap 24 seq_num 114) 2024-09-14T03:53:44.022 INFO:tasks.rados.rados.0.smithi070.stdout:1668: read oid 26 snap -1 2024-09-14T03:53:44.022 INFO:tasks.rados.rados.0.smithi070.stdout:1668: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:53:44.022 INFO:tasks.rados.rados.0.smithi070.stdout:1669: read oid 9 snap 175 2024-09-14T03:53:44.022 INFO:tasks.rados.rados.0.smithi070.stdout:1669: expect (ObjNum 507 snap 166 seq_num 507) 2024-09-14T03:53:44.022 INFO:tasks.rados.rados.0.smithi070.stdout:1670: read oid 30 snap -1 2024-09-14T03:53:44.022 INFO:tasks.rados.rados.0.smithi070.stdout:1670: expect (ObjNum 380 snap 129 seq_num 380) 2024-09-14T03:53:44.022 INFO:tasks.rados.rados.0.smithi070.stdout:1671: snap_remove snap 165 2024-09-14T03:53:44.035 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:43 smithi070 systemd[1]: Starting Ceph osd.2 for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:53:44.035 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:43 smithi070 podman[112778]: 2024-09-14 03:53:43.795765781 +0000 UTC m=+0.015504242 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:53:44.036 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:43 smithi070 podman[112778]: 2024-09-14 03:53:43.911057624 +0000 UTC m=+0.130796075 container create 700bcce3d8d8f4c612f996f8a34c4292b2a60ecd527cb2cb49c1e650788ae52a (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2-activate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=main-a278231, GIT_CLEAN=True) 2024-09-14T03:53:44.311 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:44 smithi070 ceph-mon[96704]: osdmap e402: 8 total, 7 up, 8 in 2024-09-14T03:53:44.311 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:44 smithi070 ceph-mon[96704]: osdmap e403: 8 total, 7 up, 8 in 2024-09-14T03:53:44.311 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:44 smithi070 ceph-mon[98640]: osdmap e402: 8 total, 7 up, 8 in 2024-09-14T03:53:44.311 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:44 smithi070 ceph-mon[98640]: osdmap e403: 8 total, 7 up, 8 in 2024-09-14T03:53:44.311 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:44 smithi070 podman[112778]: 2024-09-14 03:53:44.180656928 +0000 UTC m=+0.400395380 container init 700bcce3d8d8f4c612f996f8a34c4292b2a60ecd527cb2cb49c1e650788ae52a (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2-activate, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-14T03:53:44.311 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:44 smithi070 podman[112778]: 2024-09-14 03:53:44.185383036 +0000 UTC m=+0.405121497 container start 700bcce3d8d8f4c612f996f8a34c4292b2a60ecd527cb2cb49c1e650788ae52a (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2-activate, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, RELEASE=main-a278231, maintainer=Guillaume Abrioux ) 2024-09-14T03:53:44.312 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:44 smithi070 podman[112778]: 2024-09-14 03:53:44.31013394 +0000 UTC m=+0.529872401 container attach 700bcce3d8d8f4c612f996f8a34c4292b2a60ecd527cb2cb49c1e650788ae52a (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2-activate, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T03:53:44.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:44 smithi094 ceph-mon[86597]: osdmap e402: 8 total, 7 up, 8 in 2024-09-14T03:53:44.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:44 smithi094 ceph-mon[86597]: osdmap e403: 8 total, 7 up, 8 in 2024-09-14T03:53:44.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:53:44 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:53:44.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.2\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:44.783 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:44 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2-activate[112793]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:53:44.783 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:44 smithi070 bash[112778]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:53:44.783 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:44 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2-activate[112793]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:53:44.783 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:44 smithi070 bash[112778]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:53:44.783 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:44 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2-activate[112793]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-14T03:53:44.783 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:44 smithi070 bash[112778]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-14T03:53:45.023 INFO:tasks.rados.rados.0.smithi070.stdout:1671: done (5 left) 2024-09-14T03:53:45.023 INFO:tasks.rados.rados.0.smithi070.stdout:1672: snap_create 2024-09-14T03:53:45.023 INFO:tasks.rados.rados.0.smithi070.stdout:1668: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:53:45.023 INFO:tasks.rados.rados.0.smithi070.stdout:1667: expect (ObjNum 114 snap 24 seq_num 114) 2024-09-14T03:53:45.035 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:44 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2-activate[112793]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-14T03:53:45.035 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:44 smithi070 bash[112778]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-14T03:53:45.239 INFO:tasks.rados.rados.0.smithi070.stdout:1670: expect (ObjNum 380 snap 129 seq_num 380) 2024-09-14T03:53:45.401 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:45 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2-activate[112793]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-14T03:53:45.401 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:45 smithi070 bash[112778]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-14T03:53:45.401 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:45 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2-activate[112793]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-14T03:53:45.402 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:45 smithi070 bash[112778]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-14T03:53:45.402 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:45 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2-activate[112793]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-14T03:53:45.402 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:45 smithi070 bash[112778]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-14T03:53:45.402 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:45 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2-activate[112793]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-14T03:53:45.402 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:45 smithi070 bash[112778]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-14T03:53:45.402 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:45 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2-activate[112793]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-14T03:53:45.402 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:45 smithi070 bash[112778]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-14T03:53:45.402 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:45 smithi070 podman[112778]: 2024-09-14 03:53:45.108598867 +0000 UTC m=+1.328337336 container died 700bcce3d8d8f4c612f996f8a34c4292b2a60ecd527cb2cb49c1e650788ae52a (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.vendor=CentOS, ceph=True, RELEASE=main-a278231, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-14T03:53:45.403 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:45 smithi070 ceph-mon[96704]: pgmap v196: 121 pgs: 20 active+undersized, 1 active+clean+snaptrim, 5 peering, 1 stale+active+clean, 7 active+undersized+degraded, 87 active+clean; 484 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.3 MiB/s wr, 13 op/s; 94/1053 objects degraded (8.927%) 2024-09-14T03:53:45.403 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:45 smithi070 ceph-mon[96704]: Health check failed: Degraded data redundancy: 94/1053 objects degraded (8.927%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:53:45.403 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:45 smithi070 ceph-mon[96704]: osdmap e404: 8 total, 7 up, 8 in 2024-09-14T03:53:45.403 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:45 smithi070 ceph-mon[98640]: pgmap v196: 121 pgs: 20 active+undersized, 1 active+clean+snaptrim, 5 peering, 1 stale+active+clean, 7 active+undersized+degraded, 87 active+clean; 484 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.3 MiB/s wr, 13 op/s; 94/1053 objects degraded (8.927%) 2024-09-14T03:53:45.403 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:45 smithi070 ceph-mon[98640]: Health check failed: Degraded data redundancy: 94/1053 objects degraded (8.927%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:53:45.403 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:45 smithi070 ceph-mon[98640]: osdmap e404: 8 total, 7 up, 8 in 2024-09-14T03:53:45.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:45 smithi094 ceph-mon[86597]: pgmap v196: 121 pgs: 20 active+undersized, 1 active+clean+snaptrim, 5 peering, 1 stale+active+clean, 7 active+undersized+degraded, 87 active+clean; 484 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.3 MiB/s wr, 13 op/s; 94/1053 objects degraded (8.927%) 2024-09-14T03:53:45.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:45 smithi094 ceph-mon[86597]: Health check failed: Degraded data redundancy: 94/1053 objects degraded (8.927%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:53:45.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:45 smithi094 ceph-mon[86597]: osdmap e404: 8 total, 7 up, 8 in 2024-09-14T03:53:45.550 INFO:tasks.rados.rados.0.smithi070.stdout:1665: expect (ObjNum 526 snap 174 seq_num 526) 2024-09-14T03:53:46.026 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:53:45 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:53:45.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:46.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:53:45 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:53:45.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:46.035 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:45 smithi070 podman[112939]: 2024-09-14 03:53:45.709711907 +0000 UTC m=+0.591120447 container remove 700bcce3d8d8f4c612f996f8a34c4292b2a60ecd527cb2cb49c1e650788ae52a (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=main-a278231, maintainer=Guillaume Abrioux ) 2024-09-14T03:53:46.036 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:45 smithi070 podman[113003]: 2024-09-14 03:53:45.82346969 +0000 UTC m=+0.015570566 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:53:46.036 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:45 smithi070 podman[113003]: 2024-09-14 03:53:45.927346514 +0000 UTC m=+0.119447387 container create 39ef876d2fffa79521c362315dc0028c2e2e8d9d97efa6c1bca3813e494aa7b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=main-a278231, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-14T03:53:46.536 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:46 smithi070 podman[113003]: 2024-09-14 03:53:46.180100955 +0000 UTC m=+0.372201824 container init 39ef876d2fffa79521c362315dc0028c2e2e8d9d97efa6c1bca3813e494aa7b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-14T03:53:46.536 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:46 smithi070 podman[113003]: 2024-09-14 03:53:46.185391147 +0000 UTC m=+0.377492024 container start 39ef876d2fffa79521c362315dc0028c2e2e8d9d97efa6c1bca3813e494aa7b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-14T03:53:46.536 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:46 smithi070 ceph-osd[113041]: -- 172.21.15.70:0/3156415658 <== mon.0 v2:172.21.15.70:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x559107ea7a40 con 0x559108c74400 2024-09-14T03:53:46.536 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:46 smithi070 bash[113003]: 39ef876d2fffa79521c362315dc0028c2e2e8d9d97efa6c1bca3813e494aa7b3 2024-09-14T03:53:46.536 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:46 smithi070 systemd[1]: Started Ceph osd.2 for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:53:47.277 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:53:46 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:53:46.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:47.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:47 smithi094 ceph-mon[86597]: osdmap e405: 8 total, 7 up, 8 in 2024-09-14T03:53:47.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:47 smithi094 ceph-mon[86597]: pgmap v199: 121 pgs: 24 active+undersized, 1 active+clean+snaptrim, 5 peering, 8 active+undersized+degraded, 83 active+clean; 484 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 1.3 MiB/s wr, 13 op/s; 98/1053 objects degraded (9.307%) 2024-09-14T03:53:47.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:47 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:47.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:47 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:47.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:47 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:53:47.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:47 smithi070 ceph-mon[96704]: osdmap e405: 8 total, 7 up, 8 in 2024-09-14T03:53:47.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:47 smithi070 ceph-mon[96704]: pgmap v199: 121 pgs: 24 active+undersized, 1 active+clean+snaptrim, 5 peering, 8 active+undersized+degraded, 83 active+clean; 484 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 1.3 MiB/s wr, 13 op/s; 98/1053 objects degraded (9.307%) 2024-09-14T03:53:47.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:47 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:47.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:47 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:47.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:47 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:53:47.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:47 smithi070 ceph-mon[98640]: osdmap e405: 8 total, 7 up, 8 in 2024-09-14T03:53:47.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:47 smithi070 ceph-mon[98640]: pgmap v199: 121 pgs: 24 active+undersized, 1 active+clean+snaptrim, 5 peering, 8 active+undersized+degraded, 83 active+clean; 484 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 1.3 MiB/s wr, 13 op/s; 98/1053 objects degraded (9.307%) 2024-09-14T03:53:47.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:47 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:47.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:47 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:47.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:47 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:53:47.575 INFO:tasks.rados.rados.0.smithi070.stdout:1669: expect (ObjNum 507 snap 166 seq_num 507) 2024-09-14T03:53:47.618 INFO:tasks.rados.rados.0.smithi070.stdout:1665: done (5 left) 2024-09-14T03:53:47.619 INFO:tasks.rados.rados.0.smithi070.stdout:1667: done (4 left) 2024-09-14T03:53:47.619 INFO:tasks.rados.rados.0.smithi070.stdout:1668: done (3 left) 2024-09-14T03:53:47.619 INFO:tasks.rados.rados.0.smithi070.stdout:1669: done (2 left) 2024-09-14T03:53:47.619 INFO:tasks.rados.rados.0.smithi070.stdout:1670: done (1 left) 2024-09-14T03:53:47.619 INFO:tasks.rados.rados.0.smithi070.stdout:1672: done (0 left) 2024-09-14T03:53:47.619 INFO:tasks.rados.rados.0.smithi070.stdout:1673: snap_create 2024-09-14T03:53:47.886 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:47 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2[113018]: 2024-09-14T03:53:47.575+0000 7f5ca6738640 -1 Falling back to public interface 2024-09-14T03:53:48.326 INFO:tasks.rados.rados.0.smithi070.stdout:1673: done (0 left) 2024-09-14T03:53:48.327 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 40 current snap is 179 2024-09-14T03:53:48.327 INFO:tasks.rados.rados.0.smithi070.stdout:1674: seq_num 542 ranges {7979008=679936,8658944=442368,9101312=679936,9781248=81920} 2024-09-14T03:53:48.372 INFO:tasks.rados.rados.0.smithi070.stdout:1674: writing smithi07085154-40 from 7979008 to 8658944 tid 1 2024-09-14T03:53:48.374 INFO:tasks.rados.rados.0.smithi070.stdout:1674: writing smithi07085154-40 from 8658944 to 9101312 tid 2 2024-09-14T03:53:48.379 INFO:tasks.rados.rados.0.smithi070.stdout:1674: writing smithi07085154-40 from 9101312 to 9781248 tid 3 2024-09-14T03:53:48.379 INFO:tasks.rados.rados.0.smithi070.stdout:1674: writing smithi07085154-40 from 9781248 to 9863168 tid 4 2024-09-14T03:53:48.380 INFO:tasks.rados.rados.0.smithi070.stdout:1675: snap_create 2024-09-14T03:53:48.380 INFO:tasks.rados.rados.0.smithi070.stdout:1674: finishing write tid 1 to smithi07085154-40 2024-09-14T03:53:48.380 INFO:tasks.rados.rados.0.smithi070.stdout:1674: finishing write tid 2 to smithi07085154-40 2024-09-14T03:53:48.383 INFO:tasks.rados.rados.0.smithi070.stdout:1674: finishing write tid 3 to smithi07085154-40 2024-09-14T03:53:48.383 INFO:tasks.rados.rados.0.smithi070.stdout:1674: finishing write tid 4 to smithi07085154-40 2024-09-14T03:53:48.383 INFO:tasks.rados.rados.0.smithi070.stdout:1674: finishing write tid 5 to smithi07085154-40 2024-09-14T03:53:48.384 INFO:tasks.rados.rados.0.smithi070.stdout:1674: finishing write tid 6 to smithi07085154-40 2024-09-14T03:53:48.384 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 833 (ObjNum 542 snap 179 seq_num 542) dirty exists 2024-09-14T03:53:48.384 INFO:tasks.rados.rados.0.smithi070.stdout:1674: left oid 40 (ObjNum 542 snap 179 seq_num 542) 2024-09-14T03:53:48.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:48 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:48.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:48 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:48.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:48 smithi070 ceph-mon[96704]: pgmap v200: 121 pgs: 27 active+undersized, 10 active+undersized+degraded, 84 active+clean; 484 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 366 KiB/s wr, 2 op/s; 128/1047 objects degraded (12.225%) 2024-09-14T03:53:48.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:48 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:48.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:48 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:48.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:48 smithi070 ceph-mon[98640]: pgmap v200: 121 pgs: 27 active+undersized, 10 active+undersized+degraded, 84 active+clean; 484 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 366 KiB/s wr, 2 op/s; 128/1047 objects degraded (12.225%) 2024-09-14T03:53:48.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:48 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:48.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:48 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:53:48.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:48 smithi094 ceph-mon[86597]: pgmap v200: 121 pgs: 27 active+undersized, 10 active+undersized+degraded, 84 active+clean; 484 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 366 KiB/s wr, 2 op/s; 128/1047 objects degraded (12.225%) 2024-09-14T03:53:49.020 INFO:tasks.rados.rados.0.smithi070.stdout:1674: done (1 left) 2024-09-14T03:53:49.020 INFO:tasks.rados.rados.0.smithi070.stdout:1675: done (0 left) 2024-09-14T03:53:49.020 INFO:tasks.rados.rados.0.smithi070.stdout:1676: copy_from oid 50 from oid 10 current snap is 180 2024-09-14T03:53:49.020 INFO:tasks.rados.rados.0.smithi070.stdout:1677: snap_remove snap 73 2024-09-14T03:53:49.653 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:49 smithi070 ceph-mon[96704]: osdmap e406: 8 total, 7 up, 8 in 2024-09-14T03:53:49.654 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:49 smithi070 ceph-mon[96704]: osdmap e407: 8 total, 7 up, 8 in 2024-09-14T03:53:49.654 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:49 smithi070 ceph-mon[98640]: osdmap e406: 8 total, 7 up, 8 in 2024-09-14T03:53:49.654 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:49 smithi070 ceph-mon[98640]: osdmap e407: 8 total, 7 up, 8 in 2024-09-14T03:53:49.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:49 smithi094 ceph-mon[86597]: osdmap e406: 8 total, 7 up, 8 in 2024-09-14T03:53:49.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:49 smithi094 ceph-mon[86597]: osdmap e407: 8 total, 7 up, 8 in 2024-09-14T03:53:50.023 INFO:tasks.rados.rados.0.smithi070.stdout:1677: done (1 left) 2024-09-14T03:53:50.023 INFO:tasks.rados.rados.0.smithi070.stdout:1678: copy_from oid 5 from oid 2 current snap is 180 2024-09-14T03:53:50.023 INFO:tasks.rados.rados.0.smithi070.stdout:1676: finishing copy_from racing read to smithi07085154-50 2024-09-14T03:53:50.023 INFO:tasks.rados.rados.0.smithi070.stdout:1676: finishing copy_from to smithi07085154-50 2024-09-14T03:53:50.023 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 1550 (ObjNum 508 snap 166 seq_num 508) dirty exists 2024-09-14T03:53:50.023 INFO:tasks.rados.rados.0.smithi070.stdout:1676: done (1 left) 2024-09-14T03:53:50.023 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 7 current snap is 180 2024-09-14T03:53:50.023 INFO:tasks.rados.rados.0.smithi070.stdout:1679: seq_num 543 ranges {3760128=704512,4464640=729088,5193728=475136} 2024-09-14T03:53:50.048 INFO:tasks.rados.rados.0.smithi070.stdout:1679: writing smithi07085154-7 from 3760128 to 4464640 tid 1 2024-09-14T03:53:50.053 INFO:tasks.rados.rados.0.smithi070.stdout:1679: writing smithi07085154-7 from 4464640 to 5193728 tid 2 2024-09-14T03:53:50.057 INFO:tasks.rados.rados.0.smithi070.stdout:1679: writing smithi07085154-7 from 5193728 to 5668864 tid 3 2024-09-14T03:53:50.057 INFO:tasks.rados.rados.0.smithi070.stdout:1680: snap_create 2024-09-14T03:53:50.057 INFO:tasks.rados.rados.0.smithi070.stdout:1678: finishing copy_from to smithi07085154-5 2024-09-14T03:53:50.057 INFO:tasks.rados.rados.0.smithi070.stdout:1678: got expected ENOENT (src dne) 2024-09-14T03:53:50.057 INFO:tasks.rados.rados.0.smithi070.stdout:1678: finishing copy_from racing read to smithi07085154-5 2024-09-14T03:53:50.057 INFO:tasks.rados.rados.0.smithi070.stdout:1679: finishing write tid 1 to smithi07085154-7 2024-09-14T03:53:50.057 INFO:tasks.rados.rados.0.smithi070.stdout:1679: finishing write tid 2 to smithi07085154-7 2024-09-14T03:53:50.059 INFO:tasks.rados.rados.0.smithi070.stdout:1679: finishing write tid 3 to smithi07085154-7 2024-09-14T03:53:50.059 INFO:tasks.rados.rados.0.smithi070.stdout:1679: finishing write tid 4 to smithi07085154-7 2024-09-14T03:53:50.060 INFO:tasks.rados.rados.0.smithi070.stdout:1679: finishing write tid 5 to smithi07085154-7 2024-09-14T03:53:50.060 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 1240 (ObjNum 543 snap 180 seq_num 543) dirty exists 2024-09-14T03:53:50.060 INFO:tasks.rados.rados.0.smithi070.stdout:1679: left oid 7 (ObjNum 543 snap 180 seq_num 543) 2024-09-14T03:53:50.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:53:50 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:53:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:51.027 INFO:tasks.rados.rados.0.smithi070.stdout:1678: done (2 left) 2024-09-14T03:53:51.027 INFO:tasks.rados.rados.0.smithi070.stdout:1679: done (1 left) 2024-09-14T03:53:51.028 INFO:tasks.rados.rados.0.smithi070.stdout:1680: done (0 left) 2024-09-14T03:53:51.028 INFO:tasks.rados.rados.0.smithi070.stdout:1681: read oid 9 snap -1 2024-09-14T03:53:51.028 INFO:tasks.rados.rados.0.smithi070.stdout:1681: expect (ObjNum 507 snap 166 seq_num 507) 2024-09-14T03:53:51.028 INFO:tasks.rados.rados.0.smithi070.stdout:1682: read oid 50 snap -1 2024-09-14T03:53:51.028 INFO:tasks.rados.rados.0.smithi070.stdout:1682: expect (ObjNum 508 snap 166 seq_num 508) 2024-09-14T03:53:51.028 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 17 current snap is 181 2024-09-14T03:53:51.028 INFO:tasks.rados.rados.0.smithi070.stdout:1683: seq_num 544 ranges {2154496=720896,2875392=614400,3489792=589824} 2024-09-14T03:53:51.043 INFO:tasks.rados.rados.0.smithi070.stdout:1683: writing smithi07085154-17 from 2154496 to 2875392 tid 1 2024-09-14T03:53:51.048 INFO:tasks.rados.rados.0.smithi070.stdout:1683: writing smithi07085154-17 from 2875392 to 3489792 tid 2 2024-09-14T03:53:51.054 INFO:tasks.rados.rados.0.smithi070.stdout:1683: writing smithi07085154-17 from 3489792 to 4079616 tid 3 2024-09-14T03:53:51.054 INFO:tasks.rados.rados.0.smithi070.stdout:1684: rmattr oid 44 current snap is 181 2024-09-14T03:53:51.054 INFO:tasks.rados.rados.0.smithi070.stdout:1684: done (3 left) 2024-09-14T03:53:51.054 INFO:tasks.rados.rados.0.smithi070.stdout:1685: rollback oid 38 current snap is 181 2024-09-14T03:53:51.055 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 38 to 168 2024-09-14T03:53:51.055 INFO:tasks.rados.rados.0.smithi070.stdout:1681: expect (ObjNum 507 snap 166 seq_num 507) 2024-09-14T03:53:51.089 INFO:tasks.rados.rados.0.smithi070.stdout:1683: finishing write tid 1 to smithi07085154-17 2024-09-14T03:53:51.090 INFO:tasks.rados.rados.0.smithi070.stdout:1682: expect (ObjNum 508 snap 166 seq_num 508) 2024-09-14T03:53:51.183 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:50 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:51.183 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:50 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:51.183 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:50 smithi070 ceph-mon[96704]: osdmap e408: 8 total, 7 up, 8 in 2024-09-14T03:53:51.183 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:50 smithi070 ceph-mon[96704]: pgmap v204: 121 pgs: 27 active+undersized, 10 active+undersized+degraded, 84 active+clean; 484 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 453 KiB/s wr, 6 op/s; 128/1044 objects degraded (12.261%) 2024-09-14T03:53:51.183 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:50 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:51.184 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:50 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:51.184 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:50 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:51.184 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:50 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:51.184 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:50 smithi070 ceph-mon[98640]: osdmap e408: 8 total, 7 up, 8 in 2024-09-14T03:53:51.184 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:50 smithi070 ceph-mon[98640]: pgmap v204: 121 pgs: 27 active+undersized, 10 active+undersized+degraded, 84 active+clean; 484 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 453 KiB/s wr, 6 op/s; 128/1044 objects degraded (12.261%) 2024-09-14T03:53:51.184 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:50 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:51.184 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:50 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:51.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:50 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:51.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:50 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:51.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:50 smithi094 ceph-mon[86597]: osdmap e408: 8 total, 7 up, 8 in 2024-09-14T03:53:51.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:50 smithi094 ceph-mon[86597]: pgmap v204: 121 pgs: 27 active+undersized, 10 active+undersized+degraded, 84 active+clean; 484 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 453 KiB/s wr, 6 op/s; 128/1044 objects degraded (12.261%) 2024-09-14T03:53:51.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:50 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:51.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:50 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:51.369 INFO:tasks.rados.rados.0.smithi070.stdout:1683: finishing write tid 2 to smithi07085154-17 2024-09-14T03:53:51.369 INFO:tasks.rados.rados.0.smithi070.stdout:1685: finishing rollback tid 0 to smithi07085154-38 2024-09-14T03:53:51.369 INFO:tasks.rados.rados.0.smithi070.stdout:1685: finishing rollback tid 1 to smithi07085154-38 2024-09-14T03:53:51.369 INFO:tasks.rados.rados.0.smithi070.stdout:1683: finishing write tid 3 to smithi07085154-17 2024-09-14T03:53:51.370 INFO:tasks.rados.rados.0.smithi070.stdout:1685: finishing rollback tid 2 to smithi07085154-38 2024-09-14T03:53:51.370 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 38 v 1014 (ObjNum 505 snap 165 seq_num 505) dirty exists 2024-09-14T03:53:51.370 INFO:tasks.rados.rados.0.smithi070.stdout:1683: finishing write tid 4 to smithi07085154-17 2024-09-14T03:53:51.370 INFO:tasks.rados.rados.0.smithi070.stdout:1683: finishing write tid 5 to smithi07085154-17 2024-09-14T03:53:51.370 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 929 (ObjNum 544 snap 181 seq_num 544) dirty exists 2024-09-14T03:53:51.370 INFO:tasks.rados.rados.0.smithi070.stdout:1683: left oid 17 (ObjNum 544 snap 181 seq_num 544) 2024-09-14T03:53:51.370 INFO:tasks.rados.rados.0.smithi070.stdout:1681: done (3 left) 2024-09-14T03:53:51.370 INFO:tasks.rados.rados.0.smithi070.stdout:1682: done (2 left) 2024-09-14T03:53:51.370 INFO:tasks.rados.rados.0.smithi070.stdout:1683: done (1 left) 2024-09-14T03:53:51.370 INFO:tasks.rados.rados.0.smithi070.stdout:1685: done (0 left) 2024-09-14T03:53:51.370 INFO:tasks.rados.rados.0.smithi070.stdout:1686: read oid 26 snap 102 2024-09-14T03:53:51.370 INFO:tasks.rados.rados.0.smithi070.stdout:1686: expect deleted 2024-09-14T03:53:51.371 INFO:tasks.rados.rados.0.smithi070.stdout:1687: copy_from oid 19 from oid 2 current snap is 181 2024-09-14T03:53:51.371 INFO:tasks.rados.rados.0.smithi070.stdout:1688: snap_create 2024-09-14T03:53:51.371 INFO:tasks.rados.rados.0.smithi070.stdout:1687: finishing copy_from to smithi07085154-19 2024-09-14T03:53:51.371 INFO:tasks.rados.rados.0.smithi070.stdout:1687: got expected ENOENT (src dne) 2024-09-14T03:53:51.371 INFO:tasks.rados.rados.0.smithi070.stdout:1687: finishing copy_from racing read to smithi07085154-19 2024-09-14T03:53:51.535 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:51 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2[113018]: 2024-09-14T03:53:51.308+0000 7f5ca6738640 -1 osd.2 399 log_to_monitors true 2024-09-14T03:53:52.029 INFO:tasks.rados.rados.0.smithi070.stdout:1686: done (2 left) 2024-09-14T03:53:52.029 INFO:tasks.rados.rados.0.smithi070.stdout:1687: done (1 left) 2024-09-14T03:53:52.029 INFO:tasks.rados.rados.0.smithi070.stdout:1688: done (0 left) 2024-09-14T03:53:52.030 INFO:tasks.rados.rados.0.smithi070.stdout:1689: read oid 34 snap -1 2024-09-14T03:53:52.030 INFO:tasks.rados.rados.0.smithi070.stdout:1689: expect (ObjNum 517 snap 173 seq_num 517) 2024-09-14T03:53:52.030 INFO:tasks.rados.rados.0.smithi070.stdout:1690: setattr oid 5 current snap is 182 2024-09-14T03:53:52.031 INFO:tasks.rados.rados.0.smithi070.stdout:1691: snap_create 2024-09-14T03:53:52.039 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 559 (ObjNum 516 snap 173 seq_num 516) dirty exists 2024-09-14T03:53:52.052 INFO:tasks.rados.rados.0.smithi070.stdout:1689: expect (ObjNum 517 snap 173 seq_num 517) 2024-09-14T03:53:52.172 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-09-14T03:53:52.260 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:53:52 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2[113018]: 2024-09-14T03:53:52.048+0000 7f5c9e4e2640 -1 osd.2 399 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:53:52.260 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:51 smithi070 ceph-mon[98640]: Health check update: Degraded data redundancy: 128/1044 objects degraded (12.261%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T03:53:52.261 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:51 smithi070 ceph-mon[98640]: osdmap e409: 8 total, 7 up, 8 in 2024-09-14T03:53:52.261 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:51 smithi070 ceph-mon[98640]: from='osd.2 [v2:172.21.15.70:6818/2093874787,v1:172.21.15.70:6819/2093874787]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-14T03:53:52.261 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:51 smithi070 ceph-mon[96704]: Health check update: Degraded data redundancy: 128/1044 objects degraded (12.261%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T03:53:52.261 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:51 smithi070 ceph-mon[96704]: osdmap e409: 8 total, 7 up, 8 in 2024-09-14T03:53:52.261 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:51 smithi070 ceph-mon[96704]: from='osd.2 [v2:172.21.15.70:6818/2093874787,v1:172.21.15.70:6819/2093874787]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-14T03:53:52.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:51 smithi094 ceph-mon[86597]: Health check update: Degraded data redundancy: 128/1044 objects degraded (12.261%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T03:53:52.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:51 smithi094 ceph-mon[86597]: osdmap e409: 8 total, 7 up, 8 in 2024-09-14T03:53:52.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:51 smithi094 ceph-mon[86597]: from='osd.2 [v2:172.21.15.70:6818/2093874787,v1:172.21.15.70:6819/2093874787]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-14T03:53:52.746 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:53:52.746 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (3m) 2s ago 8m 18.6M - 0.27.0 11f11916f8cd 45e79e51074c 2024-09-14T03:53:52.746 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi094 *:3000 running (3m) 109s ago 8m 79.5M - 9.4.7 954c08fa6188 2240810d0368 2024-09-14T03:53:52.747 INFO:teuthology.orchestra.run.smithi070.stdout:mds.a smithi070 running (10m) 2s ago 10m 21.1M - 17.2.7-1629-geb16b97b ab51d1bebd12 52e0dbb7be50 2024-09-14T03:53:52.747 INFO:teuthology.orchestra.run.smithi070.stdout:mds.b smithi094 running (10m) 109s ago 10m 15.4M - 17.2.7-1629-geb16b97b ab51d1bebd12 e3b5e3e00189 2024-09-14T03:53:52.747 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi094 *:8443,9283,8765 running (2m) 109s ago 13m 452M - 19.3.0-4907-ga2782315 8801d2529e43 569d9c7e0f06 2024-09-14T03:53:52.747 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (3m) 2s ago 15m 520M - 19.3.0-4907-ga2782315 8801d2529e43 0ef4d8e0cdbe 2024-09-14T03:53:52.747 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (2m) 2s ago 15m 55.0M 2048M 19.3.0-4907-ga2782315 8801d2529e43 96542de31cac 2024-09-14T03:53:52.747 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi094 running (114s) 109s ago 13m 21.3M 2048M 19.3.0-4907-ga2782315 8801d2529e43 86827be98659 2024-09-14T03:53:52.747 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (2m) 2s ago 13m 45.4M 2048M 19.3.0-4907-ga2782315 8801d2529e43 5741e4fcf4b6 2024-09-14T03:53:52.747 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (3m) 2s ago 8m 10.2M - 1.7.0 72c9c2088986 e316760a30b3 2024-09-14T03:53:52.747 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi094 *:9100 running (3m) 109s ago 8m 7750k - 1.7.0 72c9c2088986 767f8a093175 2024-09-14T03:53:52.747 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (73s) 2s ago 13m 151M 2480M 19.3.0-4907-ga2782315 8801d2529e43 2345ccac2085 2024-09-14T03:53:52.748 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (39s) 2s ago 12m 145M 2480M 19.3.0-4907-ga2782315 8801d2529e43 53868856512a 2024-09-14T03:53:52.748 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (6s) 2s ago 12m 11.6M 2480M 19.3.0-4907-ga2782315 8801d2529e43 39ef876d2fff 2024-09-14T03:53:52.748 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (11m) 2s ago 11m 171M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 2e4199b58392 2024-09-14T03:53:52.748 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi094 running (11m) 109s ago 11m 180M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 52b64527e24c 2024-09-14T03:53:52.748 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi094 running (11m) 109s ago 11m 168M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 b543f152ba87 2024-09-14T03:53:52.748 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi094 running (10m) 109s ago 10m 114M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 e66b592ac940 2024-09-14T03:53:52.748 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi094 running (10m) 109s ago 10m 152M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 037c6208e9ce 2024-09-14T03:53:52.748 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi094 *:9095 running (2m) 109s ago 8m 46.3M - 2.51.0 1d3b7f56885b ce1b353828dc 2024-09-14T03:53:53.041 INFO:tasks.rados.rados.0.smithi070.stdout:1689: done (2 left) 2024-09-14T03:53:53.041 INFO:tasks.rados.rados.0.smithi070.stdout:1690: done (1 left) 2024-09-14T03:53:53.041 INFO:tasks.rados.rados.0.smithi070.stdout:1691: done (0 left) 2024-09-14T03:53:53.041 INFO:tasks.rados.rados.0.smithi070.stdout:1692: delete oid 6 current snap is 183 2024-09-14T03:53:53.048 INFO:tasks.rados.rados.0.smithi070.stdout:1692: done (0 left) 2024-09-14T03:53:53.048 INFO:tasks.rados.rados.0.smithi070.stdout:1693: snap_create 2024-09-14T03:53:53.175 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:53:53.175 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-09-14T03:53:53.175 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:53:53.175 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:53:53.175 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-09-14T03:53:53.175 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:53:53.175 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:53:53.175 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-09-14T03:53:53.176 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 5, 2024-09-14T03:53:53.176 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:53:53.176 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:53:53.176 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": { 2024-09-14T03:53:53.176 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:53:53.176 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:53:53.176 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-09-14T03:53:53.176 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 7, 2024-09-14T03:53:53.176 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 8 2024-09-14T03:53:53.176 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-09-14T03:53:53.176 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:53:53.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:53 smithi070 ceph-mon[96704]: from='osd.2 [v2:172.21.15.70:6818/2093874787,v1:172.21.15.70:6819/2093874787]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-14T03:53:53.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:53 smithi070 ceph-mon[96704]: osdmap e410: 8 total, 7 up, 8 in 2024-09-14T03:53:53.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:53 smithi070 ceph-mon[96704]: from='osd.2 [v2:172.21.15.70:6818/2093874787,v1:172.21.15.70:6819/2093874787]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi070", "root=default"]} : dispatch 2024-09-14T03:53:53.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:53 smithi070 ceph-mon[96704]: pgmap v207: 121 pgs: 27 active+undersized, 10 active+undersized+degraded, 84 active+clean; 485 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 274 B/s rd, 500 KiB/s wr, 1 op/s; 129/1041 objects degraded (12.392%) 2024-09-14T03:53:53.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:53 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:53.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:53 smithi070 ceph-mon[98640]: from='osd.2 [v2:172.21.15.70:6818/2093874787,v1:172.21.15.70:6819/2093874787]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-14T03:53:53.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:53 smithi070 ceph-mon[98640]: osdmap e410: 8 total, 7 up, 8 in 2024-09-14T03:53:53.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:53 smithi070 ceph-mon[98640]: from='osd.2 [v2:172.21.15.70:6818/2093874787,v1:172.21.15.70:6819/2093874787]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi070", "root=default"]} : dispatch 2024-09-14T03:53:53.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:53 smithi070 ceph-mon[98640]: pgmap v207: 121 pgs: 27 active+undersized, 10 active+undersized+degraded, 84 active+clean; 485 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 274 B/s rd, 500 KiB/s wr, 1 op/s; 129/1041 objects degraded (12.392%) 2024-09-14T03:53:53.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:53 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:53.480 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:53:53.480 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:53:53.480 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-09-14T03:53:53.480 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:53:53.480 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [ 2024-09-14T03:53:53.480 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr", 2024-09-14T03:53:53.480 INFO:teuthology.orchestra.run.smithi070.stdout: "mon" 2024-09-14T03:53:53.480 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2024-09-14T03:53:53.480 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "8/20 daemons upgraded", 2024-09-14T03:53:53.480 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "Currently upgrading osd daemons", 2024-09-14T03:53:53.480 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-09-14T03:53:53.481 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:53:53.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:53 smithi094 ceph-mon[86597]: from='osd.2 [v2:172.21.15.70:6818/2093874787,v1:172.21.15.70:6819/2093874787]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-14T03:53:53.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:53 smithi094 ceph-mon[86597]: osdmap e410: 8 total, 7 up, 8 in 2024-09-14T03:53:53.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:53 smithi094 ceph-mon[86597]: from='osd.2 [v2:172.21.15.70:6818/2093874787,v1:172.21.15.70:6819/2093874787]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi070", "root=default"]} : dispatch 2024-09-14T03:53:53.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:53 smithi094 ceph-mon[86597]: pgmap v207: 121 pgs: 27 active+undersized, 10 active+undersized+degraded, 84 active+clean; 485 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 274 B/s rd, 500 KiB/s wr, 1 op/s; 129/1041 objects degraded (12.392%) 2024-09-14T03:53:53.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:53 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:53.645 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:53:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:53:53] "GET /metrics HTTP/1.1" 200 36327 "" "Prometheus/2.51.0" 2024-09-14T03:53:54.035 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:53:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:53:53.794+0000 7fa4af5ba640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-14T03:53:54.045 INFO:tasks.rados.rados.0.smithi070.stdout:1693: done (0 left) 2024-09-14T03:53:54.048 INFO:tasks.rados.rados.0.smithi070.stdout:1694: snap_remove snap 99 2024-09-14T03:53:54.325 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[96704]: from='client.44210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:54.325 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[96704]: from='client.44216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:54.325 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[96704]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:54.325 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[96704]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:53:54.325 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[96704]: osd.2 [v2:172.21.15.70:6818/2093874787,v1:172.21.15.70:6819/2093874787] boot 2024-09-14T03:53:54.326 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[96704]: osdmap e411: 8 total, 8 up, 8 in 2024-09-14T03:53:54.326 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-14T03:53:54.326 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/3108405850' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:54.326 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:54.326 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:54.326 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:54.326 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:53:54.326 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:54.326 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:53:54.326 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:54.326 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:54.326 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:54.326 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-14T03:53:54.327 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[98640]: from='client.44210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:54.327 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[98640]: from='client.44216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:54.327 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[98640]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:54.327 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[98640]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:53:54.327 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[98640]: osd.2 [v2:172.21.15.70:6818/2093874787,v1:172.21.15.70:6819/2093874787] boot 2024-09-14T03:53:54.327 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[98640]: osdmap e411: 8 total, 8 up, 8 in 2024-09-14T03:53:54.327 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-14T03:53:54.327 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/3108405850' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:54.327 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:54.327 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:54.327 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:54.327 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:53:54.328 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:54.328 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:53:54.328 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:54.328 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:54.328 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:54.328 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:54 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-14T03:53:54.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:53:54 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:53:54.148Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.2\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-14T03:53:54.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:53:54 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:53:54.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.2\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:54.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:54 smithi094 ceph-mon[86597]: from='client.44210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:54.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:54 smithi094 ceph-mon[86597]: from='client.44216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:54.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:54 smithi094 ceph-mon[86597]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:54.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:54 smithi094 ceph-mon[86597]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:53:54.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:54 smithi094 ceph-mon[86597]: osd.2 [v2:172.21.15.70:6818/2093874787,v1:172.21.15.70:6819/2093874787] boot 2024-09-14T03:53:54.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:54 smithi094 ceph-mon[86597]: osdmap e411: 8 total, 8 up, 8 in 2024-09-14T03:53:54.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:54 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-14T03:53:54.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:54 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/3108405850' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:54.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:54 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:54.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:54 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:54.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:54 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:53:54.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:54 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:53:54.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:54 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:53:54.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:54 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:53:54.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:54 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:54.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:54 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:54.529 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:54 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:53:54.529 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:54 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-14T03:53:55.057 INFO:tasks.rados.rados.0.smithi070.stdout:1694: done (0 left) 2024-09-14T03:53:55.057 INFO:tasks.rados.rados.0.smithi070.stdout:1695: delete oid 32 current snap is 184 2024-09-14T03:53:55.067 INFO:tasks.rados.rados.0.smithi070.stdout:1695: done (0 left) 2024-09-14T03:53:55.067 INFO:tasks.rados.rados.0.smithi070.stdout:1696: delete oid 21 current snap is 184 2024-09-14T03:53:55.074 INFO:tasks.rados.rados.0.smithi070.stdout:1696: done (0 left) 2024-09-14T03:53:55.074 INFO:tasks.rados.rados.0.smithi070.stdout:1697: snap_remove snap 152 2024-09-14T03:53:55.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:55 smithi094 ceph-mon[86597]: from='client.44234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:55.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:55 smithi094 ceph-mon[86597]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-14T03:53:55.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:55 smithi094 ceph-mon[86597]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-14T03:53:55.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:55 smithi094 ceph-mon[86597]: osdmap e412: 8 total, 8 up, 8 in 2024-09-14T03:53:55.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:55 smithi094 ceph-mon[86597]: pgmap v210: 121 pgs: 18 peering, 12 active+undersized, 7 active+undersized+degraded, 84 active+clean; 463 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 2.5 MiB/s wr, 7 op/s; 101/1005 objects degraded (10.050%) 2024-09-14T03:53:55.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:55 smithi070 ceph-mon[96704]: from='client.44234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:55.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:55 smithi070 ceph-mon[96704]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-14T03:53:55.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:55 smithi070 ceph-mon[96704]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-14T03:53:55.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:55 smithi070 ceph-mon[96704]: osdmap e412: 8 total, 8 up, 8 in 2024-09-14T03:53:55.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:55 smithi070 ceph-mon[96704]: pgmap v210: 121 pgs: 18 peering, 12 active+undersized, 7 active+undersized+degraded, 84 active+clean; 463 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 2.5 MiB/s wr, 7 op/s; 101/1005 objects degraded (10.050%) 2024-09-14T03:53:55.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:55 smithi070 ceph-mon[98640]: from='client.44234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:55.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:55 smithi070 ceph-mon[98640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-14T03:53:55.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:55 smithi070 ceph-mon[98640]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-14T03:53:55.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:55 smithi070 ceph-mon[98640]: osdmap e412: 8 total, 8 up, 8 in 2024-09-14T03:53:55.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:55 smithi070 ceph-mon[98640]: pgmap v210: 121 pgs: 18 peering, 12 active+undersized, 7 active+undersized+degraded, 84 active+clean; 463 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 2.5 MiB/s wr, 7 op/s; 101/1005 objects degraded (10.050%) 2024-09-14T03:53:56.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:53:55 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:53:55.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:56.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:53:55 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:53:55.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:56.059 INFO:tasks.rados.rados.0.smithi070.stdout:1697: done (0 left) 2024-09-14T03:53:56.059 INFO:tasks.rados.rados.0.smithi070.stdout:1698: snap_remove snap 139 2024-09-14T03:53:56.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:56 smithi094 ceph-mon[86597]: osdmap e413: 8 total, 8 up, 8 in 2024-09-14T03:53:56.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:56 smithi070 ceph-mon[98640]: osdmap e413: 8 total, 8 up, 8 in 2024-09-14T03:53:56.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:56 smithi070 ceph-mon[96704]: osdmap e413: 8 total, 8 up, 8 in 2024-09-14T03:53:57.066 INFO:tasks.rados.rados.0.smithi070.stdout:1698: done (0 left) 2024-09-14T03:53:57.066 INFO:tasks.rados.rados.0.smithi070.stdout:1699: copy_from oid 11 from oid 16 current snap is 184 2024-09-14T03:53:57.066 INFO:tasks.rados.rados.0.smithi070.stdout:1700: rollback oid 40 current snap is 184 2024-09-14T03:53:57.067 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 40 to 168 2024-09-14T03:53:57.067 INFO:tasks.rados.rados.0.smithi070.stdout:1701: read oid 17 snap -1 2024-09-14T03:53:57.067 INFO:tasks.rados.rados.0.smithi070.stdout:1701: expect (ObjNum 544 snap 181 seq_num 544) 2024-09-14T03:53:57.067 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 14 current snap is 184 2024-09-14T03:53:57.067 INFO:tasks.rados.rados.0.smithi070.stdout:1702: seq_num 548 ranges {2326528=786432,3112960=598016,3710976=606208} 2024-09-14T03:53:57.085 INFO:tasks.rados.rados.0.smithi070.stdout:1702: writing smithi07085154-14 from 2326528 to 3112960 tid 1 2024-09-14T03:53:57.089 INFO:tasks.rados.rados.0.smithi070.stdout:1702: writing smithi07085154-14 from 3112960 to 3710976 tid 2 2024-09-14T03:53:57.093 INFO:tasks.rados.rados.0.smithi070.stdout:1702: writing smithi07085154-14 from 3710976 to 4317184 tid 3 2024-09-14T03:53:57.093 INFO:tasks.rados.rados.0.smithi070.stdout:1703: snap_remove snap 175 2024-09-14T03:53:57.277 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:53:56 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:53:56.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:53:57.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:57 smithi094 ceph-mon[86597]: pgmap v212: 121 pgs: 18 peering, 8 active+undersized, 7 active+undersized+degraded, 88 active+clean; 460 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 2.1 MiB/s wr, 5 op/s; 101/999 objects degraded (10.110%) 2024-09-14T03:53:57.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:57 smithi094 ceph-mon[86597]: osdmap e414: 8 total, 8 up, 8 in 2024-09-14T03:53:57.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:57 smithi094 ceph-mon[86597]: Health check update: Degraded data redundancy: 101/999 objects degraded (10.110%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:53:57.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:57 smithi070 ceph-mon[96704]: pgmap v212: 121 pgs: 18 peering, 8 active+undersized, 7 active+undersized+degraded, 88 active+clean; 460 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 2.1 MiB/s wr, 5 op/s; 101/999 objects degraded (10.110%) 2024-09-14T03:53:57.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:57 smithi070 ceph-mon[96704]: osdmap e414: 8 total, 8 up, 8 in 2024-09-14T03:53:57.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:57 smithi070 ceph-mon[96704]: Health check update: Degraded data redundancy: 101/999 objects degraded (10.110%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:53:57.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:57 smithi070 ceph-mon[98640]: pgmap v212: 121 pgs: 18 peering, 8 active+undersized, 7 active+undersized+degraded, 88 active+clean; 460 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 2.1 MiB/s wr, 5 op/s; 101/999 objects degraded (10.110%) 2024-09-14T03:53:57.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:57 smithi070 ceph-mon[98640]: osdmap e414: 8 total, 8 up, 8 in 2024-09-14T03:53:57.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:57 smithi070 ceph-mon[98640]: Health check update: Degraded data redundancy: 101/999 objects degraded (10.110%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:53:58.071 INFO:tasks.rados.rados.0.smithi070.stdout:1703: done (4 left) 2024-09-14T03:53:58.071 INFO:tasks.rados.rados.0.smithi070.stdout:1704: rollback oid 27 current snap is 184 2024-09-14T03:53:58.071 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 27 to 107 2024-09-14T03:53:58.071 INFO:tasks.rados.rados.0.smithi070.stdout:1700: finishing rollback tid 0 to smithi07085154-40 2024-09-14T03:53:58.071 INFO:tasks.rados.rados.0.smithi070.stdout:1700: finishing rollback tid 1 to smithi07085154-40 2024-09-14T03:53:58.071 INFO:tasks.rados.rados.0.smithi070.stdout:1700: finishing rollback tid 2 to smithi07085154-40 2024-09-14T03:53:58.071 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 861 (ObjNum 490 snap 164 seq_num 490) dirty exists 2024-09-14T03:53:58.072 INFO:tasks.rados.rados.0.smithi070.stdout:1702: finishing write tid 1 to smithi07085154-14 2024-09-14T03:53:58.072 INFO:tasks.rados.rados.0.smithi070.stdout:1702: finishing write tid 2 to smithi07085154-14 2024-09-14T03:53:58.072 INFO:tasks.rados.rados.0.smithi070.stdout:1701: expect (ObjNum 544 snap 181 seq_num 544) 2024-09-14T03:53:58.263 INFO:tasks.rados.rados.0.smithi070.stdout:1702: finishing write tid 3 to smithi07085154-14 2024-09-14T03:53:58.263 INFO:tasks.rados.rados.0.smithi070.stdout:1702: finishing write tid 4 to smithi07085154-14 2024-09-14T03:53:58.263 INFO:tasks.rados.rados.0.smithi070.stdout:1702: finishing write tid 5 to smithi07085154-14 2024-09-14T03:53:58.263 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 682 (ObjNum 548 snap 184 seq_num 548) dirty exists 2024-09-14T03:53:58.263 INFO:tasks.rados.rados.0.smithi070.stdout:1702: left oid 14 (ObjNum 548 snap 184 seq_num 548) 2024-09-14T03:53:58.263 INFO:tasks.rados.rados.0.smithi070.stdout:1699: finishing copy_from racing read to smithi07085154-11 2024-09-14T03:53:58.263 INFO:tasks.rados.rados.0.smithi070.stdout:1699: finishing copy_from to smithi07085154-11 2024-09-14T03:53:58.263 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 11 v 1667 (ObjNum 114 snap 24 seq_num 114) dirty exists 2024-09-14T03:53:58.263 INFO:tasks.rados.rados.0.smithi070.stdout:1704: finishing rollback tid 0 to smithi07085154-27 2024-09-14T03:53:58.263 INFO:tasks.rados.rados.0.smithi070.stdout:1699: done (4 left) 2024-09-14T03:53:58.263 INFO:tasks.rados.rados.0.smithi070.stdout:1700: done (3 left) 2024-09-14T03:53:58.264 INFO:tasks.rados.rados.0.smithi070.stdout:1701: done (2 left) 2024-09-14T03:53:58.264 INFO:tasks.rados.rados.0.smithi070.stdout:1702: done (1 left) 2024-09-14T03:53:58.264 INFO:tasks.rados.rados.0.smithi070.stdout:1705: copy_from oid 24 from oid 34 current snap is 184 2024-09-14T03:53:58.264 INFO:tasks.rados.rados.0.smithi070.stdout:1704: finishing rollback tid 1 to smithi07085154-27 2024-09-14T03:53:58.264 INFO:tasks.rados.rados.0.smithi070.stdout:1704: finishing rollback tid 2 to smithi07085154-27 2024-09-14T03:53:58.264 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 1796 (ObjNum 316 snap 107 seq_num 316) dirty exists 2024-09-14T03:53:58.264 INFO:tasks.rados.rados.0.smithi070.stdout:1704: done (1 left) 2024-09-14T03:53:58.264 INFO:tasks.rados.rados.0.smithi070.stdout:1706: delete oid 27 current snap is 184 2024-09-14T03:53:58.265 INFO:tasks.rados.rados.0.smithi070.stdout:1706: done (1 left) 2024-09-14T03:53:58.266 INFO:tasks.rados.rados.0.smithi070.stdout:1707: delete oid 9 current snap is 184 2024-09-14T03:53:58.268 INFO:tasks.rados.rados.0.smithi070.stdout:1707: done (1 left) 2024-09-14T03:53:58.268 INFO:tasks.rados.rados.0.smithi070.stdout:1708: snap_create 2024-09-14T03:53:58.277 INFO:tasks.rados.rados.0.smithi070.stdout:1705: finishing copy_from racing read to smithi07085154-24 2024-09-14T03:53:58.279 INFO:tasks.rados.rados.0.smithi070.stdout:1705: finishing copy_from to smithi07085154-24 2024-09-14T03:53:58.279 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 1226 (ObjNum 517 snap 173 seq_num 517) dirty exists 2024-09-14T03:53:58.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:58 smithi094 ceph-mon[86597]: osdmap e415: 8 total, 8 up, 8 in 2024-09-14T03:53:58.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:58 smithi070 ceph-mon[96704]: osdmap e415: 8 total, 8 up, 8 in 2024-09-14T03:53:58.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:58 smithi070 ceph-mon[98640]: osdmap e415: 8 total, 8 up, 8 in 2024-09-14T03:53:59.023 INFO:tasks.rados.rados.0.smithi070.stdout:1705: done (1 left) 2024-09-14T03:53:59.023 INFO:tasks.rados.rados.0.smithi070.stdout:1708: done (0 left) 2024-09-14T03:53:59.023 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 49 current snap is 185 2024-09-14T03:53:59.024 INFO:tasks.rados.rados.0.smithi070.stdout:1709: seq_num 551 ranges {5365760=434176,5799936=647168,6447104=450560,6897664=507904} 2024-09-14T03:53:59.054 INFO:tasks.rados.rados.0.smithi070.stdout:1709: writing smithi07085154-49 from 5365760 to 5799936 tid 1 2024-09-14T03:53:59.057 INFO:tasks.rados.rados.0.smithi070.stdout:1709: writing smithi07085154-49 from 5799936 to 6447104 tid 2 2024-09-14T03:53:59.061 INFO:tasks.rados.rados.0.smithi070.stdout:1709: writing smithi07085154-49 from 6447104 to 6897664 tid 3 2024-09-14T03:53:59.064 INFO:tasks.rados.rados.0.smithi070.stdout:1709: writing smithi07085154-49 from 6897664 to 7405568 tid 4 2024-09-14T03:53:59.064 INFO:tasks.rados.rados.0.smithi070.stdout:1710: setattr oid 31 current snap is 185 2024-09-14T03:53:59.064 INFO:tasks.rados.rados.0.smithi070.stdout:1709: finishing write tid 1 to smithi07085154-49 2024-09-14T03:53:59.064 INFO:tasks.rados.rados.0.smithi070.stdout:1709: finishing write tid 2 to smithi07085154-49 2024-09-14T03:53:59.064 INFO:tasks.rados.rados.0.smithi070.stdout:1709: finishing write tid 3 to smithi07085154-49 2024-09-14T03:53:59.066 INFO:tasks.rados.rados.0.smithi070.stdout:1709: finishing write tid 4 to smithi07085154-49 2024-09-14T03:53:59.066 INFO:tasks.rados.rados.0.smithi070.stdout:1709: finishing write tid 5 to smithi07085154-49 2024-09-14T03:53:59.068 INFO:tasks.rados.rados.0.smithi070.stdout:1709: finishing write tid 6 to smithi07085154-49 2024-09-14T03:53:59.068 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 49 v 1033 (ObjNum 551 snap 185 seq_num 551) dirty exists 2024-09-14T03:53:59.068 INFO:tasks.rados.rados.0.smithi070.stdout:1709: left oid 49 (ObjNum 551 snap 185 seq_num 551) 2024-09-14T03:53:59.068 INFO:tasks.rados.rados.0.smithi070.stdout:1709: done (1 left) 2024-09-14T03:53:59.068 INFO:tasks.rados.rados.0.smithi070.stdout:1711: read oid 20 snap -1 2024-09-14T03:53:59.069 INFO:tasks.rados.rados.0.smithi070.stdout:1711: expect (ObjNum 524 snap 174 seq_num 524) 2024-09-14T03:53:59.069 INFO:tasks.rados.rados.0.smithi070.stdout:1712: rollback oid 9 current snap is 185 2024-09-14T03:53:59.069 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 9 to 142 2024-09-14T03:53:59.069 INFO:tasks.rados.rados.0.smithi070.stdout:1713: rollback oid 35 current snap is 185 2024-09-14T03:53:59.069 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 35 to 167 2024-09-14T03:53:59.069 INFO:tasks.rados.rados.0.smithi070.stdout:1714: snap_create 2024-09-14T03:53:59.073 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 952 (ObjNum 540 snap 176 seq_num 540) dirty exists 2024-09-14T03:53:59.073 INFO:tasks.rados.rados.0.smithi070.stdout:1713: finishing rollback tid 1 to smithi07085154-35 2024-09-14T03:53:59.073 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 35 v 0 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-14T03:53:59.077 INFO:tasks.rados.rados.0.smithi070.stdout:1712: finishing rollback tid 1 to smithi07085154-9 2024-09-14T03:53:59.077 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 1799 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-14T03:53:59.099 INFO:tasks.rados.rados.0.smithi070.stdout:1711: expect (ObjNum 524 snap 174 seq_num 524) 2024-09-14T03:53:59.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:59 smithi094 ceph-mon[86597]: pgmap v215: 121 pgs: 121 active+clean; 445 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 483 KiB/s wr, 6 op/s; 4.4 MiB/s, 1 objects/s recovering 2024-09-14T03:53:59.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:59 smithi094 ceph-mon[86597]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 101/999 objects degraded (10.110%), 7 pgs degraded) 2024-09-14T03:53:59.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:59 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T03:53:59.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:59 smithi094 ceph-mon[86597]: osdmap e416: 8 total, 8 up, 8 in 2024-09-14T03:53:59.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:53:59 smithi094 ceph-mon[86597]: osdmap e417: 8 total, 8 up, 8 in 2024-09-14T03:53:59.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:59 smithi070 ceph-mon[96704]: pgmap v215: 121 pgs: 121 active+clean; 445 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 483 KiB/s wr, 6 op/s; 4.4 MiB/s, 1 objects/s recovering 2024-09-14T03:53:59.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:59 smithi070 ceph-mon[96704]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 101/999 objects degraded (10.110%), 7 pgs degraded) 2024-09-14T03:53:59.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:59 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T03:53:59.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:59 smithi070 ceph-mon[96704]: osdmap e416: 8 total, 8 up, 8 in 2024-09-14T03:53:59.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:53:59 smithi070 ceph-mon[96704]: osdmap e417: 8 total, 8 up, 8 in 2024-09-14T03:53:59.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:59 smithi070 ceph-mon[98640]: pgmap v215: 121 pgs: 121 active+clean; 445 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 483 KiB/s wr, 6 op/s; 4.4 MiB/s, 1 objects/s recovering 2024-09-14T03:53:59.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:59 smithi070 ceph-mon[98640]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 101/999 objects degraded (10.110%), 7 pgs degraded) 2024-09-14T03:53:59.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:59 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T03:53:59.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:59 smithi070 ceph-mon[98640]: osdmap e416: 8 total, 8 up, 8 in 2024-09-14T03:53:59.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:53:59 smithi070 ceph-mon[98640]: osdmap e417: 8 total, 8 up, 8 in 2024-09-14T03:54:00.076 INFO:tasks.rados.rados.0.smithi070.stdout:1710: done (4 left) 2024-09-14T03:54:00.076 INFO:tasks.rados.rados.0.smithi070.stdout:1711: done (3 left) 2024-09-14T03:54:00.076 INFO:tasks.rados.rados.0.smithi070.stdout:1712: done (2 left) 2024-09-14T03:54:00.076 INFO:tasks.rados.rados.0.smithi070.stdout:1713: done (1 left) 2024-09-14T03:54:00.076 INFO:tasks.rados.rados.0.smithi070.stdout:1714: done (0 left) 2024-09-14T03:54:00.076 INFO:tasks.rados.rados.0.smithi070.stdout:1715: snap_create 2024-09-14T03:54:00.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:54:00 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:54:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:01.081 INFO:tasks.rados.rados.0.smithi070.stdout:1715: done (0 left) 2024-09-14T03:54:01.081 INFO:tasks.rados.rados.0.smithi070.stdout:1716: read oid 1 snap -1 2024-09-14T03:54:01.082 INFO:tasks.rados.rados.0.smithi070.stdout:1716: expect (ObjNum 539 snap 176 seq_num 539) 2024-09-14T03:54:01.082 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 14 current snap is 187 2024-09-14T03:54:01.082 INFO:tasks.rados.rados.0.smithi070.stdout:1717: seq_num 552 ranges {4317184=450560,4767744=532480,5300224=655360,5955584=417792} 2024-09-14T03:54:01.107 INFO:tasks.rados.rados.0.smithi070.stdout:1717: writing smithi07085154-14 from 4317184 to 4767744 tid 1 2024-09-14T03:54:01.110 INFO:tasks.rados.rados.0.smithi070.stdout:1717: writing smithi07085154-14 from 4767744 to 5300224 tid 2 2024-09-14T03:54:01.114 INFO:tasks.rados.rados.0.smithi070.stdout:1717: writing smithi07085154-14 from 5300224 to 5955584 tid 3 2024-09-14T03:54:01.117 INFO:tasks.rados.rados.0.smithi070.stdout:1717: writing smithi07085154-14 from 5955584 to 6373376 tid 4 2024-09-14T03:54:01.117 INFO:tasks.rados.rados.0.smithi070.stdout:1718: snap_remove snap 116 2024-09-14T03:54:01.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:01 smithi094 ceph-mon[86597]: pgmap v218: 121 pgs: 121 active+clean; 444 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 984 KiB/s wr, 8 op/s; 4.4 MiB/s, 1 objects/s recovering 2024-09-14T03:54:01.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:01 smithi094 ceph-mon[86597]: osdmap e418: 8 total, 8 up, 8 in 2024-09-14T03:54:01.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:01 smithi070 ceph-mon[96704]: pgmap v218: 121 pgs: 121 active+clean; 444 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 984 KiB/s wr, 8 op/s; 4.4 MiB/s, 1 objects/s recovering 2024-09-14T03:54:01.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:01 smithi070 ceph-mon[96704]: osdmap e418: 8 total, 8 up, 8 in 2024-09-14T03:54:01.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:01 smithi070 ceph-mon[98640]: pgmap v218: 121 pgs: 121 active+clean; 444 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 984 KiB/s wr, 8 op/s; 4.4 MiB/s, 1 objects/s recovering 2024-09-14T03:54:01.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:01 smithi070 ceph-mon[98640]: osdmap e418: 8 total, 8 up, 8 in 2024-09-14T03:54:02.088 INFO:tasks.rados.rados.0.smithi070.stdout:1718: done (2 left) 2024-09-14T03:54:02.088 INFO:tasks.rados.rados.0.smithi070.stdout:1719: rollback oid 30 current snap is 187 2024-09-14T03:54:02.088 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 30 to 123 2024-09-14T03:54:02.088 INFO:tasks.rados.rados.0.smithi070.stdout:1716: expect (ObjNum 539 snap 176 seq_num 539) 2024-09-14T03:54:02.146 INFO:tasks.rados.rados.0.smithi070.stdout:1717: finishing write tid 1 to smithi07085154-14 2024-09-14T03:54:02.146 INFO:tasks.rados.rados.0.smithi070.stdout:1717: finishing write tid 2 to smithi07085154-14 2024-09-14T03:54:02.146 INFO:tasks.rados.rados.0.smithi070.stdout:1717: finishing write tid 3 to smithi07085154-14 2024-09-14T03:54:02.146 INFO:tasks.rados.rados.0.smithi070.stdout:1717: finishing write tid 4 to smithi07085154-14 2024-09-14T03:54:02.146 INFO:tasks.rados.rados.0.smithi070.stdout:1717: finishing write tid 5 to smithi07085154-14 2024-09-14T03:54:02.146 INFO:tasks.rados.rados.0.smithi070.stdout:1717: finishing write tid 6 to smithi07085154-14 2024-09-14T03:54:02.146 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 1231 (ObjNum 552 snap 187 seq_num 552) dirty exists 2024-09-14T03:54:02.146 INFO:tasks.rados.rados.0.smithi070.stdout:1717: left oid 14 (ObjNum 552 snap 187 seq_num 552) 2024-09-14T03:54:02.146 INFO:tasks.rados.rados.0.smithi070.stdout:1716: done (2 left) 2024-09-14T03:54:02.146 INFO:tasks.rados.rados.0.smithi070.stdout:1717: done (1 left) 2024-09-14T03:54:02.146 INFO:tasks.rados.rados.0.smithi070.stdout:1720: copy_from oid 9 from oid 25 current snap is 187 2024-09-14T03:54:02.146 INFO:tasks.rados.rados.0.smithi070.stdout:1719: finishing rollback tid 0 to smithi07085154-30 2024-09-14T03:54:02.147 INFO:tasks.rados.rados.0.smithi070.stdout:1719: finishing rollback tid 1 to smithi07085154-30 2024-09-14T03:54:02.147 INFO:tasks.rados.rados.0.smithi070.stdout:1719: finishing rollback tid 2 to smithi07085154-30 2024-09-14T03:54:02.147 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 30 v 1017 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-09-14T03:54:02.147 INFO:tasks.rados.rados.0.smithi070.stdout:1719: done (1 left) 2024-09-14T03:54:02.147 INFO:tasks.rados.rados.0.smithi070.stdout:1721: setattr oid 11 current snap is 187 2024-09-14T03:54:02.147 INFO:tasks.rados.rados.0.smithi070.stdout:1722: copy_from oid 38 from oid 31 current snap is 187 2024-09-14T03:54:02.147 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 23 current snap is 187 2024-09-14T03:54:02.148 INFO:tasks.rados.rados.0.smithi070.stdout:1723: seq_num 553 ranges {0=466944,466944=417792,884736=417792,1302528=516096,1818624=253952} 2024-09-14T03:54:02.149 INFO:tasks.rados.rados.0.smithi070.stdout:1723: writing smithi07085154-23 from 0 to 466944 tid 1 2024-09-14T03:54:02.152 INFO:tasks.rados.rados.0.smithi070.stdout:1723: writing smithi07085154-23 from 466944 to 884736 tid 2 2024-09-14T03:54:02.154 INFO:tasks.rados.rados.0.smithi070.stdout:1723: writing smithi07085154-23 from 884736 to 1302528 tid 3 2024-09-14T03:54:02.157 INFO:tasks.rados.rados.0.smithi070.stdout:1723: writing smithi07085154-23 from 1302528 to 1818624 tid 4 2024-09-14T03:54:02.159 INFO:tasks.rados.rados.0.smithi070.stdout:1723: writing smithi07085154-23 from 1818624 to 2072576 tid 5 2024-09-14T03:54:02.159 INFO:tasks.rados.rados.0.smithi070.stdout:1724: snap_remove snap 172 2024-09-14T03:54:02.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:02 smithi094 ceph-mon[86597]: osdmap e419: 8 total, 8 up, 8 in 2024-09-14T03:54:02.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:02 smithi070 ceph-mon[96704]: osdmap e419: 8 total, 8 up, 8 in 2024-09-14T03:54:02.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:02 smithi070 ceph-mon[98640]: osdmap e419: 8 total, 8 up, 8 in 2024-09-14T03:54:03.088 INFO:tasks.rados.rados.0.smithi070.stdout:1724: done (4 left) 2024-09-14T03:54:03.088 INFO:tasks.rados.rados.0.smithi070.stdout:1725: copy_from oid 18 from oid 16 current snap is 187 2024-09-14T03:54:03.088 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 11 v 1668 (ObjNum 114 snap 24 seq_num 114) dirty exists 2024-09-14T03:54:03.088 INFO:tasks.rados.rados.0.smithi070.stdout:1723: finishing write tid 1 to smithi07085154-23 2024-09-14T03:54:03.088 INFO:tasks.rados.rados.0.smithi070.stdout:1723: finishing write tid 2 to smithi07085154-23 2024-09-14T03:54:03.088 INFO:tasks.rados.rados.0.smithi070.stdout:1723: finishing write tid 3 to smithi07085154-23 2024-09-14T03:54:03.089 INFO:tasks.rados.rados.0.smithi070.stdout:1722: finishing copy_from racing read to smithi07085154-38 2024-09-14T03:54:03.089 INFO:tasks.rados.rados.0.smithi070.stdout:1723: finishing write tid 4 to smithi07085154-23 2024-09-14T03:54:03.089 INFO:tasks.rados.rados.0.smithi070.stdout:1723: finishing write tid 5 to smithi07085154-23 2024-09-14T03:54:03.089 INFO:tasks.rados.rados.0.smithi070.stdout:1723: finishing write tid 6 to smithi07085154-23 2024-09-14T03:54:03.089 INFO:tasks.rados.rados.0.smithi070.stdout:1722: finishing copy_from to smithi07085154-38 2024-09-14T03:54:03.089 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 38 v 952 (ObjNum 540 snap 176 seq_num 540) dirty exists 2024-09-14T03:54:03.089 INFO:tasks.rados.rados.0.smithi070.stdout:1723: finishing write tid 7 to smithi07085154-23 2024-09-14T03:54:03.089 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 1237 (ObjNum 553 snap 187 seq_num 553) dirty exists 2024-09-14T03:54:03.089 INFO:tasks.rados.rados.0.smithi070.stdout:1723: left oid 23 (ObjNum 553 snap 187 seq_num 553) 2024-09-14T03:54:03.089 INFO:tasks.rados.rados.0.smithi070.stdout:1720: finishing copy_from racing read to smithi07085154-9 2024-09-14T03:54:03.089 INFO:tasks.rados.rados.0.smithi070.stdout:1720: finishing copy_from to smithi07085154-9 2024-09-14T03:54:03.089 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 851 (ObjNum 459 snap 155 seq_num 459) dirty exists 2024-09-14T03:54:03.089 INFO:tasks.rados.rados.0.smithi070.stdout:1720: done (4 left) 2024-09-14T03:54:03.090 INFO:tasks.rados.rados.0.smithi070.stdout:1721: done (3 left) 2024-09-14T03:54:03.090 INFO:tasks.rados.rados.0.smithi070.stdout:1722: done (2 left) 2024-09-14T03:54:03.090 INFO:tasks.rados.rados.0.smithi070.stdout:1723: done (1 left) 2024-09-14T03:54:03.090 INFO:tasks.rados.rados.0.smithi070.stdout:1726: snap_create 2024-09-14T03:54:03.122 INFO:tasks.rados.rados.0.smithi070.stdout:1725: finishing copy_from racing read to smithi07085154-18 2024-09-14T03:54:03.123 INFO:tasks.rados.rados.0.smithi070.stdout:1725: finishing copy_from to smithi07085154-18 2024-09-14T03:54:03.123 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 1667 (ObjNum 114 snap 24 seq_num 114) dirty exists 2024-09-14T03:54:03.406 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:03 smithi070 ceph-mon[96704]: pgmap v221: 121 pgs: 121 active+clean; 449 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 1.6 MiB/s wr, 4 op/s 2024-09-14T03:54:03.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:03 smithi070 ceph-mon[96704]: osdmap e420: 8 total, 8 up, 8 in 2024-09-14T03:54:03.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:03 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:03.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:03 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:03.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:03 smithi070 ceph-mon[98640]: pgmap v221: 121 pgs: 121 active+clean; 449 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 1.6 MiB/s wr, 4 op/s 2024-09-14T03:54:03.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:03 smithi070 ceph-mon[98640]: osdmap e420: 8 total, 8 up, 8 in 2024-09-14T03:54:03.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:03 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:03.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:03 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:03.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:03 smithi094 ceph-mon[86597]: pgmap v221: 121 pgs: 121 active+clean; 449 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 1.6 MiB/s wr, 4 op/s 2024-09-14T03:54:03.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:03 smithi094 ceph-mon[86597]: osdmap e420: 8 total, 8 up, 8 in 2024-09-14T03:54:03.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:03 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:03.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:03 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:03.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:54:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:54:03] "GET /metrics HTTP/1.1" 200 36314 "" "Prometheus/2.51.0" 2024-09-14T03:54:04.024 INFO:tasks.rados.rados.0.smithi070.stdout:1725: done (1 left) 2024-09-14T03:54:04.024 INFO:tasks.rados.rados.0.smithi070.stdout:1726: done (0 left) 2024-09-14T03:54:04.025 INFO:tasks.rados.rados.0.smithi070.stdout:1727: delete oid 30 current snap is 188 2024-09-14T03:54:04.027 INFO:tasks.rados.rados.0.smithi070.stdout:1727: done (0 left) 2024-09-14T03:54:04.027 INFO:tasks.rados.rados.0.smithi070.stdout:1728: rollback oid 41 current snap is 188 2024-09-14T03:54:04.027 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 41 to 123 2024-09-14T03:54:04.027 INFO:tasks.rados.rados.0.smithi070.stdout:1729: read oid 42 snap -1 2024-09-14T03:54:04.027 INFO:tasks.rados.rados.0.smithi070.stdout:1729: expect (ObjNum 507 snap 166 seq_num 507) 2024-09-14T03:54:04.028 INFO:tasks.rados.rados.0.smithi070.stdout:1730: copy_from oid 31 from oid 50 current snap is 188 2024-09-14T03:54:04.028 INFO:tasks.rados.rados.0.smithi070.stdout:1731: snap_create 2024-09-14T03:54:04.036 INFO:tasks.rados.rados.0.smithi070.stdout:1729: expect (ObjNum 507 snap 166 seq_num 507) 2024-09-14T03:54:04.072 INFO:tasks.rados.rados.0.smithi070.stdout:1728: finishing rollback tid 0 to smithi07085154-41 2024-09-14T03:54:04.073 INFO:tasks.rados.rados.0.smithi070.stdout:1728: finishing rollback tid 1 to smithi07085154-41 2024-09-14T03:54:04.073 INFO:tasks.rados.rados.0.smithi070.stdout:1728: finishing rollback tid 2 to smithi07085154-41 2024-09-14T03:54:04.073 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 1670 (ObjNum 211 snap 60 seq_num 211) dirty exists 2024-09-14T03:54:04.073 INFO:tasks.rados.rados.0.smithi070.stdout:1730: finishing copy_from racing read to smithi07085154-31 2024-09-14T03:54:04.073 INFO:tasks.rados.rados.0.smithi070.stdout:1730: finishing copy_from to smithi07085154-31 2024-09-14T03:54:04.073 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 1550 (ObjNum 508 snap 166 seq_num 508) dirty exists 2024-09-14T03:54:04.386 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:04 smithi070 ceph-mon[96704]: osdmap e421: 8 total, 8 up, 8 in 2024-09-14T03:54:04.386 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:04 smithi070 ceph-mon[96704]: osdmap e422: 8 total, 8 up, 8 in 2024-09-14T03:54:04.386 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:04 smithi070 ceph-mon[98640]: osdmap e421: 8 total, 8 up, 8 in 2024-09-14T03:54:04.387 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:04 smithi070 ceph-mon[98640]: osdmap e422: 8 total, 8 up, 8 in 2024-09-14T03:54:04.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:04 smithi094 ceph-mon[86597]: osdmap e421: 8 total, 8 up, 8 in 2024-09-14T03:54:04.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:04 smithi094 ceph-mon[86597]: osdmap e422: 8 total, 8 up, 8 in 2024-09-14T03:54:04.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:54:04 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:54:04.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.3\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:05.029 INFO:tasks.rados.rados.0.smithi070.stdout:1728: done (3 left) 2024-09-14T03:54:05.029 INFO:tasks.rados.rados.0.smithi070.stdout:1729: done (2 left) 2024-09-14T03:54:05.030 INFO:tasks.rados.rados.0.smithi070.stdout:1730: done (1 left) 2024-09-14T03:54:05.030 INFO:tasks.rados.rados.0.smithi070.stdout:1731: done (0 left) 2024-09-14T03:54:05.030 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 47 current snap is 189 2024-09-14T03:54:05.030 INFO:tasks.rados.rados.0.smithi070.stdout:1732: seq_num 555 ranges {6569984=499712,7069696=638976,7708672=425984,8134656=540672} 2024-09-14T03:54:05.066 INFO:tasks.rados.rados.0.smithi070.stdout:1732: writing smithi07085154-47 from 6569984 to 7069696 tid 1 2024-09-14T03:54:05.070 INFO:tasks.rados.rados.0.smithi070.stdout:1732: writing smithi07085154-47 from 7069696 to 7708672 tid 2 2024-09-14T03:54:05.073 INFO:tasks.rados.rados.0.smithi070.stdout:1732: writing smithi07085154-47 from 7708672 to 8134656 tid 3 2024-09-14T03:54:05.076 INFO:tasks.rados.rados.0.smithi070.stdout:1732: writing smithi07085154-47 from 8134656 to 8675328 tid 4 2024-09-14T03:54:05.076 INFO:tasks.rados.rados.0.smithi070.stdout:1733: read oid 37 snap -1 2024-09-14T03:54:05.076 INFO:tasks.rados.rados.0.smithi070.stdout:1733: expect (ObjNum 312 snap 105 seq_num 312) 2024-09-14T03:54:05.076 INFO:tasks.rados.rados.0.smithi070.stdout:1734: snap_create 2024-09-14T03:54:05.076 INFO:tasks.rados.rados.0.smithi070.stdout:1732: finishing write tid 1 to smithi07085154-47 2024-09-14T03:54:05.076 INFO:tasks.rados.rados.0.smithi070.stdout:1732: finishing write tid 2 to smithi07085154-47 2024-09-14T03:54:05.077 INFO:tasks.rados.rados.0.smithi070.stdout:1732: finishing write tid 3 to smithi07085154-47 2024-09-14T03:54:05.079 INFO:tasks.rados.rados.0.smithi070.stdout:1732: finishing write tid 4 to smithi07085154-47 2024-09-14T03:54:05.079 INFO:tasks.rados.rados.0.smithi070.stdout:1732: finishing write tid 5 to smithi07085154-47 2024-09-14T03:54:05.080 INFO:tasks.rados.rados.0.smithi070.stdout:1732: finishing write tid 6 to smithi07085154-47 2024-09-14T03:54:05.080 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 47 v 1023 (ObjNum 555 snap 189 seq_num 555) dirty exists 2024-09-14T03:54:05.080 INFO:tasks.rados.rados.0.smithi070.stdout:1732: left oid 47 (ObjNum 555 snap 189 seq_num 555) 2024-09-14T03:54:05.081 INFO:tasks.rados.rados.0.smithi070.stdout:1733: expect (ObjNum 312 snap 105 seq_num 312) 2024-09-14T03:54:05.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:05 smithi094 ceph-mon[86597]: pgmap v225: 121 pgs: 121 active+clean; 456 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s rd, 3.5 MiB/s wr, 13 op/s 2024-09-14T03:54:05.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:05 smithi094 ceph-mon[86597]: osdmap e423: 8 total, 8 up, 8 in 2024-09-14T03:54:05.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:05 smithi070 ceph-mon[98640]: pgmap v225: 121 pgs: 121 active+clean; 456 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s rd, 3.5 MiB/s wr, 13 op/s 2024-09-14T03:54:05.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:05 smithi070 ceph-mon[98640]: osdmap e423: 8 total, 8 up, 8 in 2024-09-14T03:54:05.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:05 smithi070 ceph-mon[96704]: pgmap v225: 121 pgs: 121 active+clean; 456 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s rd, 3.5 MiB/s wr, 13 op/s 2024-09-14T03:54:05.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:05 smithi070 ceph-mon[96704]: osdmap e423: 8 total, 8 up, 8 in 2024-09-14T03:54:06.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:54:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:54:05.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:06.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:54:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:54:05.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:06.031 INFO:tasks.rados.rados.0.smithi070.stdout:1732: done (2 left) 2024-09-14T03:54:06.032 INFO:tasks.rados.rados.0.smithi070.stdout:1733: done (1 left) 2024-09-14T03:54:06.032 INFO:tasks.rados.rados.0.smithi070.stdout:1734: done (0 left) 2024-09-14T03:54:06.032 INFO:tasks.rados.rados.0.smithi070.stdout:1735: snap_create 2024-09-14T03:54:07.037 INFO:tasks.rados.rados.0.smithi070.stdout:1735: done (0 left) 2024-09-14T03:54:07.037 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 27 current snap is 191 2024-09-14T03:54:07.037 INFO:tasks.rados.rados.0.smithi070.stdout:1736: seq_num 556 ranges {0=516096,516096=524288,1040384=638976,1679360=442368} 2024-09-14T03:54:07.040 INFO:tasks.rados.rados.0.smithi070.stdout:1736: writing smithi07085154-27 from 0 to 516096 tid 1 2024-09-14T03:54:07.043 INFO:tasks.rados.rados.0.smithi070.stdout:1736: writing smithi07085154-27 from 516096 to 1040384 tid 2 2024-09-14T03:54:07.047 INFO:tasks.rados.rados.0.smithi070.stdout:1736: writing smithi07085154-27 from 1040384 to 1679360 tid 3 2024-09-14T03:54:07.050 INFO:tasks.rados.rados.0.smithi070.stdout:1736: writing smithi07085154-27 from 1679360 to 2121728 tid 4 2024-09-14T03:54:07.050 INFO:tasks.rados.rados.0.smithi070.stdout:1737: setattr oid 26 current snap is 191 2024-09-14T03:54:07.050 INFO:tasks.rados.rados.0.smithi070.stdout:1736: finishing write tid 1 to smithi07085154-27 2024-09-14T03:54:07.050 INFO:tasks.rados.rados.0.smithi070.stdout:1736: finishing write tid 2 to smithi07085154-27 2024-09-14T03:54:07.053 INFO:tasks.rados.rados.0.smithi070.stdout:1738: read oid 1 snap -1 2024-09-14T03:54:07.053 INFO:tasks.rados.rados.0.smithi070.stdout:1738: expect (ObjNum 539 snap 176 seq_num 539) 2024-09-14T03:54:07.053 INFO:tasks.rados.rados.0.smithi070.stdout:1739: rollback oid 7 current snap is 191 2024-09-14T03:54:07.053 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 7 to 160 2024-09-14T03:54:07.053 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 22 current snap is 191 2024-09-14T03:54:07.053 INFO:tasks.rados.rados.0.smithi070.stdout:1740: seq_num 557 ranges {0=532480,532480=802816,1335296=794624,2129920=8192} 2024-09-14T03:54:07.056 INFO:tasks.rados.rados.0.smithi070.stdout:1740: writing smithi07085154-22 from 0 to 532480 tid 1 2024-09-14T03:54:07.062 INFO:tasks.rados.rados.0.smithi070.stdout:1740: writing smithi07085154-22 from 532480 to 1335296 tid 2 2024-09-14T03:54:07.067 INFO:tasks.rados.rados.0.smithi070.stdout:1740: writing smithi07085154-22 from 1335296 to 2129920 tid 3 2024-09-14T03:54:07.067 INFO:tasks.rados.rados.0.smithi070.stdout:1740: writing smithi07085154-22 from 2129920 to 2138112 tid 4 2024-09-14T03:54:07.067 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 17 current snap is 191 2024-09-14T03:54:07.068 INFO:tasks.rados.rados.0.smithi070.stdout:1736: finishing write tid 3 to smithi07085154-27 2024-09-14T03:54:07.068 INFO:tasks.rados.rados.0.smithi070.stdout:1736: finishing write tid 4 to smithi07085154-27 2024-09-14T03:54:07.068 INFO:tasks.rados.rados.0.smithi070.stdout:1741: seq_num 558 ranges {4079616=548864,4628480=688128,5316608=606208,5922816=311296} 2024-09-14T03:54:07.092 INFO:tasks.rados.rados.0.smithi070.stdout:1741: writing smithi07085154-17 from 4079616 to 4628480 tid 1 2024-09-14T03:54:07.095 INFO:tasks.rados.rados.0.smithi070.stdout:1741: writing smithi07085154-17 from 4628480 to 5316608 tid 2 2024-09-14T03:54:07.099 INFO:tasks.rados.rados.0.smithi070.stdout:1741: writing smithi07085154-17 from 5316608 to 5922816 tid 3 2024-09-14T03:54:07.101 INFO:tasks.rados.rados.0.smithi070.stdout:1741: writing smithi07085154-17 from 5922816 to 6234112 tid 4 2024-09-14T03:54:07.101 INFO:tasks.rados.rados.0.smithi070.stdout:1742: delete oid 30 current snap is 191 2024-09-14T03:54:07.101 INFO:tasks.rados.rados.0.smithi070.stdout:1736: finishing write tid 5 to smithi07085154-27 2024-09-14T03:54:07.101 INFO:tasks.rados.rados.0.smithi070.stdout:1739: finishing rollback tid 0 to smithi07085154-7 2024-09-14T03:54:07.101 INFO:tasks.rados.rados.0.smithi070.stdout:1739: finishing rollback tid 1 to smithi07085154-7 2024-09-14T03:54:07.101 INFO:tasks.rados.rados.0.smithi070.stdout:1739: finishing rollback tid 2 to smithi07085154-7 2024-09-14T03:54:07.101 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 1243 (ObjNum 454 snap 154 seq_num 454) dirty exists 2024-09-14T03:54:07.101 INFO:tasks.rados.rados.0.smithi070.stdout:1736: finishing write tid 6 to smithi07085154-27 2024-09-14T03:54:07.101 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 1830 (ObjNum 556 snap 191 seq_num 556) dirty exists 2024-09-14T03:54:07.102 INFO:tasks.rados.rados.0.smithi070.stdout:1736: left oid 27 (ObjNum 556 snap 191 seq_num 556) 2024-09-14T03:54:07.102 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 1832 (ObjNum 65 snap 0 seq_num 0) dirty exists 2024-09-14T03:54:07.102 INFO:tasks.rados.rados.0.smithi070.stdout:1740: finishing write tid 1 to smithi07085154-22 2024-09-14T03:54:07.102 INFO:tasks.rados.rados.0.smithi070.stdout:1738: expect (ObjNum 539 snap 176 seq_num 539) 2024-09-14T03:54:07.148 INFO:tasks.rados.rados.0.smithi070.stdout:1740: finishing write tid 2 to smithi07085154-22 2024-09-14T03:54:07.148 INFO:tasks.rados.rados.0.smithi070.stdout:1740: finishing write tid 3 to smithi07085154-22 2024-09-14T03:54:07.148 INFO:tasks.rados.rados.0.smithi070.stdout:1740: finishing write tid 4 to smithi07085154-22 2024-09-14T03:54:07.148 INFO:tasks.rados.rados.0.smithi070.stdout:1740: finishing write tid 5 to smithi07085154-22 2024-09-14T03:54:07.148 INFO:tasks.rados.rados.0.smithi070.stdout:1740: finishing write tid 6 to smithi07085154-22 2024-09-14T03:54:07.148 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 1555 (ObjNum 557 snap 191 seq_num 557) dirty exists 2024-09-14T03:54:07.149 INFO:tasks.rados.rados.0.smithi070.stdout:1740: left oid 22 (ObjNum 557 snap 191 seq_num 557) 2024-09-14T03:54:07.149 INFO:tasks.rados.rados.0.smithi070.stdout:1741: finishing write tid 1 to smithi07085154-17 2024-09-14T03:54:07.149 INFO:tasks.rados.rados.0.smithi070.stdout:1736: done (6 left) 2024-09-14T03:54:07.149 INFO:tasks.rados.rados.0.smithi070.stdout:1737: done (5 left) 2024-09-14T03:54:07.149 INFO:tasks.rados.rados.0.smithi070.stdout:1738: done (4 left) 2024-09-14T03:54:07.149 INFO:tasks.rados.rados.0.smithi070.stdout:1739: done (3 left) 2024-09-14T03:54:07.149 INFO:tasks.rados.rados.0.smithi070.stdout:1740: done (2 left) 2024-09-14T03:54:07.149 INFO:tasks.rados.rados.0.smithi070.stdout:1742: done (1 left) 2024-09-14T03:54:07.149 INFO:tasks.rados.rados.0.smithi070.stdout:1743: delete oid 15 current snap is 191 2024-09-14T03:54:07.149 INFO:tasks.rados.rados.0.smithi070.stdout:1741: finishing write tid 2 to smithi07085154-17 2024-09-14T03:54:07.149 INFO:tasks.rados.rados.0.smithi070.stdout:1741: finishing write tid 3 to smithi07085154-17 2024-09-14T03:54:07.149 INFO:tasks.rados.rados.0.smithi070.stdout:1741: finishing write tid 4 to smithi07085154-17 2024-09-14T03:54:07.149 INFO:tasks.rados.rados.0.smithi070.stdout:1741: finishing write tid 5 to smithi07085154-17 2024-09-14T03:54:07.149 INFO:tasks.rados.rados.0.smithi070.stdout:1741: finishing write tid 6 to smithi07085154-17 2024-09-14T03:54:07.150 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 1555 (ObjNum 558 snap 191 seq_num 558) dirty exists 2024-09-14T03:54:07.150 INFO:tasks.rados.rados.0.smithi070.stdout:1741: left oid 17 (ObjNum 558 snap 191 seq_num 558) 2024-09-14T03:54:07.150 INFO:tasks.rados.rados.0.smithi070.stdout:1741: done (1 left) 2024-09-14T03:54:07.151 INFO:tasks.rados.rados.0.smithi070.stdout:1743: done (0 left) 2024-09-14T03:54:07.151 INFO:tasks.rados.rados.0.smithi070.stdout:1744: copy_from oid 27 from oid 15 current snap is 191 2024-09-14T03:54:07.151 INFO:tasks.rados.rados.0.smithi070.stdout:1745: read oid 30 snap 188 2024-09-14T03:54:07.151 INFO:tasks.rados.rados.0.smithi070.stdout:1745: expect deleted 2024-09-14T03:54:07.151 INFO:tasks.rados.rados.0.smithi070.stdout:1746: snap_remove snap 161 2024-09-14T03:54:07.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:07 smithi094 ceph-mon[86597]: osdmap e424: 8 total, 8 up, 8 in 2024-09-14T03:54:07.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:07 smithi094 ceph-mon[86597]: pgmap v228: 121 pgs: 121 active+clean; 456 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 2.4 MiB/s wr, 10 op/s 2024-09-14T03:54:07.277 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:54:06 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:54:06.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:07.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:07 smithi070 ceph-mon[98640]: osdmap e424: 8 total, 8 up, 8 in 2024-09-14T03:54:07.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:07 smithi070 ceph-mon[98640]: pgmap v228: 121 pgs: 121 active+clean; 456 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 2.4 MiB/s wr, 10 op/s 2024-09-14T03:54:07.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:07 smithi070 ceph-mon[96704]: osdmap e424: 8 total, 8 up, 8 in 2024-09-14T03:54:07.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:07 smithi070 ceph-mon[96704]: pgmap v228: 121 pgs: 121 active+clean; 456 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 2.4 MiB/s wr, 10 op/s 2024-09-14T03:54:08.045 INFO:tasks.rados.rados.0.smithi070.stdout:1746: done (2 left) 2024-09-14T03:54:08.045 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 7 current snap is 191 2024-09-14T03:54:08.045 INFO:tasks.rados.rados.0.smithi070.stdout:1747: seq_num 561 ranges {5513216=598016,6111232=745472,6856704=778240,7634944=81920} 2024-09-14T03:54:08.080 INFO:tasks.rados.rados.0.smithi070.stdout:1747: writing smithi07085154-7 from 5513216 to 6111232 tid 1 2024-09-14T03:54:08.084 INFO:tasks.rados.rados.0.smithi070.stdout:1747: writing smithi07085154-7 from 6111232 to 6856704 tid 2 2024-09-14T03:54:08.088 INFO:tasks.rados.rados.0.smithi070.stdout:1747: writing smithi07085154-7 from 6856704 to 7634944 tid 3 2024-09-14T03:54:08.089 INFO:tasks.rados.rados.0.smithi070.stdout:1747: writing smithi07085154-7 from 7634944 to 7716864 tid 4 2024-09-14T03:54:08.089 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 40 current snap is 191 2024-09-14T03:54:08.089 INFO:tasks.rados.rados.0.smithi070.stdout:1748: seq_num 562 ranges {1015808=622592,1638400=630784,2269184=581632,2850816=393216} 2024-09-14T03:54:08.098 INFO:tasks.rados.rados.0.smithi070.stdout:1748: writing smithi07085154-40 from 1015808 to 1638400 tid 1 2024-09-14T03:54:08.102 INFO:tasks.rados.rados.0.smithi070.stdout:1748: writing smithi07085154-40 from 1638400 to 2269184 tid 2 2024-09-14T03:54:08.105 INFO:tasks.rados.rados.0.smithi070.stdout:1748: writing smithi07085154-40 from 2269184 to 2850816 tid 3 2024-09-14T03:54:08.108 INFO:tasks.rados.rados.0.smithi070.stdout:1748: writing smithi07085154-40 from 2850816 to 3244032 tid 4 2024-09-14T03:54:08.108 INFO:tasks.rados.rados.0.smithi070.stdout:1749: read oid 47 snap -1 2024-09-14T03:54:08.108 INFO:tasks.rados.rados.0.smithi070.stdout:1749: expect (ObjNum 555 snap 189 seq_num 555) 2024-09-14T03:54:08.108 INFO:tasks.rados.rados.0.smithi070.stdout:1750: snap_create 2024-09-14T03:54:08.108 INFO:tasks.rados.rados.0.smithi070.stdout:1744: finishing copy_from to smithi07085154-27 2024-09-14T03:54:08.108 INFO:tasks.rados.rados.0.smithi070.stdout:1744: got expected ENOENT (src dne) 2024-09-14T03:54:08.108 INFO:tasks.rados.rados.0.smithi070.stdout:1744: finishing copy_from racing read to smithi07085154-27 2024-09-14T03:54:08.109 INFO:tasks.rados.rados.0.smithi070.stdout:1747: finishing write tid 1 to smithi07085154-7 2024-09-14T03:54:08.109 INFO:tasks.rados.rados.0.smithi070.stdout:1747: finishing write tid 2 to smithi07085154-7 2024-09-14T03:54:08.109 INFO:tasks.rados.rados.0.smithi070.stdout:1747: finishing write tid 3 to smithi07085154-7 2024-09-14T03:54:08.109 INFO:tasks.rados.rados.0.smithi070.stdout:1747: finishing write tid 4 to smithi07085154-7 2024-09-14T03:54:08.109 INFO:tasks.rados.rados.0.smithi070.stdout:1747: finishing write tid 5 to smithi07085154-7 2024-09-14T03:54:08.109 INFO:tasks.rados.rados.0.smithi070.stdout:1747: finishing write tid 6 to smithi07085154-7 2024-09-14T03:54:08.109 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 1248 (ObjNum 561 snap 191 seq_num 561) dirty exists 2024-09-14T03:54:08.109 INFO:tasks.rados.rados.0.smithi070.stdout:1747: left oid 7 (ObjNum 561 snap 191 seq_num 561) 2024-09-14T03:54:08.109 INFO:tasks.rados.rados.0.smithi070.stdout:1748: finishing write tid 1 to smithi07085154-40 2024-09-14T03:54:08.109 INFO:tasks.rados.rados.0.smithi070.stdout:1748: finishing write tid 2 to smithi07085154-40 2024-09-14T03:54:08.109 INFO:tasks.rados.rados.0.smithi070.stdout:1748: finishing write tid 3 to smithi07085154-40 2024-09-14T03:54:08.111 INFO:tasks.rados.rados.0.smithi070.stdout:1748: finishing write tid 4 to smithi07085154-40 2024-09-14T03:54:08.111 INFO:tasks.rados.rados.0.smithi070.stdout:1748: finishing write tid 5 to smithi07085154-40 2024-09-14T03:54:08.112 INFO:tasks.rados.rados.0.smithi070.stdout:1748: finishing write tid 6 to smithi07085154-40 2024-09-14T03:54:08.113 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 891 (ObjNum 562 snap 191 seq_num 562) dirty exists 2024-09-14T03:54:08.113 INFO:tasks.rados.rados.0.smithi070.stdout:1748: left oid 40 (ObjNum 562 snap 191 seq_num 562) 2024-09-14T03:54:08.146 INFO:tasks.rados.rados.0.smithi070.stdout:1749: expect (ObjNum 555 snap 189 seq_num 555) 2024-09-14T03:54:08.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:08 smithi070 ceph-mon[96704]: osdmap e425: 8 total, 8 up, 8 in 2024-09-14T03:54:08.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:08 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:08.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:08 smithi070 ceph-mon[98640]: osdmap e425: 8 total, 8 up, 8 in 2024-09-14T03:54:08.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:08 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:08.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:08 smithi094 ceph-mon[86597]: osdmap e425: 8 total, 8 up, 8 in 2024-09-14T03:54:08.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:08 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:09.025 INFO:tasks.rados.rados.0.smithi070.stdout:1744: done (5 left) 2024-09-14T03:54:09.025 INFO:tasks.rados.rados.0.smithi070.stdout:1745: done (4 left) 2024-09-14T03:54:09.025 INFO:tasks.rados.rados.0.smithi070.stdout:1747: done (3 left) 2024-09-14T03:54:09.025 INFO:tasks.rados.rados.0.smithi070.stdout:1748: done (2 left) 2024-09-14T03:54:09.025 INFO:tasks.rados.rados.0.smithi070.stdout:1749: done (1 left) 2024-09-14T03:54:09.025 INFO:tasks.rados.rados.0.smithi070.stdout:1750: done (0 left) 2024-09-14T03:54:09.025 INFO:tasks.rados.rados.0.smithi070.stdout:1751: delete oid 20 current snap is 192 2024-09-14T03:54:09.031 INFO:tasks.rados.rados.0.smithi070.stdout:1751: done (0 left) 2024-09-14T03:54:09.031 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 29 current snap is 192 2024-09-14T03:54:09.032 INFO:tasks.rados.rados.0.smithi070.stdout:1752: seq_num 564 ranges {6578176=655360,7233536=442368,7675904=598016,8273920=565248} 2024-09-14T03:54:09.071 INFO:tasks.rados.rados.0.smithi070.stdout:1752: writing smithi07085154-29 from 6578176 to 7233536 tid 1 2024-09-14T03:54:09.073 INFO:tasks.rados.rados.0.smithi070.stdout:1752: writing smithi07085154-29 from 7233536 to 7675904 tid 2 2024-09-14T03:54:09.077 INFO:tasks.rados.rados.0.smithi070.stdout:1752: writing smithi07085154-29 from 7675904 to 8273920 tid 3 2024-09-14T03:54:09.080 INFO:tasks.rados.rados.0.smithi070.stdout:1752: writing smithi07085154-29 from 8273920 to 8839168 tid 4 2024-09-14T03:54:09.080 INFO:tasks.rados.rados.0.smithi070.stdout:1753: rmattr oid 24 current snap is 192 2024-09-14T03:54:09.080 INFO:tasks.rados.rados.0.smithi070.stdout:1753: done (1 left) 2024-09-14T03:54:09.080 INFO:tasks.rados.rados.0.smithi070.stdout:1754: snap_remove snap 163 2024-09-14T03:54:09.298 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:09 smithi070 ceph-mon[96704]: osdmap e426: 8 total, 8 up, 8 in 2024-09-14T03:54:09.298 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:09 smithi070 ceph-mon[96704]: pgmap v231: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 480 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 6.0 MiB/s wr, 12 op/s 2024-09-14T03:54:09.298 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:09 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-14T03:54:09.298 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:09 smithi070 ceph-mon[96704]: osdmap e427: 8 total, 8 up, 8 in 2024-09-14T03:54:09.298 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:09 smithi070 ceph-mon[98640]: osdmap e426: 8 total, 8 up, 8 in 2024-09-14T03:54:09.298 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:09 smithi070 ceph-mon[98640]: pgmap v231: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 480 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 6.0 MiB/s wr, 12 op/s 2024-09-14T03:54:09.298 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:09 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-14T03:54:09.298 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:09 smithi070 ceph-mon[98640]: osdmap e427: 8 total, 8 up, 8 in 2024-09-14T03:54:09.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:09 smithi094 ceph-mon[86597]: osdmap e426: 8 total, 8 up, 8 in 2024-09-14T03:54:09.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:09 smithi094 ceph-mon[86597]: pgmap v231: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 480 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 6.0 MiB/s wr, 12 op/s 2024-09-14T03:54:09.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:09 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-14T03:54:09.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:09 smithi094 ceph-mon[86597]: osdmap e427: 8 total, 8 up, 8 in 2024-09-14T03:54:10.049 INFO:tasks.rados.rados.0.smithi070.stdout:1754: done (1 left) 2024-09-14T03:54:10.049 INFO:tasks.rados.rados.0.smithi070.stdout:1755: delete oid 8 current snap is 192 2024-09-14T03:54:10.049 INFO:tasks.rados.rados.0.smithi070.stdout:1752: finishing write tid 1 to smithi07085154-29 2024-09-14T03:54:10.049 INFO:tasks.rados.rados.0.smithi070.stdout:1752: finishing write tid 2 to smithi07085154-29 2024-09-14T03:54:10.049 INFO:tasks.rados.rados.0.smithi070.stdout:1752: finishing write tid 3 to smithi07085154-29 2024-09-14T03:54:10.049 INFO:tasks.rados.rados.0.smithi070.stdout:1752: finishing write tid 4 to smithi07085154-29 2024-09-14T03:54:10.049 INFO:tasks.rados.rados.0.smithi070.stdout:1752: finishing write tid 5 to smithi07085154-29 2024-09-14T03:54:10.049 INFO:tasks.rados.rados.0.smithi070.stdout:1752: finishing write tid 6 to smithi07085154-29 2024-09-14T03:54:10.049 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 1560 (ObjNum 564 snap 192 seq_num 564) dirty exists 2024-09-14T03:54:10.049 INFO:tasks.rados.rados.0.smithi070.stdout:1752: left oid 29 (ObjNum 564 snap 192 seq_num 564) 2024-09-14T03:54:10.065 INFO:tasks.rados.rados.0.smithi070.stdout:1752: done (1 left) 2024-09-14T03:54:10.065 INFO:tasks.rados.rados.0.smithi070.stdout:1755: done (0 left) 2024-09-14T03:54:10.065 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 50 current snap is 192 2024-09-14T03:54:10.065 INFO:tasks.rados.rados.0.smithi070.stdout:1756: seq_num 566 ranges {4964352=688128,5652480=614400,6266880=565248,6832128=425984} 2024-09-14T03:54:10.097 INFO:tasks.rados.rados.0.smithi070.stdout:1756: writing smithi07085154-50 from 4964352 to 5652480 tid 1 2024-09-14T03:54:10.100 INFO:tasks.rados.rados.0.smithi070.stdout:1756: writing smithi07085154-50 from 5652480 to 6266880 tid 2 2024-09-14T03:54:10.103 INFO:tasks.rados.rados.0.smithi070.stdout:1756: writing smithi07085154-50 from 6266880 to 6832128 tid 3 2024-09-14T03:54:10.106 INFO:tasks.rados.rados.0.smithi070.stdout:1756: writing smithi07085154-50 from 6832128 to 7258112 tid 4 2024-09-14T03:54:10.106 INFO:tasks.rados.rados.0.smithi070.stdout:1757: snap_remove snap 134 2024-09-14T03:54:10.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:10 smithi070 ceph-mon[98640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-14T03:54:10.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:10 smithi070 ceph-mon[98640]: Upgrade: osd.3 is safe to restart 2024-09-14T03:54:10.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:10 smithi070 ceph-mon[96704]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-14T03:54:10.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:10 smithi070 ceph-mon[96704]: Upgrade: osd.3 is safe to restart 2024-09-14T03:54:10.428 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:10 smithi094 ceph-mon[86597]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-14T03:54:10.428 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:10 smithi094 ceph-mon[86597]: Upgrade: osd.3 is safe to restart 2024-09-14T03:54:10.429 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:54:10 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:54:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:11.065 INFO:tasks.rados.rados.0.smithi070.stdout:1757: done (1 left) 2024-09-14T03:54:11.065 INFO:tasks.rados.rados.0.smithi070.stdout:1758: read oid 37 snap -1 2024-09-14T03:54:11.065 INFO:tasks.rados.rados.0.smithi070.stdout:1758: expect (ObjNum 312 snap 105 seq_num 312) 2024-09-14T03:54:11.066 INFO:tasks.rados.rados.0.smithi070.stdout:1759: delete oid 26 current snap is 192 2024-09-14T03:54:11.066 INFO:tasks.rados.rados.0.smithi070.stdout:1756: finishing write tid 1 to smithi07085154-50 2024-09-14T03:54:11.066 INFO:tasks.rados.rados.0.smithi070.stdout:1756: finishing write tid 2 to smithi07085154-50 2024-09-14T03:54:11.066 INFO:tasks.rados.rados.0.smithi070.stdout:1756: finishing write tid 3 to smithi07085154-50 2024-09-14T03:54:11.066 INFO:tasks.rados.rados.0.smithi070.stdout:1756: finishing write tid 4 to smithi07085154-50 2024-09-14T03:54:11.066 INFO:tasks.rados.rados.0.smithi070.stdout:1756: finishing write tid 5 to smithi07085154-50 2024-09-14T03:54:11.066 INFO:tasks.rados.rados.0.smithi070.stdout:1756: finishing write tid 6 to smithi07085154-50 2024-09-14T03:54:11.066 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 1565 (ObjNum 566 snap 192 seq_num 566) dirty exists 2024-09-14T03:54:11.066 INFO:tasks.rados.rados.0.smithi070.stdout:1756: left oid 50 (ObjNum 566 snap 192 seq_num 566) 2024-09-14T03:54:11.078 INFO:tasks.rados.rados.0.smithi070.stdout:1758: expect (ObjNum 312 snap 105 seq_num 312) 2024-09-14T03:54:11.109 INFO:tasks.rados.rados.0.smithi070.stdout:1756: done (2 left) 2024-09-14T03:54:11.109 INFO:tasks.rados.rados.0.smithi070.stdout:1758: done (1 left) 2024-09-14T03:54:11.110 INFO:tasks.rados.rados.0.smithi070.stdout:1759: done (0 left) 2024-09-14T03:54:11.110 INFO:tasks.rados.rados.0.smithi070.stdout:1760: read oid 39 snap 123 2024-09-14T03:54:11.110 INFO:tasks.rados.rados.0.smithi070.stdout:1760: expect (ObjNum 202 snap 56 seq_num 202) 2024-09-14T03:54:11.110 INFO:tasks.rados.rados.0.smithi070.stdout:1761: copy_from oid 24 from oid 22 current snap is 192 2024-09-14T03:54:11.110 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 31 current snap is 192 2024-09-14T03:54:11.110 INFO:tasks.rados.rados.0.smithi070.stdout:1762: seq_num 568 ranges {4964352=720896,5685248=786432,6471680=581632,7053312=237568} 2024-09-14T03:54:11.139 INFO:tasks.rados.rados.0.smithi070.stdout:1762: writing smithi07085154-31 from 4964352 to 5685248 tid 1 2024-09-14T03:54:11.144 INFO:tasks.rados.rados.0.smithi070.stdout:1762: writing smithi07085154-31 from 5685248 to 6471680 tid 2 2024-09-14T03:54:11.147 INFO:tasks.rados.rados.0.smithi070.stdout:1762: writing smithi07085154-31 from 6471680 to 7053312 tid 3 2024-09-14T03:54:11.148 INFO:tasks.rados.rados.0.smithi070.stdout:1762: writing smithi07085154-31 from 7053312 to 7290880 tid 4 2024-09-14T03:54:11.149 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 2 current snap is 192 2024-09-14T03:54:11.149 INFO:tasks.rados.rados.0.smithi070.stdout:1763: seq_num 569 ranges {0=737280,737280=671744,1409024=737280,2146304=196608} 2024-09-14T03:54:11.152 INFO:tasks.rados.rados.0.smithi070.stdout:1763: writing smithi07085154-2 from 0 to 737280 tid 1 2024-09-14T03:54:11.156 INFO:tasks.rados.rados.0.smithi070.stdout:1763: writing smithi07085154-2 from 737280 to 1409024 tid 2 2024-09-14T03:54:11.160 INFO:tasks.rados.rados.0.smithi070.stdout:1763: writing smithi07085154-2 from 1409024 to 2146304 tid 3 2024-09-14T03:54:11.161 INFO:tasks.rados.rados.0.smithi070.stdout:1763: writing smithi07085154-2 from 2146304 to 2342912 tid 4 2024-09-14T03:54:11.162 INFO:tasks.rados.rados.0.smithi070.stdout:1764: snap_remove snap 190 2024-09-14T03:54:11.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:11 smithi070 ceph-mon[96704]: osdmap e428: 8 total, 8 up, 8 in 2024-09-14T03:54:11.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:11 smithi070 ceph-mon[96704]: pgmap v234: 121 pgs: 6 active+clean+snaptrim, 115 active+clean; 487 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 7.7 MiB/s wr, 15 op/s 2024-09-14T03:54:11.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:11 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:11.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:11 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-14T03:54:11.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:11 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:54:11.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:11 smithi070 ceph-mon[98640]: osdmap e428: 8 total, 8 up, 8 in 2024-09-14T03:54:11.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:11 smithi070 ceph-mon[98640]: pgmap v234: 121 pgs: 6 active+clean+snaptrim, 115 active+clean; 487 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 7.7 MiB/s wr, 15 op/s 2024-09-14T03:54:11.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:11 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:11.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:11 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-14T03:54:11.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:11 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:54:11.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:11 smithi094 ceph-mon[86597]: osdmap e428: 8 total, 8 up, 8 in 2024-09-14T03:54:11.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:11 smithi094 ceph-mon[86597]: pgmap v234: 121 pgs: 6 active+clean+snaptrim, 115 active+clean; 487 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 7.7 MiB/s wr, 15 op/s 2024-09-14T03:54:11.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:11 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:11.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:11 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-14T03:54:11.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:11 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:54:12.070 INFO:tasks.rados.rados.0.smithi070.stdout:1764: done (4 left) 2024-09-14T03:54:12.070 INFO:tasks.rados.rados.0.smithi070.stdout:1765: rollback oid 27 current snap is 192 2024-09-14T03:54:12.070 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 27 to 173 2024-09-14T03:54:12.070 INFO:tasks.rados.rados.0.smithi070.stdout:1766: delete oid 34 current snap is 192 2024-09-14T03:54:12.070 INFO:tasks.rados.rados.0.smithi070.stdout:1761: finishing copy_from racing read to smithi07085154-24 2024-09-14T03:54:12.070 INFO:tasks.rados.rados.0.smithi070.stdout:1760: expect (ObjNum 202 snap 56 seq_num 202) 2024-09-14T03:54:12.148 INFO:tasks.rados.rados.0.smithi070.stdout:1761: finishing copy_from to smithi07085154-24 2024-09-14T03:54:12.148 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 1555 (ObjNum 557 snap 191 seq_num 557) dirty exists 2024-09-14T03:54:12.148 INFO:tasks.rados.rados.0.smithi070.stdout:1762: finishing write tid 1 to smithi07085154-31 2024-09-14T03:54:12.148 INFO:tasks.rados.rados.0.smithi070.stdout:1762: finishing write tid 2 to smithi07085154-31 2024-09-14T03:54:12.148 INFO:tasks.rados.rados.0.smithi070.stdout:1760: done (5 left) 2024-09-14T03:54:12.148 INFO:tasks.rados.rados.0.smithi070.stdout:1761: done (4 left) 2024-09-14T03:54:12.148 INFO:tasks.rados.rados.0.smithi070.stdout:1766: done (3 left) 2024-09-14T03:54:12.148 INFO:tasks.rados.rados.0.smithi070.stdout:1767: snap_create 2024-09-14T03:54:12.148 INFO:tasks.rados.rados.0.smithi070.stdout:1762: finishing write tid 3 to smithi07085154-31 2024-09-14T03:54:12.148 INFO:tasks.rados.rados.0.smithi070.stdout:1762: finishing write tid 4 to smithi07085154-31 2024-09-14T03:54:12.148 INFO:tasks.rados.rados.0.smithi070.stdout:1762: finishing write tid 5 to smithi07085154-31 2024-09-14T03:54:12.149 INFO:tasks.rados.rados.0.smithi070.stdout:1762: finishing write tid 6 to smithi07085154-31 2024-09-14T03:54:12.149 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 1560 (ObjNum 568 snap 192 seq_num 568) dirty exists 2024-09-14T03:54:12.149 INFO:tasks.rados.rados.0.smithi070.stdout:1762: left oid 31 (ObjNum 568 snap 192 seq_num 568) 2024-09-14T03:54:12.149 INFO:tasks.rados.rados.0.smithi070.stdout:1763: finishing write tid 1 to smithi07085154-2 2024-09-14T03:54:12.149 INFO:tasks.rados.rados.0.smithi070.stdout:1763: finishing write tid 2 to smithi07085154-2 2024-09-14T03:54:12.149 INFO:tasks.rados.rados.0.smithi070.stdout:1763: finishing write tid 3 to smithi07085154-2 2024-09-14T03:54:12.149 INFO:tasks.rados.rados.0.smithi070.stdout:1763: finishing write tid 4 to smithi07085154-2 2024-09-14T03:54:12.149 INFO:tasks.rados.rados.0.smithi070.stdout:1763: finishing write tid 5 to smithi07085154-2 2024-09-14T03:54:12.149 INFO:tasks.rados.rados.0.smithi070.stdout:1763: finishing write tid 6 to smithi07085154-2 2024-09-14T03:54:12.149 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 2 v 1877 (ObjNum 569 snap 192 seq_num 569) dirty exists 2024-09-14T03:54:12.149 INFO:tasks.rados.rados.0.smithi070.stdout:1763: left oid 2 (ObjNum 569 snap 192 seq_num 569) 2024-09-14T03:54:12.149 INFO:tasks.rados.rados.0.smithi070.stdout:1765: finishing rollback tid 0 to smithi07085154-27 2024-09-14T03:54:12.149 INFO:tasks.rados.rados.0.smithi070.stdout:1765: finishing rollback tid 1 to smithi07085154-27 2024-09-14T03:54:12.150 INFO:tasks.rados.rados.0.smithi070.stdout:1765: finishing rollback tid 2 to smithi07085154-27 2024-09-14T03:54:12.150 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 1881 (ObjNum 459 snap 155 seq_num 459) dirty exists 2024-09-14T03:54:12.345 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:12 smithi070 ceph-mon[96704]: Upgrade: Updating osd.3 2024-09-14T03:54:12.346 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:12 smithi070 ceph-mon[96704]: Deploying daemon osd.3 on smithi070 2024-09-14T03:54:12.346 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:12 smithi070 ceph-mon[96704]: osdmap e429: 8 total, 8 up, 8 in 2024-09-14T03:54:12.346 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:12 smithi070 ceph-mon[98640]: Upgrade: Updating osd.3 2024-09-14T03:54:12.346 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:12 smithi070 ceph-mon[98640]: Deploying daemon osd.3 on smithi070 2024-09-14T03:54:12.346 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:12 smithi070 ceph-mon[98640]: osdmap e429: 8 total, 8 up, 8 in 2024-09-14T03:54:12.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:12 smithi094 ceph-mon[86597]: Upgrade: Updating osd.3 2024-09-14T03:54:12.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:12 smithi094 ceph-mon[86597]: Deploying daemon osd.3 on smithi070 2024-09-14T03:54:12.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:12 smithi094 ceph-mon[86597]: osdmap e429: 8 total, 8 up, 8 in 2024-09-14T03:54:13.072 INFO:tasks.rados.rados.0.smithi070.stdout:1762: done (3 left) 2024-09-14T03:54:13.072 INFO:tasks.rados.rados.0.smithi070.stdout:1763: done (2 left) 2024-09-14T03:54:13.073 INFO:tasks.rados.rados.0.smithi070.stdout:1765: done (1 left) 2024-09-14T03:54:13.073 INFO:tasks.rados.rados.0.smithi070.stdout:1767: done (0 left) 2024-09-14T03:54:13.073 INFO:tasks.rados.rados.0.smithi070.stdout:1768: read oid 21 snap 123 2024-09-14T03:54:13.073 INFO:tasks.rados.rados.0.smithi070.stdout:1768: expect (ObjNum 114 snap 24 seq_num 114) 2024-09-14T03:54:13.073 INFO:tasks.rados.rados.0.smithi070.stdout:1769: delete oid 43 current snap is 193 2024-09-14T03:54:13.077 INFO:tasks.rados.rados.0.smithi070.stdout:1769: done (1 left) 2024-09-14T03:54:13.077 INFO:tasks.rados.rados.0.smithi070.stdout:1770: snap_remove snap 185 2024-09-14T03:54:13.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:13 smithi070 ceph-mon[96704]: pgmap v236: 121 pgs: 6 active+clean+snaptrim, 115 active+clean; 498 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 5.4 MiB/s wr, 19 op/s 2024-09-14T03:54:13.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:13 smithi070 ceph-mon[96704]: osdmap e430: 8 total, 8 up, 8 in 2024-09-14T03:54:13.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:13 smithi070 ceph-mon[98640]: pgmap v236: 121 pgs: 6 active+clean+snaptrim, 115 active+clean; 498 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 5.4 MiB/s wr, 19 op/s 2024-09-14T03:54:13.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:13 smithi070 ceph-mon[98640]: osdmap e430: 8 total, 8 up, 8 in 2024-09-14T03:54:13.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:13 smithi094 ceph-mon[86597]: pgmap v236: 121 pgs: 6 active+clean+snaptrim, 115 active+clean; 498 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 5.4 MiB/s wr, 19 op/s 2024-09-14T03:54:13.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:13 smithi094 ceph-mon[86597]: osdmap e430: 8 total, 8 up, 8 in 2024-09-14T03:54:13.605 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:54:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:54:13] "GET /metrics HTTP/1.1" 200 36314 "" "Prometheus/2.51.0" 2024-09-14T03:54:13.606 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:13 smithi070 systemd[1]: Stopping Ceph osd.3 for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:54:14.027 INFO:tasks.rados.rados.0.smithi070.stdout:1770: done (1 left) 2024-09-14T03:54:14.027 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 34 current snap is 193 2024-09-14T03:54:14.027 INFO:tasks.rados.rados.0.smithi070.stdout:1771: seq_num 572 ranges {0=786432,786432=770048,1556480=557056,2113536=278528} 2024-09-14T03:54:14.032 INFO:tasks.rados.rados.0.smithi070.stdout:1771: writing smithi07085154-34 from 0 to 786432 tid 1 2024-09-14T03:54:14.035 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3[61924]: 2024-09-14T03:54:13.604+0000 7fd7cc0d0640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:54:14.035 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3[61924]: 2024-09-14T03:54:13.604+0000 7fd7cc0d0640 -1 osd.3 431 *** Got signal Terminated *** 2024-09-14T03:54:14.036 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3[61924]: 2024-09-14T03:54:13.604+0000 7fd7cc0d0640 -1 osd.3 431 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T03:54:14.037 INFO:tasks.rados.rados.0.smithi070.stdout:1771: writing smithi07085154-34 from 786432 to 1556480 tid 2 2024-09-14T03:54:14.041 INFO:tasks.rados.rados.0.smithi070.stdout:1771: writing smithi07085154-34 from 1556480 to 2113536 tid 3 2024-09-14T03:54:14.043 INFO:tasks.rados.rados.0.smithi070.stdout:1771: writing smithi07085154-34 from 2113536 to 2392064 tid 4 2024-09-14T03:54:14.044 INFO:tasks.rados.rados.0.smithi070.stdout:1768: expect (ObjNum 114 snap 24 seq_num 114) 2024-09-14T03:54:14.241 INFO:tasks.rados.rados.0.smithi070.stdout:1771: finishing write tid 1 to smithi07085154-34 2024-09-14T03:54:14.241 INFO:tasks.rados.rados.0.smithi070.stdout:1771: finishing write tid 2 to smithi07085154-34 2024-09-14T03:54:14.241 INFO:tasks.rados.rados.0.smithi070.stdout:1771: finishing write tid 3 to smithi07085154-34 2024-09-14T03:54:14.241 INFO:tasks.rados.rados.0.smithi070.stdout:1771: finishing write tid 4 to smithi07085154-34 2024-09-14T03:54:14.241 INFO:tasks.rados.rados.0.smithi070.stdout:1768: done (1 left) 2024-09-14T03:54:14.241 INFO:tasks.rados.rados.0.smithi070.stdout:1772: setattr oid 12 current snap is 193 2024-09-14T03:54:14.242 INFO:tasks.rados.rados.0.smithi070.stdout:1771: finishing write tid 5 to smithi07085154-34 2024-09-14T03:54:14.242 INFO:tasks.rados.rados.0.smithi070.stdout:1771: finishing write tid 6 to smithi07085154-34 2024-09-14T03:54:14.242 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 1571 (ObjNum 572 snap 193 seq_num 572) dirty exists 2024-09-14T03:54:14.242 INFO:tasks.rados.rados.0.smithi070.stdout:1771: left oid 34 (ObjNum 572 snap 193 seq_num 572) 2024-09-14T03:54:14.242 INFO:tasks.rados.rados.0.smithi070.stdout:1771: done (1 left) 2024-09-14T03:54:14.242 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 21 current snap is 193 2024-09-14T03:54:14.242 INFO:tasks.rados.rados.0.smithi070.stdout:1773: seq_num 573 ranges {0=802816,802816=655360,1458176=720896,2179072=229376} 2024-09-14T03:54:14.246 INFO:tasks.rados.rados.0.smithi070.stdout:1773: writing smithi07085154-21 from 0 to 802816 tid 1 2024-09-14T03:54:14.249 INFO:tasks.rados.rados.0.smithi070.stdout:1773: writing smithi07085154-21 from 802816 to 1458176 tid 2 2024-09-14T03:54:14.253 INFO:tasks.rados.rados.0.smithi070.stdout:1773: writing smithi07085154-21 from 1458176 to 2179072 tid 3 2024-09-14T03:54:14.255 INFO:tasks.rados.rados.0.smithi070.stdout:1773: writing smithi07085154-21 from 2179072 to 2408448 tid 4 2024-09-14T03:54:14.255 INFO:tasks.rados.rados.0.smithi070.stdout:1774: copy_from oid 45 from oid 29 current snap is 193 2024-09-14T03:54:14.255 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 563 (ObjNum 534 snap 176 seq_num 534) dirty exists 2024-09-14T03:54:14.255 INFO:tasks.rados.rados.0.smithi070.stdout:1772: done (2 left) 2024-09-14T03:54:14.256 INFO:tasks.rados.rados.0.smithi070.stdout:1775: rollback oid 3 current snap is 193 2024-09-14T03:54:14.256 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 3 to 191 2024-09-14T03:54:14.256 INFO:tasks.rados.rados.0.smithi070.stdout:1776: delete oid 28 current snap is 193 2024-09-14T03:54:14.257 INFO:tasks.rados.rados.0.smithi070.stdout:1775: finishing rollback tid 1 to smithi07085154-3 2024-09-14T03:54:14.257 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 3 v 1009 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-14T03:54:14.258 INFO:tasks.rados.rados.0.smithi070.stdout:1775: done (3 left) 2024-09-14T03:54:14.258 INFO:tasks.rados.rados.0.smithi070.stdout:1776: done (2 left) 2024-09-14T03:54:14.258 INFO:tasks.rados.rados.0.smithi070.stdout:1777: copy_from oid 22 from oid 3 current snap is 193 2024-09-14T03:54:14.258 INFO:tasks.rados.rados.0.smithi070.stdout:1778: copy_from oid 24 from oid 42 current snap is 193 2024-09-14T03:54:14.258 INFO:tasks.rados.rados.0.smithi070.stdout:1779: read oid 2 snap -1 2024-09-14T03:54:14.258 INFO:tasks.rados.rados.0.smithi070.stdout:1779: expect (ObjNum 569 snap 192 seq_num 569) 2024-09-14T03:54:14.259 INFO:tasks.rados.rados.0.smithi070.stdout:1780: delete oid 46 current snap is 193 2024-09-14T03:54:14.266 INFO:tasks.rados.rados.0.smithi070.stdout:1777: finishing copy_from to smithi07085154-22 2024-09-14T03:54:14.266 INFO:tasks.rados.rados.0.smithi070.stdout:1777: got expected ENOENT (src dne) 2024-09-14T03:54:14.266 INFO:tasks.rados.rados.0.smithi070.stdout:1777: finishing copy_from racing read to smithi07085154-22 2024-09-14T03:54:14.267 INFO:tasks.rados.rados.0.smithi070.stdout:1777: done (5 left) 2024-09-14T03:54:14.267 INFO:tasks.rados.rados.0.smithi070.stdout:1780: done (4 left) 2024-09-14T03:54:14.268 INFO:tasks.rados.rados.0.smithi070.stdout:1781: rmattr oid 35 current snap is 193 2024-09-14T03:54:14.268 INFO:tasks.rados.rados.0.smithi070.stdout:1781: done (4 left) 2024-09-14T03:54:14.268 INFO:tasks.rados.rados.0.smithi070.stdout:1782: read oid 7 snap -1 2024-09-14T03:54:14.268 INFO:tasks.rados.rados.0.smithi070.stdout:1782: expect (ObjNum 561 snap 191 seq_num 561) 2024-09-14T03:54:14.268 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 46 current snap is 193 2024-09-14T03:54:14.268 INFO:tasks.rados.rados.0.smithi070.stdout:1783: seq_num 576 ranges {0=49152} 2024-09-14T03:54:14.268 INFO:tasks.rados.rados.0.smithi070.stdout:1783: writing smithi07085154-46 from 0 to 49152 tid 1 2024-09-14T03:54:14.268 INFO:tasks.rados.rados.0.smithi070.stdout:1784: delete oid 43 current snap is 193 2024-09-14T03:54:14.269 INFO:tasks.rados.rados.0.smithi070.stdout:1779: expect (ObjNum 569 snap 192 seq_num 569) 2024-09-14T03:54:14.341 INFO:tasks.rados.rados.0.smithi070.stdout:1783: finishing write tid 1 to smithi07085154-46 2024-09-14T03:54:14.342 INFO:tasks.rados.rados.0.smithi070.stdout:1778: finishing copy_from racing read to smithi07085154-24 2024-09-14T03:54:14.342 INFO:tasks.rados.rados.0.smithi070.stdout:1778: finishing copy_from to smithi07085154-24 2024-09-14T03:54:14.342 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 1545 (ObjNum 507 snap 166 seq_num 507) dirty exists 2024-09-14T03:54:14.342 INFO:tasks.rados.rados.0.smithi070.stdout:1783: finishing write tid 2 to smithi07085154-46 2024-09-14T03:54:14.342 INFO:tasks.rados.rados.0.smithi070.stdout:1783: finishing write tid 3 to smithi07085154-46 2024-09-14T03:54:14.342 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 46 v 1889 (ObjNum 576 snap 193 seq_num 576) dirty exists 2024-09-14T03:54:14.342 INFO:tasks.rados.rados.0.smithi070.stdout:1783: left oid 46 (ObjNum 576 snap 193 seq_num 576) 2024-09-14T03:54:14.342 INFO:tasks.rados.rados.0.smithi070.stdout:1782: expect (ObjNum 561 snap 191 seq_num 561) 2024-09-14T03:54:14.379 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:14 smithi070 ceph-mon[96704]: osdmap e431: 8 total, 8 up, 8 in 2024-09-14T03:54:14.380 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:14 smithi070 ceph-mon[96704]: osd.3 marked itself down and dead 2024-09-14T03:54:14.380 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:14 smithi070 ceph-mon[96704]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:54:14.380 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:14 smithi070 ceph-mon[96704]: osdmap e432: 8 total, 7 up, 8 in 2024-09-14T03:54:14.380 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:14 smithi070 ceph-mon[98640]: osdmap e431: 8 total, 8 up, 8 in 2024-09-14T03:54:14.380 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:14 smithi070 ceph-mon[98640]: osd.3 marked itself down and dead 2024-09-14T03:54:14.380 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:14 smithi070 ceph-mon[98640]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:54:14.380 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:14 smithi070 ceph-mon[98640]: osdmap e432: 8 total, 7 up, 8 in 2024-09-14T03:54:14.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:14 smithi094 ceph-mon[86597]: osdmap e431: 8 total, 8 up, 8 in 2024-09-14T03:54:14.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:14 smithi094 ceph-mon[86597]: osd.3 marked itself down and dead 2024-09-14T03:54:14.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:14 smithi094 ceph-mon[86597]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:54:14.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:14 smithi094 ceph-mon[86597]: osdmap e432: 8 total, 7 up, 8 in 2024-09-14T03:54:14.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:54:14 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:54:14.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.3\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:14.663 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:14 smithi070 podman[117119]: 2024-09-14 03:54:14.379015012 +0000 UTC m=+0.881855698 container died 2e4199b58392ff3254e892156cb7a00f45e98b59008be8fca534a10b7188ac4e (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-eb16b97, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-14T03:54:14.855 INFO:tasks.rados.rados.0.smithi070.stdout:1774: finishing copy_from racing read to smithi07085154-45 2024-09-14T03:54:14.855 INFO:tasks.rados.rados.0.smithi070.stdout:1774: finishing copy_from to smithi07085154-45 2024-09-14T03:54:14.855 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 45 v 1560 (ObjNum 564 snap 192 seq_num 564) dirty exists 2024-09-14T03:54:14.921 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:14 smithi070 podman[117119]: 2024-09-14 03:54:14.752574372 +0000 UTC m=+1.255415047 container cleanup 2e4199b58392ff3254e892156cb7a00f45e98b59008be8fca534a10b7188ac4e (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-eb16b97, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-14T03:54:14.922 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:14 smithi070 bash[117119]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3 2024-09-14T03:54:15.045 INFO:tasks.rados.rados.0.smithi070.stdout:1774: done (6 left) 2024-09-14T03:54:15.046 INFO:tasks.rados.rados.0.smithi070.stdout:1778: done (5 left) 2024-09-14T03:54:15.046 INFO:tasks.rados.rados.0.smithi070.stdout:1779: done (4 left) 2024-09-14T03:54:15.046 INFO:tasks.rados.rados.0.smithi070.stdout:1782: done (3 left) 2024-09-14T03:54:15.046 INFO:tasks.rados.rados.0.smithi070.stdout:1783: done (2 left) 2024-09-14T03:54:15.046 INFO:tasks.rados.rados.0.smithi070.stdout:1784: done (1 left) 2024-09-14T03:54:15.046 INFO:tasks.rados.rados.0.smithi070.stdout:1785: snap_remove snap 178 2024-09-14T03:54:15.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:15 smithi070 ceph-mon[98640]: pgmap v240: 121 pgs: 16 stale+active+clean, 105 active+clean; 508 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 4.3 MiB/s wr, 20 op/s 2024-09-14T03:54:15.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:15 smithi070 ceph-mon[98640]: osdmap e433: 8 total, 7 up, 8 in 2024-09-14T03:54:15.286 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:14 smithi070 podman[117138]: 2024-09-14 03:54:14.920903556 +0000 UTC m=+0.542274177 container remove 2e4199b58392ff3254e892156cb7a00f45e98b59008be8fca534a10b7188ac4e (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=quincy-eb16b97, GIT_CLEAN=True) 2024-09-14T03:54:15.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:15 smithi070 ceph-mon[96704]: pgmap v240: 121 pgs: 16 stale+active+clean, 105 active+clean; 508 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 4.3 MiB/s wr, 20 op/s 2024-09-14T03:54:15.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:15 smithi070 ceph-mon[96704]: osdmap e433: 8 total, 7 up, 8 in 2024-09-14T03:54:15.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:15 smithi094 ceph-mon[86597]: pgmap v240: 121 pgs: 16 stale+active+clean, 105 active+clean; 508 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 4.3 MiB/s wr, 20 op/s 2024-09-14T03:54:15.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:15 smithi094 ceph-mon[86597]: osdmap e433: 8 total, 7 up, 8 in 2024-09-14T03:54:15.684 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:15 smithi070 podman[117232]: 2024-09-14 03:54:15.238526631 +0000 UTC m=+0.015839074 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:54:15.684 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:15 smithi070 podman[117232]: 2024-09-14 03:54:15.338589495 +0000 UTC m=+0.115901925 container create f00b081bcce8d28f53e4443794d265e6d8431f20a5ccb0707ca6481c5a39374d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0) 2024-09-14T03:54:15.684 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:15 smithi070 podman[117232]: 2024-09-14 03:54:15.574465739 +0000 UTC m=+0.351778168 container init f00b081bcce8d28f53e4443794d265e6d8431f20a5ccb0707ca6481c5a39374d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=main-a278231) 2024-09-14T03:54:15.684 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:15 smithi070 podman[117232]: 2024-09-14 03:54:15.579296709 +0000 UTC m=+0.356609141 container start f00b081bcce8d28f53e4443794d265e6d8431f20a5ccb0707ca6481c5a39374d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=main-a278231, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-14T03:54:15.684 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:15 smithi070 podman[117232]: 2024-09-14 03:54:15.682730525 +0000 UTC m=+0.460042973 container attach f00b081bcce8d28f53e4443794d265e6d8431f20a5ccb0707ca6481c5a39374d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3-deactivate, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-14T03:54:16.026 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:54:15 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:54:15.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:16.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:54:15 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:54:15.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:16.034 INFO:tasks.rados.rados.0.smithi070.stdout:1785: done (1 left) 2024-09-14T03:54:16.034 INFO:tasks.rados.rados.0.smithi070.stdout:1786: delete oid 50 current snap is 193 2024-09-14T03:54:16.034 INFO:tasks.rados.rados.0.smithi070.stdout:1773: finishing write tid 1 to smithi07085154-21 2024-09-14T03:54:16.034 INFO:tasks.rados.rados.0.smithi070.stdout:1773: finishing write tid 2 to smithi07085154-21 2024-09-14T03:54:16.034 INFO:tasks.rados.rados.0.smithi070.stdout:1773: finishing write tid 3 to smithi07085154-21 2024-09-14T03:54:16.034 INFO:tasks.rados.rados.0.smithi070.stdout:1773: finishing write tid 4 to smithi07085154-21 2024-09-14T03:54:16.034 INFO:tasks.rados.rados.0.smithi070.stdout:1773: finishing write tid 5 to smithi07085154-21 2024-09-14T03:54:16.034 INFO:tasks.rados.rados.0.smithi070.stdout:1773: finishing write tid 6 to smithi07085154-21 2024-09-14T03:54:16.035 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 555 (ObjNum 573 snap 193 seq_num 573) dirty exists 2024-09-14T03:54:16.035 INFO:tasks.rados.rados.0.smithi070.stdout:1773: left oid 21 (ObjNum 573 snap 193 seq_num 573) 2024-09-14T03:54:16.035 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:15 smithi070 podman[117232]: 2024-09-14 03:54:15.767808953 +0000 UTC m=+0.545121396 container died f00b081bcce8d28f53e4443794d265e6d8431f20a5ccb0707ca6481c5a39374d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=main-a278231, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:54:16.044 INFO:tasks.rados.rados.0.smithi070.stdout:1773: done (1 left) 2024-09-14T03:54:16.044 INFO:tasks.rados.rados.0.smithi070.stdout:1786: done (0 left) 2024-09-14T03:54:16.045 INFO:tasks.rados.rados.0.smithi070.stdout:1787: delete oid 23 current snap is 193 2024-09-14T03:54:16.048 INFO:tasks.rados.rados.0.smithi070.stdout:1787: done (0 left) 2024-09-14T03:54:16.048 INFO:tasks.rados.rados.0.smithi070.stdout:1788: copy_from oid 6 from oid 30 current snap is 193 2024-09-14T03:54:16.049 INFO:tasks.rados.rados.0.smithi070.stdout:1789: snap_remove snap 137 2024-09-14T03:54:16.786 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:16 smithi070 podman[117275]: 2024-09-14 03:54:16.37843647 +0000 UTC m=+0.604995559 container remove f00b081bcce8d28f53e4443794d265e6d8431f20a5ccb0707ca6481c5a39374d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-14T03:54:16.786 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:16 smithi070 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.3.service: Deactivated successfully. 2024-09-14T03:54:16.786 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:16 smithi070 systemd[1]: Stopped Ceph osd.3 for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:54:16.786 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:16 smithi070 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.3.service: Consumed 11.688s CPU time. 2024-09-14T03:54:16.786 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:16 smithi070 systemd[1]: Starting Ceph osd.3 for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:54:17.039 INFO:tasks.rados.rados.0.smithi070.stdout:1789: done (1 left) 2024-09-14T03:54:17.039 INFO:tasks.rados.rados.0.smithi070.stdout:1790: snap_create 2024-09-14T03:54:17.039 INFO:tasks.rados.rados.0.smithi070.stdout:1788: finishing copy_from to smithi07085154-6 2024-09-14T03:54:17.039 INFO:tasks.rados.rados.0.smithi070.stdout:1788: got expected ENOENT (src dne) 2024-09-14T03:54:17.039 INFO:tasks.rados.rados.0.smithi070.stdout:1788: finishing copy_from racing read to smithi07085154-6 2024-09-14T03:54:17.116 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:17 smithi070 ceph-mon[98640]: osdmap e434: 8 total, 7 up, 8 in 2024-09-14T03:54:17.116 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:17 smithi070 ceph-mon[98640]: pgmap v243: 121 pgs: 8 peering, 14 stale+active+clean, 99 active+clean; 508 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 2.7 MiB/s wr, 10 op/s 2024-09-14T03:54:17.116 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:17 smithi070 ceph-mon[96704]: osdmap e434: 8 total, 7 up, 8 in 2024-09-14T03:54:17.117 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:17 smithi070 ceph-mon[96704]: pgmap v243: 121 pgs: 8 peering, 14 stale+active+clean, 99 active+clean; 508 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 2.7 MiB/s wr, 10 op/s 2024-09-14T03:54:17.117 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:16 smithi070 podman[117380]: 2024-09-14 03:54:16.712237235 +0000 UTC m=+0.015772210 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:54:17.117 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:16 smithi070 podman[117380]: 2024-09-14 03:54:16.813135118 +0000 UTC m=+0.116670081 container create 660143018c5e0fe725f9412003a12503a40c3c06caf383bb29c1888da37a6477 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3-activate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:54:17.117 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:17 smithi070 podman[117380]: 2024-09-14 03:54:17.015579523 +0000 UTC m=+0.319114486 container init 660143018c5e0fe725f9412003a12503a40c3c06caf383bb29c1888da37a6477 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3-activate, CEPH_POINT_RELEASE=, RELEASE=main-a278231, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True) 2024-09-14T03:54:17.117 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:17 smithi070 podman[117380]: 2024-09-14 03:54:17.020450728 +0000 UTC m=+0.323985704 container start 660143018c5e0fe725f9412003a12503a40c3c06caf383bb29c1888da37a6477 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.build-date=20240731, RELEASE=main-a278231, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-14T03:54:17.117 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:17 smithi070 podman[117380]: 2024-09-14 03:54:17.115652259 +0000 UTC m=+0.419187225 container attach 660143018c5e0fe725f9412003a12503a40c3c06caf383bb29c1888da37a6477 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.license=GPLv2, RELEASE=main-a278231, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-14T03:54:17.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:17 smithi094 ceph-mon[86597]: osdmap e434: 8 total, 7 up, 8 in 2024-09-14T03:54:17.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:17 smithi094 ceph-mon[86597]: pgmap v243: 121 pgs: 8 peering, 14 stale+active+clean, 99 active+clean; 508 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 2.7 MiB/s wr, 10 op/s 2024-09-14T03:54:17.277 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:54:16 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:54:16.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:17.535 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:17 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3-activate[117395]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:54:17.536 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:17 smithi070 bash[117380]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:54:17.536 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:17 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3-activate[117395]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:54:17.536 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:17 smithi070 bash[117380]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:54:17.926 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:17 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3-activate[117395]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-14T03:54:17.926 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:17 smithi070 bash[117380]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-14T03:54:17.926 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:17 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3-activate[117395]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-14T03:54:17.927 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:17 smithi070 bash[117380]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-14T03:54:17.927 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:17 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3-activate[117395]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-14T03:54:17.927 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:17 smithi070 bash[117380]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-14T03:54:18.039 INFO:tasks.rados.rados.0.smithi070.stdout:1788: done (1 left) 2024-09-14T03:54:18.039 INFO:tasks.rados.rados.0.smithi070.stdout:1790: done (0 left) 2024-09-14T03:54:18.039 INFO:tasks.rados.rados.0.smithi070.stdout:1791: delete oid 46 current snap is 194 2024-09-14T03:54:18.044 INFO:tasks.rados.rados.0.smithi070.stdout:1791: done (0 left) 2024-09-14T03:54:18.045 INFO:tasks.rados.rados.0.smithi070.stdout:1792: rollback oid 6 current snap is 194 2024-09-14T03:54:18.045 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 6 to 180 2024-09-14T03:54:18.045 INFO:tasks.rados.rados.0.smithi070.stdout:1793: read oid 17 snap -1 2024-09-14T03:54:18.045 INFO:tasks.rados.rados.0.smithi070.stdout:1793: expect (ObjNum 558 snap 191 seq_num 558) 2024-09-14T03:54:18.045 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 8 current snap is 194 2024-09-14T03:54:18.045 INFO:tasks.rados.rados.0.smithi070.stdout:1794: seq_num 581 ranges {0=139264} 2024-09-14T03:54:18.046 INFO:tasks.rados.rados.0.smithi070.stdout:1794: writing smithi07085154-8 from 0 to 139264 tid 1 2024-09-14T03:54:18.046 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 24 current snap is 194 2024-09-14T03:54:18.046 INFO:tasks.rados.rados.0.smithi070.stdout:1795: seq_num 582 ranges {1302528=155648} 2024-09-14T03:54:18.054 INFO:tasks.rados.rados.0.smithi070.stdout:1795: writing smithi07085154-24 from 1302528 to 1458176 tid 1 2024-09-14T03:54:18.054 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 25 current snap is 194 2024-09-14T03:54:18.054 INFO:tasks.rados.rados.0.smithi070.stdout:1794: finishing write tid 1 to smithi07085154-8 2024-09-14T03:54:18.054 INFO:tasks.rados.rados.0.smithi070.stdout:1796: seq_num 583 ranges {16171008=172032} 2024-09-14T03:54:18.140 INFO:tasks.rados.rados.0.smithi070.stdout:1796: writing smithi07085154-25 from 16171008 to 16343040 tid 1 2024-09-14T03:54:18.141 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 22 current snap is 194 2024-09-14T03:54:18.141 INFO:tasks.rados.rados.0.smithi070.stdout:1797: seq_num 584 ranges {2138112=188416} 2024-09-14T03:54:18.153 INFO:tasks.rados.rados.0.smithi070.stdout:1797: writing smithi07085154-22 from 2138112 to 2326528 tid 1 2024-09-14T03:54:18.153 INFO:tasks.rados.rados.0.smithi070.stdout:1798: delete oid 49 current snap is 194 2024-09-14T03:54:18.153 INFO:tasks.rados.rados.0.smithi070.stdout:1794: finishing write tid 2 to smithi07085154-8 2024-09-14T03:54:18.153 INFO:tasks.rados.rados.0.smithi070.stdout:1794: finishing write tid 3 to smithi07085154-8 2024-09-14T03:54:18.154 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 8 v 1675 (ObjNum 581 snap 194 seq_num 581) dirty exists 2024-09-14T03:54:18.154 INFO:tasks.rados.rados.0.smithi070.stdout:1794: left oid 8 (ObjNum 581 snap 194 seq_num 581) 2024-09-14T03:54:18.154 INFO:tasks.rados.rados.0.smithi070.stdout:1795: finishing write tid 1 to smithi07085154-24 2024-09-14T03:54:18.154 INFO:tasks.rados.rados.0.smithi070.stdout:1795: finishing write tid 2 to smithi07085154-24 2024-09-14T03:54:18.154 INFO:tasks.rados.rados.0.smithi070.stdout:1795: finishing write tid 3 to smithi07085154-24 2024-09-14T03:54:18.154 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 1558 (ObjNum 582 snap 194 seq_num 582) dirty exists 2024-09-14T03:54:18.154 INFO:tasks.rados.rados.0.smithi070.stdout:1795: left oid 24 (ObjNum 582 snap 194 seq_num 582) 2024-09-14T03:54:18.154 INFO:tasks.rados.rados.0.smithi070.stdout:1792: finishing rollback tid 1 to smithi07085154-6 2024-09-14T03:54:18.154 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 855 (ObjNum 449 snap 0 seq_num 2930985232) dirty dne 2024-09-14T03:54:18.154 INFO:tasks.rados.rados.0.smithi070.stdout:1793: expect (ObjNum 558 snap 191 seq_num 558) 2024-09-14T03:54:18.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:18 smithi070 ceph-mon[96704]: osdmap e435: 8 total, 7 up, 8 in 2024-09-14T03:54:18.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:18 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:18.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:18 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:18.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:18 smithi070 ceph-mon[98640]: osdmap e435: 8 total, 7 up, 8 in 2024-09-14T03:54:18.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:18 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:18.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:18 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:18.286 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:17 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3-activate[117395]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-14T03:54:18.286 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:17 smithi070 bash[117380]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-14T03:54:18.286 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:17 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3-activate[117395]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-14T03:54:18.287 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:17 smithi070 bash[117380]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-14T03:54:18.287 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:17 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3-activate[117395]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-14T03:54:18.287 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:17 smithi070 bash[117380]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-14T03:54:18.287 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:17 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3-activate[117395]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-14T03:54:18.287 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:17 smithi070 bash[117380]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-14T03:54:18.287 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:17 smithi070 podman[117380]: 2024-09-14 03:54:17.953861286 +0000 UTC m=+1.257396267 container died 660143018c5e0fe725f9412003a12503a40c3c06caf383bb29c1888da37a6477 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3-activate, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:54:18.462 INFO:tasks.rados.rados.0.smithi070.stdout:1796: finishing write tid 1 to smithi07085154-25 2024-09-14T03:54:18.462 INFO:tasks.rados.rados.0.smithi070.stdout:1796: finishing write tid 2 to smithi07085154-25 2024-09-14T03:54:18.462 INFO:tasks.rados.rados.0.smithi070.stdout:1796: finishing write tid 3 to smithi07085154-25 2024-09-14T03:54:18.462 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 1562 (ObjNum 583 snap 194 seq_num 583) dirty exists 2024-09-14T03:54:18.463 INFO:tasks.rados.rados.0.smithi070.stdout:1796: left oid 25 (ObjNum 583 snap 194 seq_num 583) 2024-09-14T03:54:18.463 INFO:tasks.rados.rados.0.smithi070.stdout:1792: done (6 left) 2024-09-14T03:54:18.463 INFO:tasks.rados.rados.0.smithi070.stdout:1793: done (5 left) 2024-09-14T03:54:18.463 INFO:tasks.rados.rados.0.smithi070.stdout:1794: done (4 left) 2024-09-14T03:54:18.463 INFO:tasks.rados.rados.0.smithi070.stdout:1795: done (3 left) 2024-09-14T03:54:18.463 INFO:tasks.rados.rados.0.smithi070.stdout:1796: done (2 left) 2024-09-14T03:54:18.463 INFO:tasks.rados.rados.0.smithi070.stdout:1798: done (1 left) 2024-09-14T03:54:18.463 INFO:tasks.rados.rados.0.smithi070.stdout:1799: rollback oid 49 current snap is 194 2024-09-14T03:54:18.463 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 49 to 126 2024-09-14T03:54:18.463 INFO:tasks.rados.rados.0.smithi070.stdout:1797: finishing write tid 1 to smithi07085154-22 2024-09-14T03:54:18.463 INFO:tasks.rados.rados.0.smithi070.stdout:1797: finishing write tid 2 to smithi07085154-22 2024-09-14T03:54:18.463 INFO:tasks.rados.rados.0.smithi070.stdout:1797: finishing write tid 3 to smithi07085154-22 2024-09-14T03:54:18.463 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 1574 (ObjNum 584 snap 194 seq_num 584) dirty exists 2024-09-14T03:54:18.464 INFO:tasks.rados.rados.0.smithi070.stdout:1797: left oid 22 (ObjNum 584 snap 194 seq_num 584) 2024-09-14T03:54:18.464 INFO:tasks.rados.rados.0.smithi070.stdout:1797: done (1 left) 2024-09-14T03:54:18.464 INFO:tasks.rados.rados.0.smithi070.stdout:1800: snap_remove snap 171 2024-09-14T03:54:18.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:18 smithi094 ceph-mon[86597]: osdmap e435: 8 total, 7 up, 8 in 2024-09-14T03:54:18.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:18 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:18.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:18 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:18.895 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:18 smithi070 podman[117555]: 2024-09-14 03:54:18.569633441 +0000 UTC m=+0.610077861 container remove 660143018c5e0fe725f9412003a12503a40c3c06caf383bb29c1888da37a6477 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS) 2024-09-14T03:54:18.895 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:18 smithi070 podman[117622]: 2024-09-14 03:54:18.676614269 +0000 UTC m=+0.016262964 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:54:18.895 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:18 smithi070 podman[117622]: 2024-09-14 03:54:18.779052127 +0000 UTC m=+0.118700864 container create 8efc112edbe957b36f8c9922f10b8f838a422c29e4a1aa44728640497b3cf806 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3, org.label-schema.schema-version=1.0, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-14T03:54:19.028 INFO:tasks.rados.rados.0.smithi070.stdout:1800: done (1 left) 2024-09-14T03:54:19.028 INFO:tasks.rados.rados.0.smithi070.stdout:1801: rmattr oid 4 current snap is 194 2024-09-14T03:54:19.028 INFO:tasks.rados.rados.0.smithi070.stdout:1801: done (1 left) 2024-09-14T03:54:19.028 INFO:tasks.rados.rados.0.smithi070.stdout:1802: snap_create 2024-09-14T03:54:19.028 INFO:tasks.rados.rados.0.smithi070.stdout:1799: finishing rollback tid 1 to smithi07085154-49 2024-09-14T03:54:19.029 INFO:tasks.rados.rados.0.smithi070.stdout:1799: finishing rollback tid 2 to smithi07085154-49 2024-09-14T03:54:19.029 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 49 v 1563 (ObjNum 317 snap 107 seq_num 317) dirty exists 2024-09-14T03:54:19.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:19 smithi070 ceph-mon[96704]: osdmap e436: 8 total, 7 up, 8 in 2024-09-14T03:54:19.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:19 smithi070 ceph-mon[96704]: pgmap v246: 121 pgs: 1 active+clean+snaptrim, 25 active+undersized, 1 active+clean+snaptrim_wait, 8 peering, 3 stale+active+clean, 4 active+undersized+degraded, 79 active+clean; 529 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.5 MiB/s wr, 13 op/s; 13/987 objects degraded (1.317%) 2024-09-14T03:54:19.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:19 smithi070 ceph-mon[96704]: osdmap e437: 8 total, 7 up, 8 in 2024-09-14T03:54:19.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:19 smithi070 ceph-mon[98640]: osdmap e436: 8 total, 7 up, 8 in 2024-09-14T03:54:19.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:19 smithi070 ceph-mon[98640]: pgmap v246: 121 pgs: 1 active+clean+snaptrim, 25 active+undersized, 1 active+clean+snaptrim_wait, 8 peering, 3 stale+active+clean, 4 active+undersized+degraded, 79 active+clean; 529 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.5 MiB/s wr, 13 op/s; 13/987 objects degraded (1.317%) 2024-09-14T03:54:19.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:19 smithi070 ceph-mon[98640]: osdmap e437: 8 total, 7 up, 8 in 2024-09-14T03:54:19.286 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:19 smithi070 podman[117622]: 2024-09-14 03:54:19.023355483 +0000 UTC m=+0.363004199 container init 8efc112edbe957b36f8c9922f10b8f838a422c29e4a1aa44728640497b3cf806 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-14T03:54:19.287 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:19 smithi070 podman[117622]: 2024-09-14 03:54:19.028759531 +0000 UTC m=+0.368408227 container start 8efc112edbe957b36f8c9922f10b8f838a422c29e4a1aa44728640497b3cf806 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3, CEPH_POINT_RELEASE=, ceph=True, RELEASE=main-a278231, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS) 2024-09-14T03:54:19.287 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:19 smithi070 ceph-osd[117661]: -- 172.21.15.70:0/565257811 <== mon.0 v2:172.21.15.70:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x564b69819a40 con 0x564b697f7c00 2024-09-14T03:54:19.287 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:19 smithi070 bash[117622]: 8efc112edbe957b36f8c9922f10b8f838a422c29e4a1aa44728640497b3cf806 2024-09-14T03:54:19.287 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:19 smithi070 systemd[1]: Started Ceph osd.3 for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:54:19.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:19 smithi094 ceph-mon[86597]: osdmap e436: 8 total, 7 up, 8 in 2024-09-14T03:54:19.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:19 smithi094 ceph-mon[86597]: pgmap v246: 121 pgs: 1 active+clean+snaptrim, 25 active+undersized, 1 active+clean+snaptrim_wait, 8 peering, 3 stale+active+clean, 4 active+undersized+degraded, 79 active+clean; 529 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.5 MiB/s wr, 13 op/s; 13/987 objects degraded (1.317%) 2024-09-14T03:54:19.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:19 smithi094 ceph-mon[86597]: osdmap e437: 8 total, 7 up, 8 in 2024-09-14T03:54:20.029 INFO:tasks.rados.rados.0.smithi070.stdout:1799: done (1 left) 2024-09-14T03:54:20.030 INFO:tasks.rados.rados.0.smithi070.stdout:1802: done (0 left) 2024-09-14T03:54:20.030 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 32 current snap is 195 2024-09-14T03:54:20.030 INFO:tasks.rados.rados.0.smithi070.stdout:1803: seq_num 586 ranges {0=221184} 2024-09-14T03:54:20.031 INFO:tasks.rados.rados.0.smithi070.stdout:1803: writing smithi07085154-32 from 0 to 221184 tid 1 2024-09-14T03:54:20.031 INFO:tasks.rados.rados.0.smithi070.stdout:1804: rollback oid 4 current snap is 195 2024-09-14T03:54:20.031 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 4 to 181 2024-09-14T03:54:20.031 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 17 current snap is 195 2024-09-14T03:54:20.031 INFO:tasks.rados.rados.0.smithi070.stdout:1805: seq_num 587 ranges {6234112=237568} 2024-09-14T03:54:20.066 INFO:tasks.rados.rados.0.smithi070.stdout:1805: writing smithi07085154-17 from 6234112 to 6471680 tid 1 2024-09-14T03:54:20.066 INFO:tasks.rados.rados.0.smithi070.stdout:1806: rollback oid 34 current snap is 195 2024-09-14T03:54:20.067 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 34 to 177 2024-09-14T03:54:20.067 INFO:tasks.rados.rados.0.smithi070.stdout:1803: finishing write tid 1 to smithi07085154-32 2024-09-14T03:54:20.067 INFO:tasks.rados.rados.0.smithi070.stdout:1803: finishing write tid 2 to smithi07085154-32 2024-09-14T03:54:20.067 INFO:tasks.rados.rados.0.smithi070.stdout:1804: finishing rollback tid 0 to smithi07085154-4 2024-09-14T03:54:20.067 INFO:tasks.rados.rados.0.smithi070.stdout:1804: finishing rollback tid 1 to smithi07085154-4 2024-09-14T03:54:20.067 INFO:tasks.rados.rados.0.smithi070.stdout:1804: finishing rollback tid 2 to smithi07085154-4 2024-09-14T03:54:20.067 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 1253 (ObjNum 517 snap 173 seq_num 517) dirty exists 2024-09-14T03:54:20.067 INFO:tasks.rados.rados.0.smithi070.stdout:1803: finishing write tid 3 to smithi07085154-32 2024-09-14T03:54:20.067 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 1670 (ObjNum 586 snap 195 seq_num 586) dirty exists 2024-09-14T03:54:20.067 INFO:tasks.rados.rados.0.smithi070.stdout:1803: left oid 32 (ObjNum 586 snap 195 seq_num 586) 2024-09-14T03:54:20.067 INFO:tasks.rados.rados.0.smithi070.stdout:1803: done (3 left) 2024-09-14T03:54:20.067 INFO:tasks.rados.rados.0.smithi070.stdout:1804: done (2 left) 2024-09-14T03:54:20.067 INFO:tasks.rados.rados.0.smithi070.stdout:1807: setattr oid 16 current snap is 195 2024-09-14T03:54:20.068 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 41 current snap is 195 2024-09-14T03:54:20.068 INFO:tasks.rados.rados.0.smithi070.stdout:1808: seq_num 588 ranges {2195456=253952} 2024-09-14T03:54:20.081 INFO:tasks.rados.rados.0.smithi070.stdout:1808: writing smithi07085154-41 from 2195456 to 2449408 tid 1 2024-09-14T03:54:20.081 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 12 current snap is 195 2024-09-14T03:54:20.081 INFO:tasks.rados.rados.0.smithi070.stdout:1809: seq_num 589 ranges {5718016=270336} 2024-09-14T03:54:20.114 INFO:tasks.rados.rados.0.smithi070.stdout:1809: writing smithi07085154-12 from 5718016 to 5988352 tid 1 2024-09-14T03:54:20.114 INFO:tasks.rados.rados.0.smithi070.stdout:1810: read oid 5 snap -1 2024-09-14T03:54:20.114 INFO:tasks.rados.rados.0.smithi070.stdout:1810: expect (ObjNum 516 snap 173 seq_num 516) 2024-09-14T03:54:20.114 INFO:tasks.rados.rados.0.smithi070.stdout:1811: copy_from oid 26 from oid 19 current snap is 195 2024-09-14T03:54:20.114 INFO:tasks.rados.rados.0.smithi070.stdout:1805: finishing write tid 1 to smithi07085154-17 2024-09-14T03:54:20.115 INFO:tasks.rados.rados.0.smithi070.stdout:1805: finishing write tid 2 to smithi07085154-17 2024-09-14T03:54:20.115 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 1927 (ObjNum 114 snap 24 seq_num 114) dirty exists 2024-09-14T03:54:20.115 INFO:tasks.rados.rados.0.smithi070.stdout:1806: finishing rollback tid 0 to smithi07085154-34 2024-09-14T03:54:20.115 INFO:tasks.rados.rados.0.smithi070.stdout:1806: finishing rollback tid 1 to smithi07085154-34 2024-09-14T03:54:20.115 INFO:tasks.rados.rados.0.smithi070.stdout:1806: finishing rollback tid 2 to smithi07085154-34 2024-09-14T03:54:20.115 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 1577 (ObjNum 517 snap 173 seq_num 517) dirty exists 2024-09-14T03:54:20.115 INFO:tasks.rados.rados.0.smithi070.stdout:1805: finishing write tid 3 to smithi07085154-17 2024-09-14T03:54:20.115 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 1564 (ObjNum 587 snap 195 seq_num 587) dirty exists 2024-09-14T03:54:20.115 INFO:tasks.rados.rados.0.smithi070.stdout:1805: left oid 17 (ObjNum 587 snap 195 seq_num 587) 2024-09-14T03:54:20.115 INFO:tasks.rados.rados.0.smithi070.stdout:1808: finishing write tid 1 to smithi07085154-41 2024-09-14T03:54:20.116 INFO:tasks.rados.rados.0.smithi070.stdout:1805: done (6 left) 2024-09-14T03:54:20.116 INFO:tasks.rados.rados.0.smithi070.stdout:1806: done (5 left) 2024-09-14T03:54:20.116 INFO:tasks.rados.rados.0.smithi070.stdout:1807: done (4 left) 2024-09-14T03:54:20.116 INFO:tasks.rados.rados.0.smithi070.stdout:1812: rollback oid 11 current snap is 195 2024-09-14T03:54:20.116 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 11 to 167 2024-09-14T03:54:20.116 INFO:tasks.rados.rados.0.smithi070.stdout:1808: finishing write tid 2 to smithi07085154-41 2024-09-14T03:54:20.116 INFO:tasks.rados.rados.0.smithi070.stdout:1808: finishing write tid 3 to smithi07085154-41 2024-09-14T03:54:20.116 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 1677 (ObjNum 588 snap 195 seq_num 588) dirty exists 2024-09-14T03:54:20.116 INFO:tasks.rados.rados.0.smithi070.stdout:1808: left oid 41 (ObjNum 588 snap 195 seq_num 588) 2024-09-14T03:54:20.116 INFO:tasks.rados.rados.0.smithi070.stdout:1808: done (4 left) 2024-09-14T03:54:20.116 INFO:tasks.rados.rados.0.smithi070.stdout:1813: copy_from oid 3 from oid 32 current snap is 195 2024-09-14T03:54:20.116 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 27 current snap is 195 2024-09-14T03:54:20.116 INFO:tasks.rados.rados.0.smithi070.stdout:1814: seq_num 590 ranges {16171008=286720} 2024-09-14T03:54:20.202 INFO:tasks.rados.rados.0.smithi070.stdout:1814: writing smithi07085154-27 from 16171008 to 16457728 tid 1 2024-09-14T03:54:20.202 INFO:tasks.rados.rados.0.smithi070.stdout:1815: delete oid 43 current snap is 195 2024-09-14T03:54:20.202 INFO:tasks.rados.rados.0.smithi070.stdout:1809: finishing write tid 1 to smithi07085154-12 2024-09-14T03:54:20.202 INFO:tasks.rados.rados.0.smithi070.stdout:1812: finishing rollback tid 0 to smithi07085154-11 2024-09-14T03:54:20.202 INFO:tasks.rados.rados.0.smithi070.stdout:1809: finishing write tid 2 to smithi07085154-12 2024-09-14T03:54:20.202 INFO:tasks.rados.rados.0.smithi070.stdout:1812: finishing rollback tid 1 to smithi07085154-11 2024-09-14T03:54:20.202 INFO:tasks.rados.rados.0.smithi070.stdout:1812: finishing rollback tid 2 to smithi07085154-11 2024-09-14T03:54:20.202 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 11 v 1673 (ObjNum 506 snap 165 seq_num 506) dirty exists 2024-09-14T03:54:20.202 INFO:tasks.rados.rados.0.smithi070.stdout:1809: finishing write tid 3 to smithi07085154-12 2024-09-14T03:54:20.202 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 565 (ObjNum 589 snap 195 seq_num 589) dirty exists 2024-09-14T03:54:20.203 INFO:tasks.rados.rados.0.smithi070.stdout:1809: left oid 12 (ObjNum 589 snap 195 seq_num 589) 2024-09-14T03:54:20.203 INFO:tasks.rados.rados.0.smithi070.stdout:1810: expect (ObjNum 516 snap 173 seq_num 516) 2024-09-14T03:54:20.266 INFO:tasks.rados.rados.0.smithi070.stdout:1813: finishing copy_from racing read to smithi07085154-3 2024-09-14T03:54:20.266 INFO:tasks.rados.rados.0.smithi070.stdout:1813: finishing copy_from to smithi07085154-3 2024-09-14T03:54:20.266 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 3 v 1670 (ObjNum 586 snap 195 seq_num 586) dirty exists 2024-09-14T03:54:20.266 INFO:tasks.rados.rados.0.smithi070.stdout:1811: finishing copy_from racing read to smithi07085154-26 2024-09-14T03:54:20.266 INFO:tasks.rados.rados.0.smithi070.stdout:1811: finishing copy_from to smithi07085154-26 2024-09-14T03:54:20.267 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 1547 (ObjNum 262 snap 76 seq_num 262) dirty exists 2024-09-14T03:54:20.267 INFO:tasks.rados.rados.0.smithi070.stdout:1809: done (6 left) 2024-09-14T03:54:20.267 INFO:tasks.rados.rados.0.smithi070.stdout:1810: done (5 left) 2024-09-14T03:54:20.267 INFO:tasks.rados.rados.0.smithi070.stdout:1811: done (4 left) 2024-09-14T03:54:20.267 INFO:tasks.rados.rados.0.smithi070.stdout:1812: done (3 left) 2024-09-14T03:54:20.267 INFO:tasks.rados.rados.0.smithi070.stdout:1813: done (2 left) 2024-09-14T03:54:20.267 INFO:tasks.rados.rados.0.smithi070.stdout:1815: done (1 left) 2024-09-14T03:54:20.267 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 47 current snap is 195 2024-09-14T03:54:20.267 INFO:tasks.rados.rados.0.smithi070.stdout:1814: finishing write tid 1 to smithi07085154-27 2024-09-14T03:54:20.267 INFO:tasks.rados.rados.0.smithi070.stdout:1816: seq_num 592 ranges {8675328=319488} 2024-09-14T03:54:20.316 INFO:tasks.rados.rados.0.smithi070.stdout:1816: writing smithi07085154-47 from 8675328 to 8994816 tid 1 2024-09-14T03:54:20.316 INFO:tasks.rados.rados.0.smithi070.stdout:1817: rollback oid 25 current snap is 195 2024-09-14T03:54:20.316 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 25 to 194 2024-09-14T03:54:20.316 INFO:tasks.rados.rados.0.smithi070.stdout:1814: finishing write tid 2 to smithi07085154-27 2024-09-14T03:54:20.316 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 12 current snap is 195 2024-09-14T03:54:20.316 INFO:tasks.rados.rados.0.smithi070.stdout:1814: finishing write tid 3 to smithi07085154-27 2024-09-14T03:54:20.316 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 1931 (ObjNum 590 snap 195 seq_num 590) dirty exists 2024-09-14T03:54:20.316 INFO:tasks.rados.rados.0.smithi070.stdout:1814: left oid 27 (ObjNum 590 snap 195 seq_num 590) 2024-09-14T03:54:20.316 INFO:tasks.rados.rados.0.smithi070.stdout:1818: seq_num 593 ranges {5988352=335872} 2024-09-14T03:54:20.350 INFO:tasks.rados.rados.0.smithi070.stdout:1818: writing smithi07085154-12 from 5988352 to 6324224 tid 1 2024-09-14T03:54:20.350 INFO:tasks.rados.rados.0.smithi070.stdout:1814: done (3 left) 2024-09-14T03:54:20.351 INFO:tasks.rados.rados.0.smithi070.stdout:1819: setattr oid 14 current snap is 195 2024-09-14T03:54:20.351 INFO:tasks.rados.rados.0.smithi070.stdout:1816: finishing write tid 1 to smithi07085154-47 2024-09-14T03:54:20.351 INFO:tasks.rados.rados.0.smithi070.stdout:1816: finishing write tid 2 to smithi07085154-47 2024-09-14T03:54:20.351 INFO:tasks.rados.rados.0.smithi070.stdout:1817: finishing rollback tid 0 to smithi07085154-25 2024-09-14T03:54:20.351 INFO:tasks.rados.rados.0.smithi070.stdout:1817: finishing rollback tid 1 to smithi07085154-25 2024-09-14T03:54:20.351 INFO:tasks.rados.rados.0.smithi070.stdout:1816: finishing write tid 3 to smithi07085154-47 2024-09-14T03:54:20.351 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 47 v 1672 (ObjNum 592 snap 195 seq_num 592) dirty exists 2024-09-14T03:54:20.351 INFO:tasks.rados.rados.0.smithi070.stdout:1816: left oid 47 (ObjNum 592 snap 195 seq_num 592) 2024-09-14T03:54:20.351 INFO:tasks.rados.rados.0.smithi070.stdout:1817: finishing rollback tid 2 to smithi07085154-25 2024-09-14T03:54:20.351 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 1567 (ObjNum 583 snap 194 seq_num 583) dirty exists 2024-09-14T03:54:20.352 INFO:tasks.rados.rados.0.smithi070.stdout:1816: done (3 left) 2024-09-14T03:54:20.352 INFO:tasks.rados.rados.0.smithi070.stdout:1817: done (2 left) 2024-09-14T03:54:20.352 INFO:tasks.rados.rados.0.smithi070.stdout:1820: setattr oid 18 current snap is 195 2024-09-14T03:54:20.353 INFO:tasks.rados.rados.0.smithi070.stdout:1818: finishing write tid 1 to smithi07085154-12 2024-09-14T03:54:20.353 INFO:tasks.rados.rados.0.smithi070.stdout:1818: finishing write tid 2 to smithi07085154-12 2024-09-14T03:54:20.354 INFO:tasks.rados.rados.0.smithi070.stdout:1818: finishing write tid 3 to smithi07085154-12 2024-09-14T03:54:20.354 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 567 (ObjNum 593 snap 195 seq_num 593) dirty exists 2024-09-14T03:54:20.355 INFO:tasks.rados.rados.0.smithi070.stdout:1818: left oid 12 (ObjNum 593 snap 195 seq_num 593) 2024-09-14T03:54:20.355 INFO:tasks.rados.rados.0.smithi070.stdout:1818: done (2 left) 2024-09-14T03:54:20.355 INFO:tasks.rados.rados.0.smithi070.stdout:1821: copy_from oid 40 from oid 3 current snap is 195 2024-09-14T03:54:20.355 INFO:tasks.rados.rados.0.smithi070.stdout:1822: read oid 10 snap -1 2024-09-14T03:54:20.355 INFO:tasks.rados.rados.0.smithi070.stdout:1822: expect (ObjNum 508 snap 166 seq_num 508) 2024-09-14T03:54:20.355 INFO:tasks.rados.rados.0.smithi070.stdout:1823: rollback oid 7 current snap is 195 2024-09-14T03:54:20.355 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 7 to 160 2024-09-14T03:54:20.355 INFO:tasks.rados.rados.0.smithi070.stdout:1824: copy_from oid 42 from oid 17 current snap is 195 2024-09-14T03:54:20.355 INFO:tasks.rados.rados.0.smithi070.stdout:1825: rollback oid 1 current snap is 195 2024-09-14T03:54:20.355 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 1 to 160 2024-09-14T03:54:20.355 INFO:tasks.rados.rados.0.smithi070.stdout:1826: snap_create 2024-09-14T03:54:20.356 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 1559 (ObjNum 552 snap 187 seq_num 552) dirty exists 2024-09-14T03:54:20.358 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 1678 (ObjNum 114 snap 24 seq_num 114) dirty exists 2024-09-14T03:54:20.359 INFO:tasks.rados.rados.0.smithi070.stdout:1823: finishing rollback tid 0 to smithi07085154-7 2024-09-14T03:54:20.359 INFO:tasks.rados.rados.0.smithi070.stdout:1823: finishing rollback tid 1 to smithi07085154-7 2024-09-14T03:54:20.360 INFO:tasks.rados.rados.0.smithi070.stdout:1823: finishing rollback tid 2 to smithi07085154-7 2024-09-14T03:54:20.360 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 1256 (ObjNum 454 snap 154 seq_num 454) dirty exists 2024-09-14T03:54:20.360 INFO:tasks.rados.rados.0.smithi070.stdout:1821: finishing copy_from racing read to smithi07085154-40 2024-09-14T03:54:20.360 INFO:tasks.rados.rados.0.smithi070.stdout:1825: finishing rollback tid 0 to smithi07085154-1 2024-09-14T03:54:20.360 INFO:tasks.rados.rados.0.smithi070.stdout:1825: finishing rollback tid 1 to smithi07085154-1 2024-09-14T03:54:20.360 INFO:tasks.rados.rados.0.smithi070.stdout:1825: finishing rollback tid 2 to smithi07085154-1 2024-09-14T03:54:20.361 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 1 v 911 (ObjNum 470 snap 160 seq_num 470) dirty exists 2024-09-14T03:54:20.361 INFO:tasks.rados.rados.0.smithi070.stdout:1821: finishing copy_from to smithi07085154-40 2024-09-14T03:54:20.361 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 1670 (ObjNum 586 snap 195 seq_num 586) dirty exists 2024-09-14T03:54:20.373 INFO:tasks.rados.rados.0.smithi070.stdout:1822: expect (ObjNum 508 snap 166 seq_num 508) 2024-09-14T03:54:20.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:20 smithi094 ceph-mon[86597]: Health check failed: Degraded data redundancy: 13/987 objects degraded (1.317%), 4 pgs degraded (PG_DEGRADED) 2024-09-14T03:54:20.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:20 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:20.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:20 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:20.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:20 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:54:20.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:20 smithi094 ceph-mon[86597]: osdmap e438: 8 total, 7 up, 8 in 2024-09-14T03:54:20.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:54:20 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:54:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:20.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:20 smithi070 ceph-mon[98640]: Health check failed: Degraded data redundancy: 13/987 objects degraded (1.317%), 4 pgs degraded (PG_DEGRADED) 2024-09-14T03:54:20.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:20 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:20.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:20 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:20.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:20 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:54:20.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:20 smithi070 ceph-mon[98640]: osdmap e438: 8 total, 7 up, 8 in 2024-09-14T03:54:20.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:20 smithi070 ceph-mon[96704]: Health check failed: Degraded data redundancy: 13/987 objects degraded (1.317%), 4 pgs degraded (PG_DEGRADED) 2024-09-14T03:54:20.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:20 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:20.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:20 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:20.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:20 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:54:20.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:20 smithi070 ceph-mon[96704]: osdmap e438: 8 total, 7 up, 8 in 2024-09-14T03:54:20.653 INFO:tasks.rados.rados.0.smithi070.stdout:1824: finishing copy_from racing read to smithi07085154-42 2024-09-14T03:54:20.653 INFO:tasks.rados.rados.0.smithi070.stdout:1824: finishing copy_from to smithi07085154-42 2024-09-14T03:54:20.653 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 1564 (ObjNum 587 snap 195 seq_num 587) dirty exists 2024-09-14T03:54:21.053 INFO:tasks.rados.rados.0.smithi070.stdout:1819: done (7 left) 2024-09-14T03:54:21.053 INFO:tasks.rados.rados.0.smithi070.stdout:1820: done (6 left) 2024-09-14T03:54:21.053 INFO:tasks.rados.rados.0.smithi070.stdout:1821: done (5 left) 2024-09-14T03:54:21.053 INFO:tasks.rados.rados.0.smithi070.stdout:1822: done (4 left) 2024-09-14T03:54:21.053 INFO:tasks.rados.rados.0.smithi070.stdout:1823: done (3 left) 2024-09-14T03:54:21.053 INFO:tasks.rados.rados.0.smithi070.stdout:1824: done (2 left) 2024-09-14T03:54:21.053 INFO:tasks.rados.rados.0.smithi070.stdout:1825: done (1 left) 2024-09-14T03:54:21.053 INFO:tasks.rados.rados.0.smithi070.stdout:1826: done (0 left) 2024-09-14T03:54:21.053 INFO:tasks.rados.rados.0.smithi070.stdout:1827: snap_create 2024-09-14T03:54:21.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:21 smithi070 ceph-mon[98640]: pgmap v249: 121 pgs: 1 active+clean+snaptrim, 34 active+undersized, 1 active+clean+snaptrim_wait, 7 active+undersized+degraded, 78 active+clean; 539 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.7 MiB/s rd, 3.6 MiB/s wr, 16 op/s; 82/990 objects degraded (8.283%) 2024-09-14T03:54:21.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:21 smithi070 ceph-mon[96704]: pgmap v249: 121 pgs: 1 active+clean+snaptrim, 34 active+undersized, 1 active+clean+snaptrim_wait, 7 active+undersized+degraded, 78 active+clean; 539 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.7 MiB/s rd, 3.6 MiB/s wr, 16 op/s; 82/990 objects degraded (8.283%) 2024-09-14T03:54:21.286 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:20 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3[117637]: 2024-09-14T03:54:20.906+0000 7f4b3bd6a640 -1 Falling back to public interface 2024-09-14T03:54:21.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:21 smithi094 ceph-mon[86597]: pgmap v249: 121 pgs: 1 active+clean+snaptrim, 34 active+undersized, 1 active+clean+snaptrim_wait, 7 active+undersized+degraded, 78 active+clean; 539 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.7 MiB/s rd, 3.6 MiB/s wr, 16 op/s; 82/990 objects degraded (8.283%) 2024-09-14T03:54:22.055 INFO:tasks.rados.rados.0.smithi070.stdout:1827: done (0 left) 2024-09-14T03:54:22.055 INFO:tasks.rados.rados.0.smithi070.stdout:1828: read oid 37 snap -1 2024-09-14T03:54:22.055 INFO:tasks.rados.rados.0.smithi070.stdout:1828: expect (ObjNum 312 snap 105 seq_num 312) 2024-09-14T03:54:22.055 INFO:tasks.rados.rados.0.smithi070.stdout:1829: read oid 17 snap -1 2024-09-14T03:54:22.055 INFO:tasks.rados.rados.0.smithi070.stdout:1829: expect (ObjNum 587 snap 195 seq_num 587) 2024-09-14T03:54:22.055 INFO:tasks.rados.rados.0.smithi070.stdout:1830: snap_remove snap 183 2024-09-14T03:54:22.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:22 smithi070 ceph-mon[96704]: osdmap e439: 8 total, 7 up, 8 in 2024-09-14T03:54:22.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:22 smithi070 ceph-mon[98640]: osdmap e439: 8 total, 7 up, 8 in 2024-09-14T03:54:22.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:22 smithi094 ceph-mon[86597]: osdmap e439: 8 total, 7 up, 8 in 2024-09-14T03:54:23.064 INFO:tasks.rados.rados.0.smithi070.stdout:1830: done (2 left) 2024-09-14T03:54:23.064 INFO:tasks.rados.rados.0.smithi070.stdout:1831: copy_from oid 36 from oid 27 current snap is 197 2024-09-14T03:54:23.064 INFO:tasks.rados.rados.0.smithi070.stdout:1828: expect (ObjNum 312 snap 105 seq_num 312) 2024-09-14T03:54:23.103 INFO:tasks.rados.rados.0.smithi070.stdout:1829: expect (ObjNum 587 snap 195 seq_num 587) 2024-09-14T03:54:23.353 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:23 smithi070 ceph-mon[96704]: osdmap e440: 8 total, 7 up, 8 in 2024-09-14T03:54:23.353 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:23 smithi070 ceph-mon[96704]: pgmap v252: 121 pgs: 34 active+undersized, 7 active+undersized+degraded, 80 active+clean; 561 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 5.5 MiB/s wr, 13 op/s; 82/1005 objects degraded (8.159%) 2024-09-14T03:54:23.353 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:23 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:23.353 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:23 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:23.353 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:23 smithi070 ceph-mon[98640]: osdmap e440: 8 total, 7 up, 8 in 2024-09-14T03:54:23.353 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:23 smithi070 ceph-mon[98640]: pgmap v252: 121 pgs: 34 active+undersized, 7 active+undersized+degraded, 80 active+clean; 561 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 5.5 MiB/s wr, 13 op/s; 82/1005 objects degraded (8.159%) 2024-09-14T03:54:23.353 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:23 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:23.353 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:23 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:23.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:23 smithi094 ceph-mon[86597]: osdmap e440: 8 total, 7 up, 8 in 2024-09-14T03:54:23.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:23 smithi094 ceph-mon[86597]: pgmap v252: 121 pgs: 34 active+undersized, 7 active+undersized+degraded, 80 active+clean; 561 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 5.5 MiB/s wr, 13 op/s; 82/1005 objects degraded (8.159%) 2024-09-14T03:54:23.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:23 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:23.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:23 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:23.530 INFO:tasks.rados.rados.0.smithi070.stdout:1831: finishing copy_from racing read to smithi07085154-36 2024-09-14T03:54:23.530 INFO:tasks.rados.rados.0.smithi070.stdout:1831: finishing copy_from to smithi07085154-36 2024-09-14T03:54:23.530 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 36 v 1931 (ObjNum 590 snap 195 seq_num 590) dirty exists 2024-09-14T03:54:23.530 INFO:tasks.rados.rados.0.smithi070.stdout:1828: done (2 left) 2024-09-14T03:54:23.530 INFO:tasks.rados.rados.0.smithi070.stdout:1829: done (1 left) 2024-09-14T03:54:23.530 INFO:tasks.rados.rados.0.smithi070.stdout:1831: done (0 left) 2024-09-14T03:54:23.530 INFO:tasks.rados.rados.0.smithi070.stdout:1832: snap_remove snap 102 2024-09-14T03:54:23.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:54:23 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:54:23] "GET /metrics HTTP/1.1" 200 36311 "" "Prometheus/2.51.0" 2024-09-14T03:54:23.812 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-09-14T03:54:24.030 INFO:tasks.rados.rados.0.smithi070.stdout:1832: done (0 left) 2024-09-14T03:54:24.030 INFO:tasks.rados.rados.0.smithi070.stdout:1833: read oid 27 snap -1 2024-09-14T03:54:24.030 INFO:tasks.rados.rados.0.smithi070.stdout:1833: expect (ObjNum 590 snap 195 seq_num 590) 2024-09-14T03:54:24.030 INFO:tasks.rados.rados.0.smithi070.stdout:1834: copy_from oid 49 from oid 13 current snap is 197 2024-09-14T03:54:24.031 INFO:tasks.rados.rados.0.smithi070.stdout:1835: copy_from oid 17 from oid 41 current snap is 197 2024-09-14T03:54:24.031 INFO:tasks.rados.rados.0.smithi070.stdout:1836: read oid 25 snap 174 2024-09-14T03:54:24.031 INFO:tasks.rados.rados.0.smithi070.stdout:1836: expect (ObjNum 459 snap 155 seq_num 459) 2024-09-14T03:54:24.031 INFO:tasks.rados.rados.0.smithi070.stdout:1837: snap_remove snap 191 2024-09-14T03:54:24.064 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 03:54:24 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0[103808]: 2024-09-14T03:54:24.055+0000 7f4ac4cf4640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap b2 after trimming was started 2024-09-14T03:54:24.394 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:24 smithi070 ceph-mon[96704]: osdmap e441: 8 total, 7 up, 8 in 2024-09-14T03:54:24.394 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:24 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:24.394 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:24 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:24.395 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:24 smithi070 ceph-mon[96704]: osdmap e442: 8 total, 7 up, 8 in 2024-09-14T03:54:24.395 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:24 smithi070 ceph-mon[98640]: osdmap e441: 8 total, 7 up, 8 in 2024-09-14T03:54:24.395 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:24 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:24.395 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:24 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:24.395 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:24 smithi070 ceph-mon[98640]: osdmap e442: 8 total, 7 up, 8 in 2024-09-14T03:54:24.520 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:54:24.520 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (4m) 1s ago 9m 21.2M - 0.27.0 11f11916f8cd 45e79e51074c 2024-09-14T03:54:24.520 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi094 *:3000 running (3m) 2m ago 8m 79.5M - 9.4.7 954c08fa6188 2240810d0368 2024-09-14T03:54:24.520 INFO:teuthology.orchestra.run.smithi070.stdout:mds.a smithi070 running (10m) 1s ago 10m 21.4M - 17.2.7-1629-geb16b97b ab51d1bebd12 52e0dbb7be50 2024-09-14T03:54:24.520 INFO:teuthology.orchestra.run.smithi070.stdout:mds.b smithi094 running (10m) 2m ago 10m 15.4M - 17.2.7-1629-geb16b97b ab51d1bebd12 e3b5e3e00189 2024-09-14T03:54:24.520 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi094 *:8443,9283,8765 running (3m) 2m ago 13m 452M - 19.3.0-4907-ga2782315 8801d2529e43 569d9c7e0f06 2024-09-14T03:54:24.520 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (3m) 1s ago 15m 521M - 19.3.0-4907-ga2782315 8801d2529e43 0ef4d8e0cdbe 2024-09-14T03:54:24.520 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (3m) 1s ago 15m 57.7M 2048M 19.3.0-4907-ga2782315 8801d2529e43 96542de31cac 2024-09-14T03:54:24.520 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi094 running (2m) 2m ago 14m 21.3M 2048M 19.3.0-4907-ga2782315 8801d2529e43 86827be98659 2024-09-14T03:54:24.520 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (2m) 1s ago 14m 51.9M 2048M 19.3.0-4907-ga2782315 8801d2529e43 5741e4fcf4b6 2024-09-14T03:54:24.521 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (4m) 1s ago 8m 12.2M - 1.7.0 72c9c2088986 e316760a30b3 2024-09-14T03:54:24.521 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi094 *:9100 running (3m) 2m ago 8m 7750k - 1.7.0 72c9c2088986 767f8a093175 2024-09-14T03:54:24.521 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (104s) 1s ago 13m 180M 2480M 19.3.0-4907-ga2782315 8801d2529e43 2345ccac2085 2024-09-14T03:54:24.521 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (71s) 1s ago 13m 177M 2480M 19.3.0-4907-ga2782315 8801d2529e43 53868856512a 2024-09-14T03:54:24.521 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (38s) 1s ago 12m 104M 2480M 19.3.0-4907-ga2782315 8801d2529e43 39ef876d2fff 2024-09-14T03:54:24.521 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (5s) 1s ago 12m 14.5M 2480M 19.3.0-4907-ga2782315 8801d2529e43 8efc112edbe9 2024-09-14T03:54:24.521 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi094 running (12m) 2m ago 12m 180M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 52b64527e24c 2024-09-14T03:54:24.521 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi094 running (11m) 2m ago 11m 168M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 b543f152ba87 2024-09-14T03:54:24.521 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi094 running (11m) 2m ago 11m 114M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 e66b592ac940 2024-09-14T03:54:24.521 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi094 running (10m) 2m ago 10m 152M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 037c6208e9ce 2024-09-14T03:54:24.521 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi094 *:9095 running (3m) 2m ago 9m 46.3M - 2.51.0 1d3b7f56885b ce1b353828dc 2024-09-14T03:54:24.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:24 smithi094 ceph-mon[86597]: osdmap e441: 8 total, 7 up, 8 in 2024-09-14T03:54:24.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:24 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:24.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:24 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:24.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:24 smithi094 ceph-mon[86597]: osdmap e442: 8 total, 7 up, 8 in 2024-09-14T03:54:24.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:54:24 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:54:24.147Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.3\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-14T03:54:24.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:54:24 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:54:24.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.3\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:24.785 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:24 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3[117637]: 2024-09-14T03:54:24.588+0000 7f4b3bd6a640 -1 osd.3 431 log_to_monitors true 2024-09-14T03:54:24.859 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:54:24.860 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-09-14T03:54:24.860 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:54:24.860 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:54:24.860 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-09-14T03:54:24.860 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:54:24.860 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:54:24.860 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-09-14T03:54:24.860 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 4, 2024-09-14T03:54:24.860 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:54:24.860 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:54:24.860 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": { 2024-09-14T03:54:24.860 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:54:24.860 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:54:24.861 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-09-14T03:54:24.861 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 6, 2024-09-14T03:54:24.861 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 8 2024-09-14T03:54:24.861 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-09-14T03:54:24.861 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:54:25.035 INFO:tasks.rados.rados.0.smithi070.stdout:1837: done (4 left) 2024-09-14T03:54:25.035 INFO:tasks.rados.rados.0.smithi070.stdout:1838: read oid 18 snap 144 2024-09-14T03:54:25.035 INFO:tasks.rados.rados.0.smithi070.stdout:1838: expect deleted 2024-09-14T03:54:25.035 INFO:tasks.rados.rados.0.smithi070.stdout:1839: read oid 47 snap -1 2024-09-14T03:54:25.036 INFO:tasks.rados.rados.0.smithi070.stdout:1839: expect (ObjNum 592 snap 195 seq_num 592) 2024-09-14T03:54:25.036 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 34 current snap is 197 2024-09-14T03:54:25.036 INFO:tasks.rados.rados.0.smithi070.stdout:1840: seq_num 594 ranges {3973120=352256} 2024-09-14T03:54:25.061 INFO:tasks.rados.rados.0.smithi070.stdout:1840: writing smithi07085154-34 from 3973120 to 4325376 tid 1 2024-09-14T03:54:25.062 INFO:tasks.rados.rados.0.smithi070.stdout:1841: rollback oid 12 current snap is 197 2024-09-14T03:54:25.062 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 12 to 184 2024-09-14T03:54:25.062 INFO:tasks.rados.rados.0.smithi070.stdout:1835: finishing copy_from racing read to smithi07085154-17 2024-09-14T03:54:25.062 INFO:tasks.rados.rados.0.smithi070.stdout:1834: finishing copy_from racing read to smithi07085154-49 2024-09-14T03:54:25.062 INFO:tasks.rados.rados.0.smithi070.stdout:1834: finishing copy_from to smithi07085154-49 2024-09-14T03:54:25.062 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 49 v 1186 (ObjNum 403 snap 140 seq_num 403) dirty exists 2024-09-14T03:54:25.062 INFO:tasks.rados.rados.0.smithi070.stdout:1835: finishing copy_from to smithi07085154-17 2024-09-14T03:54:25.062 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 1677 (ObjNum 588 snap 195 seq_num 588) dirty exists 2024-09-14T03:54:25.062 INFO:tasks.rados.rados.0.smithi070.stdout:1833: expect (ObjNum 590 snap 195 seq_num 590) 2024-09-14T03:54:25.140 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 03:54:25 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3[117637]: 2024-09-14T03:54:25.066+0000 7f4b33b14640 -1 osd.3 431 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:54:25.141 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:25 smithi070 ceph-mon[96704]: from='client.34329 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:54:25.141 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:25 smithi070 ceph-mon[96704]: pgmap v255: 121 pgs: 34 active+undersized, 7 active+undersized+degraded, 80 active+clean; 570 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 3.9 MiB/s wr, 25 op/s; 84/1038 objects degraded (8.092%) 2024-09-14T03:54:25.141 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:25 smithi070 ceph-mon[96704]: from='osd.3 [v2:172.21.15.70:6826/3491567009,v1:172.21.15.70:6827/3491567009]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-14T03:54:25.141 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:25 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/2443325751' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:54:25.141 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:25 smithi070 ceph-mon[96704]: Health check update: Degraded data redundancy: 84/1038 objects degraded (8.092%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:54:25.141 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:25 smithi070 ceph-mon[96704]: from='osd.3 [v2:172.21.15.70:6826/3491567009,v1:172.21.15.70:6827/3491567009]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-14T03:54:25.141 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:25 smithi070 ceph-mon[96704]: osdmap e443: 8 total, 7 up, 8 in 2024-09-14T03:54:25.141 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:25 smithi070 ceph-mon[96704]: from='osd.3 [v2:172.21.15.70:6826/3491567009,v1:172.21.15.70:6827/3491567009]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi070", "root=default"]} : dispatch 2024-09-14T03:54:25.141 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:25 smithi070 ceph-mon[98640]: from='client.34329 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:54:25.141 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:25 smithi070 ceph-mon[98640]: pgmap v255: 121 pgs: 34 active+undersized, 7 active+undersized+degraded, 80 active+clean; 570 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 3.9 MiB/s wr, 25 op/s; 84/1038 objects degraded (8.092%) 2024-09-14T03:54:25.142 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:25 smithi070 ceph-mon[98640]: from='osd.3 [v2:172.21.15.70:6826/3491567009,v1:172.21.15.70:6827/3491567009]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-14T03:54:25.142 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:25 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/2443325751' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:54:25.142 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:25 smithi070 ceph-mon[98640]: Health check update: Degraded data redundancy: 84/1038 objects degraded (8.092%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:54:25.142 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:25 smithi070 ceph-mon[98640]: from='osd.3 [v2:172.21.15.70:6826/3491567009,v1:172.21.15.70:6827/3491567009]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-14T03:54:25.142 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:25 smithi070 ceph-mon[98640]: osdmap e443: 8 total, 7 up, 8 in 2024-09-14T03:54:25.142 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:25 smithi070 ceph-mon[98640]: from='osd.3 [v2:172.21.15.70:6826/3491567009,v1:172.21.15.70:6827/3491567009]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi070", "root=default"]} : dispatch 2024-09-14T03:54:25.172 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:54:25.172 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:54:25.172 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-09-14T03:54:25.172 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:54:25.173 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [ 2024-09-14T03:54:25.173 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr", 2024-09-14T03:54:25.173 INFO:teuthology.orchestra.run.smithi070.stdout: "mon" 2024-09-14T03:54:25.173 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2024-09-14T03:54:25.173 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "9/20 daemons upgraded", 2024-09-14T03:54:25.173 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "Currently upgrading osd daemons", 2024-09-14T03:54:25.173 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-09-14T03:54:25.173 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:54:25.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:25 smithi094 ceph-mon[86597]: from='client.34329 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:54:25.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:25 smithi094 ceph-mon[86597]: pgmap v255: 121 pgs: 34 active+undersized, 7 active+undersized+degraded, 80 active+clean; 570 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 3.9 MiB/s wr, 25 op/s; 84/1038 objects degraded (8.092%) 2024-09-14T03:54:25.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:25 smithi094 ceph-mon[86597]: from='osd.3 [v2:172.21.15.70:6826/3491567009,v1:172.21.15.70:6827/3491567009]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-14T03:54:25.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:25 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/2443325751' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:54:25.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:25 smithi094 ceph-mon[86597]: Health check update: Degraded data redundancy: 84/1038 objects degraded (8.092%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:54:25.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:25 smithi094 ceph-mon[86597]: from='osd.3 [v2:172.21.15.70:6826/3491567009,v1:172.21.15.70:6827/3491567009]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-14T03:54:25.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:25 smithi094 ceph-mon[86597]: osdmap e443: 8 total, 7 up, 8 in 2024-09-14T03:54:25.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:25 smithi094 ceph-mon[86597]: from='osd.3 [v2:172.21.15.70:6826/3491567009,v1:172.21.15.70:6827/3491567009]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi070", "root=default"]} : dispatch 2024-09-14T03:54:26.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:54:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:54:25.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:26.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:54:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:54:25.742Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:26.515 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:26 smithi070 ceph-mon[96704]: from='client.34335 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:54:26.515 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:26 smithi070 ceph-mon[96704]: from='client.44255 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:54:26.515 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:26 smithi070 ceph-mon[98640]: from='client.34335 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:54:26.515 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:26 smithi070 ceph-mon[98640]: from='client.44255 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:54:26.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:26 smithi094 ceph-mon[86597]: from='client.34335 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:54:26.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:26 smithi094 ceph-mon[86597]: from='client.44255 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:54:27.039 INFO:tasks.rados.rados.0.smithi070.stdout:1836: expect (ObjNum 459 snap 155 seq_num 459) 2024-09-14T03:54:27.276 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:54:26 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:54:26.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:27.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:27 smithi094 ceph-mon[86597]: from='client.44270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:54:27.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:27 smithi094 ceph-mon[86597]: pgmap v257: 121 pgs: 34 active+undersized, 7 active+undersized+degraded, 80 active+clean; 570 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.2 MiB/s rd, 579 KiB/s wr, 20 op/s; 86/1044 objects degraded (8.238%) 2024-09-14T03:54:27.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:27 smithi094 ceph-mon[86597]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:54:27.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:27 smithi094 ceph-mon[86597]: osd.3 [v2:172.21.15.70:6826/3491567009,v1:172.21.15.70:6827/3491567009] boot 2024-09-14T03:54:27.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:27 smithi094 ceph-mon[86597]: osdmap e444: 8 total, 8 up, 8 in 2024-09-14T03:54:27.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:27 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-14T03:54:27.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:27 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:27.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:27 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:27.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:27 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:54:27.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:27 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:54:27.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:27 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:27.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:27 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:54:27.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:27 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:54:27.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:27 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:54:27.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:27 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:54:27.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:27 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-14T03:54:27.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:27 smithi094 ceph-mon[86597]: osdmap e445: 8 total, 8 up, 8 in 2024-09-14T03:54:27.285 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:54:27.013+0000 7fa4af5ba640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-14T03:54:27.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-mon[98640]: from='client.44270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:54:27.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-mon[98640]: pgmap v257: 121 pgs: 34 active+undersized, 7 active+undersized+degraded, 80 active+clean; 570 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.2 MiB/s rd, 579 KiB/s wr, 20 op/s; 86/1044 objects degraded (8.238%) 2024-09-14T03:54:27.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-mon[98640]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:54:27.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-mon[98640]: osd.3 [v2:172.21.15.70:6826/3491567009,v1:172.21.15.70:6827/3491567009] boot 2024-09-14T03:54:27.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-mon[98640]: osdmap e444: 8 total, 8 up, 8 in 2024-09-14T03:54:27.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-14T03:54:27.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:27.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:27.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:54:27.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:54:27.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:27.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:54:27.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:54:27.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:54:27.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:54:27.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-14T03:54:27.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-mon[98640]: osdmap e445: 8 total, 8 up, 8 in 2024-09-14T03:54:27.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-mon[96704]: from='client.44270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:54:27.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-mon[96704]: pgmap v257: 121 pgs: 34 active+undersized, 7 active+undersized+degraded, 80 active+clean; 570 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.2 MiB/s rd, 579 KiB/s wr, 20 op/s; 86/1044 objects degraded (8.238%) 2024-09-14T03:54:27.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-mon[96704]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:54:27.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-mon[96704]: osd.3 [v2:172.21.15.70:6826/3491567009,v1:172.21.15.70:6827/3491567009] boot 2024-09-14T03:54:27.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-mon[96704]: osdmap e444: 8 total, 8 up, 8 in 2024-09-14T03:54:27.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-14T03:54:27.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:27.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:27.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:54:27.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:54:27.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:27.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:54:27.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:54:27.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:54:27.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:54:27.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-14T03:54:27.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:27 smithi070 ceph-mon[96704]: osdmap e445: 8 total, 8 up, 8 in 2024-09-14T03:54:28.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:28 smithi094 ceph-mon[86597]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-14T03:54:28.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:28 smithi094 ceph-mon[86597]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-14T03:54:28.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:28 smithi070 ceph-mon[96704]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-14T03:54:28.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:28 smithi070 ceph-mon[96704]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-14T03:54:28.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:28 smithi070 ceph-mon[98640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-14T03:54:28.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:28 smithi070 ceph-mon[98640]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-14T03:54:28.713 INFO:tasks.rados.rados.0.smithi070.stdout:1840: finishing write tid 1 to smithi07085154-34 2024-09-14T03:54:28.713 INFO:tasks.rados.rados.0.smithi070.stdout:1841: finishing rollback tid 0 to smithi07085154-12 2024-09-14T03:54:28.713 INFO:tasks.rados.rados.0.smithi070.stdout:1841: finishing rollback tid 1 to smithi07085154-12 2024-09-14T03:54:28.713 INFO:tasks.rados.rados.0.smithi070.stdout:1841: finishing rollback tid 2 to smithi07085154-12 2024-09-14T03:54:28.713 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 570 (ObjNum 534 snap 176 seq_num 534) dirty exists 2024-09-14T03:54:28.713 INFO:tasks.rados.rados.0.smithi070.stdout:1840: finishing write tid 2 to smithi07085154-34 2024-09-14T03:54:28.713 INFO:tasks.rados.rados.0.smithi070.stdout:1833: done (7 left) 2024-09-14T03:54:28.713 INFO:tasks.rados.rados.0.smithi070.stdout:1834: done (6 left) 2024-09-14T03:54:28.714 INFO:tasks.rados.rados.0.smithi070.stdout:1835: done (5 left) 2024-09-14T03:54:28.714 INFO:tasks.rados.rados.0.smithi070.stdout:1836: done (4 left) 2024-09-14T03:54:28.714 INFO:tasks.rados.rados.0.smithi070.stdout:1838: done (3 left) 2024-09-14T03:54:28.714 INFO:tasks.rados.rados.0.smithi070.stdout:1841: done (2 left) 2024-09-14T03:54:28.714 INFO:tasks.rados.rados.0.smithi070.stdout:1842: snap_create 2024-09-14T03:54:28.714 INFO:tasks.rados.rados.0.smithi070.stdout:1840: finishing write tid 3 to smithi07085154-34 2024-09-14T03:54:28.714 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 1579 (ObjNum 594 snap 197 seq_num 594) dirty exists 2024-09-14T03:54:28.714 INFO:tasks.rados.rados.0.smithi070.stdout:1840: left oid 34 (ObjNum 594 snap 197 seq_num 594) 2024-09-14T03:54:28.714 INFO:tasks.rados.rados.0.smithi070.stdout:1839: expect (ObjNum 592 snap 195 seq_num 592) 2024-09-14T03:54:29.409 INFO:tasks.rados.rados.0.smithi070.stdout:1839: done (2 left) 2024-09-14T03:54:29.409 INFO:tasks.rados.rados.0.smithi070.stdout:1840: done (1 left) 2024-09-14T03:54:29.409 INFO:tasks.rados.rados.0.smithi070.stdout:1842: done (0 left) 2024-09-14T03:54:29.409 INFO:tasks.rados.rados.0.smithi070.stdout:1843: rmattr oid 18 current snap is 198 2024-09-14T03:54:29.409 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 14 current snap is 198 2024-09-14T03:54:29.409 INFO:tasks.rados.rados.0.smithi070.stdout:1844: seq_num 595 ranges {6373376=368640} 2024-09-14T03:54:29.444 INFO:tasks.rados.rados.0.smithi070.stdout:1844: writing smithi07085154-14 from 6373376 to 6742016 tid 1 2024-09-14T03:54:29.444 INFO:tasks.rados.rados.0.smithi070.stdout:1845: snap_create 2024-09-14T03:54:29.445 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 1679 (ObjNum 114 snap 24 seq_num 114) dirty exists 2024-09-14T03:54:29.449 INFO:tasks.rados.rados.0.smithi070.stdout:1844: finishing write tid 1 to smithi07085154-14 2024-09-14T03:54:29.450 INFO:tasks.rados.rados.0.smithi070.stdout:1844: finishing write tid 2 to smithi07085154-14 2024-09-14T03:54:29.451 INFO:tasks.rados.rados.0.smithi070.stdout:1844: finishing write tid 3 to smithi07085154-14 2024-09-14T03:54:29.451 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 1561 (ObjNum 595 snap 198 seq_num 595) dirty exists 2024-09-14T03:54:29.451 INFO:tasks.rados.rados.0.smithi070.stdout:1844: left oid 14 (ObjNum 595 snap 198 seq_num 595) 2024-09-14T03:54:29.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:29 smithi094 ceph-mon[86597]: pgmap v260: 121 pgs: 9 peering, 20 active+undersized, 2 active+undersized+degraded, 90 active+clean; 584 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s rd, 4.6 MiB/s wr, 11 op/s; 43/1023 objects degraded (4.203%); 5.9 MiB/s, 1 objects/s recovering 2024-09-14T03:54:29.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:29 smithi070 ceph-mon[98640]: pgmap v260: 121 pgs: 9 peering, 20 active+undersized, 2 active+undersized+degraded, 90 active+clean; 584 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s rd, 4.6 MiB/s wr, 11 op/s; 43/1023 objects degraded (4.203%); 5.9 MiB/s, 1 objects/s recovering 2024-09-14T03:54:29.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:29 smithi070 ceph-mon[96704]: pgmap v260: 121 pgs: 9 peering, 20 active+undersized, 2 active+undersized+degraded, 90 active+clean; 584 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s rd, 4.6 MiB/s wr, 11 op/s; 43/1023 objects degraded (4.203%); 5.9 MiB/s, 1 objects/s recovering 2024-09-14T03:54:30.094 INFO:tasks.rados.rados.0.smithi070.stdout:1843: done (2 left) 2024-09-14T03:54:30.094 INFO:tasks.rados.rados.0.smithi070.stdout:1844: done (1 left) 2024-09-14T03:54:30.094 INFO:tasks.rados.rados.0.smithi070.stdout:1845: done (0 left) 2024-09-14T03:54:30.094 INFO:tasks.rados.rados.0.smithi070.stdout:1846: copy_from oid 34 from oid 41 current snap is 199 2024-09-14T03:54:30.094 INFO:tasks.rados.rados.0.smithi070.stdout:1847: rollback oid 2 current snap is 199 2024-09-14T03:54:30.094 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 2 to 107 2024-09-14T03:54:30.095 INFO:tasks.rados.rados.0.smithi070.stdout:1848: rollback oid 26 current snap is 199 2024-09-14T03:54:30.095 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 26 to 168 2024-09-14T03:54:30.095 INFO:tasks.rados.rados.0.smithi070.stdout:1849: read oid 9 snap -1 2024-09-14T03:54:30.095 INFO:tasks.rados.rados.0.smithi070.stdout:1849: expect (ObjNum 459 snap 155 seq_num 459) 2024-09-14T03:54:30.095 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 18 current snap is 199 2024-09-14T03:54:30.095 INFO:tasks.rados.rados.0.smithi070.stdout:1850: seq_num 596 ranges {4784128=385024} 2024-09-14T03:54:30.122 INFO:tasks.rados.rados.0.smithi070.stdout:1850: writing smithi07085154-18 from 4784128 to 5169152 tid 1 2024-09-14T03:54:30.122 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 33 current snap is 199 2024-09-14T03:54:30.123 INFO:tasks.rados.rados.0.smithi070.stdout:1847: finishing rollback tid 0 to smithi07085154-2 2024-09-14T03:54:30.123 INFO:tasks.rados.rados.0.smithi070.stdout:1847: finishing rollback tid 1 to smithi07085154-2 2024-09-14T03:54:30.123 INFO:tasks.rados.rados.0.smithi070.stdout:1851: seq_num 597 ranges {4096000=401408} 2024-09-14T03:54:30.148 INFO:tasks.rados.rados.0.smithi070.stdout:1851: writing smithi07085154-33 from 4096000 to 4497408 tid 1 2024-09-14T03:54:30.148 INFO:tasks.rados.rados.0.smithi070.stdout:1852: delete oid 6 current snap is 199 2024-09-14T03:54:30.148 INFO:tasks.rados.rados.0.smithi070.stdout:1847: finishing rollback tid 2 to smithi07085154-2 2024-09-14T03:54:30.148 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 2 v 1973 (ObjNum 307 snap 101 seq_num 307) dirty exists 2024-09-14T03:54:30.148 INFO:tasks.rados.rados.0.smithi070.stdout:1848: finishing rollback tid 0 to smithi07085154-26 2024-09-14T03:54:30.148 INFO:tasks.rados.rados.0.smithi070.stdout:1848: finishing rollback tid 1 to smithi07085154-26 2024-09-14T03:54:30.149 INFO:tasks.rados.rados.0.smithi070.stdout:1848: finishing rollback tid 2 to smithi07085154-26 2024-09-14T03:54:30.149 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 1977 (ObjNum 177 snap 0 seq_num 4214099018) dirty exists 2024-09-14T03:54:30.149 INFO:tasks.rados.rados.0.smithi070.stdout:1846: finishing copy_from racing read to smithi07085154-34 2024-09-14T03:54:30.149 INFO:tasks.rados.rados.0.smithi070.stdout:1846: finishing copy_from to smithi07085154-34 2024-09-14T03:54:30.149 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 1677 (ObjNum 588 snap 195 seq_num 588) dirty exists 2024-09-14T03:54:30.149 INFO:tasks.rados.rados.0.smithi070.stdout:1850: finishing write tid 1 to smithi07085154-18 2024-09-14T03:54:30.149 INFO:tasks.rados.rados.0.smithi070.stdout:1850: finishing write tid 2 to smithi07085154-18 2024-09-14T03:54:30.149 INFO:tasks.rados.rados.0.smithi070.stdout:1850: finishing write tid 3 to smithi07085154-18 2024-09-14T03:54:30.149 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 1681 (ObjNum 596 snap 199 seq_num 596) dirty exists 2024-09-14T03:54:30.149 INFO:tasks.rados.rados.0.smithi070.stdout:1850: left oid 18 (ObjNum 596 snap 199 seq_num 596) 2024-09-14T03:54:30.152 INFO:tasks.rados.rados.0.smithi070.stdout:1846: done (6 left) 2024-09-14T03:54:30.152 INFO:tasks.rados.rados.0.smithi070.stdout:1847: done (5 left) 2024-09-14T03:54:30.152 INFO:tasks.rados.rados.0.smithi070.stdout:1848: done (4 left) 2024-09-14T03:54:30.152 INFO:tasks.rados.rados.0.smithi070.stdout:1850: done (3 left) 2024-09-14T03:54:30.152 INFO:tasks.rados.rados.0.smithi070.stdout:1852: done (2 left) 2024-09-14T03:54:30.152 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 8 current snap is 199 2024-09-14T03:54:30.152 INFO:tasks.rados.rados.0.smithi070.stdout:1851: finishing write tid 1 to smithi07085154-33 2024-09-14T03:54:30.152 INFO:tasks.rados.rados.0.smithi070.stdout:1853: seq_num 599 ranges {139264=434176} 2024-09-14T03:54:30.156 INFO:tasks.rados.rados.0.smithi070.stdout:1853: writing smithi07085154-8 from 139264 to 573440 tid 1 2024-09-14T03:54:30.156 INFO:tasks.rados.rados.0.smithi070.stdout:1854: rmattr oid 25 current snap is 199 2024-09-14T03:54:30.156 INFO:tasks.rados.rados.0.smithi070.stdout:1851: finishing write tid 2 to smithi07085154-33 2024-09-14T03:54:30.156 INFO:tasks.rados.rados.0.smithi070.stdout:1851: finishing write tid 3 to smithi07085154-33 2024-09-14T03:54:30.156 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 33 v 1565 (ObjNum 597 snap 199 seq_num 597) dirty exists 2024-09-14T03:54:30.156 INFO:tasks.rados.rados.0.smithi070.stdout:1851: left oid 33 (ObjNum 597 snap 199 seq_num 597) 2024-09-14T03:54:30.157 INFO:tasks.rados.rados.0.smithi070.stdout:1851: done (3 left) 2024-09-14T03:54:30.157 INFO:tasks.rados.rados.0.smithi070.stdout:1855: copy_from oid 37 from oid 49 current snap is 199 2024-09-14T03:54:30.157 INFO:tasks.rados.rados.0.smithi070.stdout:1856: copy_from oid 26 from oid 45 current snap is 199 2024-09-14T03:54:30.157 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 19 current snap is 199 2024-09-14T03:54:30.157 INFO:tasks.rados.rados.0.smithi070.stdout:1857: seq_num 600 ranges {7315456=450560} 2024-09-14T03:54:30.198 INFO:tasks.rados.rados.0.smithi070.stdout:1857: writing smithi07085154-19 from 7315456 to 7766016 tid 1 2024-09-14T03:54:30.199 INFO:tasks.rados.rados.0.smithi070.stdout:1858: delete oid 17 current snap is 199 2024-09-14T03:54:30.199 INFO:tasks.rados.rados.0.smithi070.stdout:1849: expect (ObjNum 459 snap 155 seq_num 459) 2024-09-14T03:54:30.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:30 smithi094 ceph-mon[86597]: osdmap e446: 8 total, 8 up, 8 in 2024-09-14T03:54:30.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:54:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:54:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:30.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:30 smithi070 ceph-mon[98640]: osdmap e446: 8 total, 8 up, 8 in 2024-09-14T03:54:30.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:30 smithi070 ceph-mon[96704]: osdmap e446: 8 total, 8 up, 8 in 2024-09-14T03:54:31.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:31 smithi094 ceph-mon[86597]: pgmap v262: 121 pgs: 9 peering, 112 active+clean; 585 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 5.4 MiB/s wr, 5 op/s; 7.4 MiB/s, 1 objects/s recovering 2024-09-14T03:54:31.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:31 smithi094 ceph-mon[86597]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 43/1023 objects degraded (4.203%), 2 pgs degraded) 2024-09-14T03:54:31.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:31 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T03:54:31.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:31 smithi094 ceph-mon[86597]: osdmap e447: 8 total, 8 up, 8 in 2024-09-14T03:54:31.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:31 smithi070 ceph-mon[96704]: pgmap v262: 121 pgs: 9 peering, 112 active+clean; 585 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 5.4 MiB/s wr, 5 op/s; 7.4 MiB/s, 1 objects/s recovering 2024-09-14T03:54:31.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:31 smithi070 ceph-mon[96704]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 43/1023 objects degraded (4.203%), 2 pgs degraded) 2024-09-14T03:54:31.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:31 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T03:54:31.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:31 smithi070 ceph-mon[96704]: osdmap e447: 8 total, 8 up, 8 in 2024-09-14T03:54:31.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:31 smithi070 ceph-mon[98640]: pgmap v262: 121 pgs: 9 peering, 112 active+clean; 585 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 5.4 MiB/s wr, 5 op/s; 7.4 MiB/s, 1 objects/s recovering 2024-09-14T03:54:31.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:31 smithi070 ceph-mon[98640]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 43/1023 objects degraded (4.203%), 2 pgs degraded) 2024-09-14T03:54:31.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:31 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T03:54:31.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:31 smithi070 ceph-mon[98640]: osdmap e447: 8 total, 8 up, 8 in 2024-09-14T03:54:31.853 INFO:tasks.rados.rados.0.smithi070.stdout:1853: finishing write tid 1 to smithi07085154-8 2024-09-14T03:54:31.853 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 1678 (ObjNum 583 snap 194 seq_num 583) dirty exists 2024-09-14T03:54:31.853 INFO:tasks.rados.rados.0.smithi070.stdout:1853: finishing write tid 2 to smithi07085154-8 2024-09-14T03:54:31.853 INFO:tasks.rados.rados.0.smithi070.stdout:1853: finishing write tid 3 to smithi07085154-8 2024-09-14T03:54:31.853 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 8 v 1683 (ObjNum 599 snap 199 seq_num 599) dirty exists 2024-09-14T03:54:31.853 INFO:tasks.rados.rados.0.smithi070.stdout:1853: left oid 8 (ObjNum 599 snap 199 seq_num 599) 2024-09-14T03:54:31.853 INFO:tasks.rados.rados.0.smithi070.stdout:1856: finishing copy_from racing read to smithi07085154-26 2024-09-14T03:54:31.853 INFO:tasks.rados.rados.0.smithi070.stdout:1856: finishing copy_from to smithi07085154-26 2024-09-14T03:54:31.853 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 1560 (ObjNum 564 snap 192 seq_num 564) dirty exists 2024-09-14T03:54:31.853 INFO:tasks.rados.rados.0.smithi070.stdout:1857: finishing write tid 1 to smithi07085154-19 2024-09-14T03:54:31.854 INFO:tasks.rados.rados.0.smithi070.stdout:1849: done (6 left) 2024-09-14T03:54:31.854 INFO:tasks.rados.rados.0.smithi070.stdout:1853: done (5 left) 2024-09-14T03:54:31.854 INFO:tasks.rados.rados.0.smithi070.stdout:1854: done (4 left) 2024-09-14T03:54:31.854 INFO:tasks.rados.rados.0.smithi070.stdout:1856: done (3 left) 2024-09-14T03:54:31.854 INFO:tasks.rados.rados.0.smithi070.stdout:1858: done (2 left) 2024-09-14T03:54:31.854 INFO:tasks.rados.rados.0.smithi070.stdout:1859: read oid 16 snap -1 2024-09-14T03:54:31.854 INFO:tasks.rados.rados.0.smithi070.stdout:1859: expect (ObjNum 114 snap 24 seq_num 114) 2024-09-14T03:54:31.854 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 40 current snap is 199 2024-09-14T03:54:31.854 INFO:tasks.rados.rados.0.smithi070.stdout:1857: finishing write tid 2 to smithi07085154-19 2024-09-14T03:54:31.854 INFO:tasks.rados.rados.0.smithi070.stdout:1855: finishing copy_from racing read to smithi07085154-37 2024-09-14T03:54:31.854 INFO:tasks.rados.rados.0.smithi070.stdout:1855: finishing copy_from to smithi07085154-37 2024-09-14T03:54:31.854 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 37 v 1186 (ObjNum 403 snap 140 seq_num 403) dirty exists 2024-09-14T03:54:31.854 INFO:tasks.rados.rados.0.smithi070.stdout:1857: finishing write tid 3 to smithi07085154-19 2024-09-14T03:54:31.855 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 19 v 1679 (ObjNum 600 snap 199 seq_num 600) dirty exists 2024-09-14T03:54:31.855 INFO:tasks.rados.rados.0.smithi070.stdout:1857: left oid 19 (ObjNum 600 snap 199 seq_num 600) 2024-09-14T03:54:31.855 INFO:tasks.rados.rados.0.smithi070.stdout:1860: seq_num 602 ranges {221184=491520} 2024-09-14T03:54:31.856 INFO:tasks.rados.rados.0.smithi070.stdout:1860: writing smithi07085154-40 from 221184 to 712704 tid 1 2024-09-14T03:54:31.857 INFO:tasks.rados.rados.0.smithi070.stdout:1855: done (3 left) 2024-09-14T03:54:31.857 INFO:tasks.rados.rados.0.smithi070.stdout:1857: done (2 left) 2024-09-14T03:54:31.857 INFO:tasks.rados.rados.0.smithi070.stdout:1861: snap_create 2024-09-14T03:54:31.862 INFO:tasks.rados.rados.0.smithi070.stdout:1860: finishing write tid 1 to smithi07085154-40 2024-09-14T03:54:31.863 INFO:tasks.rados.rados.0.smithi070.stdout:1860: finishing write tid 2 to smithi07085154-40 2024-09-14T03:54:31.863 INFO:tasks.rados.rados.0.smithi070.stdout:1860: finishing write tid 3 to smithi07085154-40 2024-09-14T03:54:31.863 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 1933 (ObjNum 602 snap 199 seq_num 602) dirty exists 2024-09-14T03:54:31.864 INFO:tasks.rados.rados.0.smithi070.stdout:1860: left oid 40 (ObjNum 602 snap 199 seq_num 602) 2024-09-14T03:54:31.867 INFO:tasks.rados.rados.0.smithi070.stdout:1859: expect (ObjNum 114 snap 24 seq_num 114) 2024-09-14T03:54:32.098 INFO:tasks.rados.rados.0.smithi070.stdout:1859: done (2 left) 2024-09-14T03:54:32.098 INFO:tasks.rados.rados.0.smithi070.stdout:1860: done (1 left) 2024-09-14T03:54:32.098 INFO:tasks.rados.rados.0.smithi070.stdout:1861: done (0 left) 2024-09-14T03:54:32.099 INFO:tasks.rados.rados.0.smithi070.stdout:1862: rollback oid 36 current snap is 200 2024-09-14T03:54:32.099 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 36 to 198 2024-09-14T03:54:32.099 INFO:tasks.rados.rados.0.smithi070.stdout:1863: snap_create 2024-09-14T03:54:32.102 INFO:tasks.rados.rados.0.smithi070.stdout:1862: finishing rollback tid 0 to smithi07085154-36 2024-09-14T03:54:32.103 INFO:tasks.rados.rados.0.smithi070.stdout:1862: finishing rollback tid 1 to smithi07085154-36 2024-09-14T03:54:32.103 INFO:tasks.rados.rados.0.smithi070.stdout:1862: finishing rollback tid 2 to smithi07085154-36 2024-09-14T03:54:32.103 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 36 v 1936 (ObjNum 590 snap 195 seq_num 590) dirty exists 2024-09-14T03:54:33.100 INFO:tasks.rados.rados.0.smithi070.stdout:1862: done (1 left) 2024-09-14T03:54:33.101 INFO:tasks.rados.rados.0.smithi070.stdout:1863: done (0 left) 2024-09-14T03:54:33.101 INFO:tasks.rados.rados.0.smithi070.stdout:1864: read oid 4 snap 121 2024-09-14T03:54:33.101 INFO:tasks.rados.rados.0.smithi070.stdout:1864: expect (ObjNum 351 snap 116 seq_num 351) 2024-09-14T03:54:33.101 INFO:tasks.rados.rados.0.smithi070.stdout:1865: snap_remove snap 167 2024-09-14T03:54:33.406 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:33 smithi070 ceph-mon[98640]: pgmap v264: 121 pgs: 121 active+clean; 594 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s rd, 5.9 MiB/s wr, 6 op/s; 6.6 MiB/s, 1 objects/s recovering 2024-09-14T03:54:33.406 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:33 smithi070 ceph-mon[98640]: osdmap e448: 8 total, 8 up, 8 in 2024-09-14T03:54:33.406 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:33 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:33.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:33 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:33.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:33 smithi070 ceph-mon[96704]: pgmap v264: 121 pgs: 121 active+clean; 594 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s rd, 5.9 MiB/s wr, 6 op/s; 6.6 MiB/s, 1 objects/s recovering 2024-09-14T03:54:33.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:33 smithi070 ceph-mon[96704]: osdmap e448: 8 total, 8 up, 8 in 2024-09-14T03:54:33.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:33 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:33.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:33 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:33.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:33 smithi094 ceph-mon[86597]: pgmap v264: 121 pgs: 121 active+clean; 594 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s rd, 5.9 MiB/s wr, 6 op/s; 6.6 MiB/s, 1 objects/s recovering 2024-09-14T03:54:33.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:33 smithi094 ceph-mon[86597]: osdmap e448: 8 total, 8 up, 8 in 2024-09-14T03:54:33.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:33 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:33.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:33 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:33.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:54:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:54:33] "GET /metrics HTTP/1.1" 200 36326 "" "Prometheus/2.51.0" 2024-09-14T03:54:34.033 INFO:tasks.rados.rados.0.smithi070.stdout:1865: done (1 left) 2024-09-14T03:54:34.033 INFO:tasks.rados.rados.0.smithi070.stdout:1866: setattr oid 49 current snap is 201 2024-09-14T03:54:34.033 INFO:tasks.rados.rados.0.smithi070.stdout:1864: expect (ObjNum 351 snap 116 seq_num 351) 2024-09-14T03:54:34.204 INFO:tasks.rados.rados.0.smithi070.stdout:1864: done (1 left) 2024-09-14T03:54:34.204 INFO:tasks.rados.rados.0.smithi070.stdout:1867: snap_remove snap 108 2024-09-14T03:54:34.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:34 smithi070 ceph-mon[96704]: osdmap e449: 8 total, 8 up, 8 in 2024-09-14T03:54:34.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:34 smithi070 ceph-mon[96704]: osdmap e450: 8 total, 8 up, 8 in 2024-09-14T03:54:34.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:34 smithi070 ceph-mon[98640]: osdmap e449: 8 total, 8 up, 8 in 2024-09-14T03:54:34.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:34 smithi070 ceph-mon[98640]: osdmap e450: 8 total, 8 up, 8 in 2024-09-14T03:54:34.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:34 smithi094 ceph-mon[86597]: osdmap e449: 8 total, 8 up, 8 in 2024-09-14T03:54:34.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:34 smithi094 ceph-mon[86597]: osdmap e450: 8 total, 8 up, 8 in 2024-09-14T03:54:34.529 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:54:34 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:54:34.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.4\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.94\", device_class=\"ssd\", hostname=\"smithi094\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.94\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.94\", device_class=\"ssd\", hostname=\"smithi094\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.94\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:35.107 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 49 v 1566 (ObjNum 403 snap 140 seq_num 403) dirty exists 2024-09-14T03:54:35.107 INFO:tasks.rados.rados.0.smithi070.stdout:1866: done (1 left) 2024-09-14T03:54:35.107 INFO:tasks.rados.rados.0.smithi070.stdout:1867: done (0 left) 2024-09-14T03:54:35.107 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 46 current snap is 201 2024-09-14T03:54:35.107 INFO:tasks.rados.rados.0.smithi070.stdout:1868: seq_num 603 ranges {0=507904} 2024-09-14T03:54:35.110 INFO:tasks.rados.rados.0.smithi070.stdout:1868: writing smithi07085154-46 from 0 to 507904 tid 1 2024-09-14T03:54:35.110 INFO:tasks.rados.rados.0.smithi070.stdout:1869: copy_from oid 29 from oid 48 current snap is 201 2024-09-14T03:54:35.111 INFO:tasks.rados.rados.0.smithi070.stdout:1870: snap_remove snap 198 2024-09-14T03:54:35.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:35 smithi094 ceph-mon[86597]: pgmap v268: 121 pgs: 121 active+clean; 598 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 3.0 MiB/s wr, 11 op/s; 588 KiB/s, 0 objects/s recovering 2024-09-14T03:54:35.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:35 smithi070 ceph-mon[98640]: pgmap v268: 121 pgs: 121 active+clean; 598 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 3.0 MiB/s wr, 11 op/s; 588 KiB/s, 0 objects/s recovering 2024-09-14T03:54:35.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:35 smithi070 ceph-mon[96704]: pgmap v268: 121 pgs: 121 active+clean; 598 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 3.0 MiB/s wr, 11 op/s; 588 KiB/s, 0 objects/s recovering 2024-09-14T03:54:36.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:54:35 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:54:35.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:36.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:54:35 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:54:35.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:36.118 INFO:tasks.rados.rados.0.smithi070.stdout:1870: done (2 left) 2024-09-14T03:54:36.119 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 49 current snap is 201 2024-09-14T03:54:36.119 INFO:tasks.rados.rados.0.smithi070.stdout:1871: seq_num 604 ranges {9109504=524288} 2024-09-14T03:54:36.171 INFO:tasks.rados.rados.0.smithi070.stdout:1871: writing smithi07085154-49 from 9109504 to 9633792 tid 1 2024-09-14T03:54:36.171 INFO:tasks.rados.rados.0.smithi070.stdout:1872: read oid 30 snap -1 2024-09-14T03:54:36.171 INFO:tasks.rados.rados.0.smithi070.stdout:1872: expect deleted 2024-09-14T03:54:36.171 INFO:tasks.rados.rados.0.smithi070.stdout:1873: snap_create 2024-09-14T03:54:36.171 INFO:tasks.rados.rados.0.smithi070.stdout:1868: finishing write tid 1 to smithi07085154-46 2024-09-14T03:54:36.171 INFO:tasks.rados.rados.0.smithi070.stdout:1868: finishing write tid 2 to smithi07085154-46 2024-09-14T03:54:36.171 INFO:tasks.rados.rados.0.smithi070.stdout:1869: finishing copy_from racing read to smithi07085154-29 2024-09-14T03:54:36.171 INFO:tasks.rados.rados.0.smithi070.stdout:1868: finishing write tid 3 to smithi07085154-46 2024-09-14T03:54:36.171 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 46 v 1992 (ObjNum 603 snap 201 seq_num 603) dirty exists 2024-09-14T03:54:36.171 INFO:tasks.rados.rados.0.smithi070.stdout:1868: left oid 46 (ObjNum 603 snap 201 seq_num 603) 2024-09-14T03:54:36.171 INFO:tasks.rados.rados.0.smithi070.stdout:1869: finishing copy_from to smithi07085154-29 2024-09-14T03:54:36.171 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 562 (ObjNum 313 snap 105 seq_num 313) dirty exists 2024-09-14T03:54:36.173 INFO:tasks.rados.rados.0.smithi070.stdout:1871: finishing write tid 1 to smithi07085154-49 2024-09-14T03:54:36.174 INFO:tasks.rados.rados.0.smithi070.stdout:1871: finishing write tid 2 to smithi07085154-49 2024-09-14T03:54:36.175 INFO:tasks.rados.rados.0.smithi070.stdout:1871: finishing write tid 3 to smithi07085154-49 2024-09-14T03:54:36.175 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 49 v 1568 (ObjNum 604 snap 201 seq_num 604) dirty exists 2024-09-14T03:54:36.175 INFO:tasks.rados.rados.0.smithi070.stdout:1871: left oid 49 (ObjNum 604 snap 201 seq_num 604) 2024-09-14T03:54:36.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:36 smithi094 ceph-mon[86597]: osdmap e451: 8 total, 8 up, 8 in 2024-09-14T03:54:36.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:36 smithi070 ceph-mon[96704]: osdmap e451: 8 total, 8 up, 8 in 2024-09-14T03:54:36.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:36 smithi070 ceph-mon[98640]: osdmap e451: 8 total, 8 up, 8 in 2024-09-14T03:54:37.121 INFO:tasks.rados.rados.0.smithi070.stdout:1868: done (4 left) 2024-09-14T03:54:37.122 INFO:tasks.rados.rados.0.smithi070.stdout:1869: done (3 left) 2024-09-14T03:54:37.122 INFO:tasks.rados.rados.0.smithi070.stdout:1871: done (2 left) 2024-09-14T03:54:37.122 INFO:tasks.rados.rados.0.smithi070.stdout:1872: done (1 left) 2024-09-14T03:54:37.122 INFO:tasks.rados.rados.0.smithi070.stdout:1873: done (0 left) 2024-09-14T03:54:37.122 INFO:tasks.rados.rados.0.smithi070.stdout:1874: setattr oid 37 current snap is 202 2024-09-14T03:54:37.123 INFO:tasks.rados.rados.0.smithi070.stdout:1875: copy_from oid 34 from oid 33 current snap is 202 2024-09-14T03:54:37.123 INFO:tasks.rados.rados.0.smithi070.stdout:1876: copy_from oid 16 from oid 43 current snap is 202 2024-09-14T03:54:37.123 INFO:tasks.rados.rados.0.smithi070.stdout:1877: rollback oid 13 current snap is 202 2024-09-14T03:54:37.123 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 13 to 189 2024-09-14T03:54:37.123 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 24 current snap is 202 2024-09-14T03:54:37.123 INFO:tasks.rados.rados.0.smithi070.stdout:1878: seq_num 605 ranges {1458176=540672} 2024-09-14T03:54:37.135 INFO:tasks.rados.rados.0.smithi070.stdout:1878: writing smithi07085154-24 from 1458176 to 1998848 tid 1 2024-09-14T03:54:37.135 INFO:tasks.rados.rados.0.smithi070.stdout:1879: copy_from oid 38 from oid 11 current snap is 202 2024-09-14T03:54:37.135 INFO:tasks.rados.rados.0.smithi070.stdout:1876: finishing copy_from to smithi07085154-16 2024-09-14T03:54:37.135 INFO:tasks.rados.rados.0.smithi070.stdout:1876: got expected ENOENT (src dne) 2024-09-14T03:54:37.135 INFO:tasks.rados.rados.0.smithi070.stdout:1876: finishing copy_from racing read to smithi07085154-16 2024-09-14T03:54:37.135 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 37 v 1684 (ObjNum 403 snap 140 seq_num 403) dirty exists 2024-09-14T03:54:37.135 INFO:tasks.rados.rados.0.smithi070.stdout:1874: done (5 left) 2024-09-14T03:54:37.135 INFO:tasks.rados.rados.0.smithi070.stdout:1876: done (4 left) 2024-09-14T03:54:37.135 INFO:tasks.rados.rados.0.smithi070.stdout:1880: rollback oid 26 current snap is 202 2024-09-14T03:54:37.135 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 26 to 107 2024-09-14T03:54:37.135 INFO:tasks.rados.rados.0.smithi070.stdout:1881: snap_create 2024-09-14T03:54:37.135 INFO:tasks.rados.rados.0.smithi070.stdout:1877: finishing rollback tid 0 to smithi07085154-13 2024-09-14T03:54:37.136 INFO:tasks.rados.rados.0.smithi070.stdout:1877: finishing rollback tid 1 to smithi07085154-13 2024-09-14T03:54:37.136 INFO:tasks.rados.rados.0.smithi070.stdout:1877: finishing rollback tid 2 to smithi07085154-13 2024-09-14T03:54:37.136 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 1676 (ObjNum 403 snap 140 seq_num 403) dirty exists 2024-09-14T03:54:37.141 INFO:tasks.rados.rados.0.smithi070.stdout:1878: finishing write tid 1 to smithi07085154-24 2024-09-14T03:54:37.141 INFO:tasks.rados.rados.0.smithi070.stdout:1880: finishing rollback tid 0 to smithi07085154-26 2024-09-14T03:54:37.141 INFO:tasks.rados.rados.0.smithi070.stdout:1878: finishing write tid 2 to smithi07085154-24 2024-09-14T03:54:37.141 INFO:tasks.rados.rados.0.smithi070.stdout:1880: finishing rollback tid 1 to smithi07085154-26 2024-09-14T03:54:37.141 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 2013 (ObjNum 289 snap 0 seq_num 1436215854) dirty dne 2024-09-14T03:54:37.142 INFO:tasks.rados.rados.0.smithi070.stdout:1878: finishing write tid 3 to smithi07085154-24 2024-09-14T03:54:37.142 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 1563 (ObjNum 605 snap 202 seq_num 605) dirty exists 2024-09-14T03:54:37.142 INFO:tasks.rados.rados.0.smithi070.stdout:1878: left oid 24 (ObjNum 605 snap 202 seq_num 605) 2024-09-14T03:54:37.155 INFO:tasks.rados.rados.0.smithi070.stdout:1875: finishing copy_from racing read to smithi07085154-34 2024-09-14T03:54:37.158 INFO:tasks.rados.rados.0.smithi070.stdout:1875: finishing copy_from to smithi07085154-34 2024-09-14T03:54:37.158 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 1565 (ObjNum 597 snap 199 seq_num 597) dirty exists 2024-09-14T03:54:37.162 INFO:tasks.rados.rados.0.smithi070.stdout:1879: finishing copy_from racing read to smithi07085154-38 2024-09-14T03:54:37.164 INFO:tasks.rados.rados.0.smithi070.stdout:1879: finishing copy_from to smithi07085154-38 2024-09-14T03:54:37.164 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 38 v 1673 (ObjNum 506 snap 165 seq_num 506) dirty exists 2024-09-14T03:54:37.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:37 smithi094 ceph-mon[86597]: pgmap v270: 121 pgs: 121 active+clean; 608 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 3.4 MiB/s wr, 10 op/s 2024-09-14T03:54:37.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:37 smithi094 ceph-mon[86597]: osdmap e452: 8 total, 8 up, 8 in 2024-09-14T03:54:37.277 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:54:36 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:54:36.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:37.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:37 smithi070 ceph-mon[96704]: pgmap v270: 121 pgs: 121 active+clean; 608 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 3.4 MiB/s wr, 10 op/s 2024-09-14T03:54:37.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:37 smithi070 ceph-mon[96704]: osdmap e452: 8 total, 8 up, 8 in 2024-09-14T03:54:37.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:37 smithi070 ceph-mon[98640]: pgmap v270: 121 pgs: 121 active+clean; 608 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 3.4 MiB/s wr, 10 op/s 2024-09-14T03:54:37.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:37 smithi070 ceph-mon[98640]: osdmap e452: 8 total, 8 up, 8 in 2024-09-14T03:54:38.129 INFO:tasks.rados.rados.0.smithi070.stdout:1875: done (5 left) 2024-09-14T03:54:38.129 INFO:tasks.rados.rados.0.smithi070.stdout:1877: done (4 left) 2024-09-14T03:54:38.129 INFO:tasks.rados.rados.0.smithi070.stdout:1878: done (3 left) 2024-09-14T03:54:38.129 INFO:tasks.rados.rados.0.smithi070.stdout:1879: done (2 left) 2024-09-14T03:54:38.129 INFO:tasks.rados.rados.0.smithi070.stdout:1880: done (1 left) 2024-09-14T03:54:38.129 INFO:tasks.rados.rados.0.smithi070.stdout:1881: done (0 left) 2024-09-14T03:54:38.129 INFO:tasks.rados.rados.0.smithi070.stdout:1882: rmattr oid 9 current snap is 203 2024-09-14T03:54:38.129 INFO:tasks.rados.rados.0.smithi070.stdout:1883: copy_from oid 11 from oid 13 current snap is 203 2024-09-14T03:54:38.129 INFO:tasks.rados.rados.0.smithi070.stdout:1884: delete oid 35 current snap is 203 2024-09-14T03:54:38.133 INFO:tasks.rados.rados.0.smithi070.stdout:1884: done (2 left) 2024-09-14T03:54:38.133 INFO:tasks.rados.rados.0.smithi070.stdout:1885: rollback oid 49 current snap is 203 2024-09-14T03:54:38.133 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 49 to 193 2024-09-14T03:54:38.133 INFO:tasks.rados.rados.0.smithi070.stdout:1886: snap_create 2024-09-14T03:54:38.138 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 2015 (ObjNum 459 snap 155 seq_num 459) dirty exists 2024-09-14T03:54:38.140 INFO:tasks.rados.rados.0.smithi070.stdout:1885: finishing rollback tid 0 to smithi07085154-49 2024-09-14T03:54:38.142 INFO:tasks.rados.rados.0.smithi070.stdout:1885: finishing rollback tid 1 to smithi07085154-49 2024-09-14T03:54:38.142 INFO:tasks.rados.rados.0.smithi070.stdout:1885: finishing rollback tid 2 to smithi07085154-49 2024-09-14T03:54:38.142 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 49 v 1571 (ObjNum 551 snap 185 seq_num 551) dirty exists 2024-09-14T03:54:38.168 INFO:tasks.rados.rados.0.smithi070.stdout:1883: finishing copy_from racing read to smithi07085154-11 2024-09-14T03:54:38.170 INFO:tasks.rados.rados.0.smithi070.stdout:1883: finishing copy_from to smithi07085154-11 2024-09-14T03:54:38.170 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 11 v 1676 (ObjNum 403 snap 140 seq_num 403) dirty exists 2024-09-14T03:54:38.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:38 smithi094 ceph-mon[86597]: osdmap e453: 8 total, 8 up, 8 in 2024-09-14T03:54:38.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:38 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:38.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:38 smithi094 ceph-mon[86597]: pgmap v273: 121 pgs: 121 active+clean; 617 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 4.7 MiB/s wr, 6 op/s 2024-09-14T03:54:38.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:38 smithi070 ceph-mon[96704]: osdmap e453: 8 total, 8 up, 8 in 2024-09-14T03:54:38.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:38 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:38.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:38 smithi070 ceph-mon[96704]: pgmap v273: 121 pgs: 121 active+clean; 617 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 4.7 MiB/s wr, 6 op/s 2024-09-14T03:54:38.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:38 smithi070 ceph-mon[98640]: osdmap e453: 8 total, 8 up, 8 in 2024-09-14T03:54:38.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:38 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:38.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:38 smithi070 ceph-mon[98640]: pgmap v273: 121 pgs: 121 active+clean; 617 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 4.7 MiB/s wr, 6 op/s 2024-09-14T03:54:39.035 INFO:tasks.rados.rados.0.smithi070.stdout:1882: done (3 left) 2024-09-14T03:54:39.035 INFO:tasks.rados.rados.0.smithi070.stdout:1883: done (2 left) 2024-09-14T03:54:39.035 INFO:tasks.rados.rados.0.smithi070.stdout:1885: done (1 left) 2024-09-14T03:54:39.036 INFO:tasks.rados.rados.0.smithi070.stdout:1886: done (0 left) 2024-09-14T03:54:39.036 INFO:tasks.rados.rados.0.smithi070.stdout:1887: read oid 30 snap 144 2024-09-14T03:54:39.036 INFO:tasks.rados.rados.0.smithi070.stdout:1887: expect (ObjNum 380 snap 129 seq_num 380) 2024-09-14T03:54:39.036 INFO:tasks.rados.rados.0.smithi070.stdout:1888: rollback oid 17 current snap is 204 2024-09-14T03:54:39.036 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 17 to 195 2024-09-14T03:54:39.036 INFO:tasks.rados.rados.0.smithi070.stdout:1889: read oid 14 snap 184 2024-09-14T03:54:39.036 INFO:tasks.rados.rados.0.smithi070.stdout:1889: expect (ObjNum 548 snap 184 seq_num 548) 2024-09-14T03:54:39.036 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 45 current snap is 204 2024-09-14T03:54:39.036 INFO:tasks.rados.rados.0.smithi070.stdout:1890: seq_num 607 ranges {8839168=573440} 2024-09-14T03:54:39.088 INFO:tasks.rados.rados.0.smithi070.stdout:1890: writing smithi07085154-45 from 8839168 to 9412608 tid 1 2024-09-14T03:54:39.088 INFO:tasks.rados.rados.0.smithi070.stdout:1891: rmattr oid 7 current snap is 204 2024-09-14T03:54:39.088 INFO:tasks.rados.rados.0.smithi070.stdout:1891: done (4 left) 2024-09-14T03:54:39.088 INFO:tasks.rados.rados.0.smithi070.stdout:1892: copy_from oid 33 from oid 36 current snap is 204 2024-09-14T03:54:39.088 INFO:tasks.rados.rados.0.smithi070.stdout:1888: finishing rollback tid 1 to smithi07085154-17 2024-09-14T03:54:39.088 INFO:tasks.rados.rados.0.smithi070.stdout:1888: finishing rollback tid 2 to smithi07085154-17 2024-09-14T03:54:39.088 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 1681 (ObjNum 587 snap 195 seq_num 587) dirty exists 2024-09-14T03:54:39.089 INFO:tasks.rados.rados.0.smithi070.stdout:1889: expect (ObjNum 548 snap 184 seq_num 548) 2024-09-14T03:54:39.248 INFO:tasks.rados.rados.0.smithi070.stdout:1887: expect (ObjNum 380 snap 129 seq_num 380) 2024-09-14T03:54:39.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:39 smithi094 ceph-mon[86597]: osdmap e454: 8 total, 8 up, 8 in 2024-09-14T03:54:39.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:39 smithi094 ceph-mon[86597]: osdmap e455: 8 total, 8 up, 8 in 2024-09-14T03:54:39.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:39 smithi070 ceph-mon[96704]: osdmap e454: 8 total, 8 up, 8 in 2024-09-14T03:54:39.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:39 smithi070 ceph-mon[96704]: osdmap e455: 8 total, 8 up, 8 in 2024-09-14T03:54:39.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:39 smithi070 ceph-mon[98640]: osdmap e454: 8 total, 8 up, 8 in 2024-09-14T03:54:39.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:39 smithi070 ceph-mon[98640]: osdmap e455: 8 total, 8 up, 8 in 2024-09-14T03:54:39.554 INFO:tasks.rados.rados.0.smithi070.stdout:1890: finishing write tid 1 to smithi07085154-45 2024-09-14T03:54:39.555 INFO:tasks.rados.rados.0.smithi070.stdout:1890: finishing write tid 2 to smithi07085154-45 2024-09-14T03:54:39.555 INFO:tasks.rados.rados.0.smithi070.stdout:1887: done (4 left) 2024-09-14T03:54:39.555 INFO:tasks.rados.rados.0.smithi070.stdout:1888: done (3 left) 2024-09-14T03:54:39.555 INFO:tasks.rados.rados.0.smithi070.stdout:1889: done (2 left) 2024-09-14T03:54:39.555 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 3 current snap is 204 2024-09-14T03:54:39.555 INFO:tasks.rados.rados.0.smithi070.stdout:1890: finishing write tid 3 to smithi07085154-45 2024-09-14T03:54:39.555 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 45 v 1573 (ObjNum 607 snap 204 seq_num 607) dirty exists 2024-09-14T03:54:39.555 INFO:tasks.rados.rados.0.smithi070.stdout:1890: left oid 45 (ObjNum 607 snap 204 seq_num 607) 2024-09-14T03:54:39.555 INFO:tasks.rados.rados.0.smithi070.stdout:1892: finishing copy_from racing read to smithi07085154-33 2024-09-14T03:54:39.555 INFO:tasks.rados.rados.0.smithi070.stdout:1892: finishing copy_from to smithi07085154-33 2024-09-14T03:54:39.555 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 33 v 1936 (ObjNum 590 snap 195 seq_num 590) dirty exists 2024-09-14T03:54:39.555 INFO:tasks.rados.rados.0.smithi070.stdout:1893: seq_num 608 ranges {221184=589824} 2024-09-14T03:54:39.558 INFO:tasks.rados.rados.0.smithi070.stdout:1893: writing smithi07085154-3 from 221184 to 811008 tid 1 2024-09-14T03:54:39.558 INFO:tasks.rados.rados.0.smithi070.stdout:1890: done (2 left) 2024-09-14T03:54:39.558 INFO:tasks.rados.rados.0.smithi070.stdout:1892: done (1 left) 2024-09-14T03:54:39.559 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 2 current snap is 204 2024-09-14T03:54:39.559 INFO:tasks.rados.rados.0.smithi070.stdout:1894: seq_num 609 ranges {352256=606208} 2024-09-14T03:54:39.564 INFO:tasks.rados.rados.0.smithi070.stdout:1894: writing smithi07085154-2 from 352256 to 958464 tid 1 2024-09-14T03:54:39.564 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 17 current snap is 204 2024-09-14T03:54:39.564 INFO:tasks.rados.rados.0.smithi070.stdout:1893: finishing write tid 1 to smithi07085154-3 2024-09-14T03:54:39.564 INFO:tasks.rados.rados.0.smithi070.stdout:1893: finishing write tid 2 to smithi07085154-3 2024-09-14T03:54:39.564 INFO:tasks.rados.rados.0.smithi070.stdout:1893: finishing write tid 3 to smithi07085154-3 2024-09-14T03:54:39.564 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 3 v 1675 (ObjNum 608 snap 204 seq_num 608) dirty exists 2024-09-14T03:54:39.564 INFO:tasks.rados.rados.0.smithi070.stdout:1893: left oid 3 (ObjNum 608 snap 204 seq_num 608) 2024-09-14T03:54:39.564 INFO:tasks.rados.rados.0.smithi070.stdout:1895: seq_num 610 ranges {6471680=622592} 2024-09-14T03:54:39.601 INFO:tasks.rados.rados.0.smithi070.stdout:1895: writing smithi07085154-17 from 6471680 to 7094272 tid 1 2024-09-14T03:54:39.601 INFO:tasks.rados.rados.0.smithi070.stdout:1893: done (2 left) 2024-09-14T03:54:39.602 INFO:tasks.rados.rados.0.smithi070.stdout:1896: snap_create 2024-09-14T03:54:39.602 INFO:tasks.rados.rados.0.smithi070.stdout:1894: finishing write tid 1 to smithi07085154-2 2024-09-14T03:54:39.602 INFO:tasks.rados.rados.0.smithi070.stdout:1894: finishing write tid 2 to smithi07085154-2 2024-09-14T03:54:39.602 INFO:tasks.rados.rados.0.smithi070.stdout:1894: finishing write tid 3 to smithi07085154-2 2024-09-14T03:54:39.602 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 2 v 2018 (ObjNum 609 snap 204 seq_num 609) dirty exists 2024-09-14T03:54:39.602 INFO:tasks.rados.rados.0.smithi070.stdout:1894: left oid 2 (ObjNum 609 snap 204 seq_num 609) 2024-09-14T03:54:39.605 INFO:tasks.rados.rados.0.smithi070.stdout:1895: finishing write tid 1 to smithi07085154-17 2024-09-14T03:54:39.605 INFO:tasks.rados.rados.0.smithi070.stdout:1895: finishing write tid 2 to smithi07085154-17 2024-09-14T03:54:39.607 INFO:tasks.rados.rados.0.smithi070.stdout:1895: finishing write tid 3 to smithi07085154-17 2024-09-14T03:54:39.607 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 1683 (ObjNum 610 snap 204 seq_num 610) dirty exists 2024-09-14T03:54:39.607 INFO:tasks.rados.rados.0.smithi070.stdout:1895: left oid 17 (ObjNum 610 snap 204 seq_num 610) 2024-09-14T03:54:40.135 INFO:tasks.rados.rados.0.smithi070.stdout:1894: done (2 left) 2024-09-14T03:54:40.135 INFO:tasks.rados.rados.0.smithi070.stdout:1895: done (1 left) 2024-09-14T03:54:40.136 INFO:tasks.rados.rados.0.smithi070.stdout:1896: done (0 left) 2024-09-14T03:54:40.136 INFO:tasks.rados.rados.0.smithi070.stdout:1897: delete oid 40 current snap is 205 2024-09-14T03:54:40.140 INFO:tasks.rados.rados.0.smithi070.stdout:1897: done (0 left) 2024-09-14T03:54:40.140 INFO:tasks.rados.rados.0.smithi070.stdout:1898: read oid 17 snap 189 2024-09-14T03:54:40.140 INFO:tasks.rados.rados.0.smithi070.stdout:1898: expect (ObjNum 544 snap 181 seq_num 544) 2024-09-14T03:54:40.140 INFO:tasks.rados.rados.0.smithi070.stdout:1899: rollback oid 27 current snap is 205 2024-09-14T03:54:40.140 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 27 to 173 2024-09-14T03:54:40.140 INFO:tasks.rados.rados.0.smithi070.stdout:1900: snap_remove snap 119 2024-09-14T03:54:40.428 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:40 smithi094 ceph-mon[86597]: pgmap v276: 121 pgs: 121 active+clean; 624 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 15 op/s 2024-09-14T03:54:40.428 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:54:40 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:54:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:40.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:40 smithi070 ceph-mon[96704]: pgmap v276: 121 pgs: 121 active+clean; 624 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 15 op/s 2024-09-14T03:54:40.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:40 smithi070 ceph-mon[98640]: pgmap v276: 121 pgs: 121 active+clean; 624 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 15 op/s 2024-09-14T03:54:41.138 INFO:tasks.rados.rados.0.smithi070.stdout:1900: done (2 left) 2024-09-14T03:54:41.138 INFO:tasks.rados.rados.0.smithi070.stdout:1901: read oid 25 snap -1 2024-09-14T03:54:41.138 INFO:tasks.rados.rados.0.smithi070.stdout:1901: expect (ObjNum 583 snap 194 seq_num 583) 2024-09-14T03:54:41.139 INFO:tasks.rados.rados.0.smithi070.stdout:1902: copy_from oid 46 from oid 30 current snap is 205 2024-09-14T03:54:41.139 INFO:tasks.rados.rados.0.smithi070.stdout:1899: finishing rollback tid 0 to smithi07085154-27 2024-09-14T03:54:41.139 INFO:tasks.rados.rados.0.smithi070.stdout:1899: finishing rollback tid 1 to smithi07085154-27 2024-09-14T03:54:41.139 INFO:tasks.rados.rados.0.smithi070.stdout:1899: finishing rollback tid 2 to smithi07085154-27 2024-09-14T03:54:41.139 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 2022 (ObjNum 459 snap 155 seq_num 459) dirty exists 2024-09-14T03:54:41.139 INFO:tasks.rados.rados.0.smithi070.stdout:1898: expect (ObjNum 544 snap 181 seq_num 544) 2024-09-14T03:54:41.335 INFO:tasks.rados.rados.0.smithi070.stdout:1902: finishing copy_from to smithi07085154-46 2024-09-14T03:54:41.335 INFO:tasks.rados.rados.0.smithi070.stdout:1902: got expected ENOENT (src dne) 2024-09-14T03:54:41.335 INFO:tasks.rados.rados.0.smithi070.stdout:1902: finishing copy_from racing read to smithi07085154-46 2024-09-14T03:54:41.336 INFO:tasks.rados.rados.0.smithi070.stdout:1901: expect (ObjNum 583 snap 194 seq_num 583) 2024-09-14T03:54:41.444 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:41 smithi094 ceph-mon[86597]: osdmap e456: 8 total, 8 up, 8 in 2024-09-14T03:54:41.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:41 smithi070 ceph-mon[96704]: osdmap e456: 8 total, 8 up, 8 in 2024-09-14T03:54:41.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:41 smithi070 ceph-mon[98640]: osdmap e456: 8 total, 8 up, 8 in 2024-09-14T03:54:42.471 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:42 smithi094 ceph-mon[86597]: osdmap e457: 8 total, 8 up, 8 in 2024-09-14T03:54:42.472 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:42 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-14T03:54:42.472 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:42 smithi094 ceph-mon[86597]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-14T03:54:42.472 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:42 smithi094 ceph-mon[86597]: Upgrade: osd.4 is safe to restart 2024-09-14T03:54:42.472 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:42 smithi094 ceph-mon[86597]: pgmap v279: 121 pgs: 121 active+clean; 649 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 2.6 MiB/s wr, 19 op/s 2024-09-14T03:54:42.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:42 smithi070 ceph-mon[98640]: osdmap e457: 8 total, 8 up, 8 in 2024-09-14T03:54:42.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:42 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-14T03:54:42.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:42 smithi070 ceph-mon[98640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-14T03:54:42.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:42 smithi070 ceph-mon[98640]: Upgrade: osd.4 is safe to restart 2024-09-14T03:54:42.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:42 smithi070 ceph-mon[98640]: pgmap v279: 121 pgs: 121 active+clean; 649 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 2.6 MiB/s wr, 19 op/s 2024-09-14T03:54:42.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:42 smithi070 ceph-mon[96704]: osdmap e457: 8 total, 8 up, 8 in 2024-09-14T03:54:42.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:42 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-14T03:54:42.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:42 smithi070 ceph-mon[96704]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-14T03:54:42.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:42 smithi070 ceph-mon[96704]: Upgrade: osd.4 is safe to restart 2024-09-14T03:54:42.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:42 smithi070 ceph-mon[96704]: pgmap v279: 121 pgs: 121 active+clean; 649 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 2.6 MiB/s wr, 19 op/s 2024-09-14T03:54:43.236 INFO:tasks.rados.rados.0.smithi070.stdout:1898: done (3 left) 2024-09-14T03:54:43.236 INFO:tasks.rados.rados.0.smithi070.stdout:1899: done (2 left) 2024-09-14T03:54:43.236 INFO:tasks.rados.rados.0.smithi070.stdout:1901: done (1 left) 2024-09-14T03:54:43.236 INFO:tasks.rados.rados.0.smithi070.stdout:1902: done (0 left) 2024-09-14T03:54:43.236 INFO:tasks.rados.rados.0.smithi070.stdout:1903: copy_from oid 13 from oid 27 current snap is 205 2024-09-14T03:54:43.237 INFO:tasks.rados.rados.0.smithi070.stdout:1904: copy_from oid 32 from oid 49 current snap is 205 2024-09-14T03:54:43.237 INFO:tasks.rados.rados.0.smithi070.stdout:1905: snap_create 2024-09-14T03:54:43.279 INFO:tasks.rados.rados.0.smithi070.stdout:1904: finishing copy_from racing read to smithi07085154-32 2024-09-14T03:54:43.292 INFO:tasks.rados.rados.0.smithi070.stdout:1905: done (2 left) 2024-09-14T03:54:43.292 INFO:tasks.rados.rados.0.smithi070.stdout:1906: read oid 5 snap -1 2024-09-14T03:54:43.292 INFO:tasks.rados.rados.0.smithi070.stdout:1906: expect (ObjNum 516 snap 173 seq_num 516) 2024-09-14T03:54:43.292 INFO:tasks.rados.rados.0.smithi070.stdout:1907: snap_remove snap 142 2024-09-14T03:54:43.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:54:43 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:54:43] "GET /metrics HTTP/1.1" 200 36326 "" "Prometheus/2.51.0" 2024-09-14T03:54:44.035 INFO:tasks.rados.rados.0.smithi070.stdout:1907: done (3 left) 2024-09-14T03:54:44.035 INFO:tasks.rados.rados.0.smithi070.stdout:1908: read oid 46 snap 173 2024-09-14T03:54:44.036 INFO:tasks.rados.rados.0.smithi070.stdout:1908: expect (ObjNum 384 snap 130 seq_num 384) 2024-09-14T03:54:44.036 INFO:tasks.rados.rados.0.smithi070.stdout:1903: finishing copy_from racing read to smithi07085154-13 2024-09-14T03:54:44.036 INFO:tasks.rados.rados.0.smithi070.stdout:1904: finishing copy_from to smithi07085154-32 2024-09-14T03:54:44.036 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 1571 (ObjNum 551 snap 185 seq_num 551) dirty exists 2024-09-14T03:54:44.036 INFO:tasks.rados.rados.0.smithi070.stdout:1903: finishing copy_from to smithi07085154-13 2024-09-14T03:54:44.036 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 2022 (ObjNum 459 snap 155 seq_num 459) dirty exists 2024-09-14T03:54:44.036 INFO:tasks.rados.rados.0.smithi070.stdout:1906: expect (ObjNum 516 snap 173 seq_num 516) 2024-09-14T03:54:44.109 INFO:tasks.rados.rados.0.smithi070.stdout:1908: expect (ObjNum 384 snap 130 seq_num 384) 2024-09-14T03:54:44.241 INFO:tasks.rados.rados.0.smithi070.stdout:1903: done (3 left) 2024-09-14T03:54:44.241 INFO:tasks.rados.rados.0.smithi070.stdout:1904: done (2 left) 2024-09-14T03:54:44.241 INFO:tasks.rados.rados.0.smithi070.stdout:1906: done (1 left) 2024-09-14T03:54:44.241 INFO:tasks.rados.rados.0.smithi070.stdout:1908: done (0 left) 2024-09-14T03:54:44.241 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 6 current snap is 206 2024-09-14T03:54:44.242 INFO:tasks.rados.rados.0.smithi070.stdout:1909: seq_num 612 ranges {0=655360} 2024-09-14T03:54:44.244 INFO:tasks.rados.rados.0.smithi070.stdout:1909: writing smithi07085154-6 from 0 to 655360 tid 1 2024-09-14T03:54:44.244 INFO:tasks.rados.rados.0.smithi070.stdout:1910: snap_create 2024-09-14T03:54:44.248 INFO:tasks.rados.rados.0.smithi070.stdout:1909: finishing write tid 1 to smithi07085154-6 2024-09-14T03:54:44.248 INFO:tasks.rados.rados.0.smithi070.stdout:1909: finishing write tid 2 to smithi07085154-6 2024-09-14T03:54:44.249 INFO:tasks.rados.rados.0.smithi070.stdout:1909: finishing write tid 3 to smithi07085154-6 2024-09-14T03:54:44.249 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 1685 (ObjNum 612 snap 206 seq_num 612) dirty exists 2024-09-14T03:54:44.249 INFO:tasks.rados.rados.0.smithi070.stdout:1909: left oid 6 (ObjNum 612 snap 206 seq_num 612) 2024-09-14T03:54:44.521 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:54:44 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:54:44.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.4\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.94\", device_class=\"ssd\", hostname=\"smithi094\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.94\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.94\", device_class=\"ssd\", hostname=\"smithi094\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.94\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:44.522 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:44 smithi094 ceph-mon[86597]: osdmap e458: 8 total, 8 up, 8 in 2024-09-14T03:54:44.522 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:44 smithi094 ceph-mon[86597]: Upgrade: Updating osd.4 2024-09-14T03:54:44.522 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:44 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:44.522 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:44 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-14T03:54:44.522 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:44 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:54:44.522 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:44 smithi094 ceph-mon[86597]: Deploying daemon osd.4 on smithi094 2024-09-14T03:54:44.522 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:44 smithi094 ceph-mon[86597]: osdmap e459: 8 total, 8 up, 8 in 2024-09-14T03:54:44.522 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:44 smithi094 ceph-mon[86597]: pgmap v282: 121 pgs: 121 active+clean; 657 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 2.6 MiB/s wr, 13 op/s 2024-09-14T03:54:44.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:44 smithi070 ceph-mon[96704]: osdmap e458: 8 total, 8 up, 8 in 2024-09-14T03:54:44.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:44 smithi070 ceph-mon[96704]: Upgrade: Updating osd.4 2024-09-14T03:54:44.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:44 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:44.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:44 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-14T03:54:44.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:44 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:54:44.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:44 smithi070 ceph-mon[96704]: Deploying daemon osd.4 on smithi094 2024-09-14T03:54:44.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:44 smithi070 ceph-mon[96704]: osdmap e459: 8 total, 8 up, 8 in 2024-09-14T03:54:44.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:44 smithi070 ceph-mon[96704]: pgmap v282: 121 pgs: 121 active+clean; 657 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 2.6 MiB/s wr, 13 op/s 2024-09-14T03:54:44.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:44 smithi070 ceph-mon[98640]: osdmap e458: 8 total, 8 up, 8 in 2024-09-14T03:54:44.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:44 smithi070 ceph-mon[98640]: Upgrade: Updating osd.4 2024-09-14T03:54:44.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:44 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:44.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:44 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-14T03:54:44.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:44 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:54:44.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:44 smithi070 ceph-mon[98640]: Deploying daemon osd.4 on smithi094 2024-09-14T03:54:44.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:44 smithi070 ceph-mon[98640]: osdmap e459: 8 total, 8 up, 8 in 2024-09-14T03:54:44.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:44 smithi070 ceph-mon[98640]: pgmap v282: 121 pgs: 121 active+clean; 657 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 2.6 MiB/s wr, 13 op/s 2024-09-14T03:54:45.041 INFO:tasks.rados.rados.0.smithi070.stdout:1909: done (1 left) 2024-09-14T03:54:45.041 INFO:tasks.rados.rados.0.smithi070.stdout:1910: done (0 left) 2024-09-14T03:54:45.041 INFO:tasks.rados.rados.0.smithi070.stdout:1911: rmattr oid 25 current snap is 207 2024-09-14T03:54:45.041 INFO:tasks.rados.rados.0.smithi070.stdout:1912: copy_from oid 11 from oid 23 current snap is 207 2024-09-14T03:54:45.041 INFO:tasks.rados.rados.0.smithi070.stdout:1913: rollback oid 39 current snap is 207 2024-09-14T03:54:45.042 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 39 to 184 2024-09-14T03:54:45.042 INFO:tasks.rados.rados.0.smithi070.stdout:1914: read oid 18 snap -1 2024-09-14T03:54:45.042 INFO:tasks.rados.rados.0.smithi070.stdout:1914: expect (ObjNum 596 snap 199 seq_num 596) 2024-09-14T03:54:45.042 INFO:tasks.rados.rados.0.smithi070.stdout:1915: setattr oid 29 current snap is 207 2024-09-14T03:54:45.042 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 35 current snap is 207 2024-09-14T03:54:45.042 INFO:tasks.rados.rados.0.smithi070.stdout:1916: seq_num 613 ranges {0=671744} 2024-09-14T03:54:45.047 INFO:tasks.rados.rados.0.smithi070.stdout:1916: writing smithi07085154-35 from 0 to 671744 tid 1 2024-09-14T03:54:45.047 INFO:tasks.rados.rados.0.smithi070.stdout:1917: delete oid 16 current snap is 207 2024-09-14T03:54:45.047 INFO:tasks.rados.rados.0.smithi070.stdout:1913: finishing rollback tid 1 to smithi07085154-39 2024-09-14T03:54:45.047 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 39 v 332 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-14T03:54:45.047 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 1686 (ObjNum 583 snap 194 seq_num 583) dirty exists 2024-09-14T03:54:45.048 INFO:tasks.rados.rados.0.smithi070.stdout:1912: finishing copy_from to smithi07085154-11 2024-09-14T03:54:45.048 INFO:tasks.rados.rados.0.smithi070.stdout:1912: got expected ENOENT (src dne) 2024-09-14T03:54:45.048 INFO:tasks.rados.rados.0.smithi070.stdout:1912: finishing copy_from racing read to smithi07085154-11 2024-09-14T03:54:45.049 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 1680 (ObjNum 313 snap 105 seq_num 313) dirty exists 2024-09-14T03:54:45.054 INFO:tasks.rados.rados.0.smithi070.stdout:1916: finishing write tid 1 to smithi07085154-35 2024-09-14T03:54:45.054 INFO:tasks.rados.rados.0.smithi070.stdout:1911: done (6 left) 2024-09-14T03:54:45.054 INFO:tasks.rados.rados.0.smithi070.stdout:1912: done (5 left) 2024-09-14T03:54:45.054 INFO:tasks.rados.rados.0.smithi070.stdout:1913: done (4 left) 2024-09-14T03:54:45.054 INFO:tasks.rados.rados.0.smithi070.stdout:1915: done (3 left) 2024-09-14T03:54:45.055 INFO:tasks.rados.rados.0.smithi070.stdout:1917: done (2 left) 2024-09-14T03:54:45.055 INFO:tasks.rados.rados.0.smithi070.stdout:1918: read oid 41 snap -1 2024-09-14T03:54:45.055 INFO:tasks.rados.rados.0.smithi070.stdout:1918: expect (ObjNum 588 snap 195 seq_num 588) 2024-09-14T03:54:45.055 INFO:tasks.rados.rados.0.smithi070.stdout:1919: read oid 36 snap -1 2024-09-14T03:54:45.055 INFO:tasks.rados.rados.0.smithi070.stdout:1919: expect (ObjNum 590 snap 195 seq_num 590) 2024-09-14T03:54:45.055 INFO:tasks.rados.rados.0.smithi070.stdout:1920: snap_remove snap 121 2024-09-14T03:54:45.999 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:54:45 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:54:45.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:45.999 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:54:45 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:54:45.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:46.047 INFO:tasks.rados.rados.0.smithi070.stdout:1920: done (4 left) 2024-09-14T03:54:46.047 INFO:tasks.rados.rados.0.smithi070.stdout:1921: rollback oid 45 current snap is 207 2024-09-14T03:54:46.047 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 45 to 176 2024-09-14T03:54:46.047 INFO:tasks.rados.rados.0.smithi070.stdout:1916: finishing write tid 2 to smithi07085154-35 2024-09-14T03:54:46.047 INFO:tasks.rados.rados.0.smithi070.stdout:1916: finishing write tid 3 to smithi07085154-35 2024-09-14T03:54:46.048 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 35 v 41 (ObjNum 613 snap 207 seq_num 613) dirty exists 2024-09-14T03:54:46.048 INFO:tasks.rados.rados.0.smithi070.stdout:1916: left oid 35 (ObjNum 613 snap 207 seq_num 613) 2024-09-14T03:54:46.048 INFO:tasks.rados.rados.0.smithi070.stdout:1914: expect (ObjNum 596 snap 199 seq_num 596) 2024-09-14T03:54:46.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:46 smithi094 ceph-mon[86597]: osdmap e460: 8 total, 8 up, 8 in 2024-09-14T03:54:46.338 INFO:tasks.rados.rados.0.smithi070.stdout:1918: expect (ObjNum 588 snap 195 seq_num 588) 2024-09-14T03:54:46.451 INFO:tasks.rados.rados.0.smithi070.stdout:1919: expect (ObjNum 590 snap 195 seq_num 590) 2024-09-14T03:54:46.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:46 smithi070 ceph-mon[96704]: osdmap e460: 8 total, 8 up, 8 in 2024-09-14T03:54:46.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:46 smithi070 ceph-mon[98640]: osdmap e460: 8 total, 8 up, 8 in 2024-09-14T03:54:47.026 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:54:46 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:54:46.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:47.027 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:46 smithi094 systemd[1]: Stopping Ceph osd.4 for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:54:47.027 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:46 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4[36388]: 2024-09-14T03:54:46.955+0000 7f986ca76640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:54:47.027 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:46 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4[36388]: 2024-09-14T03:54:46.955+0000 7f986ca76640 -1 osd.4 461 *** Got signal Terminated *** 2024-09-14T03:54:47.027 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:46 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4[36388]: 2024-09-14T03:54:46.955+0000 7f986ca76640 -1 osd.4 461 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T03:54:47.468 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:47 smithi094 ceph-mon[86597]: osdmap e461: 8 total, 8 up, 8 in 2024-09-14T03:54:47.469 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:47 smithi094 ceph-mon[86597]: pgmap v285: 121 pgs: 121 active+clean; 670 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 27 MiB/s rd, 4.2 MiB/s wr, 8 op/s 2024-09-14T03:54:47.469 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:47 smithi094 ceph-mon[86597]: osd.4 marked itself down and dead 2024-09-14T03:54:47.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:47 smithi070 ceph-mon[96704]: osdmap e461: 8 total, 8 up, 8 in 2024-09-14T03:54:47.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:47 smithi070 ceph-mon[96704]: pgmap v285: 121 pgs: 121 active+clean; 670 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 27 MiB/s rd, 4.2 MiB/s wr, 8 op/s 2024-09-14T03:54:47.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:47 smithi070 ceph-mon[96704]: osd.4 marked itself down and dead 2024-09-14T03:54:47.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:47 smithi070 ceph-mon[98640]: osdmap e461: 8 total, 8 up, 8 in 2024-09-14T03:54:47.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:47 smithi070 ceph-mon[98640]: pgmap v285: 121 pgs: 121 active+clean; 670 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 27 MiB/s rd, 4.2 MiB/s wr, 8 op/s 2024-09-14T03:54:47.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:47 smithi070 ceph-mon[98640]: osd.4 marked itself down and dead 2024-09-14T03:54:47.776 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:47 smithi094 podman[91496]: 2024-09-14 03:54:47.46785081 +0000 UTC m=+0.608204793 container died 52b64527e24c9a88ce217e56decae58ba9891931b6d823792d5290eb2a1974c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-eb16b97, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-14T03:54:48.198 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:48 smithi094 ceph-mon[86597]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:54:48.199 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:48 smithi094 ceph-mon[86597]: osdmap e462: 8 total, 7 up, 8 in 2024-09-14T03:54:48.199 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:48 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:48.199 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:48 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:48.199 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:47 smithi094 podman[91496]: 2024-09-14 03:54:47.962872746 +0000 UTC m=+1.103226726 container cleanup 52b64527e24c9a88ce217e56decae58ba9891931b6d823792d5290eb2a1974c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=quincy-eb16b97, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-14T03:54:48.199 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:47 smithi094 bash[91496]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4 2024-09-14T03:54:48.361 INFO:tasks.rados.rados.0.smithi070.stdout:1921: finishing rollback tid 0 to smithi07085154-45 2024-09-14T03:54:48.361 INFO:tasks.rados.rados.0.smithi070.stdout:1921: finishing rollback tid 1 to smithi07085154-45 2024-09-14T03:54:48.361 INFO:tasks.rados.rados.0.smithi070.stdout:1921: finishing rollback tid 2 to smithi07085154-45 2024-09-14T03:54:48.361 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 45 v 1939 (ObjNum 312 snap 105 seq_num 312) dirty exists 2024-09-14T03:54:48.361 INFO:tasks.rados.rados.0.smithi070.stdout:1914: done (4 left) 2024-09-14T03:54:48.361 INFO:tasks.rados.rados.0.smithi070.stdout:1916: done (3 left) 2024-09-14T03:54:48.361 INFO:tasks.rados.rados.0.smithi070.stdout:1918: done (2 left) 2024-09-14T03:54:48.361 INFO:tasks.rados.rados.0.smithi070.stdout:1919: done (1 left) 2024-09-14T03:54:48.361 INFO:tasks.rados.rados.0.smithi070.stdout:1921: done (0 left) 2024-09-14T03:54:48.362 INFO:tasks.rados.rados.0.smithi070.stdout:1922: read oid 10 snap -1 2024-09-14T03:54:48.362 INFO:tasks.rados.rados.0.smithi070.stdout:1922: expect (ObjNum 508 snap 166 seq_num 508) 2024-09-14T03:54:48.362 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 44 current snap is 207 2024-09-14T03:54:48.362 INFO:tasks.rados.rados.0.smithi070.stdout:1923: seq_num 615 ranges {2588672=704512} 2024-09-14T03:54:48.379 INFO:tasks.rados.rados.0.smithi070.stdout:1923: writing smithi07085154-44 from 2588672 to 3293184 tid 1 2024-09-14T03:54:48.379 INFO:tasks.rados.rados.0.smithi070.stdout:1924: rollback oid 40 current snap is 207 2024-09-14T03:54:48.379 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 40 to 126 2024-09-14T03:54:48.379 INFO:tasks.rados.rados.0.smithi070.stdout:1922: expect (ObjNum 508 snap 166 seq_num 508) 2024-09-14T03:54:48.521 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:48 smithi094 podman[91518]: 2024-09-14 03:54:48.197976418 +0000 UTC m=+0.730366469 container remove 52b64527e24c9a88ce217e56decae58ba9891931b6d823792d5290eb2a1974c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4, RELEASE=quincy-eb16b97, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-14T03:54:48.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:48 smithi070 ceph-mon[96704]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:54:48.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:48 smithi070 ceph-mon[96704]: osdmap e462: 8 total, 7 up, 8 in 2024-09-14T03:54:48.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:48 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:48.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:48 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:48.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:48 smithi070 ceph-mon[98640]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:54:48.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:48 smithi070 ceph-mon[98640]: osdmap e462: 8 total, 7 up, 8 in 2024-09-14T03:54:48.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:48 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:48.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:48 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:54:48.639 INFO:tasks.rados.rados.0.smithi070.stdout:1923: finishing write tid 1 to smithi07085154-44 2024-09-14T03:54:48.639 INFO:tasks.rados.rados.0.smithi070.stdout:1923: finishing write tid 2 to smithi07085154-44 2024-09-14T03:54:48.639 INFO:tasks.rados.rados.0.smithi070.stdout:1922: done (2 left) 2024-09-14T03:54:48.639 INFO:tasks.rados.rados.0.smithi070.stdout:1925: snap_remove snap 205 2024-09-14T03:54:48.777 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:48 smithi094 podman[91613]: 2024-09-14 03:54:48.423195738 +0000 UTC m=+0.015745337 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:54:48.777 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:48 smithi094 podman[91613]: 2024-09-14 03:54:48.541010785 +0000 UTC m=+0.133560413 container create 841cbc845a2d5fe135b94f9125f3b0231b3871df1069ae51cf82d2fbf78fa243 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True) 2024-09-14T03:54:48.777 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:48 smithi094 podman[91613]: 2024-09-14 03:54:48.752941273 +0000 UTC m=+0.345490857 container init 841cbc845a2d5fe135b94f9125f3b0231b3871df1069ae51cf82d2fbf78fa243 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-14T03:54:48.777 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:48 smithi094 podman[91613]: 2024-09-14 03:54:48.757683554 +0000 UTC m=+0.350233140 container start 841cbc845a2d5fe135b94f9125f3b0231b3871df1069ae51cf82d2fbf78fa243 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=main-a278231, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-14T03:54:49.063 INFO:tasks.rados.rados.0.smithi070.stdout:1925: done (2 left) 2024-09-14T03:54:49.063 INFO:tasks.rados.rados.0.smithi070.stdout:1926: rollback oid 35 current snap is 207 2024-09-14T03:54:49.063 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 35 to 174 2024-09-14T03:54:49.064 INFO:tasks.rados.rados.0.smithi070.stdout:1924: finishing rollback tid 1 to smithi07085154-40 2024-09-14T03:54:49.064 INFO:tasks.rados.rados.0.smithi070.stdout:1924: finishing rollback tid 2 to smithi07085154-40 2024-09-14T03:54:49.064 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 1941 (ObjNum 353 snap 116 seq_num 353) dirty exists 2024-09-14T03:54:49.064 INFO:tasks.rados.rados.0.smithi070.stdout:1923: finishing write tid 3 to smithi07085154-44 2024-09-14T03:54:49.064 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 1939 (ObjNum 615 snap 207 seq_num 615) dirty exists 2024-09-14T03:54:49.064 INFO:tasks.rados.rados.0.smithi070.stdout:1923: left oid 44 (ObjNum 615 snap 207 seq_num 615) 2024-09-14T03:54:49.064 INFO:tasks.rados.rados.0.smithi070.stdout:1923: done (2 left) 2024-09-14T03:54:49.064 INFO:tasks.rados.rados.0.smithi070.stdout:1924: done (1 left) 2024-09-14T03:54:49.064 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 1 current snap is 207 2024-09-14T03:54:49.064 INFO:tasks.rados.rados.0.smithi070.stdout:1927: seq_num 616 ranges {3063808=720896} 2024-09-14T03:54:49.084 INFO:tasks.rados.rados.0.smithi070.stdout:1927: writing smithi07085154-1 from 3063808 to 3784704 tid 1 2024-09-14T03:54:49.084 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 8 current snap is 207 2024-09-14T03:54:49.084 INFO:tasks.rados.rados.0.smithi070.stdout:1926: finishing rollback tid 0 to smithi07085154-35 2024-09-14T03:54:49.084 INFO:tasks.rados.rados.0.smithi070.stdout:1926: finishing rollback tid 1 to smithi07085154-35 2024-09-14T03:54:49.084 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 35 v 43 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-14T03:54:49.084 INFO:tasks.rados.rados.0.smithi070.stdout:1928: seq_num 617 ranges {573440=737280} 2024-09-14T03:54:49.092 INFO:tasks.rados.rados.0.smithi070.stdout:1928: writing smithi07085154-8 from 573440 to 1310720 tid 1 2024-09-14T03:54:49.092 INFO:tasks.rados.rados.0.smithi070.stdout:1926: done (2 left) 2024-09-14T03:54:49.092 INFO:tasks.rados.rados.0.smithi070.stdout:1929: snap_create 2024-09-14T03:54:49.092 INFO:tasks.rados.rados.0.smithi070.stdout:1927: finishing write tid 1 to smithi07085154-1 2024-09-14T03:54:49.092 INFO:tasks.rados.rados.0.smithi070.stdout:1927: finishing write tid 2 to smithi07085154-1 2024-09-14T03:54:49.092 INFO:tasks.rados.rados.0.smithi070.stdout:1927: finishing write tid 3 to smithi07085154-1 2024-09-14T03:54:49.092 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 1 v 1943 (ObjNum 616 snap 207 seq_num 616) dirty exists 2024-09-14T03:54:49.092 INFO:tasks.rados.rados.0.smithi070.stdout:1927: left oid 1 (ObjNum 616 snap 207 seq_num 616) 2024-09-14T03:54:49.095 INFO:tasks.rados.rados.0.smithi070.stdout:1928: finishing write tid 1 to smithi07085154-8 2024-09-14T03:54:49.095 INFO:tasks.rados.rados.0.smithi070.stdout:1928: finishing write tid 2 to smithi07085154-8 2024-09-14T03:54:49.096 INFO:tasks.rados.rados.0.smithi070.stdout:1928: finishing write tid 3 to smithi07085154-8 2024-09-14T03:54:49.096 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 8 v 1686 (ObjNum 617 snap 207 seq_num 617) dirty exists 2024-09-14T03:54:49.096 INFO:tasks.rados.rados.0.smithi070.stdout:1928: left oid 8 (ObjNum 617 snap 207 seq_num 617) 2024-09-14T03:54:49.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:49 smithi094 ceph-mon[86597]: osdmap e463: 8 total, 7 up, 8 in 2024-09-14T03:54:49.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:49 smithi094 ceph-mon[86597]: pgmap v288: 121 pgs: 5 activating+undersized, 5 peering, 10 active+undersized, 5 stale+active+clean, 2 active+undersized+degraded, 94 active+clean; 694 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 9.9 MiB/s wr, 15 op/s; 41/1101 objects degraded (3.724%) 2024-09-14T03:54:49.277 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:48 smithi094 podman[91613]: 2024-09-14 03:54:48.852919653 +0000 UTC m=+0.445469249 container attach 841cbc845a2d5fe135b94f9125f3b0231b3871df1069ae51cf82d2fbf78fa243 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=main-a278231) 2024-09-14T03:54:49.277 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:48 smithi094 podman[91613]: 2024-09-14 03:54:48.943315363 +0000 UTC m=+0.535864962 container died 841cbc845a2d5fe135b94f9125f3b0231b3871df1069ae51cf82d2fbf78fa243 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=main-a278231, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-14T03:54:49.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:49 smithi070 ceph-mon[96704]: osdmap e463: 8 total, 7 up, 8 in 2024-09-14T03:54:49.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:49 smithi070 ceph-mon[96704]: pgmap v288: 121 pgs: 5 activating+undersized, 5 peering, 10 active+undersized, 5 stale+active+clean, 2 active+undersized+degraded, 94 active+clean; 694 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 9.9 MiB/s wr, 15 op/s; 41/1101 objects degraded (3.724%) 2024-09-14T03:54:49.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:49 smithi070 ceph-mon[98640]: osdmap e463: 8 total, 7 up, 8 in 2024-09-14T03:54:49.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:49 smithi070 ceph-mon[98640]: pgmap v288: 121 pgs: 5 activating+undersized, 5 peering, 10 active+undersized, 5 stale+active+clean, 2 active+undersized+degraded, 94 active+clean; 694 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 9.9 MiB/s wr, 15 op/s; 41/1101 objects degraded (3.724%) 2024-09-14T03:54:49.991 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:49 smithi094 podman[91656]: 2024-09-14 03:54:49.557789914 +0000 UTC m=+0.608915519 container remove 841cbc845a2d5fe135b94f9125f3b0231b3871df1069ae51cf82d2fbf78fa243 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=main-a278231, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-14T03:54:49.991 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:49 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.4.service: Deactivated successfully. 2024-09-14T03:54:49.991 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:49 smithi094 systemd[1]: Stopped Ceph osd.4 for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:54:49.991 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:49 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.4.service: Consumed 16.924s CPU time. 2024-09-14T03:54:49.991 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:49 smithi094 systemd[1]: Starting Ceph osd.4 for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:54:50.068 INFO:tasks.rados.rados.0.smithi070.stdout:1927: done (2 left) 2024-09-14T03:54:50.068 INFO:tasks.rados.rados.0.smithi070.stdout:1928: done (1 left) 2024-09-14T03:54:50.068 INFO:tasks.rados.rados.0.smithi070.stdout:1929: done (0 left) 2024-09-14T03:54:50.068 INFO:tasks.rados.rados.0.smithi070.stdout:1930: delete oid 50 current snap is 208 2024-09-14T03:54:50.070 INFO:tasks.rados.rados.0.smithi070.stdout:1930: done (0 left) 2024-09-14T03:54:50.070 INFO:tasks.rados.rados.0.smithi070.stdout:1931: rmattr oid 47 current snap is 208 2024-09-14T03:54:50.070 INFO:tasks.rados.rados.0.smithi070.stdout:1932: delete oid 15 current snap is 208 2024-09-14T03:54:50.073 INFO:tasks.rados.rados.0.smithi070.stdout:1932: done (1 left) 2024-09-14T03:54:50.073 INFO:tasks.rados.rados.0.smithi070.stdout:1933: snap_remove snap 196 2024-09-14T03:54:50.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:50 smithi094 ceph-mon[86597]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T03:54:50.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:50 smithi094 ceph-mon[86597]: Health check failed: Degraded data redundancy: 41/1101 objects degraded (3.724%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T03:54:50.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:50 smithi094 ceph-mon[86597]: osdmap e464: 8 total, 7 up, 8 in 2024-09-14T03:54:50.277 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:54:50 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:54:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:50.277 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:49 smithi094 podman[91760]: 2024-09-14 03:54:49.892686725 +0000 UTC m=+0.015836396 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:54:50.278 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:50 smithi094 podman[91760]: 2024-09-14 03:54:50.014266331 +0000 UTC m=+0.137415987 container create 55b88a9dd87900c0da313b6d7623da464be04e1c41d3fe2616f5df563f8cf7a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4-activate, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=main-a278231, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T03:54:50.278 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:50 smithi094 podman[91760]: 2024-09-14 03:54:50.22103353 +0000 UTC m=+0.344183188 container init 55b88a9dd87900c0da313b6d7623da464be04e1c41d3fe2616f5df563f8cf7a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, RELEASE=main-a278231, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-14T03:54:50.278 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:50 smithi094 podman[91760]: 2024-09-14 03:54:50.225797203 +0000 UTC m=+0.348946864 container start 55b88a9dd87900c0da313b6d7623da464be04e1c41d3fe2616f5df563f8cf7a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=main-a278231, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T03:54:50.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:50 smithi070 ceph-mon[98640]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T03:54:50.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:50 smithi070 ceph-mon[98640]: Health check failed: Degraded data redundancy: 41/1101 objects degraded (3.724%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T03:54:50.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:50 smithi070 ceph-mon[98640]: osdmap e464: 8 total, 7 up, 8 in 2024-09-14T03:54:50.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:50 smithi070 ceph-mon[96704]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T03:54:50.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:50 smithi070 ceph-mon[96704]: Health check failed: Degraded data redundancy: 41/1101 objects degraded (3.724%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T03:54:50.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:50 smithi070 ceph-mon[96704]: osdmap e464: 8 total, 7 up, 8 in 2024-09-14T03:54:50.777 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:50 smithi094 podman[91760]: 2024-09-14 03:54:50.304266696 +0000 UTC m=+0.427416367 container attach 55b88a9dd87900c0da313b6d7623da464be04e1c41d3fe2616f5df563f8cf7a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4-activate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, RELEASE=main-a278231, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-14T03:54:50.777 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:50 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4-activate[91776]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:54:50.777 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:50 smithi094 bash[91760]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:54:50.777 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:50 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4-activate[91776]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:54:50.777 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:50 smithi094 bash[91760]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:54:51.076 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 47 v 1676 (ObjNum 592 snap 195 seq_num 592) dirty exists 2024-09-14T03:54:51.076 INFO:tasks.rados.rados.0.smithi070.stdout:1931: done (1 left) 2024-09-14T03:54:51.076 INFO:tasks.rados.rados.0.smithi070.stdout:1933: done (0 left) 2024-09-14T03:54:51.076 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 23 current snap is 208 2024-09-14T03:54:51.076 INFO:tasks.rados.rados.0.smithi070.stdout:1934: seq_num 620 ranges {0=794624} 2024-09-14T03:54:51.081 INFO:tasks.rados.rados.0.smithi070.stdout:1934: writing smithi07085154-23 from 0 to 794624 tid 1 2024-09-14T03:54:51.082 INFO:tasks.rados.rados.0.smithi070.stdout:1935: read oid 15 snap -1 2024-09-14T03:54:51.082 INFO:tasks.rados.rados.0.smithi070.stdout:1935: expect deleted 2024-09-14T03:54:51.082 INFO:tasks.rados.rados.0.smithi070.stdout:1936: setattr oid 36 current snap is 208 2024-09-14T03:54:51.083 INFO:tasks.rados.rados.0.smithi070.stdout:1937: snap_create 2024-09-14T03:54:51.088 INFO:tasks.rados.rados.0.smithi070.stdout:1934: finishing write tid 1 to smithi07085154-23 2024-09-14T03:54:51.088 INFO:tasks.rados.rados.0.smithi070.stdout:1934: finishing write tid 2 to smithi07085154-23 2024-09-14T03:54:51.089 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 36 v 1944 (ObjNum 590 snap 195 seq_num 590) dirty exists 2024-09-14T03:54:51.089 INFO:tasks.rados.rados.0.smithi070.stdout:1934: finishing write tid 3 to smithi07085154-23 2024-09-14T03:54:51.089 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 1565 (ObjNum 620 snap 208 seq_num 620) dirty exists 2024-09-14T03:54:51.089 INFO:tasks.rados.rados.0.smithi070.stdout:1934: left oid 23 (ObjNum 620 snap 208 seq_num 620) 2024-09-14T03:54:51.115 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:51 smithi094 ceph-mon[86597]: pgmap v291: 121 pgs: 5 activating+undersized, 5 peering, 14 active+undersized, 3 stale+active+clean, 6 active+undersized+degraded, 88 active+clean; 695 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.7 MiB/s rd, 5.9 MiB/s wr, 14 op/s; 63/1104 objects degraded (5.707%) 2024-09-14T03:54:51.115 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:51 smithi094 ceph-mon[86597]: osdmap e465: 8 total, 7 up, 8 in 2024-09-14T03:54:51.115 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:50 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4-activate[91776]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-14T03:54:51.115 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:50 smithi094 bash[91760]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-14T03:54:51.115 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:50 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4-activate[91776]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-14T03:54:51.115 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:50 smithi094 bash[91760]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-14T03:54:51.116 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4-activate[91776]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-14T03:54:51.116 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:51 smithi094 bash[91760]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-14T03:54:51.476 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4-activate[91776]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-14T03:54:51.477 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:51 smithi094 bash[91760]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-14T03:54:51.477 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4-activate[91776]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-14T03:54:51.477 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:51 smithi094 bash[91760]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-14T03:54:51.477 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4-activate[91776]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-14T03:54:51.477 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:51 smithi094 bash[91760]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-14T03:54:51.477 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:51 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4-activate[91776]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-14T03:54:51.477 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:51 smithi094 bash[91760]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-14T03:54:51.477 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:51 smithi094 podman[91760]: 2024-09-14 03:54:51.141768573 +0000 UTC m=+1.264918240 container died 55b88a9dd87900c0da313b6d7623da464be04e1c41d3fe2616f5df563f8cf7a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4-activate, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-14T03:54:51.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:51 smithi070 ceph-mon[98640]: pgmap v291: 121 pgs: 5 activating+undersized, 5 peering, 14 active+undersized, 3 stale+active+clean, 6 active+undersized+degraded, 88 active+clean; 695 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.7 MiB/s rd, 5.9 MiB/s wr, 14 op/s; 63/1104 objects degraded (5.707%) 2024-09-14T03:54:51.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:51 smithi070 ceph-mon[98640]: osdmap e465: 8 total, 7 up, 8 in 2024-09-14T03:54:51.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:51 smithi070 ceph-mon[96704]: pgmap v291: 121 pgs: 5 activating+undersized, 5 peering, 14 active+undersized, 3 stale+active+clean, 6 active+undersized+degraded, 88 active+clean; 695 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.7 MiB/s rd, 5.9 MiB/s wr, 14 op/s; 63/1104 objects degraded (5.707%) 2024-09-14T03:54:51.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:51 smithi070 ceph-mon[96704]: osdmap e465: 8 total, 7 up, 8 in 2024-09-14T03:54:52.076 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:51 smithi094 podman[91937]: 2024-09-14 03:54:51.792689252 +0000 UTC m=+0.645022016 container remove 55b88a9dd87900c0da313b6d7623da464be04e1c41d3fe2616f5df563f8cf7a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4-activate, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-14T03:54:52.076 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:52 smithi094 podman[92001]: 2024-09-14 03:54:51.912339623 +0000 UTC m=+0.016650526 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:54:52.076 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:52 smithi094 podman[92001]: 2024-09-14 03:54:52.03541225 +0000 UTC m=+0.139723153 container create 09f583ed6c7702dd334d1ec70d731d1ecc9590be5d663b896b6b21c81a35f481 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4, RELEASE=main-a278231, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:54:52.079 INFO:tasks.rados.rados.0.smithi070.stdout:1934: done (3 left) 2024-09-14T03:54:52.079 INFO:tasks.rados.rados.0.smithi070.stdout:1935: done (2 left) 2024-09-14T03:54:52.080 INFO:tasks.rados.rados.0.smithi070.stdout:1936: done (1 left) 2024-09-14T03:54:52.080 INFO:tasks.rados.rados.0.smithi070.stdout:1937: done (0 left) 2024-09-14T03:54:52.080 INFO:tasks.rados.rados.0.smithi070.stdout:1938: delete oid 2 current snap is 209 2024-09-14T03:54:52.086 INFO:tasks.rados.rados.0.smithi070.stdout:1938: done (0 left) 2024-09-14T03:54:52.086 INFO:tasks.rados.rados.0.smithi070.stdout:1939: copy_from oid 29 from oid 30 current snap is 209 2024-09-14T03:54:52.086 INFO:tasks.rados.rados.0.smithi070.stdout:1940: read oid 1 snap 195 2024-09-14T03:54:52.086 INFO:tasks.rados.rados.0.smithi070.stdout:1940: expect (ObjNum 470 snap 160 seq_num 470) 2024-09-14T03:54:52.086 INFO:tasks.rados.rados.0.smithi070.stdout:1941: rollback oid 17 current snap is 209 2024-09-14T03:54:52.086 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 17 to 182 2024-09-14T03:54:52.087 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 49 current snap is 209 2024-09-14T03:54:52.087 INFO:tasks.rados.rados.0.smithi070.stdout:1942: seq_num 622 ranges {7405568=425984,7831552=401408} 2024-09-14T03:54:52.128 INFO:tasks.rados.rados.0.smithi070.stdout:1942: writing smithi07085154-49 from 7405568 to 7831552 tid 1 2024-09-14T03:54:52.130 INFO:tasks.rados.rados.0.smithi070.stdout:1942: writing smithi07085154-49 from 7831552 to 8232960 tid 2 2024-09-14T03:54:52.130 INFO:tasks.rados.rados.0.smithi070.stdout:1943: setattr oid 5 current snap is 209 2024-09-14T03:54:52.131 INFO:tasks.rados.rados.0.smithi070.stdout:1939: finishing copy_from to smithi07085154-29 2024-09-14T03:54:52.131 INFO:tasks.rados.rados.0.smithi070.stdout:1939: got expected ENOENT (src dne) 2024-09-14T03:54:52.131 INFO:tasks.rados.rados.0.smithi070.stdout:1939: finishing copy_from racing read to smithi07085154-29 2024-09-14T03:54:52.131 INFO:tasks.rados.rados.0.smithi070.stdout:1941: finishing rollback tid 0 to smithi07085154-17 2024-09-14T03:54:52.131 INFO:tasks.rados.rados.0.smithi070.stdout:1941: finishing rollback tid 1 to smithi07085154-17 2024-09-14T03:54:52.131 INFO:tasks.rados.rados.0.smithi070.stdout:1941: finishing rollback tid 2 to smithi07085154-17 2024-09-14T03:54:52.131 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 1689 (ObjNum 544 snap 181 seq_num 544) dirty exists 2024-09-14T03:54:52.131 INFO:tasks.rados.rados.0.smithi070.stdout:1940: expect (ObjNum 470 snap 160 seq_num 470) 2024-09-14T03:54:52.247 INFO:tasks.rados.rados.0.smithi070.stdout:1942: finishing write tid 1 to smithi07085154-49 2024-09-14T03:54:52.247 INFO:tasks.rados.rados.0.smithi070.stdout:1942: finishing write tid 2 to smithi07085154-49 2024-09-14T03:54:52.247 INFO:tasks.rados.rados.0.smithi070.stdout:1942: finishing write tid 3 to smithi07085154-49 2024-09-14T03:54:52.247 INFO:tasks.rados.rados.0.smithi070.stdout:1942: finishing write tid 4 to smithi07085154-49 2024-09-14T03:54:52.247 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 49 v 1942 (ObjNum 622 snap 209 seq_num 622) dirty exists 2024-09-14T03:54:52.247 INFO:tasks.rados.rados.0.smithi070.stdout:1942: left oid 49 (ObjNum 622 snap 209 seq_num 622) 2024-09-14T03:54:52.247 INFO:tasks.rados.rados.0.smithi070.stdout:1939: done (4 left) 2024-09-14T03:54:52.247 INFO:tasks.rados.rados.0.smithi070.stdout:1940: done (3 left) 2024-09-14T03:54:52.247 INFO:tasks.rados.rados.0.smithi070.stdout:1941: done (2 left) 2024-09-14T03:54:52.247 INFO:tasks.rados.rados.0.smithi070.stdout:1942: done (1 left) 2024-09-14T03:54:52.247 INFO:tasks.rados.rados.0.smithi070.stdout:1944: read oid 41 snap 187 2024-09-14T03:54:52.247 INFO:tasks.rados.rados.0.smithi070.stdout:1944: expect (ObjNum 514 snap 171 seq_num 514) 2024-09-14T03:54:52.248 INFO:tasks.rados.rados.0.smithi070.stdout:1945: setattr oid 31 current snap is 209 2024-09-14T03:54:52.248 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 29 current snap is 209 2024-09-14T03:54:52.248 INFO:tasks.rados.rados.0.smithi070.stdout:1946: seq_num 623 ranges {450560=442368,892928=401408} 2024-09-14T03:54:52.253 INFO:tasks.rados.rados.0.smithi070.stdout:1946: writing smithi07085154-29 from 450560 to 892928 tid 1 2024-09-14T03:54:52.255 INFO:tasks.rados.rados.0.smithi070.stdout:1946: writing smithi07085154-29 from 892928 to 1294336 tid 2 2024-09-14T03:54:52.256 INFO:tasks.rados.rados.0.smithi070.stdout:1947: snap_create 2024-09-14T03:54:52.256 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 560 (ObjNum 516 snap 173 seq_num 516) dirty exists 2024-09-14T03:54:52.256 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 1690 (ObjNum 568 snap 192 seq_num 568) dirty exists 2024-09-14T03:54:52.258 INFO:tasks.rados.rados.0.smithi070.stdout:1944: expect (ObjNum 514 snap 171 seq_num 514) 2024-09-14T03:54:52.364 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:52 smithi094 ceph-mon[86597]: osdmap e466: 8 total, 7 up, 8 in 2024-09-14T03:54:52.364 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:52 smithi094 podman[92001]: 2024-09-14 03:54:52.271720513 +0000 UTC m=+0.376031417 container init 09f583ed6c7702dd334d1ec70d731d1ecc9590be5d663b896b6b21c81a35f481 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:54:52.364 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:52 smithi094 podman[92001]: 2024-09-14 03:54:52.276430296 +0000 UTC m=+0.380741192 container start 09f583ed6c7702dd334d1ec70d731d1ecc9590be5d663b896b6b21c81a35f481 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, RELEASE=main-a278231, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-09-14T03:54:52.364 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:52 smithi094 ceph-osd[92039]: -- 172.21.15.94:0/2926987770 <== mon.1 v2:172.21.15.94:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x556b1f717a40 con 0x556b1f6f5c00 2024-09-14T03:54:52.405 INFO:tasks.rados.rados.0.smithi070.stdout:1946: finishing write tid 1 to smithi07085154-29 2024-09-14T03:54:52.405 INFO:tasks.rados.rados.0.smithi070.stdout:1946: finishing write tid 2 to smithi07085154-29 2024-09-14T03:54:52.405 INFO:tasks.rados.rados.0.smithi070.stdout:1946: finishing write tid 3 to smithi07085154-29 2024-09-14T03:54:52.405 INFO:tasks.rados.rados.0.smithi070.stdout:1946: finishing write tid 4 to smithi07085154-29 2024-09-14T03:54:52.405 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 1683 (ObjNum 623 snap 209 seq_num 623) dirty exists 2024-09-14T03:54:52.405 INFO:tasks.rados.rados.0.smithi070.stdout:1946: left oid 29 (ObjNum 623 snap 209 seq_num 623) 2024-09-14T03:54:52.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:52 smithi070 ceph-mon[98640]: osdmap e466: 8 total, 7 up, 8 in 2024-09-14T03:54:52.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:52 smithi070 ceph-mon[96704]: osdmap e466: 8 total, 7 up, 8 in 2024-09-14T03:54:52.745 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:52 smithi094 bash[92001]: 09f583ed6c7702dd334d1ec70d731d1ecc9590be5d663b896b6b21c81a35f481 2024-09-14T03:54:52.745 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:52 smithi094 systemd[1]: Started Ceph osd.4 for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:54:53.084 INFO:tasks.rados.rados.0.smithi070.stdout:1943: done (4 left) 2024-09-14T03:54:53.084 INFO:tasks.rados.rados.0.smithi070.stdout:1944: done (3 left) 2024-09-14T03:54:53.084 INFO:tasks.rados.rados.0.smithi070.stdout:1945: done (2 left) 2024-09-14T03:54:53.084 INFO:tasks.rados.rados.0.smithi070.stdout:1946: done (1 left) 2024-09-14T03:54:53.084 INFO:tasks.rados.rados.0.smithi070.stdout:1947: done (0 left) 2024-09-14T03:54:53.084 INFO:tasks.rados.rados.0.smithi070.stdout:1948: delete oid 43 current snap is 210 2024-09-14T03:54:53.089 INFO:tasks.rados.rados.0.smithi070.stdout:1948: done (0 left) 2024-09-14T03:54:53.089 INFO:tasks.rados.rados.0.smithi070.stdout:1949: read oid 30 snap -1 2024-09-14T03:54:53.089 INFO:tasks.rados.rados.0.smithi070.stdout:1949: expect deleted 2024-09-14T03:54:53.089 INFO:tasks.rados.rados.0.smithi070.stdout:1950: rollback oid 41 current snap is 210 2024-09-14T03:54:53.089 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 41 to 187 2024-09-14T03:54:53.089 INFO:tasks.rados.rados.0.smithi070.stdout:1951: read oid 12 snap -1 2024-09-14T03:54:53.089 INFO:tasks.rados.rados.0.smithi070.stdout:1951: expect (ObjNum 534 snap 176 seq_num 534) 2024-09-14T03:54:53.089 INFO:tasks.rados.rados.0.smithi070.stdout:1952: rollback oid 20 current snap is 210 2024-09-14T03:54:53.090 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 20 to 179 2024-09-14T03:54:53.090 INFO:tasks.rados.rados.0.smithi070.stdout:1953: read oid 22 snap -1 2024-09-14T03:54:53.090 INFO:tasks.rados.rados.0.smithi070.stdout:1953: expect (ObjNum 584 snap 194 seq_num 584) 2024-09-14T03:54:53.090 INFO:tasks.rados.rados.0.smithi070.stdout:1954: snap_remove snap 200 2024-09-14T03:54:53.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:53 smithi094 ceph-mon[86597]: pgmap v293: 121 pgs: 5 activating+undersized, 5 peering, 23 active+undersized, 13 active+undersized+degraded, 75 active+clean; 704 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 685 KiB/s wr, 15 op/s; 168/1110 objects degraded (15.135%) 2024-09-14T03:54:53.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:53 smithi094 ceph-mon[86597]: osdmap e467: 8 total, 7 up, 8 in 2024-09-14T03:54:53.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:53 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:53.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:53 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:53.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:53 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:54:53.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:53 smithi070 ceph-mon[96704]: pgmap v293: 121 pgs: 5 activating+undersized, 5 peering, 23 active+undersized, 13 active+undersized+degraded, 75 active+clean; 704 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 685 KiB/s wr, 15 op/s; 168/1110 objects degraded (15.135%) 2024-09-14T03:54:53.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:53 smithi070 ceph-mon[96704]: osdmap e467: 8 total, 7 up, 8 in 2024-09-14T03:54:53.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:53 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:53.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:53 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:53.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:53 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:54:53.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:53 smithi070 ceph-mon[98640]: pgmap v293: 121 pgs: 5 activating+undersized, 5 peering, 23 active+undersized, 13 active+undersized+degraded, 75 active+clean; 704 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 685 KiB/s wr, 15 op/s; 168/1110 objects degraded (15.135%) 2024-09-14T03:54:53.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:53 smithi070 ceph-mon[98640]: osdmap e467: 8 total, 7 up, 8 in 2024-09-14T03:54:53.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:53 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:53.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:53 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:53.409 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:53 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:54:53.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:54:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:54:53] "GET /metrics HTTP/1.1" 200 36311 "" "Prometheus/2.51.0" 2024-09-14T03:54:54.026 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:53 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4[92016]: 2024-09-14T03:54:53.919+0000 7f6d5e487640 -1 Falling back to public interface 2024-09-14T03:54:54.040 INFO:tasks.rados.rados.0.smithi070.stdout:1954: done (5 left) 2024-09-14T03:54:54.040 INFO:tasks.rados.rados.0.smithi070.stdout:1955: copy_from oid 27 from oid 33 current snap is 210 2024-09-14T03:54:54.040 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 42 current snap is 210 2024-09-14T03:54:54.040 INFO:tasks.rados.rados.0.smithi070.stdout:1956: seq_num 625 ranges {6471680=475136,6946816=401408} 2024-09-14T03:54:54.079 INFO:tasks.rados.rados.0.smithi070.stdout:1956: writing smithi07085154-42 from 6471680 to 6946816 tid 1 2024-09-14T03:54:54.082 INFO:tasks.rados.rados.0.smithi070.stdout:1956: writing smithi07085154-42 from 6946816 to 7348224 tid 2 2024-09-14T03:54:54.082 INFO:tasks.rados.rados.0.smithi070.stdout:1950: finishing rollback tid 0 to smithi07085154-41 2024-09-14T03:54:54.082 INFO:tasks.rados.rados.0.smithi070.stdout:1952: finishing rollback tid 1 to smithi07085154-20 2024-09-14T03:54:54.082 INFO:tasks.rados.rados.0.smithi070.stdout:1950: finishing rollback tid 1 to smithi07085154-41 2024-09-14T03:54:54.082 INFO:tasks.rados.rados.0.smithi070.stdout:1952: finishing rollback tid 2 to smithi07085154-20 2024-09-14T03:54:54.083 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 20 v 1258 (ObjNum 524 snap 174 seq_num 524) dirty exists 2024-09-14T03:54:54.083 INFO:tasks.rados.rados.0.smithi070.stdout:1950: finishing rollback tid 2 to smithi07085154-41 2024-09-14T03:54:54.083 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 1689 (ObjNum 514 snap 171 seq_num 514) dirty exists 2024-09-14T03:54:54.083 INFO:tasks.rados.rados.0.smithi070.stdout:1953: expect (ObjNum 584 snap 194 seq_num 584) 2024-09-14T03:54:54.186 INFO:tasks.rados.rados.0.smithi070.stdout:1951: expect (ObjNum 534 snap 176 seq_num 534) 2024-09-14T03:54:54.431 INFO:tasks.rados.rados.0.smithi070.stdout:1956: finishing write tid 1 to smithi07085154-42 2024-09-14T03:54:54.431 INFO:tasks.rados.rados.0.smithi070.stdout:1956: finishing write tid 2 to smithi07085154-42 2024-09-14T03:54:54.431 INFO:tasks.rados.rados.0.smithi070.stdout:1956: finishing write tid 3 to smithi07085154-42 2024-09-14T03:54:54.431 INFO:tasks.rados.rados.0.smithi070.stdout:1949: done (6 left) 2024-09-14T03:54:54.431 INFO:tasks.rados.rados.0.smithi070.stdout:1950: done (5 left) 2024-09-14T03:54:54.431 INFO:tasks.rados.rados.0.smithi070.stdout:1951: done (4 left) 2024-09-14T03:54:54.431 INFO:tasks.rados.rados.0.smithi070.stdout:1952: done (3 left) 2024-09-14T03:54:54.431 INFO:tasks.rados.rados.0.smithi070.stdout:1953: done (2 left) 2024-09-14T03:54:54.431 INFO:tasks.rados.rados.0.smithi070.stdout:1957: read oid 47 snap -1 2024-09-14T03:54:54.431 INFO:tasks.rados.rados.0.smithi070.stdout:1957: expect (ObjNum 592 snap 195 seq_num 592) 2024-09-14T03:54:54.432 INFO:tasks.rados.rados.0.smithi070.stdout:1958: delete oid 28 current snap is 210 2024-09-14T03:54:54.432 INFO:tasks.rados.rados.0.smithi070.stdout:1955: finishing copy_from racing read to smithi07085154-27 2024-09-14T03:54:54.432 INFO:tasks.rados.rados.0.smithi070.stdout:1956: finishing write tid 4 to smithi07085154-42 2024-09-14T03:54:54.432 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 2094 (ObjNum 625 snap 210 seq_num 625) dirty exists 2024-09-14T03:54:54.432 INFO:tasks.rados.rados.0.smithi070.stdout:1956: left oid 42 (ObjNum 625 snap 210 seq_num 625) 2024-09-14T03:54:54.432 INFO:tasks.rados.rados.0.smithi070.stdout:1955: finishing copy_from to smithi07085154-27 2024-09-14T03:54:54.432 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 1936 (ObjNum 590 snap 195 seq_num 590) dirty exists 2024-09-14T03:54:54.432 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:54 smithi070 ceph-mon[96704]: osdmap e468: 8 total, 7 up, 8 in 2024-09-14T03:54:54.432 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:54 smithi070 ceph-mon[96704]: osdmap e469: 8 total, 7 up, 8 in 2024-09-14T03:54:54.433 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:54 smithi070 ceph-mon[98640]: osdmap e468: 8 total, 7 up, 8 in 2024-09-14T03:54:54.433 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:54 smithi070 ceph-mon[98640]: osdmap e469: 8 total, 7 up, 8 in 2024-09-14T03:54:54.433 INFO:tasks.rados.rados.0.smithi070.stdout:1955: done (3 left) 2024-09-14T03:54:54.433 INFO:tasks.rados.rados.0.smithi070.stdout:1956: done (2 left) 2024-09-14T03:54:54.433 INFO:tasks.rados.rados.0.smithi070.stdout:1958: done (1 left) 2024-09-14T03:54:54.433 INFO:tasks.rados.rados.0.smithi070.stdout:1959: copy_from oid 24 from oid 11 current snap is 210 2024-09-14T03:54:54.434 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 39 current snap is 210 2024-09-14T03:54:54.434 INFO:tasks.rados.rados.0.smithi070.stdout:1960: seq_num 627 ranges {0=507904,507904=401408} 2024-09-14T03:54:54.436 INFO:tasks.rados.rados.0.smithi070.stdout:1960: writing smithi07085154-39 from 0 to 507904 tid 1 2024-09-14T03:54:54.438 INFO:tasks.rados.rados.0.smithi070.stdout:1960: writing smithi07085154-39 from 507904 to 909312 tid 2 2024-09-14T03:54:54.438 INFO:tasks.rados.rados.0.smithi070.stdout:1961: copy_from oid 15 from oid 25 current snap is 210 2024-09-14T03:54:54.438 INFO:tasks.rados.rados.0.smithi070.stdout:1962: delete oid 7 current snap is 210 2024-09-14T03:54:54.442 INFO:tasks.rados.rados.0.smithi070.stdout:1962: done (4 left) 2024-09-14T03:54:54.442 INFO:tasks.rados.rados.0.smithi070.stdout:1963: read oid 17 snap -1 2024-09-14T03:54:54.442 INFO:tasks.rados.rados.0.smithi070.stdout:1963: expect (ObjNum 544 snap 181 seq_num 544) 2024-09-14T03:54:54.442 INFO:tasks.rados.rados.0.smithi070.stdout:1960: finishing write tid 1 to smithi07085154-39 2024-09-14T03:54:54.442 INFO:tasks.rados.rados.0.smithi070.stdout:1964: rmattr oid 10 current snap is 210 2024-09-14T03:54:54.443 INFO:tasks.rados.rados.0.smithi070.stdout:1964: done (5 left) 2024-09-14T03:54:54.443 INFO:tasks.rados.rados.0.smithi070.stdout:1965: read oid 41 snap 182 2024-09-14T03:54:54.443 INFO:tasks.rados.rados.0.smithi070.stdout:1965: expect (ObjNum 514 snap 171 seq_num 514) 2024-09-14T03:54:54.443 INFO:tasks.rados.rados.0.smithi070.stdout:1966: snap_remove snap 184 2024-09-14T03:54:54.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:54 smithi094 ceph-mon[86597]: osdmap e468: 8 total, 7 up, 8 in 2024-09-14T03:54:54.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:54 smithi094 ceph-mon[86597]: osdmap e469: 8 total, 7 up, 8 in 2024-09-14T03:54:54.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:54:54 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:54:54.147Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.4\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.94\", device_class=\"ssd\", hostname=\"smithi094\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.94\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.94\", device_class=\"ssd\", hostname=\"smithi094\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.94\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-14T03:54:54.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:54:54 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:54:54.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.4\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.94\", device_class=\"ssd\", hostname=\"smithi094\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.94\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.94\", device_class=\"ssd\", hostname=\"smithi094\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.94\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:55.089 INFO:tasks.rados.rados.0.smithi070.stdout:1966: done (6 left) 2024-09-14T03:54:55.090 INFO:tasks.rados.rados.0.smithi070.stdout:1967: snap_create 2024-09-14T03:54:55.090 INFO:tasks.rados.rados.0.smithi070.stdout:1960: finishing write tid 2 to smithi07085154-39 2024-09-14T03:54:55.090 INFO:tasks.rados.rados.0.smithi070.stdout:1960: finishing write tid 3 to smithi07085154-39 2024-09-14T03:54:55.090 INFO:tasks.rados.rados.0.smithi070.stdout:1960: finishing write tid 4 to smithi07085154-39 2024-09-14T03:54:55.090 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 39 v 335 (ObjNum 627 snap 210 seq_num 627) dirty exists 2024-09-14T03:54:55.090 INFO:tasks.rados.rados.0.smithi070.stdout:1960: left oid 39 (ObjNum 627 snap 210 seq_num 627) 2024-09-14T03:54:55.090 INFO:tasks.rados.rados.0.smithi070.stdout:1965: expect (ObjNum 514 snap 171 seq_num 514) 2024-09-14T03:54:55.249 INFO:tasks.rados.rados.0.smithi070.stdout:1959: finishing copy_from racing read to smithi07085154-24 2024-09-14T03:54:55.249 INFO:tasks.rados.rados.0.smithi070.stdout:1957: expect (ObjNum 592 snap 195 seq_num 592) 2024-09-14T03:54:55.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:55 smithi094 ceph-mon[86597]: pgmap v297: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 709 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 928 KiB/s wr, 15 op/s; 170/1122 objects degraded (15.152%) 2024-09-14T03:54:55.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:55 smithi094 ceph-mon[86597]: Health check update: Degraded data redundancy: 170/1122 objects degraded (15.152%), 14 pgs degraded (PG_DEGRADED) 2024-09-14T03:54:55.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:55 smithi094 ceph-mon[86597]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-14T03:54:55.479 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:55 smithi070 ceph-mon[96704]: pgmap v297: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 709 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 928 KiB/s wr, 15 op/s; 170/1122 objects degraded (15.152%) 2024-09-14T03:54:55.480 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:55 smithi070 ceph-mon[96704]: Health check update: Degraded data redundancy: 170/1122 objects degraded (15.152%), 14 pgs degraded (PG_DEGRADED) 2024-09-14T03:54:55.480 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:55 smithi070 ceph-mon[96704]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-14T03:54:55.480 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:55 smithi070 ceph-mon[98640]: pgmap v297: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 709 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 928 KiB/s wr, 15 op/s; 170/1122 objects degraded (15.152%) 2024-09-14T03:54:55.480 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:55 smithi070 ceph-mon[98640]: Health check update: Degraded data redundancy: 170/1122 objects degraded (15.152%), 14 pgs degraded (PG_DEGRADED) 2024-09-14T03:54:55.480 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:55 smithi070 ceph-mon[98640]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-14T03:54:55.480 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-09-14T03:54:55.973 INFO:tasks.rados.rados.0.smithi070.stdout:1959: finishing copy_from to smithi07085154-24 2024-09-14T03:54:55.973 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 1676 (ObjNum 403 snap 140 seq_num 403) dirty exists 2024-09-14T03:54:55.973 INFO:tasks.rados.rados.0.smithi070.stdout:1963: expect (ObjNum 544 snap 181 seq_num 544) 2024-09-14T03:54:56.026 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:54:55 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:54:55.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:56.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:54:55 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:54:55.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:56.046 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:54:56.047 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (4m) 33s ago 9m 21.2M - 0.27.0 11f11916f8cd 45e79e51074c 2024-09-14T03:54:56.047 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi094 *:3000 running (4m) 2m ago 9m 79.5M - 9.4.7 954c08fa6188 2240810d0368 2024-09-14T03:54:56.047 INFO:teuthology.orchestra.run.smithi070.stdout:mds.a smithi070 running (11m) 33s ago 11m 21.4M - 17.2.7-1629-geb16b97b ab51d1bebd12 52e0dbb7be50 2024-09-14T03:54:56.047 INFO:teuthology.orchestra.run.smithi070.stdout:mds.b smithi094 running (11m) 2m ago 11m 15.4M - 17.2.7-1629-geb16b97b ab51d1bebd12 e3b5e3e00189 2024-09-14T03:54:56.047 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi094 *:8443,9283,8765 running (3m) 2m ago 14m 452M - 19.3.0-4907-ga2782315 8801d2529e43 569d9c7e0f06 2024-09-14T03:54:56.047 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (4m) 33s ago 16m 521M - 19.3.0-4907-ga2782315 8801d2529e43 0ef4d8e0cdbe 2024-09-14T03:54:56.047 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (3m) 33s ago 16m 57.7M 2048M 19.3.0-4907-ga2782315 8801d2529e43 96542de31cac 2024-09-14T03:54:56.047 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi094 running (2m) 2m ago 14m 21.3M 2048M 19.3.0-4907-ga2782315 8801d2529e43 86827be98659 2024-09-14T03:54:56.047 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (3m) 33s ago 14m 51.9M 2048M 19.3.0-4907-ga2782315 8801d2529e43 5741e4fcf4b6 2024-09-14T03:54:56.047 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (4m) 33s ago 9m 12.2M - 1.7.0 72c9c2088986 e316760a30b3 2024-09-14T03:54:56.047 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi094 *:9100 running (4m) 2m ago 9m 7750k - 1.7.0 72c9c2088986 767f8a093175 2024-09-14T03:54:56.048 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (2m) 33s ago 14m 180M 2480M 19.3.0-4907-ga2782315 8801d2529e43 2345ccac2085 2024-09-14T03:54:56.048 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (102s) 33s ago 13m 177M 2480M 19.3.0-4907-ga2782315 8801d2529e43 53868856512a 2024-09-14T03:54:56.048 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (70s) 33s ago 13m 104M 2480M 19.3.0-4907-ga2782315 8801d2529e43 39ef876d2fff 2024-09-14T03:54:56.048 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (37s) 33s ago 12m 14.5M 2480M 19.3.0-4907-ga2782315 8801d2529e43 8efc112edbe9 2024-09-14T03:54:56.048 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi094 starting - - - 2480M 2024-09-14T03:54:56.048 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi094 running (12m) 2m ago 12m 168M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 b543f152ba87 2024-09-14T03:54:56.048 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi094 running (11m) 2m ago 11m 114M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 e66b592ac940 2024-09-14T03:54:56.048 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi094 running (11m) 2m ago 11m 152M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 037c6208e9ce 2024-09-14T03:54:56.048 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi094 *:9095 running (3m) 2m ago 9m 46.3M - 2.51.0 1d3b7f56885b ce1b353828dc 2024-09-14T03:54:56.168 INFO:tasks.rados.rados.0.smithi070.stdout:1961: finishing copy_from racing read to smithi07085154-15 2024-09-14T03:54:56.168 INFO:tasks.rados.rados.0.smithi070.stdout:1961: finishing copy_from to smithi07085154-15 2024-09-14T03:54:56.168 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 15 v 1686 (ObjNum 583 snap 194 seq_num 583) dirty exists 2024-09-14T03:54:56.168 INFO:tasks.rados.rados.0.smithi070.stdout:1957: done (6 left) 2024-09-14T03:54:56.168 INFO:tasks.rados.rados.0.smithi070.stdout:1959: done (5 left) 2024-09-14T03:54:56.168 INFO:tasks.rados.rados.0.smithi070.stdout:1960: done (4 left) 2024-09-14T03:54:56.168 INFO:tasks.rados.rados.0.smithi070.stdout:1961: done (3 left) 2024-09-14T03:54:56.168 INFO:tasks.rados.rados.0.smithi070.stdout:1963: done (2 left) 2024-09-14T03:54:56.168 INFO:tasks.rados.rados.0.smithi070.stdout:1965: done (1 left) 2024-09-14T03:54:56.168 INFO:tasks.rados.rados.0.smithi070.stdout:1967: done (0 left) 2024-09-14T03:54:56.168 INFO:tasks.rados.rados.0.smithi070.stdout:1968: read oid 15 snap 180 2024-09-14T03:54:56.168 INFO:tasks.rados.rados.0.smithi070.stdout:1968: expect (ObjNum 541 snap 176 seq_num 541) 2024-09-14T03:54:56.169 INFO:tasks.rados.rados.0.smithi070.stdout:1969: setattr oid 3 current snap is 211 2024-09-14T03:54:56.169 INFO:tasks.rados.rados.0.smithi070.stdout:1970: snap_remove snap 189 2024-09-14T03:54:56.369 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:56 smithi094 ceph-mon[86597]: osdmap e470: 8 total, 7 up, 8 in 2024-09-14T03:54:56.389 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:54:56.389 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-09-14T03:54:56.389 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:54:56.390 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:54:56.390 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-09-14T03:54:56.390 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:54:56.390 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:54:56.390 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-09-14T03:54:56.390 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 3, 2024-09-14T03:54:56.390 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 4 2024-09-14T03:54:56.390 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:54:56.390 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": { 2024-09-14T03:54:56.390 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:54:56.390 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:54:56.390 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-09-14T03:54:56.390 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 5, 2024-09-14T03:54:56.391 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 9 2024-09-14T03:54:56.391 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-09-14T03:54:56.391 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:54:56.391 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:56 smithi070 ceph-mon[96704]: osdmap e470: 8 total, 7 up, 8 in 2024-09-14T03:54:56.391 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:56 smithi070 ceph-mon[98640]: osdmap e470: 8 total, 7 up, 8 in 2024-09-14T03:54:56.671 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:54:56.671 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:54:56.671 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-09-14T03:54:56.671 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:54:56.671 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [ 2024-09-14T03:54:56.671 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr", 2024-09-14T03:54:56.672 INFO:teuthology.orchestra.run.smithi070.stdout: "mon" 2024-09-14T03:54:56.672 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2024-09-14T03:54:56.672 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "9/20 daemons upgraded", 2024-09-14T03:54:56.672 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "Currently upgrading osd daemons", 2024-09-14T03:54:56.672 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-09-14T03:54:56.672 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:54:57.101 INFO:tasks.rados.rados.0.smithi070.stdout:1970: done (2 left) 2024-09-14T03:54:57.101 INFO:tasks.rados.rados.0.smithi070.stdout:1971: read oid 42 snap 180 2024-09-14T03:54:57.101 INFO:tasks.rados.rados.0.smithi070.stdout:1971: expect (ObjNum 507 snap 166 seq_num 507) 2024-09-14T03:54:57.102 INFO:tasks.rados.rados.0.smithi070.stdout:1972: read oid 23 snap 193 2024-09-14T03:54:57.102 INFO:tasks.rados.rados.0.smithi070.stdout:1972: expect deleted 2024-09-14T03:54:57.102 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 26 current snap is 211 2024-09-14T03:54:57.102 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 3 v 1677 (ObjNum 608 snap 204 seq_num 608) dirty exists 2024-09-14T03:54:57.102 INFO:tasks.rados.rados.0.smithi070.stdout:1973: seq_num 629 ranges {0=540672,540672=401408} 2024-09-14T03:54:57.104 INFO:tasks.rados.rados.0.smithi070.stdout:1973: writing smithi07085154-26 from 0 to 540672 tid 1 2024-09-14T03:54:57.107 INFO:tasks.rados.rados.0.smithi070.stdout:1973: writing smithi07085154-26 from 540672 to 942080 tid 2 2024-09-14T03:54:57.107 INFO:tasks.rados.rados.0.smithi070.stdout:1969: done (4 left) 2024-09-14T03:54:57.108 INFO:tasks.rados.rados.0.smithi070.stdout:1974: delete oid 2 current snap is 211 2024-09-14T03:54:57.110 INFO:tasks.rados.rados.0.smithi070.stdout:1968: expect (ObjNum 541 snap 176 seq_num 541) 2024-09-14T03:54:57.223 INFO:tasks.rados.rados.0.smithi070.stdout:1968: done (4 left) 2024-09-14T03:54:57.223 INFO:tasks.rados.rados.0.smithi070.stdout:1972: done (3 left) 2024-09-14T03:54:57.223 INFO:tasks.rados.rados.0.smithi070.stdout:1974: done (2 left) 2024-09-14T03:54:57.223 INFO:tasks.rados.rados.0.smithi070.stdout:1975: setattr oid 14 current snap is 211 2024-09-14T03:54:57.223 INFO:tasks.rados.rados.0.smithi070.stdout:1973: finishing write tid 1 to smithi07085154-26 2024-09-14T03:54:57.223 INFO:tasks.rados.rados.0.smithi070.stdout:1971: expect (ObjNum 507 snap 166 seq_num 507) 2024-09-14T03:54:57.256 INFO:tasks.rados.rados.0.smithi070.stdout:1973: finishing write tid 2 to smithi07085154-26 2024-09-14T03:54:57.257 INFO:tasks.rados.rados.0.smithi070.stdout:1973: finishing write tid 3 to smithi07085154-26 2024-09-14T03:54:57.257 INFO:tasks.rados.rados.0.smithi070.stdout:1973: finishing write tid 4 to smithi07085154-26 2024-09-14T03:54:57.257 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 2107 (ObjNum 629 snap 211 seq_num 629) dirty exists 2024-09-14T03:54:57.257 INFO:tasks.rados.rados.0.smithi070.stdout:1973: left oid 26 (ObjNum 629 snap 211 seq_num 629) 2024-09-14T03:54:57.257 INFO:tasks.rados.rados.0.smithi070.stdout:1971: done (2 left) 2024-09-14T03:54:57.257 INFO:tasks.rados.rados.0.smithi070.stdout:1973: done (1 left) 2024-09-14T03:54:57.257 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 40 current snap is 211 2024-09-14T03:54:57.257 INFO:tasks.rados.rados.0.smithi070.stdout:1976: seq_num 631 ranges {7979008=573440,8552448=401408} 2024-09-14T03:54:57.276 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:54:56 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:54:56.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:54:57.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:57 smithi094 ceph-mon[86597]: from='client.34365 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:54:57.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:57 smithi094 ceph-mon[86597]: from='client.44285 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:54:57.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:57 smithi094 ceph-mon[86597]: from='client.34377 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:54:57.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:57 smithi094 ceph-mon[86597]: pgmap v299: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 727 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s rd, 4.5 MiB/s wr, 20 op/s; 173/1131 objects degraded (15.296%) 2024-09-14T03:54:57.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:57 smithi094 ceph-mon[86597]: osdmap e471: 8 total, 7 up, 8 in 2024-09-14T03:54:57.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:57 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/3524842432' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:54:57.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:57 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:57.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:57 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:57.301 INFO:tasks.rados.rados.0.smithi070.stdout:1976: writing smithi07085154-40 from 7979008 to 8552448 tid 1 2024-09-14T03:54:57.304 INFO:tasks.rados.rados.0.smithi070.stdout:1976: writing smithi07085154-40 from 8552448 to 8953856 tid 2 2024-09-14T03:54:57.304 INFO:tasks.rados.rados.0.smithi070.stdout:1977: rollback oid 27 current snap is 211 2024-09-14T03:54:57.304 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 27 to 176 2024-09-14T03:54:57.304 INFO:tasks.rados.rados.0.smithi070.stdout:1978: copy_from oid 8 from oid 7 current snap is 211 2024-09-14T03:54:57.304 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 24 current snap is 211 2024-09-14T03:54:57.304 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 1677 (ObjNum 595 snap 198 seq_num 595) dirty exists 2024-09-14T03:54:57.304 INFO:tasks.rados.rados.0.smithi070.stdout:1979: seq_num 632 ranges {9109504=589824,9699328=401408} 2024-09-14T03:54:57.356 INFO:tasks.rados.rados.0.smithi070.stdout:1979: writing smithi07085154-24 from 9109504 to 9699328 tid 1 2024-09-14T03:54:57.358 INFO:tasks.rados.rados.0.smithi070.stdout:1979: writing smithi07085154-24 from 9699328 to 10100736 tid 2 2024-09-14T03:54:57.358 INFO:tasks.rados.rados.0.smithi070.stdout:1975: done (4 left) 2024-09-14T03:54:57.358 INFO:tasks.rados.rados.0.smithi070.stdout:1980: delete oid 39 current snap is 211 2024-09-14T03:54:57.358 INFO:tasks.rados.rados.0.smithi070.stdout:1978: finishing copy_from to smithi07085154-8 2024-09-14T03:54:57.358 INFO:tasks.rados.rados.0.smithi070.stdout:1978: got expected ENOENT (src dne) 2024-09-14T03:54:57.359 INFO:tasks.rados.rados.0.smithi070.stdout:1978: finishing copy_from racing read to smithi07085154-8 2024-09-14T03:54:57.359 INFO:tasks.rados.rados.0.smithi070.stdout:1976: finishing write tid 1 to smithi07085154-40 2024-09-14T03:54:57.359 INFO:tasks.rados.rados.0.smithi070.stdout:1976: finishing write tid 2 to smithi07085154-40 2024-09-14T03:54:57.359 INFO:tasks.rados.rados.0.smithi070.stdout:1976: finishing write tid 3 to smithi07085154-40 2024-09-14T03:54:57.359 INFO:tasks.rados.rados.0.smithi070.stdout:1977: finishing rollback tid 0 to smithi07085154-27 2024-09-14T03:54:57.359 INFO:tasks.rados.rados.0.smithi070.stdout:1977: finishing rollback tid 1 to smithi07085154-27 2024-09-14T03:54:57.359 INFO:tasks.rados.rados.0.smithi070.stdout:1977: finishing rollback tid 2 to smithi07085154-27 2024-09-14T03:54:57.359 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 2122 (ObjNum 526 snap 174 seq_num 526) dirty exists 2024-09-14T03:54:57.359 INFO:tasks.rados.rados.0.smithi070.stdout:1976: finishing write tid 4 to smithi07085154-40 2024-09-14T03:54:57.359 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 1947 (ObjNum 631 snap 211 seq_num 631) dirty exists 2024-09-14T03:54:57.359 INFO:tasks.rados.rados.0.smithi070.stdout:1976: left oid 40 (ObjNum 631 snap 211 seq_num 631) 2024-09-14T03:54:57.360 INFO:tasks.rados.rados.0.smithi070.stdout:1976: done (4 left) 2024-09-14T03:54:57.361 INFO:tasks.rados.rados.0.smithi070.stdout:1977: done (3 left) 2024-09-14T03:54:57.361 INFO:tasks.rados.rados.0.smithi070.stdout:1978: done (2 left) 2024-09-14T03:54:57.361 INFO:tasks.rados.rados.0.smithi070.stdout:1980: done (1 left) 2024-09-14T03:54:57.361 INFO:tasks.rados.rados.0.smithi070.stdout:1981: read oid 27 snap -1 2024-09-14T03:54:57.361 INFO:tasks.rados.rados.0.smithi070.stdout:1981: expect (ObjNum 526 snap 174 seq_num 526) 2024-09-14T03:54:57.361 INFO:tasks.rados.rados.0.smithi070.stdout:1982: read oid 3 snap -1 2024-09-14T03:54:57.361 INFO:tasks.rados.rados.0.smithi070.stdout:1982: expect (ObjNum 608 snap 204 seq_num 608) 2024-09-14T03:54:57.361 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 6 current snap is 211 2024-09-14T03:54:57.361 INFO:tasks.rados.rados.0.smithi070.stdout:1983: seq_num 634 ranges {655360=622592,1277952=401408} 2024-09-14T03:54:57.368 INFO:tasks.rados.rados.0.smithi070.stdout:1983: writing smithi07085154-6 from 655360 to 1277952 tid 1 2024-09-14T03:54:57.370 INFO:tasks.rados.rados.0.smithi070.stdout:1983: writing smithi07085154-6 from 1277952 to 1679360 tid 2 2024-09-14T03:54:57.370 INFO:tasks.rados.rados.0.smithi070.stdout:1984: snap_create 2024-09-14T03:54:57.370 INFO:tasks.rados.rados.0.smithi070.stdout:1979: finishing write tid 1 to smithi07085154-24 2024-09-14T03:54:57.370 INFO:tasks.rados.rados.0.smithi070.stdout:1979: finishing write tid 2 to smithi07085154-24 2024-09-14T03:54:57.370 INFO:tasks.rados.rados.0.smithi070.stdout:1979: finishing write tid 3 to smithi07085154-24 2024-09-14T03:54:57.370 INFO:tasks.rados.rados.0.smithi070.stdout:1982: expect (ObjNum 608 snap 204 seq_num 608) 2024-09-14T03:54:57.407 INFO:tasks.rados.rados.0.smithi070.stdout:1979: finishing write tid 4 to smithi07085154-24 2024-09-14T03:54:57.407 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 1680 (ObjNum 632 snap 211 seq_num 632) dirty exists 2024-09-14T03:54:57.407 INFO:tasks.rados.rados.0.smithi070.stdout:1979: left oid 24 (ObjNum 632 snap 211 seq_num 632) 2024-09-14T03:54:57.407 INFO:tasks.rados.rados.0.smithi070.stdout:1983: finishing write tid 1 to smithi07085154-6 2024-09-14T03:54:57.407 INFO:tasks.rados.rados.0.smithi070.stdout:1983: finishing write tid 2 to smithi07085154-6 2024-09-14T03:54:57.407 INFO:tasks.rados.rados.0.smithi070.stdout:1983: finishing write tid 3 to smithi07085154-6 2024-09-14T03:54:57.407 INFO:tasks.rados.rados.0.smithi070.stdout:1983: finishing write tid 4 to smithi07085154-6 2024-09-14T03:54:57.407 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 1693 (ObjNum 634 snap 211 seq_num 634) dirty exists 2024-09-14T03:54:57.407 INFO:tasks.rados.rados.0.smithi070.stdout:1983: left oid 6 (ObjNum 634 snap 211 seq_num 634) 2024-09-14T03:54:57.419 INFO:tasks.rados.rados.0.smithi070.stdout:1981: expect (ObjNum 526 snap 174 seq_num 526) 2024-09-14T03:54:57.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:57 smithi070 ceph-mon[96704]: from='client.34365 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:54:57.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:57 smithi070 ceph-mon[96704]: from='client.44285 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:54:57.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:57 smithi070 ceph-mon[96704]: from='client.34377 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:54:57.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:57 smithi070 ceph-mon[96704]: pgmap v299: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 727 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s rd, 4.5 MiB/s wr, 20 op/s; 173/1131 objects degraded (15.296%) 2024-09-14T03:54:57.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:57 smithi070 ceph-mon[96704]: osdmap e471: 8 total, 7 up, 8 in 2024-09-14T03:54:57.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:57 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/3524842432' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:54:57.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:57 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:57.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:57 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:57.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:57 smithi070 ceph-mon[98640]: from='client.34365 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:54:57.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:57 smithi070 ceph-mon[98640]: from='client.44285 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:54:57.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:57 smithi070 ceph-mon[98640]: from='client.34377 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:54:57.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:57 smithi070 ceph-mon[98640]: pgmap v299: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 727 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s rd, 4.5 MiB/s wr, 20 op/s; 173/1131 objects degraded (15.296%) 2024-09-14T03:54:57.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:57 smithi070 ceph-mon[98640]: osdmap e471: 8 total, 7 up, 8 in 2024-09-14T03:54:57.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:57 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/3524842432' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:54:57.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:57 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:57.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:57 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:58.439 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:58 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4[92016]: 2024-09-14T03:54:58.056+0000 7f6d5e487640 -1 osd.4 461 log_to_monitors true 2024-09-14T03:54:58.439 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:58 smithi094 ceph-mon[86597]: from='client.44303 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:54:58.439 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:58 smithi094 ceph-mon[86597]: osdmap e472: 8 total, 7 up, 8 in 2024-09-14T03:54:58.439 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:58 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:58.439 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:58 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:58.439 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:58 smithi094 ceph-mon[86597]: from='osd.4 [v2:172.21.15.94:6800/930498880,v1:172.21.15.94:6801/930498880]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-14T03:54:58.439 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:58 smithi094 ceph-mon[86597]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-14T03:54:58.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:58 smithi070 ceph-mon[98640]: from='client.44303 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:54:58.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:58 smithi070 ceph-mon[98640]: osdmap e472: 8 total, 7 up, 8 in 2024-09-14T03:54:58.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:58 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:58.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:58 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:58.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:58 smithi070 ceph-mon[98640]: from='osd.4 [v2:172.21.15.94:6800/930498880,v1:172.21.15.94:6801/930498880]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-14T03:54:58.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:58 smithi070 ceph-mon[98640]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-14T03:54:58.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:58 smithi070 ceph-mon[96704]: from='client.44303 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:54:58.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:58 smithi070 ceph-mon[96704]: osdmap e472: 8 total, 7 up, 8 in 2024-09-14T03:54:58.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:58 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:58.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:58 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:54:58.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:58 smithi070 ceph-mon[96704]: from='osd.4 [v2:172.21.15.94:6800/930498880,v1:172.21.15.94:6801/930498880]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-14T03:54:58.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:58 smithi070 ceph-mon[96704]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-14T03:54:58.776 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:54:58 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4[92016]: 2024-09-14T03:54:58.437+0000 7f6d52229640 -1 osd.4 461 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:54:59.182 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:59 smithi094 ceph-mon[86597]: pgmap v302: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 763 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 11 MiB/s wr, 35 op/s; 175/1164 objects degraded (15.034%) 2024-09-14T03:54:59.182 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:59 smithi094 ceph-mon[86597]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-14T03:54:59.182 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:59 smithi094 ceph-mon[86597]: from='osd.4 [v2:172.21.15.94:6800/930498880,v1:172.21.15.94:6801/930498880]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi094", "root=default"]} : dispatch 2024-09-14T03:54:59.182 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:59 smithi094 ceph-mon[86597]: osdmap e473: 8 total, 7 up, 8 in 2024-09-14T03:54:59.182 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:59 smithi094 ceph-mon[86597]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi094", "root=default"]} : dispatch 2024-09-14T03:54:59.183 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:54:59 smithi094 ceph-mon[86597]: from='osd.4 ' entity='osd.4' 2024-09-14T03:54:59.362 INFO:tasks.rados.rados.0.smithi070.stdout:1979: done (4 left) 2024-09-14T03:54:59.362 INFO:tasks.rados.rados.0.smithi070.stdout:1981: done (3 left) 2024-09-14T03:54:59.362 INFO:tasks.rados.rados.0.smithi070.stdout:1982: done (2 left) 2024-09-14T03:54:59.363 INFO:tasks.rados.rados.0.smithi070.stdout:1983: done (1 left) 2024-09-14T03:54:59.363 INFO:tasks.rados.rados.0.smithi070.stdout:1984: done (0 left) 2024-09-14T03:54:59.363 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 22 current snap is 212 2024-09-14T03:54:59.363 INFO:tasks.rados.rados.0.smithi070.stdout:1985: seq_num 635 ranges {2326528=638976,2965504=401408} 2024-09-14T03:54:59.377 INFO:tasks.rados.rados.0.smithi070.stdout:1985: writing smithi07085154-22 from 2326528 to 2965504 tid 1 2024-09-14T03:54:59.380 INFO:tasks.rados.rados.0.smithi070.stdout:1985: writing smithi07085154-22 from 2965504 to 3366912 tid 2 2024-09-14T03:54:59.380 INFO:tasks.rados.rados.0.smithi070.stdout:1986: copy_from oid 50 from oid 15 current snap is 212 2024-09-14T03:54:59.380 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 8 current snap is 212 2024-09-14T03:54:59.380 INFO:tasks.rados.rados.0.smithi070.stdout:1987: seq_num 636 ranges {1310720=655360,1966080=401408} 2024-09-14T03:54:59.391 INFO:tasks.rados.rados.0.smithi070.stdout:1987: writing smithi07085154-8 from 1310720 to 1966080 tid 1 2024-09-14T03:54:59.394 INFO:tasks.rados.rados.0.smithi070.stdout:1987: writing smithi07085154-8 from 1966080 to 2367488 tid 2 2024-09-14T03:54:59.394 INFO:tasks.rados.rados.0.smithi070.stdout:1988: copy_from oid 31 from oid 27 current snap is 212 2024-09-14T03:54:59.394 INFO:tasks.rados.rados.0.smithi070.stdout:1985: finishing write tid 1 to smithi07085154-22 2024-09-14T03:54:59.394 INFO:tasks.rados.rados.0.smithi070.stdout:1985: finishing write tid 2 to smithi07085154-22 2024-09-14T03:54:59.394 INFO:tasks.rados.rados.0.smithi070.stdout:1985: finishing write tid 3 to smithi07085154-22 2024-09-14T03:54:59.394 INFO:tasks.rados.rados.0.smithi070.stdout:1985: finishing write tid 4 to smithi07085154-22 2024-09-14T03:54:59.394 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 1686 (ObjNum 635 snap 212 seq_num 635) dirty exists 2024-09-14T03:54:59.394 INFO:tasks.rados.rados.0.smithi070.stdout:1985: left oid 22 (ObjNum 635 snap 212 seq_num 635) 2024-09-14T03:54:59.394 INFO:tasks.rados.rados.0.smithi070.stdout:1985: done (3 left) 2024-09-14T03:54:59.395 INFO:tasks.rados.rados.0.smithi070.stdout:1989: snap_create 2024-09-14T03:54:59.398 INFO:tasks.rados.rados.0.smithi070.stdout:1987: finishing write tid 1 to smithi07085154-8 2024-09-14T03:54:59.398 INFO:tasks.rados.rados.0.smithi070.stdout:1987: finishing write tid 2 to smithi07085154-8 2024-09-14T03:54:59.398 INFO:tasks.rados.rados.0.smithi070.stdout:1987: finishing write tid 3 to smithi07085154-8 2024-09-14T03:54:59.399 INFO:tasks.rados.rados.0.smithi070.stdout:1987: finishing write tid 4 to smithi07085154-8 2024-09-14T03:54:59.399 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 8 v 1692 (ObjNum 636 snap 212 seq_num 636) dirty exists 2024-09-14T03:54:59.400 INFO:tasks.rados.rados.0.smithi070.stdout:1987: left oid 8 (ObjNum 636 snap 212 seq_num 636) 2024-09-14T03:54:59.433 INFO:tasks.rados.rados.0.smithi070.stdout:1987: done (3 left) 2024-09-14T03:54:59.433 INFO:tasks.rados.rados.0.smithi070.stdout:1989: done (2 left) 2024-09-14T03:54:59.433 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 37 current snap is 213 2024-09-14T03:54:59.433 INFO:tasks.rados.rados.0.smithi070.stdout:1990: seq_num 637 ranges {9109504=671744,9781248=401408} 2024-09-14T03:54:59.486 INFO:tasks.rados.rados.0.smithi070.stdout:1990: writing smithi07085154-37 from 9109504 to 9781248 tid 1 2024-09-14T03:54:59.488 INFO:tasks.rados.rados.0.smithi070.stdout:1990: writing smithi07085154-37 from 9781248 to 10182656 tid 2 2024-09-14T03:54:59.488 INFO:tasks.rados.rados.0.smithi070.stdout:1991: delete oid 44 current snap is 213 2024-09-14T03:54:59.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:59 smithi070 ceph-mon[96704]: pgmap v302: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 763 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 11 MiB/s wr, 35 op/s; 175/1164 objects degraded (15.034%) 2024-09-14T03:54:59.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:59 smithi070 ceph-mon[96704]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-14T03:54:59.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:59 smithi070 ceph-mon[96704]: from='osd.4 [v2:172.21.15.94:6800/930498880,v1:172.21.15.94:6801/930498880]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi094", "root=default"]} : dispatch 2024-09-14T03:54:59.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:59 smithi070 ceph-mon[96704]: osdmap e473: 8 total, 7 up, 8 in 2024-09-14T03:54:59.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:59 smithi070 ceph-mon[96704]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi094", "root=default"]} : dispatch 2024-09-14T03:54:59.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:54:59 smithi070 ceph-mon[96704]: from='osd.4 ' entity='osd.4' 2024-09-14T03:54:59.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:59 smithi070 ceph-mon[98640]: pgmap v302: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 763 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 11 MiB/s wr, 35 op/s; 175/1164 objects degraded (15.034%) 2024-09-14T03:54:59.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:59 smithi070 ceph-mon[98640]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-14T03:54:59.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:59 smithi070 ceph-mon[98640]: from='osd.4 [v2:172.21.15.94:6800/930498880,v1:172.21.15.94:6801/930498880]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi094", "root=default"]} : dispatch 2024-09-14T03:54:59.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:59 smithi070 ceph-mon[98640]: osdmap e473: 8 total, 7 up, 8 in 2024-09-14T03:54:59.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:59 smithi070 ceph-mon[98640]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi094", "root=default"]} : dispatch 2024-09-14T03:54:59.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:54:59 smithi070 ceph-mon[98640]: from='osd.4 ' entity='osd.4' 2024-09-14T03:55:00.214 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:00 smithi094 ceph-mon[86597]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:55:00.215 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:00 smithi094 ceph-mon[86597]: osd.4 [v2:172.21.15.94:6800/930498880,v1:172.21.15.94:6801/930498880] boot 2024-09-14T03:55:00.215 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:00 smithi094 ceph-mon[86597]: osdmap e474: 8 total, 8 up, 8 in 2024-09-14T03:55:00.215 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:00 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-14T03:55:00.472 INFO:tasks.rados.rados.0.smithi070.stdout:1990: finishing write tid 1 to smithi07085154-37 2024-09-14T03:55:00.473 INFO:tasks.rados.rados.0.smithi070.stdout:1990: finishing write tid 2 to smithi07085154-37 2024-09-14T03:55:00.473 INFO:tasks.rados.rados.0.smithi070.stdout:1990: finishing write tid 3 to smithi07085154-37 2024-09-14T03:55:00.473 INFO:tasks.rados.rados.0.smithi070.stdout:1990: finishing write tid 4 to smithi07085154-37 2024-09-14T03:55:00.474 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 37 v 1695 (ObjNum 637 snap 213 seq_num 637) dirty exists 2024-09-14T03:55:00.474 INFO:tasks.rados.rados.0.smithi070.stdout:1990: left oid 37 (ObjNum 637 snap 213 seq_num 637) 2024-09-14T03:55:00.480 INFO:tasks.rados.rados.0.smithi070.stdout:1990: done (3 left) 2024-09-14T03:55:00.480 INFO:tasks.rados.rados.0.smithi070.stdout:1991: done (2 left) 2024-09-14T03:55:00.480 INFO:tasks.rados.rados.0.smithi070.stdout:1992: read oid 22 snap -1 2024-09-14T03:55:00.480 INFO:tasks.rados.rados.0.smithi070.stdout:1992: expect (ObjNum 635 snap 212 seq_num 635) 2024-09-14T03:55:00.480 INFO:tasks.rados.rados.0.smithi070.stdout:1993: snap_create 2024-09-14T03:55:00.500 INFO:tasks.rados.rados.0.smithi070.stdout:1992: expect (ObjNum 635 snap 212 seq_num 635) 2024-09-14T03:55:00.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:55:00 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:55:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:55:00.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:00 smithi070 ceph-mon[98640]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:55:00.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:00 smithi070 ceph-mon[98640]: osd.4 [v2:172.21.15.94:6800/930498880,v1:172.21.15.94:6801/930498880] boot 2024-09-14T03:55:00.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:00 smithi070 ceph-mon[98640]: osdmap e474: 8 total, 8 up, 8 in 2024-09-14T03:55:00.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:00 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-14T03:55:00.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:00 smithi070 ceph-mon[96704]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:55:00.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:00 smithi070 ceph-mon[96704]: osd.4 [v2:172.21.15.94:6800/930498880,v1:172.21.15.94:6801/930498880] boot 2024-09-14T03:55:00.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:00 smithi070 ceph-mon[96704]: osdmap e474: 8 total, 8 up, 8 in 2024-09-14T03:55:00.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:00 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-14T03:55:00.682 INFO:tasks.rados.rados.0.smithi070.stdout:1986: finishing copy_from racing read to smithi07085154-50 2024-09-14T03:55:00.682 INFO:tasks.rados.rados.0.smithi070.stdout:1986: finishing copy_from to smithi07085154-50 2024-09-14T03:55:00.682 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 1686 (ObjNum 583 snap 194 seq_num 583) dirty exists 2024-09-14T03:55:00.682 INFO:tasks.rados.rados.0.smithi070.stdout:1988: finishing copy_from racing read to smithi07085154-31 2024-09-14T03:55:00.682 INFO:tasks.rados.rados.0.smithi070.stdout:1988: finishing copy_from to smithi07085154-31 2024-09-14T03:55:00.682 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 2122 (ObjNum 526 snap 174 seq_num 526) dirty exists 2024-09-14T03:55:01.035 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:55:00 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:55:00.718+0000 7fa4af5ba640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-14T03:55:01.438 INFO:tasks.rados.rados.0.smithi070.stdout:1986: done (3 left) 2024-09-14T03:55:01.439 INFO:tasks.rados.rados.0.smithi070.stdout:1988: done (2 left) 2024-09-14T03:55:01.439 INFO:tasks.rados.rados.0.smithi070.stdout:1992: done (1 left) 2024-09-14T03:55:01.439 INFO:tasks.rados.rados.0.smithi070.stdout:1993: done (0 left) 2024-09-14T03:55:01.439 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 7 current snap is 214 2024-09-14T03:55:01.439 INFO:tasks.rados.rados.0.smithi070.stdout:1994: seq_num 639 ranges {0=704512,704512=401408} 2024-09-14T03:55:01.442 INFO:tasks.rados.rados.0.smithi070.stdout:1994: writing smithi07085154-7 from 0 to 704512 tid 1 2024-09-14T03:55:01.445 INFO:tasks.rados.rados.0.smithi070.stdout:1994: writing smithi07085154-7 from 704512 to 1105920 tid 2 2024-09-14T03:55:01.445 INFO:tasks.rados.rados.0.smithi070.stdout:1995: read oid 43 snap -1 2024-09-14T03:55:01.445 INFO:tasks.rados.rados.0.smithi070.stdout:1995: expect deleted 2024-09-14T03:55:01.445 INFO:tasks.rados.rados.0.smithi070.stdout:1996: snap_remove snap 160 2024-09-14T03:55:01.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:01 smithi094 ceph-mon[86597]: pgmap v305: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 763 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 7.0 MiB/s wr, 26 op/s; 175/1164 objects degraded (15.034%) 2024-09-14T03:55:01.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:01 smithi094 ceph-mon[86597]: osdmap e475: 8 total, 8 up, 8 in 2024-09-14T03:55:01.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:01 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:01.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:01 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:01.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:01 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:55:01.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:01 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:55:01.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:01 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:01.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:01 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:55:01.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:01 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:01.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:01 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:01.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:01 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:01.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:01 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-14T03:55:01.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:01 smithi070 ceph-mon[96704]: pgmap v305: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 763 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 7.0 MiB/s wr, 26 op/s; 175/1164 objects degraded (15.034%) 2024-09-14T03:55:01.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:01 smithi070 ceph-mon[96704]: osdmap e475: 8 total, 8 up, 8 in 2024-09-14T03:55:01.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:01 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:01.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:01 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:01.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:01 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:55:01.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:01 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:55:01.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:01 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:01.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:01 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:55:01.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:01 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:01.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:01 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:01.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:01 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:01.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:01 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-14T03:55:01.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:01 smithi070 ceph-mon[98640]: pgmap v305: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 763 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 7.0 MiB/s wr, 26 op/s; 175/1164 objects degraded (15.034%) 2024-09-14T03:55:01.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:01 smithi070 ceph-mon[98640]: osdmap e475: 8 total, 8 up, 8 in 2024-09-14T03:55:01.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:01 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:01.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:01 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:01.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:01 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:55:01.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:01 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:55:01.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:01 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:01.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:01 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:55:01.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:01 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:01.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:01 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:01.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:01 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:01.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:01 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-14T03:55:02.445 INFO:tasks.rados.rados.0.smithi070.stdout:1996: done (2 left) 2024-09-14T03:55:02.445 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 25 current snap is 214 2024-09-14T03:55:02.445 INFO:tasks.rados.rados.0.smithi070.stdout:1997: seq_num 640 ranges {16343040=729088,17072128=401408} 2024-09-14T03:55:02.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:02 smithi094 ceph-mon[86597]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-14T03:55:02.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:02 smithi094 ceph-mon[86597]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-14T03:55:02.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:02 smithi094 ceph-mon[86597]: osdmap e476: 8 total, 8 up, 8 in 2024-09-14T03:55:02.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:02 smithi070 ceph-mon[96704]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-14T03:55:02.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:02 smithi070 ceph-mon[96704]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-14T03:55:02.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:02 smithi070 ceph-mon[96704]: osdmap e476: 8 total, 8 up, 8 in 2024-09-14T03:55:02.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:02 smithi070 ceph-mon[98640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-14T03:55:02.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:02 smithi070 ceph-mon[98640]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-14T03:55:02.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:02 smithi070 ceph-mon[98640]: osdmap e476: 8 total, 8 up, 8 in 2024-09-14T03:55:02.537 INFO:tasks.rados.rados.0.smithi070.stdout:1997: writing smithi07085154-25 from 16343040 to 17072128 tid 1 2024-09-14T03:55:02.539 INFO:tasks.rados.rados.0.smithi070.stdout:1997: writing smithi07085154-25 from 17072128 to 17473536 tid 2 2024-09-14T03:55:02.540 INFO:tasks.rados.rados.0.smithi070.stdout:1998: snap_remove snap 180 2024-09-14T03:55:03.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:03 smithi070 ceph-mon[98640]: pgmap v308: 121 pgs: 4 peering, 24 active+undersized, 9 active+undersized+degraded, 84 active+clean; 783 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 758 KiB/s wr, 6 op/s; 103/1170 objects degraded (8.803%) 2024-09-14T03:55:03.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:03 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:03.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:03 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:03.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:03 smithi070 ceph-mon[98640]: Health check update: Degraded data redundancy: 103/1170 objects degraded (8.803%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T03:55:03.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:03 smithi070 ceph-mon[98640]: osdmap e477: 8 total, 8 up, 8 in 2024-09-14T03:55:03.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:03 smithi070 ceph-mon[96704]: pgmap v308: 121 pgs: 4 peering, 24 active+undersized, 9 active+undersized+degraded, 84 active+clean; 783 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 758 KiB/s wr, 6 op/s; 103/1170 objects degraded (8.803%) 2024-09-14T03:55:03.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:03 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:03.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:03 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:03.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:03 smithi070 ceph-mon[96704]: Health check update: Degraded data redundancy: 103/1170 objects degraded (8.803%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T03:55:03.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:03 smithi070 ceph-mon[96704]: osdmap e477: 8 total, 8 up, 8 in 2024-09-14T03:55:03.447 INFO:tasks.rados.rados.0.smithi070.stdout:1998: done (3 left) 2024-09-14T03:55:03.447 INFO:tasks.rados.rados.0.smithi070.stdout:1999: setattr oid 40 current snap is 214 2024-09-14T03:55:03.449 INFO:tasks.rados.rados.0.smithi070.stdout:1994: finishing write tid 1 to smithi07085154-7 2024-09-14T03:55:03.449 INFO:tasks.rados.rados.0.smithi070.stdout:1994: finishing write tid 2 to smithi07085154-7 2024-09-14T03:55:03.449 INFO:tasks.rados.rados.0.smithi070.stdout:1994: finishing write tid 3 to smithi07085154-7 2024-09-14T03:55:03.449 INFO:tasks.rados.rados.0.smithi070.stdout:1994: finishing write tid 4 to smithi07085154-7 2024-09-14T03:55:03.449 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 1262 (ObjNum 639 snap 214 seq_num 639) dirty exists 2024-09-14T03:55:03.449 INFO:tasks.rados.rados.0.smithi070.stdout:1994: left oid 7 (ObjNum 639 snap 214 seq_num 639) 2024-09-14T03:55:03.449 INFO:tasks.rados.rados.0.smithi070.stdout:1997: finishing write tid 1 to smithi07085154-25 2024-09-14T03:55:03.450 INFO:tasks.rados.rados.0.smithi070.stdout:1997: finishing write tid 2 to smithi07085154-25 2024-09-14T03:55:03.450 INFO:tasks.rados.rados.0.smithi070.stdout:1997: finishing write tid 3 to smithi07085154-25 2024-09-14T03:55:03.450 INFO:tasks.rados.rados.0.smithi070.stdout:1997: finishing write tid 4 to smithi07085154-25 2024-09-14T03:55:03.450 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 2125 (ObjNum 640 snap 214 seq_num 640) dirty exists 2024-09-14T03:55:03.450 INFO:tasks.rados.rados.0.smithi070.stdout:1997: left oid 25 (ObjNum 640 snap 214 seq_num 640) 2024-09-14T03:55:03.450 INFO:tasks.rados.rados.0.smithi070.stdout:1994: done (3 left) 2024-09-14T03:55:03.450 INFO:tasks.rados.rados.0.smithi070.stdout:1995: done (2 left) 2024-09-14T03:55:03.450 INFO:tasks.rados.rados.0.smithi070.stdout:1997: done (1 left) 2024-09-14T03:55:03.450 INFO:tasks.rados.rados.0.smithi070.stdout:2000: copy_from oid 22 from oid 29 current snap is 214 2024-09-14T03:55:03.450 INFO:tasks.rados.rados.0.smithi070.stdout:2001: rmattr oid 38 current snap is 214 2024-09-14T03:55:03.451 INFO:tasks.rados.rados.0.smithi070.stdout:2002: rollback oid 12 current snap is 214 2024-09-14T03:55:03.451 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 12 to 181 2024-09-14T03:55:03.451 INFO:tasks.rados.rados.0.smithi070.stdout:2003: snap_create 2024-09-14T03:55:03.460 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 38 v 1678 (ObjNum 506 snap 165 seq_num 506) dirty exists 2024-09-14T03:55:03.462 INFO:tasks.rados.rados.0.smithi070.stdout:2002: finishing rollback tid 0 to smithi07085154-12 2024-09-14T03:55:03.464 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 1949 (ObjNum 631 snap 211 seq_num 631) dirty exists 2024-09-14T03:55:03.467 INFO:tasks.rados.rados.0.smithi070.stdout:2002: finishing rollback tid 1 to smithi07085154-12 2024-09-14T03:55:03.467 INFO:tasks.rados.rados.0.smithi070.stdout:2002: finishing rollback tid 2 to smithi07085154-12 2024-09-14T03:55:03.467 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 601 (ObjNum 534 snap 176 seq_num 534) dirty exists 2024-09-14T03:55:03.472 INFO:tasks.rados.rados.0.smithi070.stdout:2000: finishing copy_from racing read to smithi07085154-22 2024-09-14T03:55:03.475 INFO:tasks.rados.rados.0.smithi070.stdout:2000: finishing copy_from to smithi07085154-22 2024-09-14T03:55:03.475 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 1683 (ObjNum 623 snap 209 seq_num 623) dirty exists 2024-09-14T03:55:03.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:03 smithi094 ceph-mon[86597]: pgmap v308: 121 pgs: 4 peering, 24 active+undersized, 9 active+undersized+degraded, 84 active+clean; 783 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 758 KiB/s wr, 6 op/s; 103/1170 objects degraded (8.803%) 2024-09-14T03:55:03.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:03 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:03.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:03 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:03.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:03 smithi094 ceph-mon[86597]: Health check update: Degraded data redundancy: 103/1170 objects degraded (8.803%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T03:55:03.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:03 smithi094 ceph-mon[86597]: osdmap e477: 8 total, 8 up, 8 in 2024-09-14T03:55:03.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:55:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:55:03] "GET /metrics HTTP/1.1" 200 36161 "" "Prometheus/2.51.0" 2024-09-14T03:55:04.448 INFO:tasks.rados.rados.0.smithi070.stdout:1999: done (4 left) 2024-09-14T03:55:04.448 INFO:tasks.rados.rados.0.smithi070.stdout:2000: done (3 left) 2024-09-14T03:55:04.449 INFO:tasks.rados.rados.0.smithi070.stdout:2001: done (2 left) 2024-09-14T03:55:04.449 INFO:tasks.rados.rados.0.smithi070.stdout:2002: done (1 left) 2024-09-14T03:55:04.449 INFO:tasks.rados.rados.0.smithi070.stdout:2003: done (0 left) 2024-09-14T03:55:04.449 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 18 current snap is 215 2024-09-14T03:55:04.449 INFO:tasks.rados.rados.0.smithi070.stdout:2004: seq_num 641 ranges {5169152=745472,5914624=401408} 2024-09-14T03:55:04.449 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:55:04 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:55:04.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.5\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.5\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.94\", device_class=\"ssd\", hostname=\"smithi094\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.94\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.5\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.94\", device_class=\"ssd\", hostname=\"smithi094\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.94\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:55:04.480 INFO:tasks.rados.rados.0.smithi070.stdout:2004: writing smithi07085154-18 from 5169152 to 5914624 tid 1 2024-09-14T03:55:04.482 INFO:tasks.rados.rados.0.smithi070.stdout:2004: writing smithi07085154-18 from 5914624 to 6316032 tid 2 2024-09-14T03:55:04.482 INFO:tasks.rados.rados.0.smithi070.stdout:2005: snap_create 2024-09-14T03:55:04.488 INFO:tasks.rados.rados.0.smithi070.stdout:2004: finishing write tid 1 to smithi07085154-18 2024-09-14T03:55:04.488 INFO:tasks.rados.rados.0.smithi070.stdout:2004: finishing write tid 2 to smithi07085154-18 2024-09-14T03:55:04.488 INFO:tasks.rados.rados.0.smithi070.stdout:2004: finishing write tid 3 to smithi07085154-18 2024-09-14T03:55:04.489 INFO:tasks.rados.rados.0.smithi070.stdout:2004: finishing write tid 4 to smithi07085154-18 2024-09-14T03:55:04.490 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 1698 (ObjNum 641 snap 215 seq_num 641) dirty exists 2024-09-14T03:55:04.490 INFO:tasks.rados.rados.0.smithi070.stdout:2004: left oid 18 (ObjNum 641 snap 215 seq_num 641) 2024-09-14T03:55:04.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:04 smithi094 ceph-mon[86597]: osdmap e478: 8 total, 8 up, 8 in 2024-09-14T03:55:04.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:04 smithi094 ceph-mon[86597]: pgmap v311: 121 pgs: 4 peering, 2 active+undersized, 4 active+undersized+degraded, 111 active+clean; 818 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 9.6 MiB/s wr, 11 op/s; 23/1179 objects degraded (1.951%); 22 MiB/s, 3 objects/s recovering 2024-09-14T03:55:04.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:04 smithi070 ceph-mon[96704]: osdmap e478: 8 total, 8 up, 8 in 2024-09-14T03:55:04.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:04 smithi070 ceph-mon[96704]: pgmap v311: 121 pgs: 4 peering, 2 active+undersized, 4 active+undersized+degraded, 111 active+clean; 818 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 9.6 MiB/s wr, 11 op/s; 23/1179 objects degraded (1.951%); 22 MiB/s, 3 objects/s recovering 2024-09-14T03:55:04.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:04 smithi070 ceph-mon[98640]: osdmap e478: 8 total, 8 up, 8 in 2024-09-14T03:55:04.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:04 smithi070 ceph-mon[98640]: pgmap v311: 121 pgs: 4 peering, 2 active+undersized, 4 active+undersized+degraded, 111 active+clean; 818 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 9.6 MiB/s wr, 11 op/s; 23/1179 objects degraded (1.951%); 22 MiB/s, 3 objects/s recovering 2024-09-14T03:55:05.453 INFO:tasks.rados.rados.0.smithi070.stdout:2004: done (1 left) 2024-09-14T03:55:05.454 INFO:tasks.rados.rados.0.smithi070.stdout:2005: done (0 left) 2024-09-14T03:55:05.454 INFO:tasks.rados.rados.0.smithi070.stdout:2006: setattr oid 44 current snap is 216 2024-09-14T03:55:05.455 INFO:tasks.rados.rados.0.smithi070.stdout:2007: rollback oid 20 current snap is 216 2024-09-14T03:55:05.455 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 20 to 174 2024-09-14T03:55:05.455 INFO:tasks.rados.rados.0.smithi070.stdout:2008: snap_remove snap 188 2024-09-14T03:55:05.742 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:05 smithi094 ceph-mon[86597]: osdmap e479: 8 total, 8 up, 8 in 2024-09-14T03:55:05.780 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:05 smithi070 ceph-mon[96704]: osdmap e479: 8 total, 8 up, 8 in 2024-09-14T03:55:05.780 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:05 smithi070 ceph-mon[98640]: osdmap e479: 8 total, 8 up, 8 in 2024-09-14T03:55:06.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:55:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:55:05.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:55:06.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:55:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:55:05.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:55:06.467 INFO:tasks.rados.rados.0.smithi070.stdout:2008: done (2 left) 2024-09-14T03:55:06.467 INFO:tasks.rados.rados.0.smithi070.stdout:2009: copy_from oid 39 from oid 18 current snap is 216 2024-09-14T03:55:06.467 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 1950 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-14T03:55:06.467 INFO:tasks.rados.rados.0.smithi070.stdout:2007: finishing rollback tid 0 to smithi07085154-20 2024-09-14T03:55:06.467 INFO:tasks.rados.rados.0.smithi070.stdout:2007: finishing rollback tid 1 to smithi07085154-20 2024-09-14T03:55:06.467 INFO:tasks.rados.rados.0.smithi070.stdout:2007: finishing rollback tid 2 to smithi07085154-20 2024-09-14T03:55:06.467 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 20 v 1265 (ObjNum 524 snap 174 seq_num 524) dirty exists 2024-09-14T03:55:06.467 INFO:tasks.rados.rados.0.smithi070.stdout:2006: done (2 left) 2024-09-14T03:55:06.467 INFO:tasks.rados.rados.0.smithi070.stdout:2007: done (1 left) 2024-09-14T03:55:06.467 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 23 current snap is 216 2024-09-14T03:55:06.467 INFO:tasks.rados.rados.0.smithi070.stdout:2010: seq_num 642 ranges {794624=761856,1556480=401408} 2024-09-14T03:55:06.476 INFO:tasks.rados.rados.0.smithi070.stdout:2010: writing smithi07085154-23 from 794624 to 1556480 tid 1 2024-09-14T03:55:06.480 INFO:tasks.rados.rados.0.smithi070.stdout:2010: writing smithi07085154-23 from 1556480 to 1957888 tid 2 2024-09-14T03:55:06.480 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 30 current snap is 216 2024-09-14T03:55:06.480 INFO:tasks.rados.rados.0.smithi070.stdout:2011: seq_num 643 ranges {0=778240,778240=401408} 2024-09-14T03:55:06.485 INFO:tasks.rados.rados.0.smithi070.stdout:2011: writing smithi07085154-30 from 0 to 778240 tid 1 2024-09-14T03:55:06.488 INFO:tasks.rados.rados.0.smithi070.stdout:2011: writing smithi07085154-30 from 778240 to 1179648 tid 2 2024-09-14T03:55:06.489 INFO:tasks.rados.rados.0.smithi070.stdout:2010: finishing write tid 1 to smithi07085154-23 2024-09-14T03:55:06.489 INFO:tasks.rados.rados.0.smithi070.stdout:2010: finishing write tid 2 to smithi07085154-23 2024-09-14T03:55:06.489 INFO:tasks.rados.rados.0.smithi070.stdout:2010: finishing write tid 3 to smithi07085154-23 2024-09-14T03:55:06.490 INFO:tasks.rados.rados.0.smithi070.stdout:2012: read oid 19 snap -1 2024-09-14T03:55:06.490 INFO:tasks.rados.rados.0.smithi070.stdout:2012: expect (ObjNum 600 snap 199 seq_num 600) 2024-09-14T03:55:06.490 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 34 current snap is 216 2024-09-14T03:55:06.490 INFO:tasks.rados.rados.0.smithi070.stdout:2013: seq_num 644 ranges {4497408=794624,5292032=401408} 2024-09-14T03:55:06.519 INFO:tasks.rados.rados.0.smithi070.stdout:2013: writing smithi07085154-34 from 4497408 to 5292032 tid 1 2024-09-14T03:55:06.522 INFO:tasks.rados.rados.0.smithi070.stdout:2013: writing smithi07085154-34 from 5292032 to 5693440 tid 2 2024-09-14T03:55:06.522 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 38 current snap is 216 2024-09-14T03:55:06.522 INFO:tasks.rados.rados.0.smithi070.stdout:2010: finishing write tid 4 to smithi07085154-23 2024-09-14T03:55:06.523 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 1683 (ObjNum 642 snap 216 seq_num 642) dirty exists 2024-09-14T03:55:06.523 INFO:tasks.rados.rados.0.smithi070.stdout:2010: left oid 23 (ObjNum 642 snap 216 seq_num 642) 2024-09-14T03:55:06.523 INFO:tasks.rados.rados.0.smithi070.stdout:2011: finishing write tid 1 to smithi07085154-30 2024-09-14T03:55:06.523 INFO:tasks.rados.rados.0.smithi070.stdout:2014: seq_num 645 ranges {3596288=409600,4005888=778240,4784128=24576} 2024-09-14T03:55:06.544 INFO:tasks.rados.rados.0.smithi070.stdout:2014: writing smithi07085154-38 from 3596288 to 4005888 tid 1 2024-09-14T03:55:06.549 INFO:tasks.rados.rados.0.smithi070.stdout:2014: writing smithi07085154-38 from 4005888 to 4784128 tid 2 2024-09-14T03:55:06.549 INFO:tasks.rados.rados.0.smithi070.stdout:2014: writing smithi07085154-38 from 4784128 to 4808704 tid 3 2024-09-14T03:55:06.549 INFO:tasks.rados.rados.0.smithi070.stdout:2010: done (5 left) 2024-09-14T03:55:06.549 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 24 current snap is 216 2024-09-14T03:55:06.549 INFO:tasks.rados.rados.0.smithi070.stdout:2011: finishing write tid 2 to smithi07085154-30 2024-09-14T03:55:06.549 INFO:tasks.rados.rados.0.smithi070.stdout:2011: finishing write tid 3 to smithi07085154-30 2024-09-14T03:55:06.549 INFO:tasks.rados.rados.0.smithi070.stdout:2015: seq_num 646 ranges {10100736=425984,10526720=655360,11182080=147456} 2024-09-14T03:55:06.604 INFO:tasks.rados.rados.0.smithi070.stdout:2015: writing smithi07085154-24 from 10100736 to 10526720 tid 1 2024-09-14T03:55:06.607 INFO:tasks.rados.rados.0.smithi070.stdout:2015: writing smithi07085154-24 from 10526720 to 11182080 tid 2 2024-09-14T03:55:06.609 INFO:tasks.rados.rados.0.smithi070.stdout:2015: writing smithi07085154-24 from 11182080 to 11329536 tid 3 2024-09-14T03:55:06.609 INFO:tasks.rados.rados.0.smithi070.stdout:2016: copy_from oid 50 from oid 7 current snap is 216 2024-09-14T03:55:06.609 INFO:tasks.rados.rados.0.smithi070.stdout:2009: finishing copy_from racing read to smithi07085154-39 2024-09-14T03:55:06.609 INFO:tasks.rados.rados.0.smithi070.stdout:2009: finishing copy_from to smithi07085154-39 2024-09-14T03:55:06.609 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 39 v 1698 (ObjNum 641 snap 215 seq_num 641) dirty exists 2024-09-14T03:55:06.609 INFO:tasks.rados.rados.0.smithi070.stdout:2011: finishing write tid 4 to smithi07085154-30 2024-09-14T03:55:06.609 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 30 v 1681 (ObjNum 643 snap 216 seq_num 643) dirty exists 2024-09-14T03:55:06.609 INFO:tasks.rados.rados.0.smithi070.stdout:2011: left oid 30 (ObjNum 643 snap 216 seq_num 643) 2024-09-14T03:55:06.609 INFO:tasks.rados.rados.0.smithi070.stdout:2012: expect (ObjNum 600 snap 199 seq_num 600) 2024-09-14T03:55:06.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:06 smithi094 ceph-mon[86597]: osdmap e480: 8 total, 8 up, 8 in 2024-09-14T03:55:06.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:06 smithi094 ceph-mon[86597]: pgmap v314: 121 pgs: 121 active+clean; 820 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 9.4 MiB/s wr, 7 op/s; 32 MiB/s, 5 objects/s recovering 2024-09-14T03:55:06.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:06 smithi070 ceph-mon[96704]: osdmap e480: 8 total, 8 up, 8 in 2024-09-14T03:55:06.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:06 smithi070 ceph-mon[96704]: pgmap v314: 121 pgs: 121 active+clean; 820 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 9.4 MiB/s wr, 7 op/s; 32 MiB/s, 5 objects/s recovering 2024-09-14T03:55:06.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:06 smithi070 ceph-mon[98640]: osdmap e480: 8 total, 8 up, 8 in 2024-09-14T03:55:06.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:06 smithi070 ceph-mon[98640]: pgmap v314: 121 pgs: 121 active+clean; 820 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 9.4 MiB/s wr, 7 op/s; 32 MiB/s, 5 objects/s recovering 2024-09-14T03:55:07.094 INFO:tasks.rados.rados.0.smithi070.stdout:2013: finishing write tid 1 to smithi07085154-34 2024-09-14T03:55:07.095 INFO:tasks.rados.rados.0.smithi070.stdout:2013: finishing write tid 2 to smithi07085154-34 2024-09-14T03:55:07.095 INFO:tasks.rados.rados.0.smithi070.stdout:2013: finishing write tid 3 to smithi07085154-34 2024-09-14T03:55:07.095 INFO:tasks.rados.rados.0.smithi070.stdout:2013: finishing write tid 4 to smithi07085154-34 2024-09-14T03:55:07.095 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 1689 (ObjNum 644 snap 216 seq_num 644) dirty exists 2024-09-14T03:55:07.095 INFO:tasks.rados.rados.0.smithi070.stdout:2013: left oid 34 (ObjNum 644 snap 216 seq_num 644) 2024-09-14T03:55:07.095 INFO:tasks.rados.rados.0.smithi070.stdout:2014: finishing write tid 1 to smithi07085154-38 2024-09-14T03:55:07.095 INFO:tasks.rados.rados.0.smithi070.stdout:2014: finishing write tid 2 to smithi07085154-38 2024-09-14T03:55:07.095 INFO:tasks.rados.rados.0.smithi070.stdout:2014: finishing write tid 3 to smithi07085154-38 2024-09-14T03:55:07.095 INFO:tasks.rados.rados.0.smithi070.stdout:2014: finishing write tid 4 to smithi07085154-38 2024-09-14T03:55:07.095 INFO:tasks.rados.rados.0.smithi070.stdout:2014: finishing write tid 5 to smithi07085154-38 2024-09-14T03:55:07.095 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 38 v 1685 (ObjNum 645 snap 216 seq_num 645) dirty exists 2024-09-14T03:55:07.095 INFO:tasks.rados.rados.0.smithi070.stdout:2014: left oid 38 (ObjNum 645 snap 216 seq_num 645) 2024-09-14T03:55:07.095 INFO:tasks.rados.rados.0.smithi070.stdout:2015: finishing write tid 1 to smithi07085154-24 2024-09-14T03:55:07.096 INFO:tasks.rados.rados.0.smithi070.stdout:2009: done (6 left) 2024-09-14T03:55:07.096 INFO:tasks.rados.rados.0.smithi070.stdout:2011: done (5 left) 2024-09-14T03:55:07.096 INFO:tasks.rados.rados.0.smithi070.stdout:2012: done (4 left) 2024-09-14T03:55:07.096 INFO:tasks.rados.rados.0.smithi070.stdout:2013: done (3 left) 2024-09-14T03:55:07.096 INFO:tasks.rados.rados.0.smithi070.stdout:2014: done (2 left) 2024-09-14T03:55:07.096 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 23 current snap is 216 2024-09-14T03:55:07.096 INFO:tasks.rados.rados.0.smithi070.stdout:2015: finishing write tid 2 to smithi07085154-24 2024-09-14T03:55:07.096 INFO:tasks.rados.rados.0.smithi070.stdout:2017: seq_num 647 ranges {1957888=442368,2400256=589824,2990080=212992} 2024-09-14T03:55:07.107 INFO:tasks.rados.rados.0.smithi070.stdout:2017: writing smithi07085154-23 from 1957888 to 2400256 tid 1 2024-09-14T03:55:07.110 INFO:tasks.rados.rados.0.smithi070.stdout:2017: writing smithi07085154-23 from 2400256 to 2990080 tid 2 2024-09-14T03:55:07.112 INFO:tasks.rados.rados.0.smithi070.stdout:2017: writing smithi07085154-23 from 2990080 to 3203072 tid 3 2024-09-14T03:55:07.112 INFO:tasks.rados.rados.0.smithi070.stdout:2018: read oid 45 snap -1 2024-09-14T03:55:07.112 INFO:tasks.rados.rados.0.smithi070.stdout:2018: expect (ObjNum 312 snap 105 seq_num 312) 2024-09-14T03:55:07.113 INFO:tasks.rados.rados.0.smithi070.stdout:2019: read oid 39 snap -1 2024-09-14T03:55:07.113 INFO:tasks.rados.rados.0.smithi070.stdout:2019: expect (ObjNum 641 snap 215 seq_num 641) 2024-09-14T03:55:07.113 INFO:tasks.rados.rados.0.smithi070.stdout:2020: snap_create 2024-09-14T03:55:07.113 INFO:tasks.rados.rados.0.smithi070.stdout:2015: finishing write tid 3 to smithi07085154-24 2024-09-14T03:55:07.113 INFO:tasks.rados.rados.0.smithi070.stdout:2015: finishing write tid 4 to smithi07085154-24 2024-09-14T03:55:07.113 INFO:tasks.rados.rados.0.smithi070.stdout:2015: finishing write tid 5 to smithi07085154-24 2024-09-14T03:55:07.113 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 1687 (ObjNum 646 snap 216 seq_num 646) dirty exists 2024-09-14T03:55:07.113 INFO:tasks.rados.rados.0.smithi070.stdout:2015: left oid 24 (ObjNum 646 snap 216 seq_num 646) 2024-09-14T03:55:07.113 INFO:tasks.rados.rados.0.smithi070.stdout:2016: finishing copy_from racing read to smithi07085154-50 2024-09-14T03:55:07.113 INFO:tasks.rados.rados.0.smithi070.stdout:2016: finishing copy_from to smithi07085154-50 2024-09-14T03:55:07.113 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 1262 (ObjNum 639 snap 214 seq_num 639) dirty exists 2024-09-14T03:55:07.113 INFO:tasks.rados.rados.0.smithi070.stdout:2017: finishing write tid 1 to smithi07085154-23 2024-09-14T03:55:07.115 INFO:tasks.rados.rados.0.smithi070.stdout:2017: finishing write tid 2 to smithi07085154-23 2024-09-14T03:55:07.115 INFO:tasks.rados.rados.0.smithi070.stdout:2018: expect (ObjNum 312 snap 105 seq_num 312) 2024-09-14T03:55:07.142 INFO:tasks.rados.rados.0.smithi070.stdout:2017: finishing write tid 3 to smithi07085154-23 2024-09-14T03:55:07.142 INFO:tasks.rados.rados.0.smithi070.stdout:2017: finishing write tid 4 to smithi07085154-23 2024-09-14T03:55:07.142 INFO:tasks.rados.rados.0.smithi070.stdout:2017: finishing write tid 5 to smithi07085154-23 2024-09-14T03:55:07.142 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 1691 (ObjNum 647 snap 216 seq_num 647) dirty exists 2024-09-14T03:55:07.142 INFO:tasks.rados.rados.0.smithi070.stdout:2017: left oid 23 (ObjNum 647 snap 216 seq_num 647) 2024-09-14T03:55:07.142 INFO:tasks.rados.rados.0.smithi070.stdout:2019: expect (ObjNum 641 snap 215 seq_num 641) 2024-09-14T03:55:07.276 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:55:06 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:55:06.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:55:07.532 INFO:tasks.rados.rados.0.smithi070.stdout:2015: done (5 left) 2024-09-14T03:55:07.532 INFO:tasks.rados.rados.0.smithi070.stdout:2016: done (4 left) 2024-09-14T03:55:07.532 INFO:tasks.rados.rados.0.smithi070.stdout:2017: done (3 left) 2024-09-14T03:55:07.532 INFO:tasks.rados.rados.0.smithi070.stdout:2018: done (2 left) 2024-09-14T03:55:07.532 INFO:tasks.rados.rados.0.smithi070.stdout:2019: done (1 left) 2024-09-14T03:55:07.532 INFO:tasks.rados.rados.0.smithi070.stdout:2020: done (0 left) 2024-09-14T03:55:07.532 INFO:tasks.rados.rados.0.smithi070.stdout:2021: snap_create 2024-09-14T03:55:07.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:07 smithi094 ceph-mon[86597]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 23/1179 objects degraded (1.951%), 4 pgs degraded) 2024-09-14T03:55:07.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:07 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T03:55:07.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:07 smithi094 ceph-mon[86597]: osdmap e481: 8 total, 8 up, 8 in 2024-09-14T03:55:07.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:07 smithi070 ceph-mon[98640]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 23/1179 objects degraded (1.951%), 4 pgs degraded) 2024-09-14T03:55:07.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:07 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T03:55:07.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:07 smithi070 ceph-mon[98640]: osdmap e481: 8 total, 8 up, 8 in 2024-09-14T03:55:07.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:07 smithi070 ceph-mon[96704]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 23/1179 objects degraded (1.951%), 4 pgs degraded) 2024-09-14T03:55:07.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:07 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T03:55:07.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:07 smithi070 ceph-mon[96704]: osdmap e481: 8 total, 8 up, 8 in 2024-09-14T03:55:08.475 INFO:tasks.rados.rados.0.smithi070.stdout:2021: done (0 left) 2024-09-14T03:55:08.475 INFO:tasks.rados.rados.0.smithi070.stdout:2022: read oid 37 snap 217 2024-09-14T03:55:08.475 INFO:tasks.rados.rados.0.smithi070.stdout:2022: expect (ObjNum 637 snap 213 seq_num 637) 2024-09-14T03:55:08.476 INFO:tasks.rados.rados.0.smithi070.stdout:2023: read oid 17 snap 182 2024-09-14T03:55:08.476 INFO:tasks.rados.rados.0.smithi070.stdout:2023: expect (ObjNum 544 snap 181 seq_num 544) 2024-09-14T03:55:08.476 INFO:tasks.rados.rados.0.smithi070.stdout:2024: read oid 15 snap -1 2024-09-14T03:55:08.476 INFO:tasks.rados.rados.0.smithi070.stdout:2024: expect (ObjNum 583 snap 194 seq_num 583) 2024-09-14T03:55:08.476 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 27 current snap is 218 2024-09-14T03:55:08.476 INFO:tasks.rados.rados.0.smithi070.stdout:2025: seq_num 648 ranges {17793024=458752,18251776=475136,18726912=327680} 2024-09-14T03:55:08.575 INFO:tasks.rados.rados.0.smithi070.stdout:2025: writing smithi07085154-27 from 17793024 to 18251776 tid 1 2024-09-14T03:55:08.578 INFO:tasks.rados.rados.0.smithi070.stdout:2025: writing smithi07085154-27 from 18251776 to 18726912 tid 2 2024-09-14T03:55:08.580 INFO:tasks.rados.rados.0.smithi070.stdout:2025: writing smithi07085154-27 from 18726912 to 19054592 tid 3 2024-09-14T03:55:08.580 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 46 current snap is 218 2024-09-14T03:55:08.580 INFO:tasks.rados.rados.0.smithi070.stdout:2023: expect (ObjNum 544 snap 181 seq_num 544) 2024-09-14T03:55:08.765 INFO:tasks.rados.rados.0.smithi070.stdout:2022: expect (ObjNum 637 snap 213 seq_num 637) 2024-09-14T03:55:08.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:08 smithi094 ceph-mon[86597]: osdmap e482: 8 total, 8 up, 8 in 2024-09-14T03:55:08.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:08 smithi094 ceph-mon[86597]: pgmap v317: 121 pgs: 121 active+clean; 843 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 4.3 MiB/s wr, 14 op/s; 10 MiB/s, 1 objects/s recovering 2024-09-14T03:55:08.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:08 smithi070 ceph-mon[96704]: osdmap e482: 8 total, 8 up, 8 in 2024-09-14T03:55:08.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:08 smithi070 ceph-mon[96704]: pgmap v317: 121 pgs: 121 active+clean; 843 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 4.3 MiB/s wr, 14 op/s; 10 MiB/s, 1 objects/s recovering 2024-09-14T03:55:08.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:08 smithi070 ceph-mon[98640]: osdmap e482: 8 total, 8 up, 8 in 2024-09-14T03:55:08.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:08 smithi070 ceph-mon[98640]: pgmap v317: 121 pgs: 121 active+clean; 843 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 4.3 MiB/s wr, 14 op/s; 10 MiB/s, 1 objects/s recovering 2024-09-14T03:55:09.502 INFO:tasks.rados.rados.0.smithi070.stdout:2024: expect (ObjNum 583 snap 194 seq_num 583) 2024-09-14T03:55:09.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:09 smithi094 ceph-mon[86597]: osdmap e483: 8 total, 8 up, 8 in 2024-09-14T03:55:09.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:09 smithi094 ceph-mon[86597]: osdmap e484: 8 total, 8 up, 8 in 2024-09-14T03:55:09.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:09 smithi070 ceph-mon[96704]: osdmap e483: 8 total, 8 up, 8 in 2024-09-14T03:55:09.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:09 smithi070 ceph-mon[96704]: osdmap e484: 8 total, 8 up, 8 in 2024-09-14T03:55:09.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:09 smithi070 ceph-mon[98640]: osdmap e483: 8 total, 8 up, 8 in 2024-09-14T03:55:09.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:09 smithi070 ceph-mon[98640]: osdmap e484: 8 total, 8 up, 8 in 2024-09-14T03:55:10.477 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:55:10 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:55:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:55:10.750 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:10 smithi094 ceph-mon[86597]: pgmap v320: 121 pgs: 121 active+clean; 843 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s rd, 3.8 MiB/s wr, 25 op/s 2024-09-14T03:55:10.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:10 smithi070 ceph-mon[96704]: pgmap v320: 121 pgs: 121 active+clean; 843 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s rd, 3.8 MiB/s wr, 25 op/s 2024-09-14T03:55:10.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:10 smithi070 ceph-mon[98640]: pgmap v320: 121 pgs: 121 active+clean; 843 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s rd, 3.8 MiB/s wr, 25 op/s 2024-09-14T03:55:11.400 INFO:tasks.rados.rados.0.smithi070.stdout:2025: finishing write tid 1 to smithi07085154-27 2024-09-14T03:55:11.400 INFO:tasks.rados.rados.0.smithi070.stdout:2025: finishing write tid 2 to smithi07085154-27 2024-09-14T03:55:11.400 INFO:tasks.rados.rados.0.smithi070.stdout:2025: finishing write tid 3 to smithi07085154-27 2024-09-14T03:55:11.400 INFO:tasks.rados.rados.0.smithi070.stdout:2025: finishing write tid 4 to smithi07085154-27 2024-09-14T03:55:11.400 INFO:tasks.rados.rados.0.smithi070.stdout:2025: finishing write tid 5 to smithi07085154-27 2024-09-14T03:55:11.400 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 2163 (ObjNum 648 snap 218 seq_num 648) dirty exists 2024-09-14T03:55:11.400 INFO:tasks.rados.rados.0.smithi070.stdout:2025: left oid 27 (ObjNum 648 snap 218 seq_num 648) 2024-09-14T03:55:11.400 INFO:tasks.rados.rados.0.smithi070.stdout:2026: seq_num 649 ranges {507904=475136,983040=761856,1744896=40960} 2024-09-14T03:55:11.405 INFO:tasks.rados.rados.0.smithi070.stdout:2026: writing smithi07085154-46 from 507904 to 983040 tid 1 2024-09-14T03:55:11.409 INFO:tasks.rados.rados.0.smithi070.stdout:2026: writing smithi07085154-46 from 983040 to 1744896 tid 2 2024-09-14T03:55:11.410 INFO:tasks.rados.rados.0.smithi070.stdout:2026: writing smithi07085154-46 from 1744896 to 1785856 tid 3 2024-09-14T03:55:11.410 INFO:tasks.rados.rados.0.smithi070.stdout:2022: done (4 left) 2024-09-14T03:55:11.410 INFO:tasks.rados.rados.0.smithi070.stdout:2023: done (3 left) 2024-09-14T03:55:11.411 INFO:tasks.rados.rados.0.smithi070.stdout:2024: done (2 left) 2024-09-14T03:55:11.411 INFO:tasks.rados.rados.0.smithi070.stdout:2025: done (1 left) 2024-09-14T03:55:11.411 INFO:tasks.rados.rados.0.smithi070.stdout:2027: read oid 3 snap -1 2024-09-14T03:55:11.411 INFO:tasks.rados.rados.0.smithi070.stdout:2027: expect (ObjNum 608 snap 204 seq_num 608) 2024-09-14T03:55:11.411 INFO:tasks.rados.rados.0.smithi070.stdout:2028: snap_remove snap 216 2024-09-14T03:55:11.480 INFO:tasks.rados.rados.0.smithi070.stdout:2028: done (2 left) 2024-09-14T03:55:11.481 INFO:tasks.rados.rados.0.smithi070.stdout:2029: setattr oid 47 current snap is 218 2024-09-14T03:55:11.481 INFO:tasks.rados.rados.0.smithi070.stdout:2026: finishing write tid 1 to smithi07085154-46 2024-09-14T03:55:11.481 INFO:tasks.rados.rados.0.smithi070.stdout:2026: finishing write tid 2 to smithi07085154-46 2024-09-14T03:55:11.481 INFO:tasks.rados.rados.0.smithi070.stdout:2027: expect (ObjNum 608 snap 204 seq_num 608) 2024-09-14T03:55:11.508 INFO:tasks.rados.rados.0.smithi070.stdout:2026: finishing write tid 3 to smithi07085154-46 2024-09-14T03:55:11.509 INFO:tasks.rados.rados.0.smithi070.stdout:2026: finishing write tid 4 to smithi07085154-46 2024-09-14T03:55:11.509 INFO:tasks.rados.rados.0.smithi070.stdout:2026: finishing write tid 5 to smithi07085154-46 2024-09-14T03:55:11.509 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 46 v 2168 (ObjNum 649 snap 218 seq_num 649) dirty exists 2024-09-14T03:55:11.509 INFO:tasks.rados.rados.0.smithi070.stdout:2026: left oid 46 (ObjNum 649 snap 218 seq_num 649) 2024-09-14T03:55:11.509 INFO:tasks.rados.rados.0.smithi070.stdout:2026: done (2 left) 2024-09-14T03:55:11.510 INFO:tasks.rados.rados.0.smithi070.stdout:2027: done (1 left) 2024-09-14T03:55:11.510 INFO:tasks.rados.rados.0.smithi070.stdout:2030: read oid 45 snap 195 2024-09-14T03:55:11.510 INFO:tasks.rados.rados.0.smithi070.stdout:2030: expect (ObjNum 564 snap 192 seq_num 564) 2024-09-14T03:55:11.510 INFO:tasks.rados.rados.0.smithi070.stdout:2031: read oid 10 snap -1 2024-09-14T03:55:11.510 INFO:tasks.rados.rados.0.smithi070.stdout:2031: expect (ObjNum 508 snap 166 seq_num 508) 2024-09-14T03:55:11.511 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 21 current snap is 218 2024-09-14T03:55:11.511 INFO:tasks.rados.rados.0.smithi070.stdout:2032: seq_num 650 ranges {2408448=491520,2899968=647168,3547136=155648} 2024-09-14T03:55:11.524 INFO:tasks.rados.rados.0.smithi070.stdout:2032: writing smithi07085154-21 from 2408448 to 2899968 tid 1 2024-09-14T03:55:11.528 INFO:tasks.rados.rados.0.smithi070.stdout:2032: writing smithi07085154-21 from 2899968 to 3547136 tid 2 2024-09-14T03:55:11.530 INFO:tasks.rados.rados.0.smithi070.stdout:2032: writing smithi07085154-21 from 3547136 to 3702784 tid 3 2024-09-14T03:55:11.530 INFO:tasks.rados.rados.0.smithi070.stdout:2033: delete oid 9 current snap is 218 2024-09-14T03:55:11.530 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 47 v 1686 (ObjNum 592 snap 195 seq_num 592) dirty exists 2024-09-14T03:55:11.530 INFO:tasks.rados.rados.0.smithi070.stdout:2032: finishing write tid 1 to smithi07085154-21 2024-09-14T03:55:11.532 INFO:tasks.rados.rados.0.smithi070.stdout:2031: expect (ObjNum 508 snap 166 seq_num 508) 2024-09-14T03:55:11.793 INFO:tasks.rados.rados.0.smithi070.stdout:2032: finishing write tid 2 to smithi07085154-21 2024-09-14T03:55:11.794 INFO:tasks.rados.rados.0.smithi070.stdout:2032: finishing write tid 3 to smithi07085154-21 2024-09-14T03:55:11.794 INFO:tasks.rados.rados.0.smithi070.stdout:2032: finishing write tid 4 to smithi07085154-21 2024-09-14T03:55:11.794 INFO:tasks.rados.rados.0.smithi070.stdout:2029: done (4 left) 2024-09-14T03:55:11.794 INFO:tasks.rados.rados.0.smithi070.stdout:2031: done (3 left) 2024-09-14T03:55:11.794 INFO:tasks.rados.rados.0.smithi070.stdout:2033: done (2 left) 2024-09-14T03:55:11.794 INFO:tasks.rados.rados.0.smithi070.stdout:2034: rollback oid 20 current snap is 218 2024-09-14T03:55:11.794 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 20 to 181 2024-09-14T03:55:11.794 INFO:tasks.rados.rados.0.smithi070.stdout:2032: finishing write tid 5 to smithi07085154-21 2024-09-14T03:55:11.794 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 559 (ObjNum 650 snap 218 seq_num 650) dirty exists 2024-09-14T03:55:11.794 INFO:tasks.rados.rados.0.smithi070.stdout:2032: left oid 21 (ObjNum 650 snap 218 seq_num 650) 2024-09-14T03:55:11.794 INFO:tasks.rados.rados.0.smithi070.stdout:2030: expect (ObjNum 564 snap 192 seq_num 564) 2024-09-14T03:55:12.494 INFO:tasks.rados.rados.0.smithi070.stdout:2034: finishing rollback tid 0 to smithi07085154-20 2024-09-14T03:55:12.494 INFO:tasks.rados.rados.0.smithi070.stdout:2034: finishing rollback tid 1 to smithi07085154-20 2024-09-14T03:55:12.494 INFO:tasks.rados.rados.0.smithi070.stdout:2034: finishing rollback tid 2 to smithi07085154-20 2024-09-14T03:55:12.494 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 20 v 1268 (ObjNum 524 snap 174 seq_num 524) dirty exists 2024-09-14T03:55:12.494 INFO:tasks.rados.rados.0.smithi070.stdout:2030: done (2 left) 2024-09-14T03:55:12.494 INFO:tasks.rados.rados.0.smithi070.stdout:2032: done (1 left) 2024-09-14T03:55:12.494 INFO:tasks.rados.rados.0.smithi070.stdout:2034: done (0 left) 2024-09-14T03:55:12.494 INFO:tasks.rados.rados.0.smithi070.stdout:2035: read oid 34 snap 208 2024-09-14T03:55:12.494 INFO:tasks.rados.rados.0.smithi070.stdout:2035: expect (ObjNum 597 snap 199 seq_num 597) 2024-09-14T03:55:12.494 INFO:tasks.rados.rados.0.smithi070.stdout:2036: copy_from oid 33 from oid 19 current snap is 218 2024-09-14T03:55:12.494 INFO:tasks.rados.rados.0.smithi070.stdout:2037: copy_from oid 31 from oid 3 current snap is 218 2024-09-14T03:55:12.494 INFO:tasks.rados.rados.0.smithi070.stdout:2038: delete oid 1 current snap is 218 2024-09-14T03:55:12.501 INFO:tasks.rados.rados.0.smithi070.stdout:2037: finishing copy_from racing read to smithi07085154-31 2024-09-14T03:55:12.503 INFO:tasks.rados.rados.0.smithi070.stdout:2037: finishing copy_from to smithi07085154-31 2024-09-14T03:55:12.503 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 1677 (ObjNum 608 snap 204 seq_num 608) dirty exists 2024-09-14T03:55:12.505 INFO:tasks.rados.rados.0.smithi070.stdout:2037: done (3 left) 2024-09-14T03:55:12.505 INFO:tasks.rados.rados.0.smithi070.stdout:2038: done (2 left) 2024-09-14T03:55:12.505 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 44 current snap is 218 2024-09-14T03:55:12.505 INFO:tasks.rados.rados.0.smithi070.stdout:2039: seq_num 653 ranges {0=540672,540672=696320,1236992=106496} 2024-09-14T03:55:12.507 INFO:tasks.rados.rados.0.smithi070.stdout:2039: writing smithi07085154-44 from 0 to 540672 tid 1 2024-09-14T03:55:12.513 INFO:tasks.rados.rados.0.smithi070.stdout:2039: writing smithi07085154-44 from 540672 to 1236992 tid 2 2024-09-14T03:55:12.513 INFO:tasks.rados.rados.0.smithi070.stdout:2039: writing smithi07085154-44 from 1236992 to 1343488 tid 3 2024-09-14T03:55:12.513 INFO:tasks.rados.rados.0.smithi070.stdout:2040: rollback oid 15 current snap is 218 2024-09-14T03:55:12.513 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 15 to 203 2024-09-14T03:55:12.513 INFO:tasks.rados.rados.0.smithi070.stdout:2041: read oid 9 snap -1 2024-09-14T03:55:12.513 INFO:tasks.rados.rados.0.smithi070.stdout:2041: expect deleted 2024-09-14T03:55:12.513 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 3 current snap is 218 2024-09-14T03:55:12.514 INFO:tasks.rados.rados.0.smithi070.stdout:2042: seq_num 654 ranges {811008=557056,1368064=630784,1998848=172032} 2024-09-14T03:55:12.520 INFO:tasks.rados.rados.0.smithi070.stdout:2042: writing smithi07085154-3 from 811008 to 1368064 tid 1 2024-09-14T03:55:12.524 INFO:tasks.rados.rados.0.smithi070.stdout:2042: writing smithi07085154-3 from 1368064 to 1998848 tid 2 2024-09-14T03:55:12.526 INFO:tasks.rados.rados.0.smithi070.stdout:2042: writing smithi07085154-3 from 1998848 to 2170880 tid 3 2024-09-14T03:55:12.526 INFO:tasks.rados.rados.0.smithi070.stdout:2043: read oid 28 snap -1 2024-09-14T03:55:12.526 INFO:tasks.rados.rados.0.smithi070.stdout:2043: expect deleted 2024-09-14T03:55:12.526 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 18 current snap is 218 2024-09-14T03:55:12.526 INFO:tasks.rados.rados.0.smithi070.stdout:2035: expect (ObjNum 597 snap 199 seq_num 597) 2024-09-14T03:55:12.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:12 smithi094 ceph-mon[86597]: osdmap e485: 8 total, 8 up, 8 in 2024-09-14T03:55:12.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:12 smithi094 ceph-mon[86597]: pgmap v322: 121 pgs: 121 active+clean; 855 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 4.1 MiB/s wr, 26 op/s 2024-09-14T03:55:12.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:12 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:12.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:12 smithi070 ceph-mon[96704]: osdmap e485: 8 total, 8 up, 8 in 2024-09-14T03:55:12.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:12 smithi070 ceph-mon[96704]: pgmap v322: 121 pgs: 121 active+clean; 855 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 4.1 MiB/s wr, 26 op/s 2024-09-14T03:55:12.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:12 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:12.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:12 smithi070 ceph-mon[98640]: osdmap e485: 8 total, 8 up, 8 in 2024-09-14T03:55:12.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:12 smithi070 ceph-mon[98640]: pgmap v322: 121 pgs: 121 active+clean; 855 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 4.1 MiB/s wr, 26 op/s 2024-09-14T03:55:12.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:12 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:12.854 INFO:tasks.rados.rados.0.smithi070.stdout:2039: finishing write tid 1 to smithi07085154-44 2024-09-14T03:55:12.854 INFO:tasks.rados.rados.0.smithi070.stdout:2039: finishing write tid 2 to smithi07085154-44 2024-09-14T03:55:12.854 INFO:tasks.rados.rados.0.smithi070.stdout:2044: seq_num 655 ranges {6316032=573440,6889472=516096,7405568=286720} 2024-09-14T03:55:12.890 INFO:tasks.rados.rados.0.smithi070.stdout:2044: writing smithi07085154-18 from 6316032 to 6889472 tid 1 2024-09-14T03:55:12.892 INFO:tasks.rados.rados.0.smithi070.stdout:2044: writing smithi07085154-18 from 6889472 to 7405568 tid 2 2024-09-14T03:55:12.894 INFO:tasks.rados.rados.0.smithi070.stdout:2044: writing smithi07085154-18 from 7405568 to 7692288 tid 3 2024-09-14T03:55:12.894 INFO:tasks.rados.rados.0.smithi070.stdout:2035: done (7 left) 2024-09-14T03:55:12.894 INFO:tasks.rados.rados.0.smithi070.stdout:2041: done (6 left) 2024-09-14T03:55:12.894 INFO:tasks.rados.rados.0.smithi070.stdout:2045: snap_remove snap 209 2024-09-14T03:55:13.488 INFO:tasks.rados.rados.0.smithi070.stdout:2045: done (6 left) 2024-09-14T03:55:13.488 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 27 current snap is 218 2024-09-14T03:55:13.488 INFO:tasks.rados.rados.0.smithi070.stdout:2039: finishing write tid 3 to smithi07085154-44 2024-09-14T03:55:13.488 INFO:tasks.rados.rados.0.smithi070.stdout:2040: finishing rollback tid 0 to smithi07085154-15 2024-09-14T03:55:13.488 INFO:tasks.rados.rados.0.smithi070.stdout:2039: finishing write tid 4 to smithi07085154-44 2024-09-14T03:55:13.488 INFO:tasks.rados.rados.0.smithi070.stdout:2040: finishing rollback tid 1 to smithi07085154-15 2024-09-14T03:55:13.488 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 15 v 1700 (ObjNum 202 snap 0 seq_num 304) dirty dne 2024-09-14T03:55:13.489 INFO:tasks.rados.rados.0.smithi070.stdout:2039: finishing write tid 5 to smithi07085154-44 2024-09-14T03:55:13.489 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 1955 (ObjNum 653 snap 218 seq_num 653) dirty exists 2024-09-14T03:55:13.489 INFO:tasks.rados.rados.0.smithi070.stdout:2039: left oid 44 (ObjNum 653 snap 218 seq_num 653) 2024-09-14T03:55:13.489 INFO:tasks.rados.rados.0.smithi070.stdout:2042: finishing write tid 1 to smithi07085154-3 2024-09-14T03:55:13.489 INFO:tasks.rados.rados.0.smithi070.stdout:2042: finishing write tid 2 to smithi07085154-3 2024-09-14T03:55:13.489 INFO:tasks.rados.rados.0.smithi070.stdout:2042: finishing write tid 3 to smithi07085154-3 2024-09-14T03:55:13.489 INFO:tasks.rados.rados.0.smithi070.stdout:2042: finishing write tid 4 to smithi07085154-3 2024-09-14T03:55:13.489 INFO:tasks.rados.rados.0.smithi070.stdout:2042: finishing write tid 5 to smithi07085154-3 2024-09-14T03:55:13.489 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 3 v 1690 (ObjNum 654 snap 218 seq_num 654) dirty exists 2024-09-14T03:55:13.489 INFO:tasks.rados.rados.0.smithi070.stdout:2042: left oid 3 (ObjNum 654 snap 218 seq_num 654) 2024-09-14T03:55:13.489 INFO:tasks.rados.rados.0.smithi070.stdout:2036: finishing copy_from racing read to smithi07085154-33 2024-09-14T03:55:13.489 INFO:tasks.rados.rados.0.smithi070.stdout:2036: finishing copy_from to smithi07085154-33 2024-09-14T03:55:13.489 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 33 v 1679 (ObjNum 600 snap 199 seq_num 600) dirty exists 2024-09-14T03:55:13.489 INFO:tasks.rados.rados.0.smithi070.stdout:2044: finishing write tid 1 to smithi07085154-18 2024-09-14T03:55:13.490 INFO:tasks.rados.rados.0.smithi070.stdout:2044: finishing write tid 2 to smithi07085154-18 2024-09-14T03:55:13.490 INFO:tasks.rados.rados.0.smithi070.stdout:2044: finishing write tid 3 to smithi07085154-18 2024-09-14T03:55:13.490 INFO:tasks.rados.rados.0.smithi070.stdout:2044: finishing write tid 4 to smithi07085154-18 2024-09-14T03:55:13.490 INFO:tasks.rados.rados.0.smithi070.stdout:2044: finishing write tid 5 to smithi07085154-18 2024-09-14T03:55:13.490 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 1704 (ObjNum 655 snap 218 seq_num 655) dirty exists 2024-09-14T03:55:13.490 INFO:tasks.rados.rados.0.smithi070.stdout:2044: left oid 18 (ObjNum 655 snap 218 seq_num 655) 2024-09-14T03:55:13.490 INFO:tasks.rados.rados.0.smithi070.stdout:2046: seq_num 656 ranges {19054592=589824,19644416=802816} 2024-09-14T03:55:13.591 INFO:tasks.rados.rados.0.smithi070.stdout:2046: writing smithi07085154-27 from 19054592 to 19644416 tid 1 2024-09-14T03:55:13.597 INFO:tasks.rados.rados.0.smithi070.stdout:2046: writing smithi07085154-27 from 19644416 to 20447232 tid 2 2024-09-14T03:55:13.597 INFO:tasks.rados.rados.0.smithi070.stdout:2036: done (6 left) 2024-09-14T03:55:13.597 INFO:tasks.rados.rados.0.smithi070.stdout:2039: done (5 left) 2024-09-14T03:55:13.597 INFO:tasks.rados.rados.0.smithi070.stdout:2040: done (4 left) 2024-09-14T03:55:13.597 INFO:tasks.rados.rados.0.smithi070.stdout:2042: done (3 left) 2024-09-14T03:55:13.597 INFO:tasks.rados.rados.0.smithi070.stdout:2043: done (2 left) 2024-09-14T03:55:13.597 INFO:tasks.rados.rados.0.smithi070.stdout:2044: done (1 left) 2024-09-14T03:55:13.597 INFO:tasks.rados.rados.0.smithi070.stdout:2047: read oid 44 snap -1 2024-09-14T03:55:13.597 INFO:tasks.rados.rados.0.smithi070.stdout:2047: expect (ObjNum 653 snap 218 seq_num 653) 2024-09-14T03:55:13.597 INFO:tasks.rados.rados.0.smithi070.stdout:2048: snap_create 2024-09-14T03:55:13.597 INFO:tasks.rados.rados.0.smithi070.stdout:2046: finishing write tid 1 to smithi07085154-27 2024-09-14T03:55:13.601 INFO:tasks.rados.rados.0.smithi070.stdout:2046: finishing write tid 2 to smithi07085154-27 2024-09-14T03:55:13.601 INFO:tasks.rados.rados.0.smithi070.stdout:2046: finishing write tid 3 to smithi07085154-27 2024-09-14T03:55:13.602 INFO:tasks.rados.rados.0.smithi070.stdout:2046: finishing write tid 4 to smithi07085154-27 2024-09-14T03:55:13.602 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 2185 (ObjNum 656 snap 218 seq_num 656) dirty exists 2024-09-14T03:55:13.602 INFO:tasks.rados.rados.0.smithi070.stdout:2046: left oid 27 (ObjNum 656 snap 218 seq_num 656) 2024-09-14T03:55:13.604 INFO:tasks.rados.rados.0.smithi070.stdout:2047: expect (ObjNum 653 snap 218 seq_num 653) 2024-09-14T03:55:13.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:55:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:55:13] "GET /metrics HTTP/1.1" 200 36161 "" "Prometheus/2.51.0" 2024-09-14T03:55:14.044 INFO:tasks.rados.rados.0.smithi070.stdout:2046: done (2 left) 2024-09-14T03:55:14.045 INFO:tasks.rados.rados.0.smithi070.stdout:2047: done (1 left) 2024-09-14T03:55:14.045 INFO:tasks.rados.rados.0.smithi070.stdout:2048: done (0 left) 2024-09-14T03:55:14.045 INFO:tasks.rados.rados.0.smithi070.stdout:2049: rollback oid 42 current snap is 219 2024-09-14T03:55:14.045 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 42 to 214 2024-09-14T03:55:14.045 INFO:tasks.rados.rados.0.smithi070.stdout:2050: rollback oid 29 current snap is 219 2024-09-14T03:55:14.045 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 29 to 174 2024-09-14T03:55:14.045 INFO:tasks.rados.rados.0.smithi070.stdout:2051: snap_create 2024-09-14T03:55:14.050 INFO:tasks.rados.rados.0.smithi070.stdout:2050: finishing rollback tid 0 to smithi07085154-29 2024-09-14T03:55:14.051 INFO:tasks.rados.rados.0.smithi070.stdout:2050: finishing rollback tid 1 to smithi07085154-29 2024-09-14T03:55:14.051 INFO:tasks.rados.rados.0.smithi070.stdout:2050: finishing rollback tid 2 to smithi07085154-29 2024-09-14T03:55:14.051 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 1692 (ObjNum 458 snap 155 seq_num 458) dirty exists 2024-09-14T03:55:14.054 INFO:tasks.rados.rados.0.smithi070.stdout:2049: finishing rollback tid 0 to smithi07085154-42 2024-09-14T03:55:14.054 INFO:tasks.rados.rados.0.smithi070.stdout:2049: finishing rollback tid 1 to smithi07085154-42 2024-09-14T03:55:14.054 INFO:tasks.rados.rados.0.smithi070.stdout:2049: finishing rollback tid 2 to smithi07085154-42 2024-09-14T03:55:14.054 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 2189 (ObjNum 625 snap 210 seq_num 625) dirty exists 2024-09-14T03:55:14.485 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:55:14 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:55:14.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.5\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.5\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.94\", device_class=\"ssd\", hostname=\"smithi094\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.94\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.5\", ceph_version=\"ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)\", cluster_addr=\"172.21.15.94\", device_class=\"ssd\", hostname=\"smithi094\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.94\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:55:14.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:14 smithi094 ceph-mon[86597]: osdmap e486: 8 total, 8 up, 8 in 2024-09-14T03:55:14.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:14 smithi094 ceph-mon[86597]: osdmap e487: 8 total, 8 up, 8 in 2024-09-14T03:55:14.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:14 smithi094 ceph-mon[86597]: pgmap v325: 121 pgs: 121 active+clean; 858 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 1.4 MiB/s wr, 15 op/s 2024-09-14T03:55:14.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:14 smithi070 ceph-mon[98640]: osdmap e486: 8 total, 8 up, 8 in 2024-09-14T03:55:14.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:14 smithi070 ceph-mon[98640]: osdmap e487: 8 total, 8 up, 8 in 2024-09-14T03:55:14.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:14 smithi070 ceph-mon[98640]: pgmap v325: 121 pgs: 121 active+clean; 858 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 1.4 MiB/s wr, 15 op/s 2024-09-14T03:55:14.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:14 smithi070 ceph-mon[96704]: osdmap e486: 8 total, 8 up, 8 in 2024-09-14T03:55:14.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:14 smithi070 ceph-mon[96704]: osdmap e487: 8 total, 8 up, 8 in 2024-09-14T03:55:14.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:14 smithi070 ceph-mon[96704]: pgmap v325: 121 pgs: 121 active+clean; 858 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 1.4 MiB/s wr, 15 op/s 2024-09-14T03:55:15.051 INFO:tasks.rados.rados.0.smithi070.stdout:2049: done (2 left) 2024-09-14T03:55:15.052 INFO:tasks.rados.rados.0.smithi070.stdout:2050: done (1 left) 2024-09-14T03:55:15.052 INFO:tasks.rados.rados.0.smithi070.stdout:2051: done (0 left) 2024-09-14T03:55:15.052 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 50 current snap is 220 2024-09-14T03:55:15.052 INFO:tasks.rados.rados.0.smithi070.stdout:2052: seq_num 657 ranges {1105920=606208,1712128=688128,2400256=114688} 2024-09-14T03:55:15.060 INFO:tasks.rados.rados.0.smithi070.stdout:2052: writing smithi07085154-50 from 1105920 to 1712128 tid 1 2024-09-14T03:55:15.064 INFO:tasks.rados.rados.0.smithi070.stdout:2052: writing smithi07085154-50 from 1712128 to 2400256 tid 2 2024-09-14T03:55:15.065 INFO:tasks.rados.rados.0.smithi070.stdout:2052: writing smithi07085154-50 from 2400256 to 2514944 tid 3 2024-09-14T03:55:15.065 INFO:tasks.rados.rados.0.smithi070.stdout:2053: read oid 28 snap 199 2024-09-14T03:55:15.065 INFO:tasks.rados.rados.0.smithi070.stdout:2053: expect deleted 2024-09-14T03:55:15.065 INFO:tasks.rados.rados.0.smithi070.stdout:2054: snap_remove snap 210 2024-09-14T03:55:16.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:55:15 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:55:15.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:55:16.027 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:55:15 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:55:15.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:55:16.057 INFO:tasks.rados.rados.0.smithi070.stdout:2054: done (2 left) 2024-09-14T03:55:16.057 INFO:tasks.rados.rados.0.smithi070.stdout:2055: read oid 36 snap -1 2024-09-14T03:55:16.057 INFO:tasks.rados.rados.0.smithi070.stdout:2055: expect (ObjNum 590 snap 195 seq_num 590) 2024-09-14T03:55:16.057 INFO:tasks.rados.rados.0.smithi070.stdout:2056: read oid 15 snap 201 2024-09-14T03:55:16.058 INFO:tasks.rados.rados.0.smithi070.stdout:2056: expect deleted 2024-09-14T03:55:16.058 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 30 current snap is 220 2024-09-14T03:55:16.058 INFO:tasks.rados.rados.0.smithi070.stdout:2052: finishing write tid 1 to smithi07085154-50 2024-09-14T03:55:16.058 INFO:tasks.rados.rados.0.smithi070.stdout:2057: seq_num 658 ranges {1179648=622592,1802240=565248,2367488=237568} 2024-09-14T03:55:16.068 INFO:tasks.rados.rados.0.smithi070.stdout:2057: writing smithi07085154-30 from 1179648 to 1802240 tid 1 2024-09-14T03:55:16.071 INFO:tasks.rados.rados.0.smithi070.stdout:2057: writing smithi07085154-30 from 1802240 to 2367488 tid 2 2024-09-14T03:55:16.074 INFO:tasks.rados.rados.0.smithi070.stdout:2057: writing smithi07085154-30 from 2367488 to 2605056 tid 3 2024-09-14T03:55:16.074 INFO:tasks.rados.rados.0.smithi070.stdout:2058: snap_remove snap 120 2024-09-14T03:55:16.336 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:16 smithi094 ceph-mon[86597]: osdmap e488: 8 total, 8 up, 8 in 2024-09-14T03:55:16.337 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:16 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-14T03:55:16.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:16 smithi070 ceph-mon[96704]: osdmap e488: 8 total, 8 up, 8 in 2024-09-14T03:55:16.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:16 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-14T03:55:16.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:16 smithi070 ceph-mon[98640]: osdmap e488: 8 total, 8 up, 8 in 2024-09-14T03:55:16.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:16 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-14T03:55:17.062 INFO:tasks.rados.rados.0.smithi070.stdout:2058: done (5 left) 2024-09-14T03:55:17.062 INFO:tasks.rados.rados.0.smithi070.stdout:2059: snap_remove snap 176 2024-09-14T03:55:17.436 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:17 smithi094 ceph-mon[86597]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-14T03:55:17.436 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:17 smithi094 ceph-mon[86597]: Upgrade: osd.5 is safe to restart 2024-09-14T03:55:17.436 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:17 smithi094 ceph-mon[86597]: osdmap e489: 8 total, 8 up, 8 in 2024-09-14T03:55:17.436 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:17 smithi094 ceph-mon[86597]: pgmap v328: 121 pgs: 121 active+clean; 867 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 3.0 MiB/s wr, 14 op/s 2024-09-14T03:55:17.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:17 smithi070 ceph-mon[96704]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-14T03:55:17.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:17 smithi070 ceph-mon[96704]: Upgrade: osd.5 is safe to restart 2024-09-14T03:55:17.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:17 smithi070 ceph-mon[96704]: osdmap e489: 8 total, 8 up, 8 in 2024-09-14T03:55:17.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:17 smithi070 ceph-mon[96704]: pgmap v328: 121 pgs: 121 active+clean; 867 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 3.0 MiB/s wr, 14 op/s 2024-09-14T03:55:17.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:17 smithi070 ceph-mon[98640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-14T03:55:17.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:17 smithi070 ceph-mon[98640]: Upgrade: osd.5 is safe to restart 2024-09-14T03:55:17.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:17 smithi070 ceph-mon[98640]: osdmap e489: 8 total, 8 up, 8 in 2024-09-14T03:55:17.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:17 smithi070 ceph-mon[98640]: pgmap v328: 121 pgs: 121 active+clean; 867 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 3.0 MiB/s wr, 14 op/s 2024-09-14T03:55:18.064 INFO:tasks.rados.rados.0.smithi070.stdout:2059: done (5 left) 2024-09-14T03:55:18.064 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 19 current snap is 220 2024-09-14T03:55:18.064 INFO:tasks.rados.rados.0.smithi070.stdout:2052: finishing write tid 2 to smithi07085154-50 2024-09-14T03:55:18.064 INFO:tasks.rados.rados.0.smithi070.stdout:2052: finishing write tid 3 to smithi07085154-50 2024-09-14T03:55:18.064 INFO:tasks.rados.rados.0.smithi070.stdout:2052: finishing write tid 4 to smithi07085154-50 2024-09-14T03:55:18.064 INFO:tasks.rados.rados.0.smithi070.stdout:2052: finishing write tid 5 to smithi07085154-50 2024-09-14T03:55:18.064 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 1696 (ObjNum 657 snap 220 seq_num 657) dirty exists 2024-09-14T03:55:18.064 INFO:tasks.rados.rados.0.smithi070.stdout:2052: left oid 50 (ObjNum 657 snap 220 seq_num 657) 2024-09-14T03:55:18.064 INFO:tasks.rados.rados.0.smithi070.stdout:2057: finishing write tid 1 to smithi07085154-30 2024-09-14T03:55:18.065 INFO:tasks.rados.rados.0.smithi070.stdout:2057: finishing write tid 2 to smithi07085154-30 2024-09-14T03:55:18.065 INFO:tasks.rados.rados.0.smithi070.stdout:2057: finishing write tid 3 to smithi07085154-30 2024-09-14T03:55:18.065 INFO:tasks.rados.rados.0.smithi070.stdout:2057: finishing write tid 4 to smithi07085154-30 2024-09-14T03:55:18.065 INFO:tasks.rados.rados.0.smithi070.stdout:2057: finishing write tid 5 to smithi07085154-30 2024-09-14T03:55:18.065 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 30 v 1694 (ObjNum 658 snap 220 seq_num 658) dirty exists 2024-09-14T03:55:18.065 INFO:tasks.rados.rados.0.smithi070.stdout:2057: left oid 30 (ObjNum 658 snap 220 seq_num 658) 2024-09-14T03:55:18.065 INFO:tasks.rados.rados.0.smithi070.stdout:2055: expect (ObjNum 590 snap 195 seq_num 590) 2024-09-14T03:55:18.308 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:18 smithi094 ceph-mon[86597]: osdmap e490: 8 total, 8 up, 8 in 2024-09-14T03:55:18.308 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:18 smithi094 ceph-mon[86597]: Upgrade: Updating osd.5 2024-09-14T03:55:18.308 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:18 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:18.308 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:18 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-14T03:55:18.308 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:18 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:55:18.308 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:18 smithi094 ceph-mon[86597]: Deploying daemon osd.5 on smithi094 2024-09-14T03:55:18.308 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:18 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:18.309 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:18 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:18.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:18 smithi070 ceph-mon[98640]: osdmap e490: 8 total, 8 up, 8 in 2024-09-14T03:55:18.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:18 smithi070 ceph-mon[98640]: Upgrade: Updating osd.5 2024-09-14T03:55:18.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:18 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:18.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:18 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-14T03:55:18.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:18 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:55:18.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:18 smithi070 ceph-mon[98640]: Deploying daemon osd.5 on smithi094 2024-09-14T03:55:18.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:18 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:18.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:18 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:18.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:18 smithi070 ceph-mon[96704]: osdmap e490: 8 total, 8 up, 8 in 2024-09-14T03:55:18.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:18 smithi070 ceph-mon[96704]: Upgrade: Updating osd.5 2024-09-14T03:55:18.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:18 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:18.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:18 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-14T03:55:18.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:18 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:55:18.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:18 smithi070 ceph-mon[96704]: Deploying daemon osd.5 on smithi094 2024-09-14T03:55:18.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:18 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:18.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:18 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:19.207 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:19 smithi094 ceph-mon[86597]: osdmap e491: 8 total, 8 up, 8 in 2024-09-14T03:55:19.208 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:19 smithi094 ceph-mon[86597]: pgmap v331: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 860 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 3.2 MiB/s wr, 7 op/s 2024-09-14T03:55:19.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:19 smithi070 ceph-mon[96704]: osdmap e491: 8 total, 8 up, 8 in 2024-09-14T03:55:19.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:19 smithi070 ceph-mon[96704]: pgmap v331: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 860 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 3.2 MiB/s wr, 7 op/s 2024-09-14T03:55:19.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:19 smithi070 ceph-mon[98640]: osdmap e491: 8 total, 8 up, 8 in 2024-09-14T03:55:19.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:19 smithi070 ceph-mon[98640]: pgmap v331: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 860 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 3.2 MiB/s wr, 7 op/s 2024-09-14T03:55:19.975 INFO:tasks.rados.rados.0.smithi070.stdout:2060: seq_num 659 ranges {7766016=647168,8413184=450560,8863744=352256} 2024-09-14T03:55:20.018 INFO:tasks.rados.rados.0.smithi070.stdout:2060: writing smithi07085154-19 from 7766016 to 8413184 tid 1 2024-09-14T03:55:20.021 INFO:tasks.rados.rados.0.smithi070.stdout:2060: writing smithi07085154-19 from 8413184 to 8863744 tid 2 2024-09-14T03:55:20.023 INFO:tasks.rados.rados.0.smithi070.stdout:2060: writing smithi07085154-19 from 8863744 to 9216000 tid 3 2024-09-14T03:55:20.023 INFO:tasks.rados.rados.0.smithi070.stdout:2052: done (5 left) 2024-09-14T03:55:20.023 INFO:tasks.rados.rados.0.smithi070.stdout:2053: done (4 left) 2024-09-14T03:55:20.023 INFO:tasks.rados.rados.0.smithi070.stdout:2055: done (3 left) 2024-09-14T03:55:20.024 INFO:tasks.rados.rados.0.smithi070.stdout:2056: done (2 left) 2024-09-14T03:55:20.024 INFO:tasks.rados.rados.0.smithi070.stdout:2057: done (1 left) 2024-09-14T03:55:20.024 INFO:tasks.rados.rados.0.smithi070.stdout:2061: setattr oid 31 current snap is 220 2024-09-14T03:55:20.024 INFO:tasks.rados.rados.0.smithi070.stdout:2060: finishing write tid 1 to smithi07085154-19 2024-09-14T03:55:20.025 INFO:tasks.rados.rados.0.smithi070.stdout:2062: rollback oid 11 current snap is 220 2024-09-14T03:55:20.025 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 11 to 213 2024-09-14T03:55:20.025 INFO:tasks.rados.rados.0.smithi070.stdout:2063: read oid 47 snap -1 2024-09-14T03:55:20.025 INFO:tasks.rados.rados.0.smithi070.stdout:2063: expect (ObjNum 592 snap 195 seq_num 592) 2024-09-14T03:55:20.025 INFO:tasks.rados.rados.0.smithi070.stdout:2064: delete oid 4 current snap is 220 2024-09-14T03:55:20.026 INFO:tasks.rados.rados.0.smithi070.stdout:2060: finishing write tid 2 to smithi07085154-19 2024-09-14T03:55:20.026 INFO:tasks.rados.rados.0.smithi070.stdout:2060: finishing write tid 3 to smithi07085154-19 2024-09-14T03:55:20.026 INFO:tasks.rados.rados.0.smithi070.stdout:2060: finishing write tid 4 to smithi07085154-19 2024-09-14T03:55:20.029 INFO:tasks.rados.rados.0.smithi070.stdout:2064: done (4 left) 2024-09-14T03:55:20.030 INFO:tasks.rados.rados.0.smithi070.stdout:2065: rmattr oid 26 current snap is 220 2024-09-14T03:55:20.030 INFO:tasks.rados.rados.0.smithi070.stdout:2065: done (4 left) 2024-09-14T03:55:20.030 INFO:tasks.rados.rados.0.smithi070.stdout:2066: rollback oid 1 current snap is 220 2024-09-14T03:55:20.030 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 1 to 195 2024-09-14T03:55:20.030 INFO:tasks.rados.rados.0.smithi070.stdout:2067: rmattr oid 14 current snap is 220 2024-09-14T03:55:20.030 INFO:tasks.rados.rados.0.smithi070.stdout:2060: finishing write tid 5 to smithi07085154-19 2024-09-14T03:55:20.030 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 19 v 1700 (ObjNum 659 snap 220 seq_num 659) dirty exists 2024-09-14T03:55:20.030 INFO:tasks.rados.rados.0.smithi070.stdout:2060: left oid 19 (ObjNum 659 snap 220 seq_num 659) 2024-09-14T03:55:20.030 INFO:tasks.rados.rados.0.smithi070.stdout:2060: done (5 left) 2024-09-14T03:55:20.030 INFO:tasks.rados.rados.0.smithi070.stdout:2068: rollback oid 26 current snap is 220 2024-09-14T03:55:20.030 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 26 to 162 2024-09-14T03:55:20.030 INFO:tasks.rados.rados.0.smithi070.stdout:2069: read oid 41 snap -1 2024-09-14T03:55:20.031 INFO:tasks.rados.rados.0.smithi070.stdout:2069: expect (ObjNum 514 snap 171 seq_num 514) 2024-09-14T03:55:20.031 INFO:tasks.rados.rados.0.smithi070.stdout:2070: copy_from oid 45 from oid 36 current snap is 220 2024-09-14T03:55:20.031 INFO:tasks.rados.rados.0.smithi070.stdout:2071: copy_from oid 12 from oid 27 current snap is 220 2024-09-14T03:55:20.031 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 16 current snap is 220 2024-09-14T03:55:20.031 INFO:tasks.rados.rados.0.smithi070.stdout:2072: seq_num 661 ranges {0=679936,679936=622592,1302528=180224} 2024-09-14T03:55:20.034 INFO:tasks.rados.rados.0.smithi070.stdout:2072: writing smithi07085154-16 from 0 to 679936 tid 1 2024-09-14T03:55:20.038 INFO:tasks.rados.rados.0.smithi070.stdout:2072: writing smithi07085154-16 from 679936 to 1302528 tid 2 2024-09-14T03:55:20.040 INFO:tasks.rados.rados.0.smithi070.stdout:2072: writing smithi07085154-16 from 1302528 to 1482752 tid 3 2024-09-14T03:55:20.040 INFO:tasks.rados.rados.0.smithi070.stdout:2073: copy_from oid 37 from oid 13 current snap is 220 2024-09-14T03:55:20.040 INFO:tasks.rados.rados.0.smithi070.stdout:2066: finishing rollback tid 1 to smithi07085154-1 2024-09-14T03:55:20.040 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 2126 (ObjNum 608 snap 204 seq_num 608) dirty exists 2024-09-14T03:55:20.040 INFO:tasks.rados.rados.0.smithi070.stdout:2062: finishing rollback tid 0 to smithi07085154-11 2024-09-14T03:55:20.040 INFO:tasks.rados.rados.0.smithi070.stdout:2062: finishing rollback tid 1 to smithi07085154-11 2024-09-14T03:55:20.040 INFO:tasks.rados.rados.0.smithi070.stdout:2062: finishing rollback tid 2 to smithi07085154-11 2024-09-14T03:55:20.040 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 11 v 2025 (ObjNum 403 snap 140 seq_num 403) dirty exists 2024-09-14T03:55:20.040 INFO:tasks.rados.rados.0.smithi070.stdout:2061: done (10 left) 2024-09-14T03:55:20.040 INFO:tasks.rados.rados.0.smithi070.stdout:2062: done (9 left) 2024-09-14T03:55:20.041 INFO:tasks.rados.rados.0.smithi070.stdout:2074: delete oid 28 current snap is 220 2024-09-14T03:55:20.041 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 1692 (ObjNum 595 snap 198 seq_num 595) dirty exists 2024-09-14T03:55:20.042 INFO:tasks.rados.rados.0.smithi070.stdout:2067: done (9 left) 2024-09-14T03:55:20.042 INFO:tasks.rados.rados.0.smithi070.stdout:2074: done (8 left) 2024-09-14T03:55:20.042 INFO:tasks.rados.rados.0.smithi070.stdout:2075: snap_remove snap 202 2024-09-14T03:55:20.079 INFO:tasks.rados.rados.0.smithi070.stdout:2075: done (8 left) 2024-09-14T03:55:20.079 INFO:tasks.rados.rados.0.smithi070.stdout:2076: snap_remove snap 206 2024-09-14T03:55:20.159 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:19 smithi094 systemd[1]: Stopping Ceph osd.5 for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:55:20.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:55:20 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:55:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:55:20.527 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:20 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[42647]: 2024-09-14T03:55:20.155+0000 7f29e740b640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:55:20.527 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:20 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[42647]: 2024-09-14T03:55:20.155+0000 7f29e740b640 -1 osd.5 492 *** Got signal Terminated *** 2024-09-14T03:55:20.527 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:20 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[42647]: 2024-09-14T03:55:20.155+0000 7f29e740b640 -1 osd.5 492 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T03:55:21.080 INFO:tasks.rados.rados.0.smithi070.stdout:2076: done (8 left) 2024-09-14T03:55:21.081 INFO:tasks.rados.rados.0.smithi070.stdout:2077: copy_from oid 28 from oid 7 current snap is 220 2024-09-14T03:55:21.081 INFO:tasks.rados.rados.0.smithi070.stdout:2069: expect (ObjNum 514 snap 171 seq_num 514) 2024-09-14T03:55:21.239 INFO:tasks.rados.rados.0.smithi070.stdout:2066: finishing rollback tid 2 to smithi07085154-1 2024-09-14T03:55:21.239 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 1 v 1957 (ObjNum 470 snap 160 seq_num 470) dirty exists 2024-09-14T03:55:21.239 INFO:tasks.rados.rados.0.smithi070.stdout:2068: finishing rollback tid 0 to smithi07085154-26 2024-09-14T03:55:21.239 INFO:tasks.rados.rados.0.smithi070.stdout:2068: finishing rollback tid 1 to smithi07085154-26 2024-09-14T03:55:21.239 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 2228 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-14T03:55:21.239 INFO:tasks.rados.rados.0.smithi070.stdout:2072: finishing write tid 1 to smithi07085154-16 2024-09-14T03:55:21.239 INFO:tasks.rados.rados.0.smithi070.stdout:2072: finishing write tid 2 to smithi07085154-16 2024-09-14T03:55:21.239 INFO:tasks.rados.rados.0.smithi070.stdout:2072: finishing write tid 3 to smithi07085154-16 2024-09-14T03:55:21.239 INFO:tasks.rados.rados.0.smithi070.stdout:2072: finishing write tid 4 to smithi07085154-16 2024-09-14T03:55:21.239 INFO:tasks.rados.rados.0.smithi070.stdout:2072: finishing write tid 5 to smithi07085154-16 2024-09-14T03:55:21.239 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 2232 (ObjNum 661 snap 220 seq_num 661) dirty exists 2024-09-14T03:55:21.240 INFO:tasks.rados.rados.0.smithi070.stdout:2072: left oid 16 (ObjNum 661 snap 220 seq_num 661) 2024-09-14T03:55:21.240 INFO:tasks.rados.rados.0.smithi070.stdout:2063: expect (ObjNum 592 snap 195 seq_num 592) 2024-09-14T03:55:21.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:21 smithi094 ceph-mon[86597]: pgmap v332: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 858 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.1 MiB/s wr, 6 op/s 2024-09-14T03:55:21.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:21 smithi094 ceph-mon[86597]: osdmap e492: 8 total, 8 up, 8 in 2024-09-14T03:55:21.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:21 smithi094 ceph-mon[86597]: osd.5 marked itself down and dead 2024-09-14T03:55:21.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:21 smithi070 ceph-mon[98640]: pgmap v332: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 858 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.1 MiB/s wr, 6 op/s 2024-09-14T03:55:21.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:21 smithi070 ceph-mon[98640]: osdmap e492: 8 total, 8 up, 8 in 2024-09-14T03:55:21.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:21 smithi070 ceph-mon[98640]: osd.5 marked itself down and dead 2024-09-14T03:55:21.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:21 smithi070 ceph-mon[96704]: pgmap v332: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 858 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.1 MiB/s wr, 6 op/s 2024-09-14T03:55:21.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:21 smithi070 ceph-mon[96704]: osdmap e492: 8 total, 8 up, 8 in 2024-09-14T03:55:21.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:21 smithi070 ceph-mon[96704]: osd.5 marked itself down and dead 2024-09-14T03:55:21.937 INFO:tasks.rados.rados.0.smithi070.stdout:2073: finishing copy_from racing read to smithi07085154-37 2024-09-14T03:55:21.938 INFO:tasks.rados.rados.0.smithi070.stdout:2073: finishing copy_from to smithi07085154-37 2024-09-14T03:55:21.938 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 37 v 2022 (ObjNum 459 snap 155 seq_num 459) dirty exists 2024-09-14T03:55:21.938 INFO:tasks.rados.rados.0.smithi070.stdout:2070: finishing copy_from racing read to smithi07085154-45 2024-09-14T03:55:21.938 INFO:tasks.rados.rados.0.smithi070.stdout:2070: finishing copy_from to smithi07085154-45 2024-09-14T03:55:21.938 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 45 v 1944 (ObjNum 590 snap 195 seq_num 590) dirty exists 2024-09-14T03:55:21.938 INFO:tasks.rados.rados.0.smithi070.stdout:2071: finishing copy_from racing read to smithi07085154-12 2024-09-14T03:55:21.938 INFO:tasks.rados.rados.0.smithi070.stdout:2071: finishing copy_from to smithi07085154-12 2024-09-14T03:55:21.938 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 2185 (ObjNum 656 snap 218 seq_num 656) dirty exists 2024-09-14T03:55:21.938 INFO:tasks.rados.rados.0.smithi070.stdout:2063: done (8 left) 2024-09-14T03:55:21.938 INFO:tasks.rados.rados.0.smithi070.stdout:2066: done (7 left) 2024-09-14T03:55:21.938 INFO:tasks.rados.rados.0.smithi070.stdout:2068: done (6 left) 2024-09-14T03:55:21.939 INFO:tasks.rados.rados.0.smithi070.stdout:2069: done (5 left) 2024-09-14T03:55:21.939 INFO:tasks.rados.rados.0.smithi070.stdout:2070: done (4 left) 2024-09-14T03:55:21.939 INFO:tasks.rados.rados.0.smithi070.stdout:2071: done (3 left) 2024-09-14T03:55:21.939 INFO:tasks.rados.rados.0.smithi070.stdout:2072: done (2 left) 2024-09-14T03:55:21.939 INFO:tasks.rados.rados.0.smithi070.stdout:2073: done (1 left) 2024-09-14T03:55:21.939 INFO:tasks.rados.rados.0.smithi070.stdout:2078: read oid 21 snap -1 2024-09-14T03:55:21.939 INFO:tasks.rados.rados.0.smithi070.stdout:2078: expect (ObjNum 650 snap 218 seq_num 650) 2024-09-14T03:55:21.939 INFO:tasks.rados.rados.0.smithi070.stdout:2079: setattr oid 29 current snap is 220 2024-09-14T03:55:21.939 INFO:tasks.rados.rados.0.smithi070.stdout:2077: finishing copy_from racing read to smithi07085154-28 2024-09-14T03:55:21.939 INFO:tasks.rados.rados.0.smithi070.stdout:2077: finishing copy_from to smithi07085154-28 2024-09-14T03:55:21.939 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 28 v 1262 (ObjNum 639 snap 214 seq_num 639) dirty exists 2024-09-14T03:55:21.940 INFO:tasks.rados.rados.0.smithi070.stdout:2077: done (2 left) 2024-09-14T03:55:21.940 INFO:tasks.rados.rados.0.smithi070.stdout:2080: snap_remove snap 187 2024-09-14T03:55:21.948 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:21 smithi094 podman[96495]: 2024-09-14 03:55:21.657972968 +0000 UTC m=+1.626911991 container died b543f152ba87a383792f0e5b35cc69b6cc59789c549466c1f89dddb51cdce635 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=quincy-eb16b97, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-14T03:55:22.090 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 1701 (ObjNum 458 snap 155 seq_num 458) dirty exists 2024-09-14T03:55:22.090 INFO:tasks.rados.rados.0.smithi070.stdout:2078: expect (ObjNum 650 snap 218 seq_num 650) 2024-09-14T03:55:22.231 INFO:tasks.rados.rados.0.smithi070.stdout:2078: done (2 left) 2024-09-14T03:55:22.231 INFO:tasks.rados.rados.0.smithi070.stdout:2079: done (1 left) 2024-09-14T03:55:22.232 INFO:tasks.rados.rados.0.smithi070.stdout:2080: done (0 left) 2024-09-14T03:55:22.232 INFO:tasks.rados.rados.0.smithi070.stdout:2081: read oid 13 snap 193 2024-09-14T03:55:22.232 INFO:tasks.rados.rados.0.smithi070.stdout:2081: expect (ObjNum 403 snap 140 seq_num 403) 2024-09-14T03:55:22.232 INFO:tasks.rados.rados.0.smithi070.stdout:2082: rmattr oid 31 current snap is 220 2024-09-14T03:55:22.232 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 22 current snap is 220 2024-09-14T03:55:22.232 INFO:tasks.rados.rados.0.smithi070.stdout:2083: seq_num 663 ranges {1294336=712704,2007040=442368,2449408=360448} 2024-09-14T03:55:22.243 INFO:tasks.rados.rados.0.smithi070.stdout:2083: writing smithi07085154-22 from 1294336 to 2007040 tid 1 2024-09-14T03:55:22.246 INFO:tasks.rados.rados.0.smithi070.stdout:2083: writing smithi07085154-22 from 2007040 to 2449408 tid 2 2024-09-14T03:55:22.249 INFO:tasks.rados.rados.0.smithi070.stdout:2083: writing smithi07085154-22 from 2449408 to 2809856 tid 3 2024-09-14T03:55:22.249 INFO:tasks.rados.rados.0.smithi070.stdout:2084: rollback oid 36 current snap is 220 2024-09-14T03:55:22.249 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 36 to 181 2024-09-14T03:55:22.249 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 2127 (ObjNum 608 snap 204 seq_num 608) dirty exists 2024-09-14T03:55:22.249 INFO:tasks.rados.rados.0.smithi070.stdout:2083: finishing write tid 1 to smithi07085154-22 2024-09-14T03:55:22.250 INFO:tasks.rados.rados.0.smithi070.stdout:2082: done (3 left) 2024-09-14T03:55:22.250 INFO:tasks.rados.rados.0.smithi070.stdout:2085: rollback oid 21 current snap is 220 2024-09-14T03:55:22.250 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 21 to 168 2024-09-14T03:55:22.250 INFO:tasks.rados.rados.0.smithi070.stdout:2086: read oid 24 snap -1 2024-09-14T03:55:22.250 INFO:tasks.rados.rados.0.smithi070.stdout:2086: expect (ObjNum 646 snap 216 seq_num 646) 2024-09-14T03:55:22.250 INFO:tasks.rados.rados.0.smithi070.stdout:2087: copy_from oid 11 from oid 35 current snap is 220 2024-09-14T03:55:22.250 INFO:tasks.rados.rados.0.smithi070.stdout:2088: rollback oid 16 current snap is 220 2024-09-14T03:55:22.250 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 16 to 177 2024-09-14T03:55:22.250 INFO:tasks.rados.rados.0.smithi070.stdout:2089: delete oid 30 current snap is 220 2024-09-14T03:55:22.252 INFO:tasks.rados.rados.0.smithi070.stdout:2083: finishing write tid 2 to smithi07085154-22 2024-09-14T03:55:22.252 INFO:tasks.rados.rados.0.smithi070.stdout:2089: done (7 left) 2024-09-14T03:55:22.252 INFO:tasks.rados.rados.0.smithi070.stdout:2090: setattr oid 17 current snap is 220 2024-09-14T03:55:22.253 INFO:tasks.rados.rados.0.smithi070.stdout:2083: finishing write tid 3 to smithi07085154-22 2024-09-14T03:55:22.254 INFO:tasks.rados.rados.0.smithi070.stdout:2083: finishing write tid 4 to smithi07085154-22 2024-09-14T03:55:22.255 INFO:tasks.rados.rados.0.smithi070.stdout:2091: rollback oid 34 current snap is 220 2024-09-14T03:55:22.255 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 34 to 168 2024-09-14T03:55:22.255 INFO:tasks.rados.rados.0.smithi070.stdout:2092: rollback oid 44 current snap is 220 2024-09-14T03:55:22.256 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 44 to 195 2024-09-14T03:55:22.256 INFO:tasks.rados.rados.0.smithi070.stdout:2093: read oid 43 snap 186 2024-09-14T03:55:22.256 INFO:tasks.rados.rados.0.smithi070.stdout:2093: expect (ObjNum 469 snap 160 seq_num 469) 2024-09-14T03:55:22.256 INFO:tasks.rados.rados.0.smithi070.stdout:2094: delete oid 40 current snap is 220 2024-09-14T03:55:22.256 INFO:tasks.rados.rados.0.smithi070.stdout:2084: finishing rollback tid 0 to smithi07085154-36 2024-09-14T03:55:22.256 INFO:tasks.rados.rados.0.smithi070.stdout:2084: finishing rollback tid 1 to smithi07085154-36 2024-09-14T03:55:22.256 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 36 v 1959 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-14T03:55:22.257 INFO:tasks.rados.rados.0.smithi070.stdout:2088: finishing rollback tid 0 to smithi07085154-16 2024-09-14T03:55:22.257 INFO:tasks.rados.rados.0.smithi070.stdout:2088: finishing rollback tid 1 to smithi07085154-16 2024-09-14T03:55:22.258 INFO:tasks.rados.rados.0.smithi070.stdout:2083: finishing write tid 5 to smithi07085154-22 2024-09-14T03:55:22.258 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 1705 (ObjNum 663 snap 220 seq_num 663) dirty exists 2024-09-14T03:55:22.258 INFO:tasks.rados.rados.0.smithi070.stdout:2083: left oid 22 (ObjNum 663 snap 220 seq_num 663) 2024-09-14T03:55:22.259 INFO:tasks.rados.rados.0.smithi070.stdout:2085: finishing rollback tid 0 to smithi07085154-21 2024-09-14T03:55:22.259 INFO:tasks.rados.rados.0.smithi070.stdout:2085: finishing rollback tid 1 to smithi07085154-21 2024-09-14T03:55:22.259 INFO:tasks.rados.rados.0.smithi070.stdout:2088: finishing rollback tid 2 to smithi07085154-16 2024-09-14T03:55:22.259 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 2247 (ObjNum 114 snap 24 seq_num 114) dirty exists 2024-09-14T03:55:22.260 INFO:tasks.rados.rados.0.smithi070.stdout:2085: finishing rollback tid 2 to smithi07085154-21 2024-09-14T03:55:22.260 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 562 (ObjNum 501 snap 165 seq_num 501) dirty exists 2024-09-14T03:55:22.261 INFO:tasks.rados.rados.0.smithi070.stdout:2091: finishing rollback tid 0 to smithi07085154-34 2024-09-14T03:55:22.262 INFO:tasks.rados.rados.0.smithi070.stdout:2091: finishing rollback tid 1 to smithi07085154-34 2024-09-14T03:55:22.262 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 1707 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-14T03:55:22.262 INFO:tasks.rados.rados.0.smithi070.stdout:2092: finishing rollback tid 0 to smithi07085154-44 2024-09-14T03:55:22.265 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 2128 (ObjNum 544 snap 181 seq_num 544) dirty exists 2024-09-14T03:55:22.265 INFO:tasks.rados.rados.0.smithi070.stdout:2092: finishing rollback tid 1 to smithi07085154-44 2024-09-14T03:55:22.266 INFO:tasks.rados.rados.0.smithi070.stdout:2081: expect (ObjNum 403 snap 140 seq_num 403) 2024-09-14T03:55:22.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:22 smithi094 ceph-mon[86597]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:55:22.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:22 smithi094 ceph-mon[86597]: osdmap e493: 8 total, 7 up, 8 in 2024-09-14T03:55:22.277 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:22 smithi094 podman[96495]: 2024-09-14 03:55:22.130281691 +0000 UTC m=+2.099220759 container cleanup b543f152ba87a383792f0e5b35cc69b6cc59789c549466c1f89dddb51cdce635 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, RELEASE=quincy-eb16b97, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:55:22.277 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:22 smithi094 bash[96495]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5 2024-09-14T03:55:22.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:22 smithi070 ceph-mon[96704]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:55:22.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:22 smithi070 ceph-mon[96704]: osdmap e493: 8 total, 7 up, 8 in 2024-09-14T03:55:22.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:22 smithi070 ceph-mon[98640]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:55:22.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:22 smithi070 ceph-mon[98640]: osdmap e493: 8 total, 7 up, 8 in 2024-09-14T03:55:22.737 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:22 smithi094 podman[96515]: 2024-09-14 03:55:22.332381295 +0000 UTC m=+0.676246309 container remove b543f152ba87a383792f0e5b35cc69b6cc59789c549466c1f89dddb51cdce635 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5, RELEASE=quincy-eb16b97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:55:22.838 INFO:tasks.rados.rados.0.smithi070.stdout:2087: finishing copy_from to smithi07085154-11 2024-09-14T03:55:22.839 INFO:tasks.rados.rados.0.smithi070.stdout:2087: got expected ENOENT (src dne) 2024-09-14T03:55:22.839 INFO:tasks.rados.rados.0.smithi070.stdout:2087: finishing copy_from racing read to smithi07085154-11 2024-09-14T03:55:22.839 INFO:tasks.rados.rados.0.smithi070.stdout:2092: finishing rollback tid 2 to smithi07085154-44 2024-09-14T03:55:22.839 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 1962 (ObjNum 536 snap 176 seq_num 536) dirty exists 2024-09-14T03:55:22.839 INFO:tasks.rados.rados.0.smithi070.stdout:2093: expect (ObjNum 469 snap 160 seq_num 469) 2024-09-14T03:55:23.026 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:22 smithi094 podman[96610]: 2024-09-14 03:55:22.639705525 +0000 UTC m=+0.016132426 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:55:23.027 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:22 smithi094 podman[96610]: 2024-09-14 03:55:22.741961316 +0000 UTC m=+0.118388199 container create a273268c59f1ea79bd5db23a990a98cc870db9439726ab83598f7d6d21c65b6b (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5-deactivate, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=main-a278231, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-14T03:55:23.027 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:22 smithi094 podman[96610]: 2024-09-14 03:55:22.969897338 +0000 UTC m=+0.346324227 container init a273268c59f1ea79bd5db23a990a98cc870db9439726ab83598f7d6d21c65b6b (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, RELEASE=main-a278231, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True) 2024-09-14T03:55:23.027 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:22 smithi094 podman[96610]: 2024-09-14 03:55:22.974705702 +0000 UTC m=+0.351132585 container start a273268c59f1ea79bd5db23a990a98cc870db9439726ab83598f7d6d21c65b6b (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, ceph=True, RELEASE=main-a278231, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-14T03:55:23.100 INFO:tasks.rados.rados.0.smithi070.stdout:2086: expect (ObjNum 646 snap 216 seq_num 646) 2024-09-14T03:55:23.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:23 smithi070 ceph-mon[96704]: pgmap v335: 121 pgs: 1 active+clean+snaptrim, 12 peering, 1 stale+active+clean+snaptrim, 15 stale+active+clean, 92 active+clean; 861 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.0 MiB/s wr, 8 op/s 2024-09-14T03:55:23.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:23 smithi070 ceph-mon[96704]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-14T03:55:23.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:23 smithi070 ceph-mon[96704]: osdmap e494: 8 total, 7 up, 8 in 2024-09-14T03:55:23.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:23 smithi070 ceph-mon[98640]: pgmap v335: 121 pgs: 1 active+clean+snaptrim, 12 peering, 1 stale+active+clean+snaptrim, 15 stale+active+clean, 92 active+clean; 861 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.0 MiB/s wr, 8 op/s 2024-09-14T03:55:23.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:23 smithi070 ceph-mon[98640]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-14T03:55:23.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:23 smithi070 ceph-mon[98640]: osdmap e494: 8 total, 7 up, 8 in 2024-09-14T03:55:23.457 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:23 smithi094 ceph-mon[86597]: pgmap v335: 121 pgs: 1 active+clean+snaptrim, 12 peering, 1 stale+active+clean+snaptrim, 15 stale+active+clean, 92 active+clean; 861 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.0 MiB/s wr, 8 op/s 2024-09-14T03:55:23.457 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:23 smithi094 ceph-mon[86597]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-14T03:55:23.457 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:23 smithi094 ceph-mon[86597]: osdmap e494: 8 total, 7 up, 8 in 2024-09-14T03:55:23.457 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:23 smithi094 podman[96610]: 2024-09-14 03:55:23.06152891 +0000 UTC m=+0.437955815 container attach a273268c59f1ea79bd5db23a990a98cc870db9439726ab83598f7d6d21c65b6b (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=main-a278231, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-14T03:55:23.457 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:23 smithi094 podman[96610]: 2024-09-14 03:55:23.170191247 +0000 UTC m=+0.546618142 container died a273268c59f1ea79bd5db23a990a98cc870db9439726ab83598f7d6d21c65b6b (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=main-a278231, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:55:23.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:55:23 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:55:23] "GET /metrics HTTP/1.1" 200 36310 "" "Prometheus/2.51.0" 2024-09-14T03:55:24.018 INFO:tasks.rados.rados.0.smithi070.stdout:2081: done (11 left) 2024-09-14T03:55:24.018 INFO:tasks.rados.rados.0.smithi070.stdout:2083: done (10 left) 2024-09-14T03:55:24.018 INFO:tasks.rados.rados.0.smithi070.stdout:2084: done (9 left) 2024-09-14T03:55:24.018 INFO:tasks.rados.rados.0.smithi070.stdout:2085: done (8 left) 2024-09-14T03:55:24.018 INFO:tasks.rados.rados.0.smithi070.stdout:2086: done (7 left) 2024-09-14T03:55:24.018 INFO:tasks.rados.rados.0.smithi070.stdout:2087: done (6 left) 2024-09-14T03:55:24.018 INFO:tasks.rados.rados.0.smithi070.stdout:2088: done (5 left) 2024-09-14T03:55:24.018 INFO:tasks.rados.rados.0.smithi070.stdout:2090: done (4 left) 2024-09-14T03:55:24.019 INFO:tasks.rados.rados.0.smithi070.stdout:2091: done (3 left) 2024-09-14T03:55:24.019 INFO:tasks.rados.rados.0.smithi070.stdout:2092: done (2 left) 2024-09-14T03:55:24.019 INFO:tasks.rados.rados.0.smithi070.stdout:2093: done (1 left) 2024-09-14T03:55:24.019 INFO:tasks.rados.rados.0.smithi070.stdout:2094: done (0 left) 2024-09-14T03:55:24.019 INFO:tasks.rados.rados.0.smithi070.stdout:2095: snap_create 2024-09-14T03:55:24.026 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:23 smithi094 podman[96654]: 2024-09-14 03:55:23.757688126 +0000 UTC m=+0.581224543 container remove a273268c59f1ea79bd5db23a990a98cc870db9439726ab83598f7d6d21c65b6b (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5-deactivate, RELEASE=main-a278231, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-14T03:55:24.027 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:23 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.5.service: Deactivated successfully. 2024-09-14T03:55:24.027 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:23 smithi094 systemd[1]: Stopped Ceph osd.5 for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:55:24.027 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:23 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.5.service: Consumed 17.422s CPU time. 2024-09-14T03:55:24.027 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:23 smithi094 systemd[1]: Starting Ceph osd.5 for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:55:24.046 INFO:tasks.rados.rados.0.smithi070.stdout:2095: done (0 left) 2024-09-14T03:55:24.046 INFO:tasks.rados.rados.0.smithi070.stdout:2096: snap_create 2024-09-14T03:55:24.454 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:24 smithi094 podman[96757]: 2024-09-14 03:55:24.090460454 +0000 UTC m=+0.016298519 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:55:24.454 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:24 smithi094 podman[96757]: 2024-09-14 03:55:24.192194849 +0000 UTC m=+0.118032913 container create 1828846ca60a0aaa39206094d903adac7f473117f2867e6a7b030accd467a92b (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=main-a278231) 2024-09-14T03:55:24.777 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:24 smithi094 podman[96757]: 2024-09-14 03:55:24.453952186 +0000 UTC m=+0.379790266 container init 1828846ca60a0aaa39206094d903adac7f473117f2867e6a7b030accd467a92b (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5-activate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:55:24.777 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:24 smithi094 podman[96757]: 2024-09-14 03:55:24.459159955 +0000 UTC m=+0.384998027 container start 1828846ca60a0aaa39206094d903adac7f473117f2867e6a7b030accd467a92b (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=main-a278231, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:55:24.777 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:24 smithi094 podman[96757]: 2024-09-14 03:55:24.545751548 +0000 UTC m=+0.471589619 container attach 1828846ca60a0aaa39206094d903adac7f473117f2867e6a7b030accd467a92b (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=main-a278231, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-14T03:55:24.777 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:24 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5-activate[96772]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:55:24.777 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:24 smithi094 bash[96757]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:55:24.777 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:24 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5-activate[96772]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:55:24.777 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:24 smithi094 bash[96757]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:55:25.052 INFO:tasks.rados.rados.0.smithi070.stdout:2096: done (0 left) 2024-09-14T03:55:25.052 INFO:tasks.rados.rados.0.smithi070.stdout:2097: rmattr oid 40 current snap is 222 2024-09-14T03:55:25.052 INFO:tasks.rados.rados.0.smithi070.stdout:2097: done (0 left) 2024-09-14T03:55:25.052 INFO:tasks.rados.rados.0.smithi070.stdout:2098: rmattr oid 32 current snap is 222 2024-09-14T03:55:25.052 INFO:tasks.rados.rados.0.smithi070.stdout:2098: done (0 left) 2024-09-14T03:55:25.053 INFO:tasks.rados.rados.0.smithi070.stdout:2099: rollback oid 31 current snap is 222 2024-09-14T03:55:25.053 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 31 to 181 2024-09-14T03:55:25.053 INFO:tasks.rados.rados.0.smithi070.stdout:2100: snap_create 2024-09-14T03:55:25.056 INFO:tasks.rados.rados.0.smithi070.stdout:2099: finishing rollback tid 0 to smithi07085154-31 2024-09-14T03:55:25.056 INFO:tasks.rados.rados.0.smithi070.stdout:2099: finishing rollback tid 1 to smithi07085154-31 2024-09-14T03:55:25.056 INFO:tasks.rados.rados.0.smithi070.stdout:2099: finishing rollback tid 2 to smithi07085154-31 2024-09-14T03:55:25.056 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 2131 (ObjNum 540 snap 176 seq_num 540) dirty exists 2024-09-14T03:55:25.344 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:25 smithi094 ceph-mon[86597]: osdmap e495: 8 total, 7 up, 8 in 2024-09-14T03:55:25.344 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:25 smithi094 ceph-mon[86597]: pgmap v338: 121 pgs: 14 active+undersized, 1 active+clean+snaptrim, 12 peering, 1 stale+active+clean+snaptrim, 7 stale+active+clean, 7 active+undersized+degraded, 79 active+clean; 868 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 1.3 MiB/s wr, 36 op/s; 115/1272 objects degraded (9.041%) 2024-09-14T03:55:25.344 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5-activate[96772]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-14T03:55:25.345 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:25 smithi094 bash[96757]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-14T03:55:25.345 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5-activate[96772]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-14T03:55:25.345 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:25 smithi094 bash[96757]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-14T03:55:25.345 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5-activate[96772]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-14T03:55:25.345 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:25 smithi094 bash[96757]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-14T03:55:25.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:25 smithi070 ceph-mon[98640]: osdmap e495: 8 total, 7 up, 8 in 2024-09-14T03:55:25.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:25 smithi070 ceph-mon[98640]: pgmap v338: 121 pgs: 14 active+undersized, 1 active+clean+snaptrim, 12 peering, 1 stale+active+clean+snaptrim, 7 stale+active+clean, 7 active+undersized+degraded, 79 active+clean; 868 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 1.3 MiB/s wr, 36 op/s; 115/1272 objects degraded (9.041%) 2024-09-14T03:55:25.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:25 smithi070 ceph-mon[96704]: osdmap e495: 8 total, 7 up, 8 in 2024-09-14T03:55:25.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:25 smithi070 ceph-mon[96704]: pgmap v338: 121 pgs: 14 active+undersized, 1 active+clean+snaptrim, 12 peering, 1 stale+active+clean+snaptrim, 7 stale+active+clean, 7 active+undersized+degraded, 79 active+clean; 868 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 1.3 MiB/s wr, 36 op/s; 115/1272 objects degraded (9.041%) 2024-09-14T03:55:25.641 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5-activate[96772]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-14T03:55:25.642 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:25 smithi094 bash[96757]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-14T03:55:25.642 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5-activate[96772]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-14T03:55:25.642 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:25 smithi094 bash[96757]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-14T03:55:25.642 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5-activate[96772]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-14T03:55:25.642 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:25 smithi094 bash[96757]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-14T03:55:25.642 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5-activate[96772]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-14T03:55:25.642 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:25 smithi094 bash[96757]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-14T03:55:25.642 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:25 smithi094 podman[96922]: 2024-09-14 03:55:25.395616155 +0000 UTC m=+0.017733194 container died 1828846ca60a0aaa39206094d903adac7f473117f2867e6a7b030accd467a92b (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True) 2024-09-14T03:55:26.053 INFO:tasks.rados.rados.0.smithi070.stdout:2099: done (1 left) 2024-09-14T03:55:26.053 INFO:tasks.rados.rados.0.smithi070.stdout:2100: done (0 left) 2024-09-14T03:55:26.053 INFO:tasks.rados.rados.0.smithi070.stdout:2101: rollback oid 31 current snap is 223 2024-09-14T03:55:26.053 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 31 to 179 2024-09-14T03:55:26.053 INFO:tasks.rados.rados.0.smithi070.stdout:2102: snap_remove snap 204 2024-09-14T03:55:26.375 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:26 smithi094 ceph-mon[86597]: Health check failed: Degraded data redundancy: 115/1272 objects degraded (9.041%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:55:26.375 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:26 smithi094 ceph-mon[86597]: osdmap e496: 8 total, 7 up, 8 in 2024-09-14T03:55:26.376 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:26 smithi094 podman[96922]: 2024-09-14 03:55:26.025964216 +0000 UTC m=+0.648081141 container remove 1828846ca60a0aaa39206094d903adac7f473117f2867e6a7b030accd467a92b (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5-activate, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-14T03:55:26.376 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:26 smithi094 podman[96984]: 2024-09-14 03:55:26.135196542 +0000 UTC m=+0.016204463 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:55:26.376 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:26 smithi094 podman[96984]: 2024-09-14 03:55:26.268871212 +0000 UTC m=+0.149879120 container create 162396f2e7b684b77df919d2a9ad5a2e79cbe98d24eaa87ab5e92a7093147ee4 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5, GIT_CLEAN=True, RELEASE=main-a278231, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-14T03:55:26.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:26 smithi070 ceph-mon[98640]: Health check failed: Degraded data redundancy: 115/1272 objects degraded (9.041%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:55:26.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:26 smithi070 ceph-mon[98640]: osdmap e496: 8 total, 7 up, 8 in 2024-09-14T03:55:26.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:26 smithi070 ceph-mon[96704]: Health check failed: Degraded data redundancy: 115/1272 objects degraded (9.041%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T03:55:26.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:26 smithi070 ceph-mon[96704]: osdmap e496: 8 total, 7 up, 8 in 2024-09-14T03:55:26.776 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:26 smithi094 podman[96984]: 2024-09-14 03:55:26.463124066 +0000 UTC m=+0.344131977 container init 162396f2e7b684b77df919d2a9ad5a2e79cbe98d24eaa87ab5e92a7093147ee4 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-14T03:55:26.777 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:26 smithi094 podman[96984]: 2024-09-14 03:55:26.467705174 +0000 UTC m=+0.348713083 container start 162396f2e7b684b77df919d2a9ad5a2e79cbe98d24eaa87ab5e92a7093147ee4 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=main-a278231, org.label-schema.vendor=CentOS) 2024-09-14T03:55:26.777 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:26 smithi094 ceph-osd[97022]: -- 172.21.15.94:0/4159909653 <== mon.1 v2:172.21.15.94:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f822775a40 con 0x55f822753c00 2024-09-14T03:55:26.777 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:26 smithi094 bash[96984]: 162396f2e7b684b77df919d2a9ad5a2e79cbe98d24eaa87ab5e92a7093147ee4 2024-09-14T03:55:26.777 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:26 smithi094 systemd[1]: Started Ceph osd.5 for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:55:26.990 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-09-14T03:55:27.067 INFO:tasks.rados.rados.0.smithi070.stdout:2102: done (1 left) 2024-09-14T03:55:27.068 INFO:tasks.rados.rados.0.smithi070.stdout:2103: rmattr oid 21 current snap is 223 2024-09-14T03:55:27.068 INFO:tasks.rados.rados.0.smithi070.stdout:2104: snap_remove snap 123 2024-09-14T03:55:27.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:27 smithi070 ceph-mon[98640]: osdmap e497: 8 total, 7 up, 8 in 2024-09-14T03:55:27.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:27 smithi070 ceph-mon[98640]: pgmap v341: 121 pgs: 27 active+undersized, 1 active+clean+snaptrim, 12 peering, 10 active+undersized+degraded, 71 active+clean; 918 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 13 MiB/s wr, 33 op/s; 133/1281 objects degraded (10.383%) 2024-09-14T03:55:27.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:27 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:27.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:27 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:27.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:27 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:55:27.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:27 smithi070 ceph-mon[96704]: osdmap e497: 8 total, 7 up, 8 in 2024-09-14T03:55:27.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:27 smithi070 ceph-mon[96704]: pgmap v341: 121 pgs: 27 active+undersized, 1 active+clean+snaptrim, 12 peering, 10 active+undersized+degraded, 71 active+clean; 918 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 13 MiB/s wr, 33 op/s; 133/1281 objects degraded (10.383%) 2024-09-14T03:55:27.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:27 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:27.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:27 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:27.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:27 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:55:27.333 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:27 smithi094 ceph-mon[86597]: osdmap e497: 8 total, 7 up, 8 in 2024-09-14T03:55:27.334 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:27 smithi094 ceph-mon[86597]: pgmap v341: 121 pgs: 27 active+undersized, 1 active+clean+snaptrim, 12 peering, 10 active+undersized+degraded, 71 active+clean; 918 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 13 MiB/s wr, 33 op/s; 133/1281 objects degraded (10.383%) 2024-09-14T03:55:27.334 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:27 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:27.334 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:27 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:27.334 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:27 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:55:27.575 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:55:27.575 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (5m) 64s ago 10m 21.2M - 0.27.0 11f11916f8cd 45e79e51074c 2024-09-14T03:55:27.575 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi094 *:3000 running (4m) 30s ago 9m 84.5M - 9.4.7 954c08fa6188 2240810d0368 2024-09-14T03:55:27.575 INFO:teuthology.orchestra.run.smithi070.stdout:mds.a smithi070 running (11m) 64s ago 11m 21.4M - 17.2.7-1629-geb16b97b ab51d1bebd12 52e0dbb7be50 2024-09-14T03:55:27.576 INFO:teuthology.orchestra.run.smithi070.stdout:mds.b smithi094 running (11m) 30s ago 11m 16.0M - 17.2.7-1629-geb16b97b ab51d1bebd12 e3b5e3e00189 2024-09-14T03:55:27.576 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi094 *:8443,9283,8765 running (4m) 30s ago 15m 454M - 19.3.0-4907-ga2782315 8801d2529e43 569d9c7e0f06 2024-09-14T03:55:27.576 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (4m) 64s ago 16m 521M - 19.3.0-4907-ga2782315 8801d2529e43 0ef4d8e0cdbe 2024-09-14T03:55:27.576 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (4m) 64s ago 16m 57.7M 2048M 19.3.0-4907-ga2782315 8801d2529e43 96542de31cac 2024-09-14T03:55:27.576 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi094 running (3m) 30s ago 15m 44.3M 2048M 19.3.0-4907-ga2782315 8801d2529e43 86827be98659 2024-09-14T03:55:27.576 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (3m) 64s ago 15m 51.9M 2048M 19.3.0-4907-ga2782315 8801d2529e43 5741e4fcf4b6 2024-09-14T03:55:27.576 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (5m) 64s ago 9m 12.2M - 1.7.0 72c9c2088986 e316760a30b3 2024-09-14T03:55:27.576 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi094 *:9100 running (4m) 30s ago 9m 12.2M - 1.7.0 72c9c2088986 767f8a093175 2024-09-14T03:55:27.576 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (2m) 64s ago 14m 180M 2480M 19.3.0-4907-ga2782315 8801d2529e43 2345ccac2085 2024-09-14T03:55:27.576 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (2m) 64s ago 14m 177M 2480M 19.3.0-4907-ga2782315 8801d2529e43 53868856512a 2024-09-14T03:55:27.576 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (101s) 64s ago 13m 104M 2480M 19.3.0-4907-ga2782315 8801d2529e43 39ef876d2fff 2024-09-14T03:55:27.576 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (68s) 64s ago 13m 14.5M 2480M 19.3.0-4907-ga2782315 8801d2529e43 8efc112edbe9 2024-09-14T03:55:27.576 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi094 running (35s) 30s ago 13m 11.2M 2480M 19.3.0-4907-ga2782315 8801d2529e43 09f583ed6c77 2024-09-14T03:55:27.577 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi094 starting - - - 2480M 2024-09-14T03:55:27.577 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi094 running (12m) 30s ago 12m 177M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 e66b592ac940 2024-09-14T03:55:27.577 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi094 running (11m) 30s ago 11m 249M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 037c6208e9ce 2024-09-14T03:55:27.577 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi094 *:9095 running (4m) 30s ago 10m 49.7M - 2.51.0 1d3b7f56885b ce1b353828dc 2024-09-14T03:55:27.905 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:55:27.905 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-09-14T03:55:27.905 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:55:27.905 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:55:27.905 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-09-14T03:55:27.905 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:55:27.906 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:55:27.906 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-09-14T03:55:27.906 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2, 2024-09-14T03:55:27.906 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 5 2024-09-14T03:55:27.906 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:55:27.906 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": { 2024-09-14T03:55:27.906 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:55:27.906 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:55:27.906 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-09-14T03:55:27.906 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 4, 2024-09-14T03:55:27.906 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 10 2024-09-14T03:55:27.906 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-09-14T03:55:27.906 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:55:28.069 INFO:tasks.rados.rados.0.smithi070.stdout:2104: done (2 left) 2024-09-14T03:55:28.069 INFO:tasks.rados.rados.0.smithi070.stdout:2105: rollback oid 44 current snap is 223 2024-09-14T03:55:28.070 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 44 to 215 2024-09-14T03:55:28.070 INFO:tasks.rados.rados.0.smithi070.stdout:2106: read oid 16 snap -1 2024-09-14T03:55:28.070 INFO:tasks.rados.rados.0.smithi070.stdout:2106: expect (ObjNum 114 snap 24 seq_num 114) 2024-09-14T03:55:28.070 INFO:tasks.rados.rados.0.smithi070.stdout:2107: snap_create 2024-09-14T03:55:28.070 INFO:tasks.rados.rados.0.smithi070.stdout:2101: finishing rollback tid 0 to smithi07085154-31 2024-09-14T03:55:28.070 INFO:tasks.rados.rados.0.smithi070.stdout:2101: finishing rollback tid 1 to smithi07085154-31 2024-09-14T03:55:28.070 INFO:tasks.rados.rados.0.smithi070.stdout:2101: finishing rollback tid 2 to smithi07085154-31 2024-09-14T03:55:28.070 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 2134 (ObjNum 540 snap 176 seq_num 540) dirty exists 2024-09-14T03:55:28.070 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 563 (ObjNum 501 snap 165 seq_num 501) dirty exists 2024-09-14T03:55:28.080 INFO:tasks.rados.rados.0.smithi070.stdout:2105: finishing rollback tid 0 to smithi07085154-44 2024-09-14T03:55:28.080 INFO:tasks.rados.rados.0.smithi070.stdout:2105: finishing rollback tid 1 to smithi07085154-44 2024-09-14T03:55:28.081 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 1965 (ObjNum 129 snap 0 seq_num 2930523856) dirty dne 2024-09-14T03:55:28.095 INFO:tasks.rados.rados.0.smithi070.stdout:2106: expect (ObjNum 114 snap 24 seq_num 114) 2024-09-14T03:55:28.209 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:55:28.209 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:55:28.209 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-09-14T03:55:28.209 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:55:28.209 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [ 2024-09-14T03:55:28.209 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr", 2024-09-14T03:55:28.209 INFO:teuthology.orchestra.run.smithi070.stdout: "mon" 2024-09-14T03:55:28.209 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2024-09-14T03:55:28.209 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "10/20 daemons upgraded", 2024-09-14T03:55:28.209 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "Currently upgrading osd daemons", 2024-09-14T03:55:28.209 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-09-14T03:55:28.210 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:55:28.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:28 smithi070 ceph-mon[96704]: from='client.44312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:28.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:28 smithi070 ceph-mon[96704]: osdmap e498: 8 total, 7 up, 8 in 2024-09-14T03:55:28.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:28 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/3229557696' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:28.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:28 smithi070 ceph-mon[98640]: from='client.44312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:28.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:28 smithi070 ceph-mon[98640]: osdmap e498: 8 total, 7 up, 8 in 2024-09-14T03:55:28.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:28 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/3229557696' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:28.340 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:28 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[96999]: 2024-09-14T03:55:28.113+0000 7f907d988640 -1 Falling back to public interface 2024-09-14T03:55:28.340 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:28 smithi094 ceph-mon[86597]: from='client.44312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:28.340 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:28 smithi094 ceph-mon[86597]: osdmap e498: 8 total, 7 up, 8 in 2024-09-14T03:55:28.340 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:28 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/3229557696' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:29.047 INFO:tasks.rados.rados.0.smithi070.stdout:2101: done (4 left) 2024-09-14T03:55:29.048 INFO:tasks.rados.rados.0.smithi070.stdout:2103: done (3 left) 2024-09-14T03:55:29.048 INFO:tasks.rados.rados.0.smithi070.stdout:2105: done (2 left) 2024-09-14T03:55:29.048 INFO:tasks.rados.rados.0.smithi070.stdout:2106: done (1 left) 2024-09-14T03:55:29.048 INFO:tasks.rados.rados.0.smithi070.stdout:2107: done (0 left) 2024-09-14T03:55:29.048 INFO:tasks.rados.rados.0.smithi070.stdout:2108: copy_from oid 21 from oid 50 current snap is 224 2024-09-14T03:55:29.048 INFO:tasks.rados.rados.0.smithi070.stdout:2109: read oid 7 snap -1 2024-09-14T03:55:29.048 INFO:tasks.rados.rados.0.smithi070.stdout:2109: expect (ObjNum 639 snap 214 seq_num 639) 2024-09-14T03:55:29.048 INFO:tasks.rados.rados.0.smithi070.stdout:2110: read oid 37 snap -1 2024-09-14T03:55:29.048 INFO:tasks.rados.rados.0.smithi070.stdout:2110: expect (ObjNum 459 snap 155 seq_num 459) 2024-09-14T03:55:29.048 INFO:tasks.rados.rados.0.smithi070.stdout:2111: snap_create 2024-09-14T03:55:29.059 INFO:tasks.rados.rados.0.smithi070.stdout:2109: expect (ObjNum 639 snap 214 seq_num 639) 2024-09-14T03:55:29.095 INFO:tasks.rados.rados.0.smithi070.stdout:2108: finishing copy_from racing read to smithi07085154-21 2024-09-14T03:55:29.095 INFO:tasks.rados.rados.0.smithi070.stdout:2108: finishing copy_from to smithi07085154-21 2024-09-14T03:55:29.095 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 1696 (ObjNum 657 snap 220 seq_num 657) dirty exists 2024-09-14T03:55:29.110 INFO:tasks.rados.rados.0.smithi070.stdout:2110: expect (ObjNum 459 snap 155 seq_num 459) 2024-09-14T03:55:29.411 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:29 smithi094 ceph-mon[86597]: from='client.54259 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:29.411 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:29 smithi094 ceph-mon[86597]: from='client.44324 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:29.411 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:29 smithi094 ceph-mon[86597]: osdmap e499: 8 total, 7 up, 8 in 2024-09-14T03:55:29.412 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:29 smithi094 ceph-mon[86597]: pgmap v344: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 927 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 13 MiB/s wr, 4 op/s; 201/1284 objects degraded (15.654%) 2024-09-14T03:55:29.412 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:29 smithi094 ceph-mon[86597]: osdmap e500: 8 total, 7 up, 8 in 2024-09-14T03:55:29.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:29 smithi070 ceph-mon[98640]: from='client.54259 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:29.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:29 smithi070 ceph-mon[98640]: from='client.44324 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:29.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:29 smithi070 ceph-mon[98640]: osdmap e499: 8 total, 7 up, 8 in 2024-09-14T03:55:29.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:29 smithi070 ceph-mon[98640]: pgmap v344: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 927 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 13 MiB/s wr, 4 op/s; 201/1284 objects degraded (15.654%) 2024-09-14T03:55:29.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:29 smithi070 ceph-mon[98640]: osdmap e500: 8 total, 7 up, 8 in 2024-09-14T03:55:29.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:29 smithi070 ceph-mon[96704]: from='client.54259 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:29.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:29 smithi070 ceph-mon[96704]: from='client.44324 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:29.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:29 smithi070 ceph-mon[96704]: osdmap e499: 8 total, 7 up, 8 in 2024-09-14T03:55:29.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:29 smithi070 ceph-mon[96704]: pgmap v344: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 927 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 13 MiB/s wr, 4 op/s; 201/1284 objects degraded (15.654%) 2024-09-14T03:55:29.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:29 smithi070 ceph-mon[96704]: osdmap e500: 8 total, 7 up, 8 in 2024-09-14T03:55:30.213 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:30 smithi094 ceph-mon[86597]: from='client.44336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:30.214 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:30 smithi094 ceph-mon[86597]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2024-09-14T03:55:30.214 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:30 smithi094 ceph-mon[86597]: osdmap e501: 8 total, 7 up, 8 in 2024-09-14T03:55:30.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:55:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:55:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:55:30.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:30 smithi070 ceph-mon[98640]: from='client.44336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:30.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:30 smithi070 ceph-mon[98640]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2024-09-14T03:55:30.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:30 smithi070 ceph-mon[98640]: osdmap e501: 8 total, 7 up, 8 in 2024-09-14T03:55:30.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:30 smithi070 ceph-mon[96704]: from='client.44336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:30.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:30 smithi070 ceph-mon[96704]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2024-09-14T03:55:30.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:30 smithi070 ceph-mon[96704]: osdmap e501: 8 total, 7 up, 8 in 2024-09-14T03:55:30.759 INFO:tasks.rados.rados.0.smithi070.stdout:2108: done (3 left) 2024-09-14T03:55:30.759 INFO:tasks.rados.rados.0.smithi070.stdout:2109: done (2 left) 2024-09-14T03:55:30.759 INFO:tasks.rados.rados.0.smithi070.stdout:2110: done (1 left) 2024-09-14T03:55:30.759 INFO:tasks.rados.rados.0.smithi070.stdout:2111: done (0 left) 2024-09-14T03:55:30.759 INFO:tasks.rados.rados.0.smithi070.stdout:2112: rollback oid 37 current snap is 225 2024-09-14T03:55:30.759 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 37 to 224 2024-09-14T03:55:30.760 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 48 current snap is 225 2024-09-14T03:55:30.760 INFO:tasks.rados.rados.0.smithi070.stdout:2113: seq_num 666 ranges {450560=761856,1212416=491520,1703936=311296} 2024-09-14T03:55:30.766 INFO:tasks.rados.rados.0.smithi070.stdout:2113: writing smithi07085154-48 from 450560 to 1212416 tid 1 2024-09-14T03:55:30.769 INFO:tasks.rados.rados.0.smithi070.stdout:2113: writing smithi07085154-48 from 1212416 to 1703936 tid 2 2024-09-14T03:55:30.771 INFO:tasks.rados.rados.0.smithi070.stdout:2113: writing smithi07085154-48 from 1703936 to 2015232 tid 3 2024-09-14T03:55:30.771 INFO:tasks.rados.rados.0.smithi070.stdout:2114: snap_create 2024-09-14T03:55:30.771 INFO:tasks.rados.rados.0.smithi070.stdout:2112: finishing rollback tid 0 to smithi07085154-37 2024-09-14T03:55:30.772 INFO:tasks.rados.rados.0.smithi070.stdout:2112: finishing rollback tid 1 to smithi07085154-37 2024-09-14T03:55:30.772 INFO:tasks.rados.rados.0.smithi070.stdout:2112: finishing rollback tid 2 to smithi07085154-37 2024-09-14T03:55:30.772 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 37 v 2025 (ObjNum 459 snap 155 seq_num 459) dirty exists 2024-09-14T03:55:30.772 INFO:tasks.rados.rados.0.smithi070.stdout:2113: finishing write tid 1 to smithi07085154-48 2024-09-14T03:55:30.773 INFO:tasks.rados.rados.0.smithi070.stdout:2113: finishing write tid 2 to smithi07085154-48 2024-09-14T03:55:30.773 INFO:tasks.rados.rados.0.smithi070.stdout:2113: finishing write tid 3 to smithi07085154-48 2024-09-14T03:55:30.774 INFO:tasks.rados.rados.0.smithi070.stdout:2113: finishing write tid 4 to smithi07085154-48 2024-09-14T03:55:30.774 INFO:tasks.rados.rados.0.smithi070.stdout:2113: finishing write tid 5 to smithi07085154-48 2024-09-14T03:55:30.775 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 2189 (ObjNum 666 snap 225 seq_num 666) dirty exists 2024-09-14T03:55:30.775 INFO:tasks.rados.rados.0.smithi070.stdout:2113: left oid 48 (ObjNum 666 snap 225 seq_num 666) 2024-09-14T03:55:31.084 INFO:tasks.rados.rados.0.smithi070.stdout:2112: done (2 left) 2024-09-14T03:55:31.084 INFO:tasks.rados.rados.0.smithi070.stdout:2113: done (1 left) 2024-09-14T03:55:31.084 INFO:tasks.rados.rados.0.smithi070.stdout:2114: done (0 left) 2024-09-14T03:55:31.084 INFO:tasks.rados.rados.0.smithi070.stdout:2115: read oid 21 snap -1 2024-09-14T03:55:31.084 INFO:tasks.rados.rados.0.smithi070.stdout:2115: expect (ObjNum 657 snap 220 seq_num 657) 2024-09-14T03:55:31.085 INFO:tasks.rados.rados.0.smithi070.stdout:2116: snap_remove snap 192 2024-09-14T03:55:31.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:31 smithi094 ceph-mon[86597]: pgmap v347: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 929 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 0 B/s wr, 4 op/s; 203/1278 objects degraded (15.884%) 2024-09-14T03:55:31.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:31 smithi094 ceph-mon[86597]: Health check update: Degraded data redundancy: 203/1278 objects degraded (15.884%), 13 pgs degraded (PG_DEGRADED) 2024-09-14T03:55:31.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:31 smithi070 ceph-mon[96704]: pgmap v347: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 929 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 0 B/s wr, 4 op/s; 203/1278 objects degraded (15.884%) 2024-09-14T03:55:31.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:31 smithi070 ceph-mon[96704]: Health check update: Degraded data redundancy: 203/1278 objects degraded (15.884%), 13 pgs degraded (PG_DEGRADED) 2024-09-14T03:55:31.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:31 smithi070 ceph-mon[98640]: pgmap v347: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 929 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 0 B/s wr, 4 op/s; 203/1278 objects degraded (15.884%) 2024-09-14T03:55:31.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:31 smithi070 ceph-mon[98640]: Health check update: Degraded data redundancy: 203/1278 objects degraded (15.884%), 13 pgs degraded (PG_DEGRADED) 2024-09-14T03:55:32.089 INFO:tasks.rados.rados.0.smithi070.stdout:2116: done (1 left) 2024-09-14T03:55:32.089 INFO:tasks.rados.rados.0.smithi070.stdout:2117: copy_from oid 1 from oid 37 current snap is 226 2024-09-14T03:55:32.089 INFO:tasks.rados.rados.0.smithi070.stdout:2115: expect (ObjNum 657 snap 220 seq_num 657) 2024-09-14T03:55:32.179 INFO:tasks.rados.rados.0.smithi070.stdout:2117: finishing copy_from racing read to smithi07085154-1 2024-09-14T03:55:32.179 INFO:tasks.rados.rados.0.smithi070.stdout:2117: finishing copy_from to smithi07085154-1 2024-09-14T03:55:32.180 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 1 v 2025 (ObjNum 459 snap 155 seq_num 459) dirty exists 2024-09-14T03:55:32.180 INFO:tasks.rados.rados.0.smithi070.stdout:2115: done (1 left) 2024-09-14T03:55:32.180 INFO:tasks.rados.rados.0.smithi070.stdout:2117: done (0 left) 2024-09-14T03:55:32.180 INFO:tasks.rados.rados.0.smithi070.stdout:2118: snap_create 2024-09-14T03:55:32.340 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:32 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[96999]: 2024-09-14T03:55:32.291+0000 7f907d988640 -1 osd.5 492 log_to_monitors true 2024-09-14T03:55:32.341 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:32 smithi094 ceph-mon[86597]: osdmap e502: 8 total, 7 up, 8 in 2024-09-14T03:55:32.341 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:32 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:32.341 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:32 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:32.341 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:32 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:32.341 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:32 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:32.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:32 smithi070 ceph-mon[96704]: osdmap e502: 8 total, 7 up, 8 in 2024-09-14T03:55:32.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:32 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:32.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:32 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:32.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:32 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:32.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:32 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:32.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:32 smithi070 ceph-mon[98640]: osdmap e502: 8 total, 7 up, 8 in 2024-09-14T03:55:32.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:32 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:32.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:32 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:32.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:32 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:32.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:32 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:33.093 INFO:tasks.rados.rados.0.smithi070.stdout:2118: done (0 left) 2024-09-14T03:55:33.093 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 21 current snap is 227 2024-09-14T03:55:33.093 INFO:tasks.rados.rados.0.smithi070.stdout:2119: seq_num 667 ranges {2514944=778240,3293184=778240,4071424=24576} 2024-09-14T03:55:33.111 INFO:tasks.rados.rados.0.smithi070.stdout:2119: writing smithi07085154-21 from 2514944 to 3293184 tid 1 2024-09-14T03:55:33.116 INFO:tasks.rados.rados.0.smithi070.stdout:2119: writing smithi07085154-21 from 3293184 to 4071424 tid 2 2024-09-14T03:55:33.116 INFO:tasks.rados.rados.0.smithi070.stdout:2119: writing smithi07085154-21 from 4071424 to 4096000 tid 3 2024-09-14T03:55:33.116 INFO:tasks.rados.rados.0.smithi070.stdout:2120: snap_remove snap 225 2024-09-14T03:55:33.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:33 smithi094 ceph-mon[86597]: pgmap v349: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 931 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 612 KiB/s wr, 8 op/s; 203/1278 objects degraded (15.884%) 2024-09-14T03:55:33.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:33 smithi094 ceph-mon[86597]: osdmap e503: 8 total, 7 up, 8 in 2024-09-14T03:55:33.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:33 smithi094 ceph-mon[86597]: from='osd.5 [v2:172.21.15.94:6808/1267079794,v1:172.21.15.94:6809/1267079794]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-14T03:55:33.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:33 smithi094 ceph-mon[86597]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-14T03:55:33.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:33 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:33.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:33 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:33.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:33 smithi070 ceph-mon[96704]: pgmap v349: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 931 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 612 KiB/s wr, 8 op/s; 203/1278 objects degraded (15.884%) 2024-09-14T03:55:33.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:33 smithi070 ceph-mon[96704]: osdmap e503: 8 total, 7 up, 8 in 2024-09-14T03:55:33.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:33 smithi070 ceph-mon[96704]: from='osd.5 [v2:172.21.15.94:6808/1267079794,v1:172.21.15.94:6809/1267079794]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-14T03:55:33.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:33 smithi070 ceph-mon[96704]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-14T03:55:33.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:33 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:33.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:33 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:33.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:33 smithi070 ceph-mon[98640]: pgmap v349: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 931 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 612 KiB/s wr, 8 op/s; 203/1278 objects degraded (15.884%) 2024-09-14T03:55:33.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:33 smithi070 ceph-mon[98640]: osdmap e503: 8 total, 7 up, 8 in 2024-09-14T03:55:33.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:33 smithi070 ceph-mon[98640]: from='osd.5 [v2:172.21.15.94:6808/1267079794,v1:172.21.15.94:6809/1267079794]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-14T03:55:33.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:33 smithi070 ceph-mon[98640]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-14T03:55:33.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:33 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:33.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:33 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:33.683 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:55:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[96999]: 2024-09-14T03:55:33.441+0000 7f907172a640 -1 osd.5 492 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:55:33.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:55:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:55:33] "GET /metrics HTTP/1.1" 200 36309 "" "Prometheus/2.51.0" 2024-09-14T03:55:34.101 INFO:tasks.rados.rados.0.smithi070.stdout:2120: done (1 left) 2024-09-14T03:55:34.102 INFO:tasks.rados.rados.0.smithi070.stdout:2121: read oid 46 snap -1 2024-09-14T03:55:34.102 INFO:tasks.rados.rados.0.smithi070.stdout:2121: expect (ObjNum 649 snap 218 seq_num 649) 2024-09-14T03:55:34.102 INFO:tasks.rados.rados.0.smithi070.stdout:2122: snap_create 2024-09-14T03:55:34.102 INFO:tasks.rados.rados.0.smithi070.stdout:2119: finishing write tid 1 to smithi07085154-21 2024-09-14T03:55:34.102 INFO:tasks.rados.rados.0.smithi070.stdout:2119: finishing write tid 2 to smithi07085154-21 2024-09-14T03:55:34.102 INFO:tasks.rados.rados.0.smithi070.stdout:2119: finishing write tid 3 to smithi07085154-21 2024-09-14T03:55:34.102 INFO:tasks.rados.rados.0.smithi070.stdout:2119: finishing write tid 4 to smithi07085154-21 2024-09-14T03:55:34.102 INFO:tasks.rados.rados.0.smithi070.stdout:2119: finishing write tid 5 to smithi07085154-21 2024-09-14T03:55:34.102 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 1700 (ObjNum 667 snap 227 seq_num 667) dirty exists 2024-09-14T03:55:34.102 INFO:tasks.rados.rados.0.smithi070.stdout:2119: left oid 21 (ObjNum 667 snap 227 seq_num 667) 2024-09-14T03:55:34.342 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:34 smithi094 ceph-mon[86597]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-14T03:55:34.343 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:34 smithi094 ceph-mon[86597]: osdmap e504: 8 total, 7 up, 8 in 2024-09-14T03:55:34.343 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:34 smithi094 ceph-mon[86597]: from='osd.5 [v2:172.21.15.94:6808/1267079794,v1:172.21.15.94:6809/1267079794]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi094", "root=default"]} : dispatch 2024-09-14T03:55:34.343 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:34 smithi094 ceph-mon[86597]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi094", "root=default"]} : dispatch 2024-09-14T03:55:34.343 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:34 smithi094 ceph-mon[86597]: from='osd.5 ' entity='osd.5' 2024-09-14T03:55:34.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:34 smithi070 ceph-mon[96704]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-14T03:55:34.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:34 smithi070 ceph-mon[96704]: osdmap e504: 8 total, 7 up, 8 in 2024-09-14T03:55:34.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:34 smithi070 ceph-mon[96704]: from='osd.5 [v2:172.21.15.94:6808/1267079794,v1:172.21.15.94:6809/1267079794]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi094", "root=default"]} : dispatch 2024-09-14T03:55:34.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:34 smithi070 ceph-mon[96704]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi094", "root=default"]} : dispatch 2024-09-14T03:55:34.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:34 smithi070 ceph-mon[96704]: from='osd.5 ' entity='osd.5' 2024-09-14T03:55:34.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:34 smithi070 ceph-mon[98640]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-14T03:55:34.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:34 smithi070 ceph-mon[98640]: osdmap e504: 8 total, 7 up, 8 in 2024-09-14T03:55:34.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:34 smithi070 ceph-mon[98640]: from='osd.5 [v2:172.21.15.94:6808/1267079794,v1:172.21.15.94:6809/1267079794]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi094", "root=default"]} : dispatch 2024-09-14T03:55:34.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:34 smithi070 ceph-mon[98640]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi094", "root=default"]} : dispatch 2024-09-14T03:55:34.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:34 smithi070 ceph-mon[98640]: from='osd.5 ' entity='osd.5' 2024-09-14T03:55:35.104 INFO:tasks.rados.rados.0.smithi070.stdout:2119: done (2 left) 2024-09-14T03:55:35.104 INFO:tasks.rados.rados.0.smithi070.stdout:2122: done (1 left) 2024-09-14T03:55:35.104 INFO:tasks.rados.rados.0.smithi070.stdout:2123: rollback oid 4 current snap is 228 2024-09-14T03:55:35.104 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 4 to 107 2024-09-14T03:55:35.104 INFO:tasks.rados.rados.0.smithi070.stdout:2124: copy_from oid 17 from oid 48 current snap is 228 2024-09-14T03:55:35.104 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 8 current snap is 228 2024-09-14T03:55:35.104 INFO:tasks.rados.rados.0.smithi070.stdout:2125: seq_num 668 ranges {2367488=794624,3162112=663552,3825664=139264} 2024-09-14T03:55:35.122 INFO:tasks.rados.rados.0.smithi070.stdout:2125: writing smithi07085154-8 from 2367488 to 3162112 tid 1 2024-09-14T03:55:35.127 INFO:tasks.rados.rados.0.smithi070.stdout:2125: writing smithi07085154-8 from 3162112 to 3825664 tid 2 2024-09-14T03:55:35.128 INFO:tasks.rados.rados.0.smithi070.stdout:2125: writing smithi07085154-8 from 3825664 to 3964928 tid 3 2024-09-14T03:55:35.128 INFO:tasks.rados.rados.0.smithi070.stdout:2126: snap_remove snap 211 2024-09-14T03:55:35.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:35 smithi094 ceph-mon[86597]: pgmap v352: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 943 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 4.4 MiB/s wr, 7 op/s; 203/1269 objects degraded (15.997%) 2024-09-14T03:55:35.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:35 smithi094 ceph-mon[86597]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:55:35.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:35 smithi094 ceph-mon[86597]: osd.5 [v2:172.21.15.94:6808/1267079794,v1:172.21.15.94:6809/1267079794] boot 2024-09-14T03:55:35.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:35 smithi094 ceph-mon[86597]: osdmap e505: 8 total, 8 up, 8 in 2024-09-14T03:55:35.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:35 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-14T03:55:35.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:35 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:35.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:35 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:35.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:35 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:55:35.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:35 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:55:35.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:35 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:35.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:35 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:55:35.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:35 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:35.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:35 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:35.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:35 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:35.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:35 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-14T03:55:35.285 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:55:35 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:55:35.023+0000 7fa4af5ba640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (17 PGs are or would become offline) 2024-09-14T03:55:35.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:35 smithi070 ceph-mon[98640]: pgmap v352: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 943 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 4.4 MiB/s wr, 7 op/s; 203/1269 objects degraded (15.997%) 2024-09-14T03:55:35.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:35 smithi070 ceph-mon[98640]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:55:35.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:35 smithi070 ceph-mon[98640]: osd.5 [v2:172.21.15.94:6808/1267079794,v1:172.21.15.94:6809/1267079794] boot 2024-09-14T03:55:35.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:35 smithi070 ceph-mon[98640]: osdmap e505: 8 total, 8 up, 8 in 2024-09-14T03:55:35.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:35 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-14T03:55:35.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:35 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:35.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:35 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:35.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:35 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:55:35.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:35 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:55:35.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:35 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:35.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:35 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:55:35.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:35 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:35.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:35 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:35.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:35 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:35.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:35 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-14T03:55:35.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:35 smithi070 ceph-mon[96704]: pgmap v352: 121 pgs: 36 active+undersized, 13 active+undersized+degraded, 72 active+clean; 943 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 4.4 MiB/s wr, 7 op/s; 203/1269 objects degraded (15.997%) 2024-09-14T03:55:35.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:35 smithi070 ceph-mon[96704]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:55:35.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:35 smithi070 ceph-mon[96704]: osd.5 [v2:172.21.15.94:6808/1267079794,v1:172.21.15.94:6809/1267079794] boot 2024-09-14T03:55:35.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:35 smithi070 ceph-mon[96704]: osdmap e505: 8 total, 8 up, 8 in 2024-09-14T03:55:35.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:35 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-14T03:55:35.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:35 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:35.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:35 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:35.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:35 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:55:35.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:35 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:55:35.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:35 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:35.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:35 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:55:35.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:35 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:35.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:35 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:35.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:35 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:55:35.288 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:35 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-14T03:55:36.110 INFO:tasks.rados.rados.0.smithi070.stdout:2123: finishing rollback tid 1 to smithi07085154-4 2024-09-14T03:55:36.110 INFO:tasks.rados.rados.0.smithi070.stdout:2123: finishing rollback tid 2 to smithi07085154-4 2024-09-14T03:55:36.110 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 1271 (ObjNum 243 snap 69 seq_num 243) dirty exists 2024-09-14T03:55:36.110 INFO:tasks.rados.rados.0.smithi070.stdout:2125: finishing write tid 1 to smithi07085154-8 2024-09-14T03:55:36.110 INFO:tasks.rados.rados.0.smithi070.stdout:2125: finishing write tid 2 to smithi07085154-8 2024-09-14T03:55:36.110 INFO:tasks.rados.rados.0.smithi070.stdout:2125: finishing write tid 3 to smithi07085154-8 2024-09-14T03:55:36.110 INFO:tasks.rados.rados.0.smithi070.stdout:2125: finishing write tid 4 to smithi07085154-8 2024-09-14T03:55:36.110 INFO:tasks.rados.rados.0.smithi070.stdout:2125: finishing write tid 5 to smithi07085154-8 2024-09-14T03:55:36.110 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 8 v 2029 (ObjNum 668 snap 228 seq_num 668) dirty exists 2024-09-14T03:55:36.110 INFO:tasks.rados.rados.0.smithi070.stdout:2125: left oid 8 (ObjNum 668 snap 228 seq_num 668) 2024-09-14T03:55:36.111 INFO:tasks.rados.rados.0.smithi070.stdout:2121: expect (ObjNum 649 snap 218 seq_num 649) 2024-09-14T03:55:36.175 INFO:tasks.rados.rados.0.smithi070.stdout:2124: finishing copy_from racing read to smithi07085154-17 2024-09-14T03:55:36.175 INFO:tasks.rados.rados.0.smithi070.stdout:2124: finishing copy_from to smithi07085154-17 2024-09-14T03:55:36.175 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 2189 (ObjNum 666 snap 225 seq_num 666) dirty exists 2024-09-14T03:55:36.175 INFO:tasks.rados.rados.0.smithi070.stdout:2121: done (4 left) 2024-09-14T03:55:36.175 INFO:tasks.rados.rados.0.smithi070.stdout:2123: done (3 left) 2024-09-14T03:55:36.175 INFO:tasks.rados.rados.0.smithi070.stdout:2124: done (2 left) 2024-09-14T03:55:36.175 INFO:tasks.rados.rados.0.smithi070.stdout:2125: done (1 left) 2024-09-14T03:55:36.175 INFO:tasks.rados.rados.0.smithi070.stdout:2126: done (0 left) 2024-09-14T03:55:36.175 INFO:tasks.rados.rados.0.smithi070.stdout:2127: read oid 39 snap -1 2024-09-14T03:55:36.175 INFO:tasks.rados.rados.0.smithi070.stdout:2127: expect (ObjNum 641 snap 215 seq_num 641) 2024-09-14T03:55:36.175 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 1 current snap is 228 2024-09-14T03:55:36.176 INFO:tasks.rados.rados.0.smithi070.stdout:2128: seq_num 669 ranges {16171008=409600,16580608=548864,17129472=655360} 2024-09-14T03:55:36.264 INFO:tasks.rados.rados.0.smithi070.stdout:2128: writing smithi07085154-1 from 16171008 to 16580608 tid 1 2024-09-14T03:55:36.264 INFO:tasks.rados.rados.0.smithi070.stdout:2128: writing smithi07085154-1 from 16580608 to 17129472 tid 2 2024-09-14T03:55:36.268 INFO:tasks.rados.rados.0.smithi070.stdout:2128: writing smithi07085154-1 from 17129472 to 17784832 tid 3 2024-09-14T03:55:36.268 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 31 current snap is 228 2024-09-14T03:55:36.268 INFO:tasks.rados.rados.0.smithi070.stdout:2129: seq_num 670 ranges {1851392=425984,2277376=483328,2760704=507904,3268608=212992} 2024-09-14T03:55:36.280 INFO:tasks.rados.rados.0.smithi070.stdout:2129: writing smithi07085154-31 from 1851392 to 2277376 tid 1 2024-09-14T03:55:36.283 INFO:tasks.rados.rados.0.smithi070.stdout:2129: writing smithi07085154-31 from 2277376 to 2760704 tid 2 2024-09-14T03:55:36.286 INFO:tasks.rados.rados.0.smithi070.stdout:2129: writing smithi07085154-31 from 2760704 to 3268608 tid 3 2024-09-14T03:55:36.287 INFO:tasks.rados.rados.0.smithi070.stdout:2129: writing smithi07085154-31 from 3268608 to 3481600 tid 4 2024-09-14T03:55:36.288 INFO:tasks.rados.rados.0.smithi070.stdout:2130: read oid 22 snap -1 2024-09-14T03:55:36.288 INFO:tasks.rados.rados.0.smithi070.stdout:2130: expect (ObjNum 663 snap 220 seq_num 663) 2024-09-14T03:55:36.288 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 18 current snap is 228 2024-09-14T03:55:36.288 INFO:tasks.rados.rados.0.smithi070.stdout:2127: expect (ObjNum 641 snap 215 seq_num 641) 2024-09-14T03:55:36.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:36 smithi094 ceph-mon[86597]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-14T03:55:36.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:36 smithi094 ceph-mon[86597]: Upgrade: unsafe to stop osd(s) at this time (17 PGs are or would become offline) 2024-09-14T03:55:36.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:36 smithi094 ceph-mon[86597]: osdmap e506: 8 total, 8 up, 8 in 2024-09-14T03:55:36.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:36 smithi070 ceph-mon[98640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-14T03:55:36.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:36 smithi070 ceph-mon[98640]: Upgrade: unsafe to stop osd(s) at this time (17 PGs are or would become offline) 2024-09-14T03:55:36.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:36 smithi070 ceph-mon[98640]: osdmap e506: 8 total, 8 up, 8 in 2024-09-14T03:55:36.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:36 smithi070 ceph-mon[96704]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-14T03:55:36.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:36 smithi070 ceph-mon[96704]: Upgrade: unsafe to stop osd(s) at this time (17 PGs are or would become offline) 2024-09-14T03:55:36.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:36 smithi070 ceph-mon[96704]: osdmap e506: 8 total, 8 up, 8 in 2024-09-14T03:55:36.676 INFO:tasks.rados.rados.0.smithi070.stdout:2128: finishing write tid 1 to smithi07085154-1 2024-09-14T03:55:36.676 INFO:tasks.rados.rados.0.smithi070.stdout:2128: finishing write tid 2 to smithi07085154-1 2024-09-14T03:55:36.676 INFO:tasks.rados.rados.0.smithi070.stdout:2128: finishing write tid 3 to smithi07085154-1 2024-09-14T03:55:36.676 INFO:tasks.rados.rados.0.smithi070.stdout:2128: finishing write tid 4 to smithi07085154-1 2024-09-14T03:55:36.676 INFO:tasks.rados.rados.0.smithi070.stdout:2128: finishing write tid 5 to smithi07085154-1 2024-09-14T03:55:36.676 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 1 v 2029 (ObjNum 669 snap 228 seq_num 669) dirty exists 2024-09-14T03:55:36.677 INFO:tasks.rados.rados.0.smithi070.stdout:2128: left oid 1 (ObjNum 669 snap 228 seq_num 669) 2024-09-14T03:55:36.677 INFO:tasks.rados.rados.0.smithi070.stdout:2129: finishing write tid 1 to smithi07085154-31 2024-09-14T03:55:36.677 INFO:tasks.rados.rados.0.smithi070.stdout:2131: seq_num 671 ranges {7692288=442368,8134656=761856,8896512=442368} 2024-09-14T03:55:36.718 INFO:tasks.rados.rados.0.smithi070.stdout:2131: writing smithi07085154-18 from 7692288 to 8134656 tid 1 2024-09-14T03:55:36.722 INFO:tasks.rados.rados.0.smithi070.stdout:2131: writing smithi07085154-18 from 8134656 to 8896512 tid 2 2024-09-14T03:55:36.725 INFO:tasks.rados.rados.0.smithi070.stdout:2131: writing smithi07085154-18 from 8896512 to 9338880 tid 3 2024-09-14T03:55:36.725 INFO:tasks.rados.rados.0.smithi070.stdout:2127: done (4 left) 2024-09-14T03:55:36.725 INFO:tasks.rados.rados.0.smithi070.stdout:2128: done (3 left) 2024-09-14T03:55:36.725 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 17 current snap is 228 2024-09-14T03:55:36.725 INFO:tasks.rados.rados.0.smithi070.stdout:2129: finishing write tid 2 to smithi07085154-31 2024-09-14T03:55:36.726 INFO:tasks.rados.rados.0.smithi070.stdout:2129: finishing write tid 3 to smithi07085154-31 2024-09-14T03:55:36.726 INFO:tasks.rados.rados.0.smithi070.stdout:2129: finishing write tid 4 to smithi07085154-31 2024-09-14T03:55:36.726 INFO:tasks.rados.rados.0.smithi070.stdout:2132: seq_num 672 ranges {2015232=458752,2473984=647168,3121152=466944,3588096=90112} 2024-09-14T03:55:36.738 INFO:tasks.rados.rados.0.smithi070.stdout:2132: writing smithi07085154-17 from 2015232 to 2473984 tid 1 2024-09-14T03:55:36.742 INFO:tasks.rados.rados.0.smithi070.stdout:2132: writing smithi07085154-17 from 2473984 to 3121152 tid 2 2024-09-14T03:55:36.745 INFO:tasks.rados.rados.0.smithi070.stdout:2132: writing smithi07085154-17 from 3121152 to 3588096 tid 3 2024-09-14T03:55:36.746 INFO:tasks.rados.rados.0.smithi070.stdout:2132: writing smithi07085154-17 from 3588096 to 3678208 tid 4 2024-09-14T03:55:36.746 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 10 current snap is 228 2024-09-14T03:55:36.746 INFO:tasks.rados.rados.0.smithi070.stdout:2129: finishing write tid 5 to smithi07085154-31 2024-09-14T03:55:36.746 INFO:tasks.rados.rados.0.smithi070.stdout:2129: finishing write tid 6 to smithi07085154-31 2024-09-14T03:55:36.746 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 2194 (ObjNum 670 snap 228 seq_num 670) dirty exists 2024-09-14T03:55:36.746 INFO:tasks.rados.rados.0.smithi070.stdout:2129: left oid 31 (ObjNum 670 snap 228 seq_num 670) 2024-09-14T03:55:36.746 INFO:tasks.rados.rados.0.smithi070.stdout:2130: expect (ObjNum 663 snap 220 seq_num 663) 2024-09-14T03:55:36.846 INFO:tasks.rados.rados.0.smithi070.stdout:2131: finishing write tid 1 to smithi07085154-18 2024-09-14T03:55:36.846 INFO:tasks.rados.rados.0.smithi070.stdout:2131: finishing write tid 2 to smithi07085154-18 2024-09-14T03:55:36.846 INFO:tasks.rados.rados.0.smithi070.stdout:2131: finishing write tid 3 to smithi07085154-18 2024-09-14T03:55:36.846 INFO:tasks.rados.rados.0.smithi070.stdout:2131: finishing write tid 4 to smithi07085154-18 2024-09-14T03:55:36.846 INFO:tasks.rados.rados.0.smithi070.stdout:2131: finishing write tid 5 to smithi07085154-18 2024-09-14T03:55:36.847 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 2033 (ObjNum 671 snap 228 seq_num 671) dirty exists 2024-09-14T03:55:36.847 INFO:tasks.rados.rados.0.smithi070.stdout:2131: left oid 18 (ObjNum 671 snap 228 seq_num 671) 2024-09-14T03:55:36.847 INFO:tasks.rados.rados.0.smithi070.stdout:2132: finishing write tid 1 to smithi07085154-17 2024-09-14T03:55:36.847 INFO:tasks.rados.rados.0.smithi070.stdout:2132: finishing write tid 2 to smithi07085154-17 2024-09-14T03:55:36.847 INFO:tasks.rados.rados.0.smithi070.stdout:2132: finishing write tid 3 to smithi07085154-17 2024-09-14T03:55:36.847 INFO:tasks.rados.rados.0.smithi070.stdout:2132: finishing write tid 4 to smithi07085154-17 2024-09-14T03:55:36.847 INFO:tasks.rados.rados.0.smithi070.stdout:2132: finishing write tid 5 to smithi07085154-17 2024-09-14T03:55:36.847 INFO:tasks.rados.rados.0.smithi070.stdout:2132: finishing write tid 6 to smithi07085154-17 2024-09-14T03:55:36.847 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 2199 (ObjNum 672 snap 228 seq_num 672) dirty exists 2024-09-14T03:55:36.847 INFO:tasks.rados.rados.0.smithi070.stdout:2132: left oid 17 (ObjNum 672 snap 228 seq_num 672) 2024-09-14T03:55:36.847 INFO:tasks.rados.rados.0.smithi070.stdout:2133: seq_num 673 ranges {4964352=475136,5439488=532480,5971968=671744} 2024-09-14T03:55:36.876 INFO:tasks.rados.rados.0.smithi070.stdout:2133: writing smithi07085154-10 from 4964352 to 5439488 tid 1 2024-09-14T03:55:36.877 INFO:tasks.rados.rados.0.smithi070.stdout:2133: writing smithi07085154-10 from 5439488 to 5971968 tid 2 2024-09-14T03:55:36.881 INFO:tasks.rados.rados.0.smithi070.stdout:2133: writing smithi07085154-10 from 5971968 to 6643712 tid 3 2024-09-14T03:55:36.881 INFO:tasks.rados.rados.0.smithi070.stdout:2129: done (4 left) 2024-09-14T03:55:36.881 INFO:tasks.rados.rados.0.smithi070.stdout:2130: done (3 left) 2024-09-14T03:55:36.882 INFO:tasks.rados.rados.0.smithi070.stdout:2131: done (2 left) 2024-09-14T03:55:36.882 INFO:tasks.rados.rados.0.smithi070.stdout:2132: done (1 left) 2024-09-14T03:55:36.882 INFO:tasks.rados.rados.0.smithi070.stdout:2134: rollback oid 9 current snap is 228 2024-09-14T03:55:36.882 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 9 to 182 2024-09-14T03:55:36.882 INFO:tasks.rados.rados.0.smithi070.stdout:2133: finishing write tid 1 to smithi07085154-10 2024-09-14T03:55:36.882 INFO:tasks.rados.rados.0.smithi070.stdout:2133: finishing write tid 2 to smithi07085154-10 2024-09-14T03:55:36.882 INFO:tasks.rados.rados.0.smithi070.stdout:2135: delete oid 24 current snap is 228 2024-09-14T03:55:36.885 INFO:tasks.rados.rados.0.smithi070.stdout:2135: done (2 left) 2024-09-14T03:55:36.885 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 39 current snap is 228 2024-09-14T03:55:36.885 INFO:tasks.rados.rados.0.smithi070.stdout:2136: seq_num 675 ranges {6316032=507904,6823936=704512,7528448=499712} 2024-09-14T03:55:36.921 INFO:tasks.rados.rados.0.smithi070.stdout:2136: writing smithi07085154-39 from 6316032 to 6823936 tid 1 2024-09-14T03:55:36.924 INFO:tasks.rados.rados.0.smithi070.stdout:2136: writing smithi07085154-39 from 6823936 to 7528448 tid 2 2024-09-14T03:55:36.928 INFO:tasks.rados.rados.0.smithi070.stdout:2136: writing smithi07085154-39 from 7528448 to 8028160 tid 3 2024-09-14T03:55:36.928 INFO:tasks.rados.rados.0.smithi070.stdout:2137: delete oid 14 current snap is 228 2024-09-14T03:55:36.928 INFO:tasks.rados.rados.0.smithi070.stdout:2133: finishing write tid 3 to smithi07085154-10 2024-09-14T03:55:36.928 INFO:tasks.rados.rados.0.smithi070.stdout:2133: finishing write tid 4 to smithi07085154-10 2024-09-14T03:55:36.928 INFO:tasks.rados.rados.0.smithi070.stdout:2133: finishing write tid 5 to smithi07085154-10 2024-09-14T03:55:36.928 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 2306 (ObjNum 673 snap 228 seq_num 673) dirty exists 2024-09-14T03:55:36.928 INFO:tasks.rados.rados.0.smithi070.stdout:2133: left oid 10 (ObjNum 673 snap 228 seq_num 673) 2024-09-14T03:55:36.928 INFO:tasks.rados.rados.0.smithi070.stdout:2134: finishing rollback tid 1 to smithi07085154-9 2024-09-14T03:55:36.928 INFO:tasks.rados.rados.0.smithi070.stdout:2134: finishing rollback tid 2 to smithi07085154-9 2024-09-14T03:55:36.928 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 2308 (ObjNum 507 snap 166 seq_num 507) dirty exists 2024-09-14T03:55:36.928 INFO:tasks.rados.rados.0.smithi070.stdout:2136: finishing write tid 1 to smithi07085154-39 2024-09-14T03:55:36.929 INFO:tasks.rados.rados.0.smithi070.stdout:2136: finishing write tid 2 to smithi07085154-39 2024-09-14T03:55:36.931 INFO:tasks.rados.rados.0.smithi070.stdout:2133: done (3 left) 2024-09-14T03:55:36.931 INFO:tasks.rados.rados.0.smithi070.stdout:2134: done (2 left) 2024-09-14T03:55:36.932 INFO:tasks.rados.rados.0.smithi070.stdout:2137: done (1 left) 2024-09-14T03:55:36.932 INFO:tasks.rados.rados.0.smithi070.stdout:2138: snap_create 2024-09-14T03:55:36.932 INFO:tasks.rados.rados.0.smithi070.stdout:2136: finishing write tid 3 to smithi07085154-39 2024-09-14T03:55:36.932 INFO:tasks.rados.rados.0.smithi070.stdout:2136: finishing write tid 4 to smithi07085154-39 2024-09-14T03:55:36.933 INFO:tasks.rados.rados.0.smithi070.stdout:2136: finishing write tid 5 to smithi07085154-39 2024-09-14T03:55:36.933 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 39 v 1702 (ObjNum 675 snap 228 seq_num 675) dirty exists 2024-09-14T03:55:36.933 INFO:tasks.rados.rados.0.smithi070.stdout:2136: left oid 39 (ObjNum 675 snap 228 seq_num 675) 2024-09-14T03:55:37.111 INFO:tasks.rados.rados.0.smithi070.stdout:2136: done (1 left) 2024-09-14T03:55:37.111 INFO:tasks.rados.rados.0.smithi070.stdout:2138: done (0 left) 2024-09-14T03:55:37.111 INFO:tasks.rados.rados.0.smithi070.stdout:2139: delete oid 1 current snap is 229 2024-09-14T03:55:37.114 INFO:tasks.rados.rados.0.smithi070.stdout:2139: done (0 left) 2024-09-14T03:55:37.114 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 40 current snap is 229 2024-09-14T03:55:37.114 INFO:tasks.rados.rados.0.smithi070.stdout:2140: seq_num 678 ranges {0=565248,565248=802816,1368064=401408} 2024-09-14T03:55:37.117 INFO:tasks.rados.rados.0.smithi070.stdout:2140: writing smithi07085154-40 from 0 to 565248 tid 1 2024-09-14T03:55:37.122 INFO:tasks.rados.rados.0.smithi070.stdout:2140: writing smithi07085154-40 from 565248 to 1368064 tid 2 2024-09-14T03:55:37.124 INFO:tasks.rados.rados.0.smithi070.stdout:2140: writing smithi07085154-40 from 1368064 to 1769472 tid 3 2024-09-14T03:55:37.124 INFO:tasks.rados.rados.0.smithi070.stdout:2141: copy_from oid 26 from oid 4 current snap is 229 2024-09-14T03:55:37.125 INFO:tasks.rados.rados.0.smithi070.stdout:2140: finishing write tid 1 to smithi07085154-40 2024-09-14T03:55:37.125 INFO:tasks.rados.rados.0.smithi070.stdout:2142: snap_create 2024-09-14T03:55:37.127 INFO:tasks.rados.rados.0.smithi070.stdout:2140: finishing write tid 2 to smithi07085154-40 2024-09-14T03:55:37.127 INFO:tasks.rados.rados.0.smithi070.stdout:2140: finishing write tid 3 to smithi07085154-40 2024-09-14T03:55:37.127 INFO:tasks.rados.rados.0.smithi070.stdout:2140: finishing write tid 4 to smithi07085154-40 2024-09-14T03:55:37.129 INFO:tasks.rados.rados.0.smithi070.stdout:2140: finishing write tid 5 to smithi07085154-40 2024-09-14T03:55:37.129 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 2034 (ObjNum 678 snap 229 seq_num 678) dirty exists 2024-09-14T03:55:37.129 INFO:tasks.rados.rados.0.smithi070.stdout:2140: left oid 40 (ObjNum 678 snap 229 seq_num 678) 2024-09-14T03:55:37.133 INFO:tasks.rados.rados.0.smithi070.stdout:2141: finishing copy_from racing read to smithi07085154-26 2024-09-14T03:55:37.136 INFO:tasks.rados.rados.0.smithi070.stdout:2141: finishing copy_from to smithi07085154-26 2024-09-14T03:55:37.136 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 1271 (ObjNum 243 snap 69 seq_num 243) dirty exists 2024-09-14T03:55:37.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:37 smithi094 ceph-mon[86597]: pgmap v355: 121 pgs: 3 peering, 33 active+undersized, 13 active+undersized+degraded, 72 active+clean; 945 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 4.2 MiB/s wr, 2 op/s; 204/1275 objects degraded (16.000%) 2024-09-14T03:55:37.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:37 smithi094 ceph-mon[86597]: Health check update: Degraded data redundancy: 204/1275 objects degraded (16.000%), 13 pgs degraded (PG_DEGRADED) 2024-09-14T03:55:37.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:37 smithi094 ceph-mon[86597]: osdmap e507: 8 total, 8 up, 8 in 2024-09-14T03:55:37.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:37 smithi070 ceph-mon[98640]: pgmap v355: 121 pgs: 3 peering, 33 active+undersized, 13 active+undersized+degraded, 72 active+clean; 945 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 4.2 MiB/s wr, 2 op/s; 204/1275 objects degraded (16.000%) 2024-09-14T03:55:37.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:37 smithi070 ceph-mon[98640]: Health check update: Degraded data redundancy: 204/1275 objects degraded (16.000%), 13 pgs degraded (PG_DEGRADED) 2024-09-14T03:55:37.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:37 smithi070 ceph-mon[98640]: osdmap e507: 8 total, 8 up, 8 in 2024-09-14T03:55:37.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:37 smithi070 ceph-mon[96704]: pgmap v355: 121 pgs: 3 peering, 33 active+undersized, 13 active+undersized+degraded, 72 active+clean; 945 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 4.2 MiB/s wr, 2 op/s; 204/1275 objects degraded (16.000%) 2024-09-14T03:55:37.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:37 smithi070 ceph-mon[96704]: Health check update: Degraded data redundancy: 204/1275 objects degraded (16.000%), 13 pgs degraded (PG_DEGRADED) 2024-09-14T03:55:37.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:37 smithi070 ceph-mon[96704]: osdmap e507: 8 total, 8 up, 8 in 2024-09-14T03:55:38.119 INFO:tasks.rados.rados.0.smithi070.stdout:2140: done (2 left) 2024-09-14T03:55:38.120 INFO:tasks.rados.rados.0.smithi070.stdout:2141: done (1 left) 2024-09-14T03:55:38.120 INFO:tasks.rados.rados.0.smithi070.stdout:2142: done (0 left) 2024-09-14T03:55:38.120 INFO:tasks.rados.rados.0.smithi070.stdout:2143: snap_remove snap 218 2024-09-14T03:55:38.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:38 smithi094 ceph-mon[86597]: osdmap e508: 8 total, 8 up, 8 in 2024-09-14T03:55:38.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:38 smithi070 ceph-mon[98640]: osdmap e508: 8 total, 8 up, 8 in 2024-09-14T03:55:38.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:38 smithi070 ceph-mon[96704]: osdmap e508: 8 total, 8 up, 8 in 2024-09-14T03:55:39.120 INFO:tasks.rados.rados.0.smithi070.stdout:2143: done (0 left) 2024-09-14T03:55:39.120 INFO:tasks.rados.rados.0.smithi070.stdout:2144: delete oid 21 current snap is 230 2024-09-14T03:55:39.130 INFO:tasks.rados.rados.0.smithi070.stdout:2144: done (0 left) 2024-09-14T03:55:39.130 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 23 current snap is 230 2024-09-14T03:55:39.130 INFO:tasks.rados.rados.0.smithi070.stdout:2145: seq_num 680 ranges {3203072=598016,3801088=573440,4374528=434176,4808704=196608} 2024-09-14T03:55:39.151 INFO:tasks.rados.rados.0.smithi070.stdout:2145: writing smithi07085154-23 from 3203072 to 3801088 tid 1 2024-09-14T03:55:39.154 INFO:tasks.rados.rados.0.smithi070.stdout:2145: writing smithi07085154-23 from 3801088 to 4374528 tid 2 2024-09-14T03:55:39.157 INFO:tasks.rados.rados.0.smithi070.stdout:2145: writing smithi07085154-23 from 4374528 to 4808704 tid 3 2024-09-14T03:55:39.158 INFO:tasks.rados.rados.0.smithi070.stdout:2145: writing smithi07085154-23 from 4808704 to 5005312 tid 4 2024-09-14T03:55:39.158 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 43 current snap is 230 2024-09-14T03:55:39.158 INFO:tasks.rados.rados.0.smithi070.stdout:2145: finishing write tid 1 to smithi07085154-23 2024-09-14T03:55:39.158 INFO:tasks.rados.rados.0.smithi070.stdout:2146: seq_num 681 ranges {0=614400,614400=458752,1073152=638976,1712128=106496} 2024-09-14T03:55:39.161 INFO:tasks.rados.rados.0.smithi070.stdout:2146: writing smithi07085154-43 from 0 to 614400 tid 1 2024-09-14T03:55:39.164 INFO:tasks.rados.rados.0.smithi070.stdout:2146: writing smithi07085154-43 from 614400 to 1073152 tid 2 2024-09-14T03:55:39.168 INFO:tasks.rados.rados.0.smithi070.stdout:2146: writing smithi07085154-43 from 1073152 to 1712128 tid 3 2024-09-14T03:55:39.169 INFO:tasks.rados.rados.0.smithi070.stdout:2146: writing smithi07085154-43 from 1712128 to 1818624 tid 4 2024-09-14T03:55:39.169 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 48 current snap is 230 2024-09-14T03:55:39.169 INFO:tasks.rados.rados.0.smithi070.stdout:2145: finishing write tid 2 to smithi07085154-23 2024-09-14T03:55:39.169 INFO:tasks.rados.rados.0.smithi070.stdout:2145: finishing write tid 3 to smithi07085154-23 2024-09-14T03:55:39.169 INFO:tasks.rados.rados.0.smithi070.stdout:2145: finishing write tid 4 to smithi07085154-23 2024-09-14T03:55:39.169 INFO:tasks.rados.rados.0.smithi070.stdout:2145: finishing write tid 5 to smithi07085154-23 2024-09-14T03:55:39.169 INFO:tasks.rados.rados.0.smithi070.stdout:2145: finishing write tid 6 to smithi07085154-23 2024-09-14T03:55:39.169 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 1699 (ObjNum 680 snap 230 seq_num 680) dirty exists 2024-09-14T03:55:39.169 INFO:tasks.rados.rados.0.smithi070.stdout:2145: left oid 23 (ObjNum 680 snap 230 seq_num 680) 2024-09-14T03:55:39.169 INFO:tasks.rados.rados.0.smithi070.stdout:2146: finishing write tid 1 to smithi07085154-43 2024-09-14T03:55:39.170 INFO:tasks.rados.rados.0.smithi070.stdout:2146: finishing write tid 2 to smithi07085154-43 2024-09-14T03:55:39.170 INFO:tasks.rados.rados.0.smithi070.stdout:2147: seq_num 682 ranges {2015232=630784,2646016=745472,3391488=442368,3833856=16384} 2024-09-14T03:55:39.183 INFO:tasks.rados.rados.0.smithi070.stdout:2147: writing smithi07085154-48 from 2015232 to 2646016 tid 1 2024-09-14T03:55:39.187 INFO:tasks.rados.rados.0.smithi070.stdout:2147: writing smithi07085154-48 from 2646016 to 3391488 tid 2 2024-09-14T03:55:39.191 INFO:tasks.rados.rados.0.smithi070.stdout:2147: writing smithi07085154-48 from 3391488 to 3833856 tid 3 2024-09-14T03:55:39.191 INFO:tasks.rados.rados.0.smithi070.stdout:2147: writing smithi07085154-48 from 3833856 to 3850240 tid 4 2024-09-14T03:55:39.191 INFO:tasks.rados.rados.0.smithi070.stdout:2145: done (2 left) 2024-09-14T03:55:39.191 INFO:tasks.rados.rados.0.smithi070.stdout:2146: finishing write tid 3 to smithi07085154-43 2024-09-14T03:55:39.191 INFO:tasks.rados.rados.0.smithi070.stdout:2146: finishing write tid 4 to smithi07085154-43 2024-09-14T03:55:39.191 INFO:tasks.rados.rados.0.smithi070.stdout:2146: finishing write tid 5 to smithi07085154-43 2024-09-14T03:55:39.191 INFO:tasks.rados.rados.0.smithi070.stdout:2146: finishing write tid 6 to smithi07085154-43 2024-09-14T03:55:39.191 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 2038 (ObjNum 681 snap 230 seq_num 681) dirty exists 2024-09-14T03:55:39.191 INFO:tasks.rados.rados.0.smithi070.stdout:2146: left oid 43 (ObjNum 681 snap 230 seq_num 681) 2024-09-14T03:55:39.191 INFO:tasks.rados.rados.0.smithi070.stdout:2147: finishing write tid 1 to smithi07085154-48 2024-09-14T03:55:39.191 INFO:tasks.rados.rados.0.smithi070.stdout:2148: read oid 11 snap -1 2024-09-14T03:55:39.191 INFO:tasks.rados.rados.0.smithi070.stdout:2148: expect (ObjNum 403 snap 140 seq_num 403) 2024-09-14T03:55:39.191 INFO:tasks.rados.rados.0.smithi070.stdout:2146: done (2 left) 2024-09-14T03:55:39.192 INFO:tasks.rados.rados.0.smithi070.stdout:2149: copy_from oid 7 from oid 41 current snap is 230 2024-09-14T03:55:39.192 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 3 current snap is 230 2024-09-14T03:55:39.192 INFO:tasks.rados.rados.0.smithi070.stdout:2150: seq_num 683 ranges {2170880=647168,2818048=630784,3448832=573440} 2024-09-14T03:55:39.206 INFO:tasks.rados.rados.0.smithi070.stdout:2150: writing smithi07085154-3 from 2170880 to 2818048 tid 1 2024-09-14T03:55:39.210 INFO:tasks.rados.rados.0.smithi070.stdout:2150: writing smithi07085154-3 from 2818048 to 3448832 tid 2 2024-09-14T03:55:39.215 INFO:tasks.rados.rados.0.smithi070.stdout:2150: writing smithi07085154-3 from 3448832 to 4022272 tid 3 2024-09-14T03:55:39.215 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 32 current snap is 230 2024-09-14T03:55:39.215 INFO:tasks.rados.rados.0.smithi070.stdout:2147: finishing write tid 2 to smithi07085154-48 2024-09-14T03:55:39.215 INFO:tasks.rados.rados.0.smithi070.stdout:2147: finishing write tid 3 to smithi07085154-48 2024-09-14T03:55:39.215 INFO:tasks.rados.rados.0.smithi070.stdout:2147: finishing write tid 4 to smithi07085154-48 2024-09-14T03:55:39.215 INFO:tasks.rados.rados.0.smithi070.stdout:2147: finishing write tid 5 to smithi07085154-48 2024-09-14T03:55:39.215 INFO:tasks.rados.rados.0.smithi070.stdout:2147: finishing write tid 6 to smithi07085154-48 2024-09-14T03:55:39.215 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 2194 (ObjNum 682 snap 230 seq_num 682) dirty exists 2024-09-14T03:55:39.215 INFO:tasks.rados.rados.0.smithi070.stdout:2147: left oid 48 (ObjNum 682 snap 230 seq_num 682) 2024-09-14T03:55:39.215 INFO:tasks.rados.rados.0.smithi070.stdout:2150: finishing write tid 1 to smithi07085154-3 2024-09-14T03:55:39.216 INFO:tasks.rados.rados.0.smithi070.stdout:2151: seq_num 684 ranges {7405568=663552,8069120=507904,8577024=409600,8986624=286720} 2024-09-14T03:55:39.258 INFO:tasks.rados.rados.0.smithi070.stdout:2151: writing smithi07085154-32 from 7405568 to 8069120 tid 1 2024-09-14T03:55:39.264 INFO:tasks.rados.rados.0.smithi070.stdout:2151: writing smithi07085154-32 from 8069120 to 8577024 tid 2 2024-09-14T03:55:39.264 INFO:tasks.rados.rados.0.smithi070.stdout:2151: writing smithi07085154-32 from 8577024 to 8986624 tid 3 2024-09-14T03:55:39.265 INFO:tasks.rados.rados.0.smithi070.stdout:2151: writing smithi07085154-32 from 8986624 to 9273344 tid 4 2024-09-14T03:55:39.266 INFO:tasks.rados.rados.0.smithi070.stdout:2147: done (4 left) 2024-09-14T03:55:39.266 INFO:tasks.rados.rados.0.smithi070.stdout:2152: snap_create 2024-09-14T03:55:39.266 INFO:tasks.rados.rados.0.smithi070.stdout:2149: finishing copy_from racing read to smithi07085154-7 2024-09-14T03:55:39.266 INFO:tasks.rados.rados.0.smithi070.stdout:2149: finishing copy_from to smithi07085154-7 2024-09-14T03:55:39.266 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 1689 (ObjNum 514 snap 171 seq_num 514) dirty exists 2024-09-14T03:55:39.266 INFO:tasks.rados.rados.0.smithi070.stdout:2150: finishing write tid 2 to smithi07085154-3 2024-09-14T03:55:39.266 INFO:tasks.rados.rados.0.smithi070.stdout:2150: finishing write tid 3 to smithi07085154-3 2024-09-14T03:55:39.266 INFO:tasks.rados.rados.0.smithi070.stdout:2150: finishing write tid 4 to smithi07085154-3 2024-09-14T03:55:39.266 INFO:tasks.rados.rados.0.smithi070.stdout:2150: finishing write tid 5 to smithi07085154-3 2024-09-14T03:55:39.266 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 3 v 1699 (ObjNum 683 snap 230 seq_num 683) dirty exists 2024-09-14T03:55:39.266 INFO:tasks.rados.rados.0.smithi070.stdout:2150: left oid 3 (ObjNum 683 snap 230 seq_num 683) 2024-09-14T03:55:39.266 INFO:tasks.rados.rados.0.smithi070.stdout:2148: expect (ObjNum 403 snap 140 seq_num 403) 2024-09-14T03:55:39.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:39 smithi094 ceph-mon[86597]: pgmap v358: 121 pgs: 3 peering, 21 active+undersized, 5 active+undersized+degraded, 92 active+clean; 942 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.2 MiB/s wr, 7 op/s; 66/1260 objects degraded (5.238%); 1.1 MiB/s, 0 objects/s recovering 2024-09-14T03:55:39.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:39 smithi094 ceph-mon[86597]: osdmap e509: 8 total, 8 up, 8 in 2024-09-14T03:55:39.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:39 smithi070 ceph-mon[96704]: pgmap v358: 121 pgs: 3 peering, 21 active+undersized, 5 active+undersized+degraded, 92 active+clean; 942 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.2 MiB/s wr, 7 op/s; 66/1260 objects degraded (5.238%); 1.1 MiB/s, 0 objects/s recovering 2024-09-14T03:55:39.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:39 smithi070 ceph-mon[96704]: osdmap e509: 8 total, 8 up, 8 in 2024-09-14T03:55:39.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:39 smithi070 ceph-mon[98640]: pgmap v358: 121 pgs: 3 peering, 21 active+undersized, 5 active+undersized+degraded, 92 active+clean; 942 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.2 MiB/s wr, 7 op/s; 66/1260 objects degraded (5.238%); 1.1 MiB/s, 0 objects/s recovering 2024-09-14T03:55:39.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:39 smithi070 ceph-mon[98640]: osdmap e509: 8 total, 8 up, 8 in 2024-09-14T03:55:39.830 INFO:tasks.rados.rados.0.smithi070.stdout:2151: finishing write tid 1 to smithi07085154-32 2024-09-14T03:55:39.830 INFO:tasks.rados.rados.0.smithi070.stdout:2151: finishing write tid 2 to smithi07085154-32 2024-09-14T03:55:39.830 INFO:tasks.rados.rados.0.smithi070.stdout:2151: finishing write tid 3 to smithi07085154-32 2024-09-14T03:55:39.831 INFO:tasks.rados.rados.0.smithi070.stdout:2151: finishing write tid 4 to smithi07085154-32 2024-09-14T03:55:39.831 INFO:tasks.rados.rados.0.smithi070.stdout:2151: finishing write tid 5 to smithi07085154-32 2024-09-14T03:55:39.831 INFO:tasks.rados.rados.0.smithi070.stdout:2151: finishing write tid 6 to smithi07085154-32 2024-09-14T03:55:39.831 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 2030 (ObjNum 684 snap 230 seq_num 684) dirty exists 2024-09-14T03:55:39.831 INFO:tasks.rados.rados.0.smithi070.stdout:2151: left oid 32 (ObjNum 684 snap 230 seq_num 684) 2024-09-14T03:55:40.131 INFO:tasks.rados.rados.0.smithi070.stdout:2148: done (4 left) 2024-09-14T03:55:40.131 INFO:tasks.rados.rados.0.smithi070.stdout:2149: done (3 left) 2024-09-14T03:55:40.131 INFO:tasks.rados.rados.0.smithi070.stdout:2150: done (2 left) 2024-09-14T03:55:40.131 INFO:tasks.rados.rados.0.smithi070.stdout:2151: done (1 left) 2024-09-14T03:55:40.131 INFO:tasks.rados.rados.0.smithi070.stdout:2152: done (0 left) 2024-09-14T03:55:40.132 INFO:tasks.rados.rados.0.smithi070.stdout:2153: copy_from oid 45 from oid 2 current snap is 231 2024-09-14T03:55:40.132 INFO:tasks.rados.rados.0.smithi070.stdout:2154: read oid 19 snap 182 2024-09-14T03:55:40.132 INFO:tasks.rados.rados.0.smithi070.stdout:2154: expect (ObjNum 262 snap 76 seq_num 262) 2024-09-14T03:55:40.132 INFO:tasks.rados.rados.0.smithi070.stdout:2155: delete oid 5 current snap is 231 2024-09-14T03:55:40.134 INFO:tasks.rados.rados.0.smithi070.stdout:2153: finishing copy_from to smithi07085154-45 2024-09-14T03:55:40.134 INFO:tasks.rados.rados.0.smithi070.stdout:2153: got expected ENOENT (src dne) 2024-09-14T03:55:40.134 INFO:tasks.rados.rados.0.smithi070.stdout:2153: finishing copy_from racing read to smithi07085154-45 2024-09-14T03:55:40.139 INFO:tasks.rados.rados.0.smithi070.stdout:2153: done (2 left) 2024-09-14T03:55:40.139 INFO:tasks.rados.rados.0.smithi070.stdout:2155: done (1 left) 2024-09-14T03:55:40.139 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 5 current snap is 231 2024-09-14T03:55:40.139 INFO:tasks.rados.rados.0.smithi070.stdout:2156: seq_num 686 ranges {0=696320,696320=729088,1425408=425984,1851392=49152} 2024-09-14T03:55:40.142 INFO:tasks.rados.rados.0.smithi070.stdout:2156: writing smithi07085154-5 from 0 to 696320 tid 1 2024-09-14T03:55:40.147 INFO:tasks.rados.rados.0.smithi070.stdout:2156: writing smithi07085154-5 from 696320 to 1425408 tid 2 2024-09-14T03:55:40.149 INFO:tasks.rados.rados.0.smithi070.stdout:2156: writing smithi07085154-5 from 1425408 to 1851392 tid 3 2024-09-14T03:55:40.150 INFO:tasks.rados.rados.0.smithi070.stdout:2156: writing smithi07085154-5 from 1851392 to 1900544 tid 4 2024-09-14T03:55:40.150 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 13 current snap is 231 2024-09-14T03:55:40.150 INFO:tasks.rados.rados.0.smithi070.stdout:2156: finishing write tid 1 to smithi07085154-5 2024-09-14T03:55:40.150 INFO:tasks.rados.rados.0.smithi070.stdout:2157: seq_num 687 ranges {16171008=712704,16883712=614400,17498112=589824} 2024-09-14T03:55:40.240 INFO:tasks.rados.rados.0.smithi070.stdout:2157: writing smithi07085154-13 from 16171008 to 16883712 tid 1 2024-09-14T03:55:40.243 INFO:tasks.rados.rados.0.smithi070.stdout:2157: writing smithi07085154-13 from 16883712 to 17498112 tid 2 2024-09-14T03:55:40.247 INFO:tasks.rados.rados.0.smithi070.stdout:2157: writing smithi07085154-13 from 17498112 to 18087936 tid 3 2024-09-14T03:55:40.247 INFO:tasks.rados.rados.0.smithi070.stdout:2158: snap_remove snap 224 2024-09-14T03:55:40.428 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:40 smithi094 ceph-mon[86597]: osdmap e510: 8 total, 8 up, 8 in 2024-09-14T03:55:40.428 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:40 smithi094 ceph-mon[86597]: pgmap v361: 121 pgs: 121 active+clean; 955 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.8 MiB/s rd, 4.5 MiB/s wr, 18 op/s; 4.7 MiB/s, 2 objects/s recovering 2024-09-14T03:55:40.428 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:55:40 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:55:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:55:40.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:40 smithi070 ceph-mon[96704]: osdmap e510: 8 total, 8 up, 8 in 2024-09-14T03:55:40.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:40 smithi070 ceph-mon[96704]: pgmap v361: 121 pgs: 121 active+clean; 955 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.8 MiB/s rd, 4.5 MiB/s wr, 18 op/s; 4.7 MiB/s, 2 objects/s recovering 2024-09-14T03:55:40.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:40 smithi070 ceph-mon[98640]: osdmap e510: 8 total, 8 up, 8 in 2024-09-14T03:55:40.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:40 smithi070 ceph-mon[98640]: pgmap v361: 121 pgs: 121 active+clean; 955 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.8 MiB/s rd, 4.5 MiB/s wr, 18 op/s; 4.7 MiB/s, 2 objects/s recovering 2024-09-14T03:55:41.136 INFO:tasks.rados.rados.0.smithi070.stdout:2158: done (3 left) 2024-09-14T03:55:41.136 INFO:tasks.rados.rados.0.smithi070.stdout:2159: setattr oid 26 current snap is 231 2024-09-14T03:55:41.136 INFO:tasks.rados.rados.0.smithi070.stdout:2156: finishing write tid 2 to smithi07085154-5 2024-09-14T03:55:41.136 INFO:tasks.rados.rados.0.smithi070.stdout:2156: finishing write tid 3 to smithi07085154-5 2024-09-14T03:55:41.136 INFO:tasks.rados.rados.0.smithi070.stdout:2156: finishing write tid 4 to smithi07085154-5 2024-09-14T03:55:41.136 INFO:tasks.rados.rados.0.smithi070.stdout:2156: finishing write tid 5 to smithi07085154-5 2024-09-14T03:55:41.137 INFO:tasks.rados.rados.0.smithi070.stdout:2156: finishing write tid 6 to smithi07085154-5 2024-09-14T03:55:41.137 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 566 (ObjNum 686 snap 231 seq_num 686) dirty exists 2024-09-14T03:55:41.137 INFO:tasks.rados.rados.0.smithi070.stdout:2156: left oid 5 (ObjNum 686 snap 231 seq_num 686) 2024-09-14T03:55:41.137 INFO:tasks.rados.rados.0.smithi070.stdout:2154: expect (ObjNum 262 snap 76 seq_num 262) 2024-09-14T03:55:41.503 INFO:tasks.rados.rados.0.smithi070.stdout:2157: finishing write tid 1 to smithi07085154-13 2024-09-14T03:55:41.503 INFO:tasks.rados.rados.0.smithi070.stdout:2157: finishing write tid 2 to smithi07085154-13 2024-09-14T03:55:41.503 INFO:tasks.rados.rados.0.smithi070.stdout:2157: finishing write tid 3 to smithi07085154-13 2024-09-14T03:55:41.503 INFO:tasks.rados.rados.0.smithi070.stdout:2157: finishing write tid 4 to smithi07085154-13 2024-09-14T03:55:41.504 INFO:tasks.rados.rados.0.smithi070.stdout:2157: finishing write tid 5 to smithi07085154-13 2024-09-14T03:55:41.504 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 2034 (ObjNum 687 snap 231 seq_num 687) dirty exists 2024-09-14T03:55:41.504 INFO:tasks.rados.rados.0.smithi070.stdout:2157: left oid 13 (ObjNum 687 snap 231 seq_num 687) 2024-09-14T03:55:41.504 INFO:tasks.rados.rados.0.smithi070.stdout:2154: done (3 left) 2024-09-14T03:55:41.504 INFO:tasks.rados.rados.0.smithi070.stdout:2156: done (2 left) 2024-09-14T03:55:41.504 INFO:tasks.rados.rados.0.smithi070.stdout:2157: done (1 left) 2024-09-14T03:55:41.505 INFO:tasks.rados.rados.0.smithi070.stdout:2160: snap_create 2024-09-14T03:55:41.507 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 2319 (ObjNum 243 snap 69 seq_num 243) dirty exists 2024-09-14T03:55:41.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:41 smithi094 ceph-mon[86597]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 66/1260 objects degraded (5.238%), 5 pgs degraded) 2024-09-14T03:55:41.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:41 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T03:55:41.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:41 smithi094 ceph-mon[86597]: osdmap e511: 8 total, 8 up, 8 in 2024-09-14T03:55:41.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:41 smithi070 ceph-mon[96704]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 66/1260 objects degraded (5.238%), 5 pgs degraded) 2024-09-14T03:55:41.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:41 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T03:55:41.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:41 smithi070 ceph-mon[96704]: osdmap e511: 8 total, 8 up, 8 in 2024-09-14T03:55:41.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:41 smithi070 ceph-mon[98640]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 66/1260 objects degraded (5.238%), 5 pgs degraded) 2024-09-14T03:55:41.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:41 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T03:55:41.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:41 smithi070 ceph-mon[98640]: osdmap e511: 8 total, 8 up, 8 in 2024-09-14T03:55:42.139 INFO:tasks.rados.rados.0.smithi070.stdout:2159: done (1 left) 2024-09-14T03:55:42.139 INFO:tasks.rados.rados.0.smithi070.stdout:2160: done (0 left) 2024-09-14T03:55:42.139 INFO:tasks.rados.rados.0.smithi070.stdout:2161: snap_remove snap 173 2024-09-14T03:55:42.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:42 smithi094 ceph-mon[86597]: osdmap e512: 8 total, 8 up, 8 in 2024-09-14T03:55:42.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:42 smithi094 ceph-mon[86597]: pgmap v364: 121 pgs: 121 active+clean; 966 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 6.3 MiB/s wr, 24 op/s; 3.6 MiB/s, 1 objects/s recovering 2024-09-14T03:55:42.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:42 smithi070 ceph-mon[96704]: osdmap e512: 8 total, 8 up, 8 in 2024-09-14T03:55:42.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:42 smithi070 ceph-mon[96704]: pgmap v364: 121 pgs: 121 active+clean; 966 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 6.3 MiB/s wr, 24 op/s; 3.6 MiB/s, 1 objects/s recovering 2024-09-14T03:55:42.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:42 smithi070 ceph-mon[98640]: osdmap e512: 8 total, 8 up, 8 in 2024-09-14T03:55:42.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:42 smithi070 ceph-mon[98640]: pgmap v364: 121 pgs: 121 active+clean; 966 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 6.3 MiB/s wr, 24 op/s; 3.6 MiB/s, 1 objects/s recovering 2024-09-14T03:55:43.142 INFO:tasks.rados.rados.0.smithi070.stdout:2161: done (0 left) 2024-09-14T03:55:43.142 INFO:tasks.rados.rados.0.smithi070.stdout:2162: read oid 8 snap -1 2024-09-14T03:55:43.142 INFO:tasks.rados.rados.0.smithi070.stdout:2162: expect (ObjNum 668 snap 228 seq_num 668) 2024-09-14T03:55:43.142 INFO:tasks.rados.rados.0.smithi070.stdout:2163: read oid 35 snap -1 2024-09-14T03:55:43.142 INFO:tasks.rados.rados.0.smithi070.stdout:2163: expect deleted 2024-09-14T03:55:43.142 INFO:tasks.rados.rados.0.smithi070.stdout:2164: read oid 6 snap -1 2024-09-14T03:55:43.142 INFO:tasks.rados.rados.0.smithi070.stdout:2164: expect (ObjNum 634 snap 211 seq_num 634) 2024-09-14T03:55:43.143 INFO:tasks.rados.rados.0.smithi070.stdout:2165: snap_remove snap 207 2024-09-14T03:55:43.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:43 smithi070 ceph-mon[96704]: osdmap e513: 8 total, 8 up, 8 in 2024-09-14T03:55:43.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:43 smithi070 ceph-mon[98640]: osdmap e513: 8 total, 8 up, 8 in 2024-09-14T03:55:43.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:43 smithi094 ceph-mon[86597]: osdmap e513: 8 total, 8 up, 8 in 2024-09-14T03:55:43.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:55:43 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:55:43] "GET /metrics HTTP/1.1" 200 36309 "" "Prometheus/2.51.0" 2024-09-14T03:55:44.052 INFO:tasks.rados.rados.0.smithi070.stdout:2165: done (3 left) 2024-09-14T03:55:44.052 INFO:tasks.rados.rados.0.smithi070.stdout:2166: snap_create 2024-09-14T03:55:44.052 INFO:tasks.rados.rados.0.smithi070.stdout:2164: expect (ObjNum 634 snap 211 seq_num 634) 2024-09-14T03:55:44.120 INFO:tasks.rados.rados.0.smithi070.stdout:2162: expect (ObjNum 668 snap 228 seq_num 668) 2024-09-14T03:55:44.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:44 smithi094 ceph-mon[86597]: osdmap e514: 8 total, 8 up, 8 in 2024-09-14T03:55:44.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:44 smithi094 ceph-mon[86597]: osdmap e515: 8 total, 8 up, 8 in 2024-09-14T03:55:44.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:44 smithi094 ceph-mon[86597]: pgmap v368: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 971 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 4.0 MiB/s wr, 18 op/s 2024-09-14T03:55:44.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:44 smithi070 ceph-mon[96704]: osdmap e514: 8 total, 8 up, 8 in 2024-09-14T03:55:44.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:44 smithi070 ceph-mon[96704]: osdmap e515: 8 total, 8 up, 8 in 2024-09-14T03:55:44.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:44 smithi070 ceph-mon[96704]: pgmap v368: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 971 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 4.0 MiB/s wr, 18 op/s 2024-09-14T03:55:44.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:44 smithi070 ceph-mon[98640]: osdmap e514: 8 total, 8 up, 8 in 2024-09-14T03:55:44.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:44 smithi070 ceph-mon[98640]: osdmap e515: 8 total, 8 up, 8 in 2024-09-14T03:55:44.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:44 smithi070 ceph-mon[98640]: pgmap v368: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 971 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 4.0 MiB/s wr, 18 op/s 2024-09-14T03:55:45.056 INFO:tasks.rados.rados.0.smithi070.stdout:2162: done (3 left) 2024-09-14T03:55:45.056 INFO:tasks.rados.rados.0.smithi070.stdout:2163: done (2 left) 2024-09-14T03:55:45.056 INFO:tasks.rados.rados.0.smithi070.stdout:2164: done (1 left) 2024-09-14T03:55:45.056 INFO:tasks.rados.rados.0.smithi070.stdout:2166: done (0 left) 2024-09-14T03:55:45.056 INFO:tasks.rados.rados.0.smithi070.stdout:2167: snap_remove snap 228 2024-09-14T03:55:46.061 INFO:tasks.rados.rados.0.smithi070.stdout:2167: done (0 left) 2024-09-14T03:55:46.061 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 6 current snap is 233 2024-09-14T03:55:46.061 INFO:tasks.rados.rados.0.smithi070.stdout:2168: seq_num 688 ranges {1679360=729088,2408448=499712,2908160=704512} 2024-09-14T03:55:46.073 INFO:tasks.rados.rados.0.smithi070.stdout:2168: writing smithi07085154-6 from 1679360 to 2408448 tid 1 2024-09-14T03:55:46.076 INFO:tasks.rados.rados.0.smithi070.stdout:2168: writing smithi07085154-6 from 2408448 to 2908160 tid 2 2024-09-14T03:55:46.081 INFO:tasks.rados.rados.0.smithi070.stdout:2168: writing smithi07085154-6 from 2908160 to 3612672 tid 3 2024-09-14T03:55:46.081 INFO:tasks.rados.rados.0.smithi070.stdout:2169: copy_from oid 20 from oid 45 current snap is 233 2024-09-14T03:55:46.081 INFO:tasks.rados.rados.0.smithi070.stdout:2168: finishing write tid 1 to smithi07085154-6 2024-09-14T03:55:46.081 INFO:tasks.rados.rados.0.smithi070.stdout:2168: finishing write tid 2 to smithi07085154-6 2024-09-14T03:55:46.081 INFO:tasks.rados.rados.0.smithi070.stdout:2170: rollback oid 39 current snap is 233 2024-09-14T03:55:46.081 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 39 to 162 2024-09-14T03:55:46.081 INFO:tasks.rados.rados.0.smithi070.stdout:2171: read oid 30 snap 201 2024-09-14T03:55:46.082 INFO:tasks.rados.rados.0.smithi070.stdout:2171: expect deleted 2024-09-14T03:55:46.082 INFO:tasks.rados.rados.0.smithi070.stdout:2172: read oid 26 snap 215 2024-09-14T03:55:46.082 INFO:tasks.rados.rados.0.smithi070.stdout:2172: expect (ObjNum 629 snap 211 seq_num 629) 2024-09-14T03:55:46.082 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 32 current snap is 233 2024-09-14T03:55:46.082 INFO:tasks.rados.rados.0.smithi070.stdout:2173: seq_num 689 ranges {9273344=745472,10018816=786432,10805248=417792} 2024-09-14T03:55:46.136 INFO:tasks.rados.rados.0.smithi070.stdout:2173: writing smithi07085154-32 from 9273344 to 10018816 tid 1 2024-09-14T03:55:46.141 INFO:tasks.rados.rados.0.smithi070.stdout:2173: writing smithi07085154-32 from 10018816 to 10805248 tid 2 2024-09-14T03:55:46.144 INFO:tasks.rados.rados.0.smithi070.stdout:2173: writing smithi07085154-32 from 10805248 to 11223040 tid 3 2024-09-14T03:55:46.144 INFO:tasks.rados.rados.0.smithi070.stdout:2174: read oid 3 snap -1 2024-09-14T03:55:46.144 INFO:tasks.rados.rados.0.smithi070.stdout:2174: expect (ObjNum 683 snap 230 seq_num 683) 2024-09-14T03:55:46.144 INFO:tasks.rados.rados.0.smithi070.stdout:2175: read oid 31 snap -1 2024-09-14T03:55:46.144 INFO:tasks.rados.rados.0.smithi070.stdout:2175: expect (ObjNum 670 snap 228 seq_num 670) 2024-09-14T03:55:46.144 INFO:tasks.rados.rados.0.smithi070.stdout:2176: snap_remove snap 168 2024-09-14T03:55:46.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:46 smithi094 ceph-mon[86597]: osdmap e516: 8 total, 8 up, 8 in 2024-09-14T03:55:46.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:46 smithi070 ceph-mon[98640]: osdmap e516: 8 total, 8 up, 8 in 2024-09-14T03:55:46.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:46 smithi070 ceph-mon[96704]: osdmap e516: 8 total, 8 up, 8 in 2024-09-14T03:55:47.071 INFO:tasks.rados.rados.0.smithi070.stdout:2176: done (8 left) 2024-09-14T03:55:47.071 INFO:tasks.rados.rados.0.smithi070.stdout:2177: delete oid 13 current snap is 233 2024-09-14T03:55:47.071 INFO:tasks.rados.rados.0.smithi070.stdout:2172: expect (ObjNum 629 snap 211 seq_num 629) 2024-09-14T03:55:47.110 INFO:tasks.rados.rados.0.smithi070.stdout:2170: finishing rollback tid 0 to smithi07085154-39 2024-09-14T03:55:47.110 INFO:tasks.rados.rados.0.smithi070.stdout:2170: finishing rollback tid 1 to smithi07085154-39 2024-09-14T03:55:47.110 INFO:tasks.rados.rados.0.smithi070.stdout:2168: finishing write tid 3 to smithi07085154-6 2024-09-14T03:55:47.110 INFO:tasks.rados.rados.0.smithi070.stdout:2168: finishing write tid 4 to smithi07085154-6 2024-09-14T03:55:47.110 INFO:tasks.rados.rados.0.smithi070.stdout:2170: finishing rollback tid 2 to smithi07085154-39 2024-09-14T03:55:47.110 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 39 v 1705 (ObjNum 460 snap 155 seq_num 460) dirty exists 2024-09-14T03:55:47.111 INFO:tasks.rados.rados.0.smithi070.stdout:2168: finishing write tid 5 to smithi07085154-6 2024-09-14T03:55:47.111 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 2203 (ObjNum 688 snap 233 seq_num 688) dirty exists 2024-09-14T03:55:47.111 INFO:tasks.rados.rados.0.smithi070.stdout:2168: left oid 6 (ObjNum 688 snap 233 seq_num 688) 2024-09-14T03:55:47.111 INFO:tasks.rados.rados.0.smithi070.stdout:2169: finishing copy_from racing read to smithi07085154-20 2024-09-14T03:55:47.111 INFO:tasks.rados.rados.0.smithi070.stdout:2169: finishing copy_from to smithi07085154-20 2024-09-14T03:55:47.111 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 20 v 1944 (ObjNum 590 snap 195 seq_num 590) dirty exists 2024-09-14T03:55:47.111 INFO:tasks.rados.rados.0.smithi070.stdout:2173: finishing write tid 1 to smithi07085154-32 2024-09-14T03:55:47.111 INFO:tasks.rados.rados.0.smithi070.stdout:2173: finishing write tid 2 to smithi07085154-32 2024-09-14T03:55:47.111 INFO:tasks.rados.rados.0.smithi070.stdout:2173: finishing write tid 3 to smithi07085154-32 2024-09-14T03:55:47.111 INFO:tasks.rados.rados.0.smithi070.stdout:2173: finishing write tid 4 to smithi07085154-32 2024-09-14T03:55:47.111 INFO:tasks.rados.rados.0.smithi070.stdout:2168: done (8 left) 2024-09-14T03:55:47.111 INFO:tasks.rados.rados.0.smithi070.stdout:2169: done (7 left) 2024-09-14T03:55:47.111 INFO:tasks.rados.rados.0.smithi070.stdout:2170: done (6 left) 2024-09-14T03:55:47.111 INFO:tasks.rados.rados.0.smithi070.stdout:2171: done (5 left) 2024-09-14T03:55:47.112 INFO:tasks.rados.rados.0.smithi070.stdout:2172: done (4 left) 2024-09-14T03:55:47.112 INFO:tasks.rados.rados.0.smithi070.stdout:2177: done (3 left) 2024-09-14T03:55:47.112 INFO:tasks.rados.rados.0.smithi070.stdout:2178: copy_from oid 47 from oid 7 current snap is 233 2024-09-14T03:55:47.112 INFO:tasks.rados.rados.0.smithi070.stdout:2173: finishing write tid 5 to smithi07085154-32 2024-09-14T03:55:47.112 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 2038 (ObjNum 689 snap 233 seq_num 689) dirty exists 2024-09-14T03:55:47.112 INFO:tasks.rados.rados.0.smithi070.stdout:2173: left oid 32 (ObjNum 689 snap 233 seq_num 689) 2024-09-14T03:55:47.112 INFO:tasks.rados.rados.0.smithi070.stdout:2175: expect (ObjNum 670 snap 228 seq_num 670) 2024-09-14T03:55:47.230 INFO:tasks.rados.rados.0.smithi070.stdout:2174: expect (ObjNum 683 snap 230 seq_num 683) 2024-09-14T03:55:47.386 INFO:tasks.rados.rados.0.smithi070.stdout:2178: finishing copy_from racing read to smithi07085154-47 2024-09-14T03:55:47.386 INFO:tasks.rados.rados.0.smithi070.stdout:2178: finishing copy_from to smithi07085154-47 2024-09-14T03:55:47.386 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 47 v 1689 (ObjNum 514 snap 171 seq_num 514) dirty exists 2024-09-14T03:55:47.386 INFO:tasks.rados.rados.0.smithi070.stdout:2173: done (3 left) 2024-09-14T03:55:47.386 INFO:tasks.rados.rados.0.smithi070.stdout:2174: done (2 left) 2024-09-14T03:55:47.386 INFO:tasks.rados.rados.0.smithi070.stdout:2175: done (1 left) 2024-09-14T03:55:47.386 INFO:tasks.rados.rados.0.smithi070.stdout:2178: done (0 left) 2024-09-14T03:55:47.386 INFO:tasks.rados.rados.0.smithi070.stdout:2179: rollback oid 7 current snap is 233 2024-09-14T03:55:47.387 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 7 to 197 2024-09-14T03:55:47.387 INFO:tasks.rados.rados.0.smithi070.stdout:2180: copy_from oid 23 from oid 13 current snap is 233 2024-09-14T03:55:47.387 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 42 current snap is 233 2024-09-14T03:55:47.387 INFO:tasks.rados.rados.0.smithi070.stdout:2181: seq_num 691 ranges {7348224=778240,8126464=548864,8675328=614400,9289728=40960} 2024-09-14T03:55:47.428 INFO:tasks.rados.rados.0.smithi070.stdout:2181: writing smithi07085154-42 from 7348224 to 8126464 tid 1 2024-09-14T03:55:47.431 INFO:tasks.rados.rados.0.smithi070.stdout:2181: writing smithi07085154-42 from 8126464 to 8675328 tid 2 2024-09-14T03:55:47.435 INFO:tasks.rados.rados.0.smithi070.stdout:2181: writing smithi07085154-42 from 8675328 to 9289728 tid 3 2024-09-14T03:55:47.436 INFO:tasks.rados.rados.0.smithi070.stdout:2181: writing smithi07085154-42 from 9289728 to 9330688 tid 4 2024-09-14T03:55:47.436 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 10 current snap is 233 2024-09-14T03:55:47.436 INFO:tasks.rados.rados.0.smithi070.stdout:2180: finishing copy_from to smithi07085154-23 2024-09-14T03:55:47.436 INFO:tasks.rados.rados.0.smithi070.stdout:2180: got expected ENOENT (src dne) 2024-09-14T03:55:47.436 INFO:tasks.rados.rados.0.smithi070.stdout:2180: finishing copy_from racing read to smithi07085154-23 2024-09-14T03:55:47.436 INFO:tasks.rados.rados.0.smithi070.stdout:2179: finishing rollback tid 0 to smithi07085154-7 2024-09-14T03:55:47.436 INFO:tasks.rados.rados.0.smithi070.stdout:2179: finishing rollback tid 1 to smithi07085154-7 2024-09-14T03:55:47.436 INFO:tasks.rados.rados.0.smithi070.stdout:2179: finishing rollback tid 2 to smithi07085154-7 2024-09-14T03:55:47.436 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 1947 (ObjNum 454 snap 154 seq_num 454) dirty exists 2024-09-14T03:55:47.436 INFO:tasks.rados.rados.0.smithi070.stdout:2181: finishing write tid 1 to smithi07085154-42 2024-09-14T03:55:47.436 INFO:tasks.rados.rados.0.smithi070.stdout:2182: seq_num 692 ranges {6643712=794624,7438336=483328,7921664=720896} 2024-09-14T03:55:47.475 INFO:tasks.rados.rados.0.smithi070.stdout:2182: writing smithi07085154-10 from 6643712 to 7438336 tid 1 2024-09-14T03:55:47.477 INFO:tasks.rados.rados.0.smithi070.stdout:2182: writing smithi07085154-10 from 7438336 to 7921664 tid 2 2024-09-14T03:55:47.483 INFO:tasks.rados.rados.0.smithi070.stdout:2182: writing smithi07085154-10 from 7921664 to 8642560 tid 3 2024-09-14T03:55:47.483 INFO:tasks.rados.rados.0.smithi070.stdout:2179: done (3 left) 2024-09-14T03:55:47.483 INFO:tasks.rados.rados.0.smithi070.stdout:2180: done (2 left) 2024-09-14T03:55:47.483 INFO:tasks.rados.rados.0.smithi070.stdout:2183: read oid 25 snap -1 2024-09-14T03:55:47.483 INFO:tasks.rados.rados.0.smithi070.stdout:2183: expect (ObjNum 640 snap 214 seq_num 640) 2024-09-14T03:55:47.483 INFO:tasks.rados.rados.0.smithi070.stdout:2184: copy_from oid 35 from oid 38 current snap is 233 2024-09-14T03:55:47.483 INFO:tasks.rados.rados.0.smithi070.stdout:2181: finishing write tid 2 to smithi07085154-42 2024-09-14T03:55:47.483 INFO:tasks.rados.rados.0.smithi070.stdout:2181: finishing write tid 3 to smithi07085154-42 2024-09-14T03:55:47.484 INFO:tasks.rados.rados.0.smithi070.stdout:2181: finishing write tid 4 to smithi07085154-42 2024-09-14T03:55:47.484 INFO:tasks.rados.rados.0.smithi070.stdout:2181: finishing write tid 5 to smithi07085154-42 2024-09-14T03:55:47.484 INFO:tasks.rados.rados.0.smithi070.stdout:2181: finishing write tid 6 to smithi07085154-42 2024-09-14T03:55:47.484 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 2365 (ObjNum 691 snap 233 seq_num 691) dirty exists 2024-09-14T03:55:47.484 INFO:tasks.rados.rados.0.smithi070.stdout:2181: left oid 42 (ObjNum 691 snap 233 seq_num 691) 2024-09-14T03:55:47.484 INFO:tasks.rados.rados.0.smithi070.stdout:2182: finishing write tid 1 to smithi07085154-10 2024-09-14T03:55:47.484 INFO:tasks.rados.rados.0.smithi070.stdout:2182: finishing write tid 2 to smithi07085154-10 2024-09-14T03:55:47.484 INFO:tasks.rados.rados.0.smithi070.stdout:2181: done (3 left) 2024-09-14T03:55:47.484 INFO:tasks.rados.rados.0.smithi070.stdout:2185: snap_create 2024-09-14T03:55:47.490 INFO:tasks.rados.rados.0.smithi070.stdout:2182: finishing write tid 3 to smithi07085154-10 2024-09-14T03:55:47.490 INFO:tasks.rados.rados.0.smithi070.stdout:2182: finishing write tid 4 to smithi07085154-10 2024-09-14T03:55:47.491 INFO:tasks.rados.rados.0.smithi070.stdout:2182: finishing write tid 5 to smithi07085154-10 2024-09-14T03:55:47.491 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 2370 (ObjNum 692 snap 233 seq_num 692) dirty exists 2024-09-14T03:55:47.491 INFO:tasks.rados.rados.0.smithi070.stdout:2182: left oid 10 (ObjNum 692 snap 233 seq_num 692) 2024-09-14T03:55:47.501 INFO:tasks.rados.rados.0.smithi070.stdout:2184: finishing copy_from racing read to smithi07085154-35 2024-09-14T03:55:47.504 INFO:tasks.rados.rados.0.smithi070.stdout:2184: finishing copy_from to smithi07085154-35 2024-09-14T03:55:47.504 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 35 v 1685 (ObjNum 645 snap 216 seq_num 645) dirty exists 2024-09-14T03:55:47.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:47 smithi094 ceph-mon[86597]: osdmap e517: 8 total, 8 up, 8 in 2024-09-14T03:55:47.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:47 smithi094 ceph-mon[86597]: pgmap v371: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 971 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 1.3 MiB/s wr, 7 op/s 2024-09-14T03:55:47.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:47 smithi070 ceph-mon[98640]: osdmap e517: 8 total, 8 up, 8 in 2024-09-14T03:55:47.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:47 smithi070 ceph-mon[98640]: pgmap v371: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 971 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 1.3 MiB/s wr, 7 op/s 2024-09-14T03:55:47.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:47 smithi070 ceph-mon[96704]: osdmap e517: 8 total, 8 up, 8 in 2024-09-14T03:55:47.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:47 smithi070 ceph-mon[96704]: pgmap v371: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 971 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 1.3 MiB/s wr, 7 op/s 2024-09-14T03:55:47.564 INFO:tasks.rados.rados.0.smithi070.stdout:2183: expect (ObjNum 640 snap 214 seq_num 640) 2024-09-14T03:55:48.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:48 smithi094 ceph-mon[86597]: osdmap e518: 8 total, 8 up, 8 in 2024-09-14T03:55:48.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:48 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:48.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:48 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:48.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:48 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:48.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:48 smithi070 ceph-mon[96704]: osdmap e518: 8 total, 8 up, 8 in 2024-09-14T03:55:48.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:48 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:48.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:48 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:48.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:48 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:48.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:48 smithi070 ceph-mon[98640]: osdmap e518: 8 total, 8 up, 8 in 2024-09-14T03:55:48.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:48 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:48.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:48 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:55:48.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:48 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:49.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:49 smithi094 ceph-mon[86597]: pgmap v373: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 990 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 4.7 MiB/s wr, 4 op/s 2024-09-14T03:55:49.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:49 smithi094 ceph-mon[86597]: osdmap e519: 8 total, 8 up, 8 in 2024-09-14T03:55:49.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:49 smithi094 ceph-mon[86597]: osdmap e520: 8 total, 8 up, 8 in 2024-09-14T03:55:49.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:49 smithi070 ceph-mon[96704]: pgmap v373: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 990 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 4.7 MiB/s wr, 4 op/s 2024-09-14T03:55:49.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:49 smithi070 ceph-mon[96704]: osdmap e519: 8 total, 8 up, 8 in 2024-09-14T03:55:49.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:49 smithi070 ceph-mon[96704]: osdmap e520: 8 total, 8 up, 8 in 2024-09-14T03:55:49.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:49 smithi070 ceph-mon[98640]: pgmap v373: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 990 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 4.7 MiB/s wr, 4 op/s 2024-09-14T03:55:49.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:49 smithi070 ceph-mon[98640]: osdmap e519: 8 total, 8 up, 8 in 2024-09-14T03:55:49.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:49 smithi070 ceph-mon[98640]: osdmap e520: 8 total, 8 up, 8 in 2024-09-14T03:55:49.755 INFO:tasks.rados.rados.0.smithi070.stdout:2182: done (3 left) 2024-09-14T03:55:49.755 INFO:tasks.rados.rados.0.smithi070.stdout:2183: done (2 left) 2024-09-14T03:55:49.755 INFO:tasks.rados.rados.0.smithi070.stdout:2184: done (1 left) 2024-09-14T03:55:49.755 INFO:tasks.rados.rados.0.smithi070.stdout:2185: done (0 left) 2024-09-14T03:55:49.755 INFO:tasks.rados.rados.0.smithi070.stdout:2186: read oid 15 snap -1 2024-09-14T03:55:49.755 INFO:tasks.rados.rados.0.smithi070.stdout:2186: expect deleted 2024-09-14T03:55:49.756 INFO:tasks.rados.rados.0.smithi070.stdout:2187: rmattr oid 36 current snap is 234 2024-09-14T03:55:49.756 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 45 current snap is 234 2024-09-14T03:55:49.756 INFO:tasks.rados.rados.0.smithi070.stdout:2188: seq_num 693 ranges {16457728=409600,16867328=770048,17637376=581632,18219008=253952} 2024-09-14T03:55:49.841 INFO:tasks.rados.rados.0.smithi070.stdout:2188: writing smithi07085154-45 from 16457728 to 16867328 tid 1 2024-09-14T03:55:49.982 INFO:tasks.rados.rados.0.smithi070.stdout:2188: writing smithi07085154-45 from 16867328 to 17637376 tid 2 2024-09-14T03:55:49.982 INFO:tasks.rados.rados.0.smithi070.stdout:2188: writing smithi07085154-45 from 17637376 to 18219008 tid 3 2024-09-14T03:55:49.982 INFO:tasks.rados.rados.0.smithi070.stdout:2188: writing smithi07085154-45 from 18219008 to 18472960 tid 4 2024-09-14T03:55:49.982 INFO:tasks.rados.rados.0.smithi070.stdout:2189: delete oid 2 current snap is 234 2024-09-14T03:55:49.982 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 36 v 0 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-14T03:55:49.983 INFO:tasks.rados.rados.0.smithi070.stdout:2188: finishing write tid 1 to smithi07085154-45 2024-09-14T03:55:49.983 INFO:tasks.rados.rados.0.smithi070.stdout:2188: finishing write tid 2 to smithi07085154-45 2024-09-14T03:55:49.983 INFO:tasks.rados.rados.0.smithi070.stdout:2188: finishing write tid 3 to smithi07085154-45 2024-09-14T03:55:49.983 INFO:tasks.rados.rados.0.smithi070.stdout:2188: finishing write tid 4 to smithi07085154-45 2024-09-14T03:55:49.983 INFO:tasks.rados.rados.0.smithi070.stdout:2188: finishing write tid 5 to smithi07085154-45 2024-09-14T03:55:49.983 INFO:tasks.rados.rados.0.smithi070.stdout:2186: done (3 left) 2024-09-14T03:55:49.983 INFO:tasks.rados.rados.0.smithi070.stdout:2187: done (2 left) 2024-09-14T03:55:49.983 INFO:tasks.rados.rados.0.smithi070.stdout:2189: done (1 left) 2024-09-14T03:55:49.983 INFO:tasks.rados.rados.0.smithi070.stdout:2190: setattr oid 46 current snap is 234 2024-09-14T03:55:49.983 INFO:tasks.rados.rados.0.smithi070.stdout:2188: finishing write tid 6 to smithi07085154-45 2024-09-14T03:55:49.983 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 45 v 1949 (ObjNum 693 snap 234 seq_num 693) dirty exists 2024-09-14T03:55:49.983 INFO:tasks.rados.rados.0.smithi070.stdout:2188: left oid 45 (ObjNum 693 snap 234 seq_num 693) 2024-09-14T03:55:49.983 INFO:tasks.rados.rados.0.smithi070.stdout:2188: done (1 left) 2024-09-14T03:55:49.984 INFO:tasks.rados.rados.0.smithi070.stdout:2191: read oid 50 snap -1 2024-09-14T03:55:49.984 INFO:tasks.rados.rados.0.smithi070.stdout:2191: expect (ObjNum 657 snap 220 seq_num 657) 2024-09-14T03:55:49.984 INFO:tasks.rados.rados.0.smithi070.stdout:2192: copy_from oid 6 from oid 18 current snap is 234 2024-09-14T03:55:49.984 INFO:tasks.rados.rados.0.smithi070.stdout:2193: read oid 16 snap -1 2024-09-14T03:55:49.984 INFO:tasks.rados.rados.0.smithi070.stdout:2193: expect (ObjNum 114 snap 24 seq_num 114) 2024-09-14T03:55:49.984 INFO:tasks.rados.rados.0.smithi070.stdout:2194: snap_create 2024-09-14T03:55:49.984 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 46 v 2373 (ObjNum 649 snap 218 seq_num 649) dirty exists 2024-09-14T03:55:49.984 INFO:tasks.rados.rados.0.smithi070.stdout:2191: expect (ObjNum 657 snap 220 seq_num 657) 2024-09-14T03:55:49.984 INFO:tasks.rados.rados.0.smithi070.stdout:2193: expect (ObjNum 114 snap 24 seq_num 114) 2024-09-14T03:55:50.152 INFO:tasks.rados.rados.0.smithi070.stdout:2192: finishing copy_from racing read to smithi07085154-6 2024-09-14T03:55:50.152 INFO:tasks.rados.rados.0.smithi070.stdout:2192: finishing copy_from to smithi07085154-6 2024-09-14T03:55:50.152 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 2033 (ObjNum 671 snap 228 seq_num 671) dirty exists 2024-09-14T03:55:50.153 INFO:tasks.rados.rados.0.smithi070.stdout:2190: done (4 left) 2024-09-14T03:55:50.153 INFO:tasks.rados.rados.0.smithi070.stdout:2191: done (3 left) 2024-09-14T03:55:50.153 INFO:tasks.rados.rados.0.smithi070.stdout:2192: done (2 left) 2024-09-14T03:55:50.153 INFO:tasks.rados.rados.0.smithi070.stdout:2193: done (1 left) 2024-09-14T03:55:50.153 INFO:tasks.rados.rados.0.smithi070.stdout:2194: done (0 left) 2024-09-14T03:55:50.153 INFO:tasks.rados.rados.0.smithi070.stdout:2195: snap_create 2024-09-14T03:55:50.391 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:50 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-14T03:55:50.391 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:55:50 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:55:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:55:50.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:50 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-14T03:55:50.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:50 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-14T03:55:51.084 INFO:tasks.rados.rados.0.smithi070.stdout:2195: done (0 left) 2024-09-14T03:55:51.084 INFO:tasks.rados.rados.0.smithi070.stdout:2196: delete oid 45 current snap is 236 2024-09-14T03:55:51.089 INFO:tasks.rados.rados.0.smithi070.stdout:2196: done (0 left) 2024-09-14T03:55:51.089 INFO:tasks.rados.rados.0.smithi070.stdout:2197: read oid 40 snap -1 2024-09-14T03:55:51.089 INFO:tasks.rados.rados.0.smithi070.stdout:2197: expect (ObjNum 678 snap 229 seq_num 678) 2024-09-14T03:55:51.089 INFO:tasks.rados.rados.0.smithi070.stdout:2198: snap_remove snap 177 2024-09-14T03:55:51.356 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:51 smithi094 ceph-mon[86597]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-14T03:55:51.356 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:51 smithi094 ceph-mon[86597]: Upgrade: osd.6 is safe to restart 2024-09-14T03:55:51.356 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:51 smithi094 ceph-mon[86597]: osdmap e521: 8 total, 8 up, 8 in 2024-09-14T03:55:51.356 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:51 smithi094 ceph-mon[86597]: pgmap v377: 121 pgs: 121 active+clean; 995 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 5.2 MiB/s wr, 6 op/s 2024-09-14T03:55:51.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:51 smithi070 ceph-mon[96704]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-14T03:55:51.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:51 smithi070 ceph-mon[96704]: Upgrade: osd.6 is safe to restart 2024-09-14T03:55:51.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:51 smithi070 ceph-mon[96704]: osdmap e521: 8 total, 8 up, 8 in 2024-09-14T03:55:51.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:51 smithi070 ceph-mon[96704]: pgmap v377: 121 pgs: 121 active+clean; 995 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 5.2 MiB/s wr, 6 op/s 2024-09-14T03:55:51.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:51 smithi070 ceph-mon[98640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-14T03:55:51.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:51 smithi070 ceph-mon[98640]: Upgrade: osd.6 is safe to restart 2024-09-14T03:55:51.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:51 smithi070 ceph-mon[98640]: osdmap e521: 8 total, 8 up, 8 in 2024-09-14T03:55:51.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:51 smithi070 ceph-mon[98640]: pgmap v377: 121 pgs: 121 active+clean; 995 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 5.2 MiB/s wr, 6 op/s 2024-09-14T03:55:52.089 INFO:tasks.rados.rados.0.smithi070.stdout:2198: done (1 left) 2024-09-14T03:55:52.089 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 36 current snap is 236 2024-09-14T03:55:52.089 INFO:tasks.rados.rados.0.smithi070.stdout:2199: seq_num 696 ranges {0=458752,458752=417792,876544=753664,1630208=434176} 2024-09-14T03:55:52.091 INFO:tasks.rados.rados.0.smithi070.stdout:2199: writing smithi07085154-36 from 0 to 458752 tid 1 2024-09-14T03:55:52.095 INFO:tasks.rados.rados.0.smithi070.stdout:2199: writing smithi07085154-36 from 458752 to 876544 tid 2 2024-09-14T03:55:52.100 INFO:tasks.rados.rados.0.smithi070.stdout:2199: writing smithi07085154-36 from 876544 to 1630208 tid 3 2024-09-14T03:55:52.103 INFO:tasks.rados.rados.0.smithi070.stdout:2199: writing smithi07085154-36 from 1630208 to 2064384 tid 4 2024-09-14T03:55:52.103 INFO:tasks.rados.rados.0.smithi070.stdout:2200: delete oid 1 current snap is 236 2024-09-14T03:55:52.103 INFO:tasks.rados.rados.0.smithi070.stdout:2197: expect (ObjNum 678 snap 229 seq_num 678) 2024-09-14T03:55:52.158 INFO:tasks.rados.rados.0.smithi070.stdout:2199: finishing write tid 1 to smithi07085154-36 2024-09-14T03:55:52.158 INFO:tasks.rados.rados.0.smithi070.stdout:2199: finishing write tid 2 to smithi07085154-36 2024-09-14T03:55:52.158 INFO:tasks.rados.rados.0.smithi070.stdout:2199: finishing write tid 3 to smithi07085154-36 2024-09-14T03:55:52.158 INFO:tasks.rados.rados.0.smithi070.stdout:2199: finishing write tid 4 to smithi07085154-36 2024-09-14T03:55:52.158 INFO:tasks.rados.rados.0.smithi070.stdout:2199: finishing write tid 5 to smithi07085154-36 2024-09-14T03:55:52.158 INFO:tasks.rados.rados.0.smithi070.stdout:2199: finishing write tid 6 to smithi07085154-36 2024-09-14T03:55:52.158 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 36 v 2039 (ObjNum 696 snap 236 seq_num 696) dirty exists 2024-09-14T03:55:52.158 INFO:tasks.rados.rados.0.smithi070.stdout:2199: left oid 36 (ObjNum 696 snap 236 seq_num 696) 2024-09-14T03:55:52.158 INFO:tasks.rados.rados.0.smithi070.stdout:2197: done (2 left) 2024-09-14T03:55:52.159 INFO:tasks.rados.rados.0.smithi070.stdout:2199: done (1 left) 2024-09-14T03:55:52.159 INFO:tasks.rados.rados.0.smithi070.stdout:2200: done (0 left) 2024-09-14T03:55:52.159 INFO:tasks.rados.rados.0.smithi070.stdout:2201: read oid 34 snap -1 2024-09-14T03:55:52.159 INFO:tasks.rados.rados.0.smithi070.stdout:2201: expect deleted 2024-09-14T03:55:52.159 INFO:tasks.rados.rados.0.smithi070.stdout:2202: delete oid 41 current snap is 236 2024-09-14T03:55:52.160 INFO:tasks.rados.rados.0.smithi070.stdout:2201: done (1 left) 2024-09-14T03:55:52.161 INFO:tasks.rados.rados.0.smithi070.stdout:2202: done (0 left) 2024-09-14T03:55:52.161 INFO:tasks.rados.rados.0.smithi070.stdout:2203: delete oid 23 current snap is 236 2024-09-14T03:55:52.163 INFO:tasks.rados.rados.0.smithi070.stdout:2203: done (0 left) 2024-09-14T03:55:52.163 INFO:tasks.rados.rados.0.smithi070.stdout:2204: read oid 37 snap 201 2024-09-14T03:55:52.163 INFO:tasks.rados.rados.0.smithi070.stdout:2204: expect (ObjNum 403 snap 140 seq_num 403) 2024-09-14T03:55:52.163 INFO:tasks.rados.rados.0.smithi070.stdout:2205: rmattr oid 18 current snap is 236 2024-09-14T03:55:52.163 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 20 current snap is 236 2024-09-14T03:55:52.163 INFO:tasks.rados.rados.0.smithi070.stdout:2206: seq_num 700 ranges {16457728=532480,16990208=409600,17399808=737280,18137088=458752} 2024-09-14T03:55:52.253 INFO:tasks.rados.rados.0.smithi070.stdout:2206: writing smithi07085154-20 from 16457728 to 16990208 tid 1 2024-09-14T03:55:52.255 INFO:tasks.rados.rados.0.smithi070.stdout:2206: writing smithi07085154-20 from 16990208 to 17399808 tid 2 2024-09-14T03:55:52.260 INFO:tasks.rados.rados.0.smithi070.stdout:2206: writing smithi07085154-20 from 17399808 to 18137088 tid 3 2024-09-14T03:55:52.263 INFO:tasks.rados.rados.0.smithi070.stdout:2206: writing smithi07085154-20 from 18137088 to 18595840 tid 4 2024-09-14T03:55:52.263 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 36 current snap is 236 2024-09-14T03:55:52.263 INFO:tasks.rados.rados.0.smithi070.stdout:2207: seq_num 701 ranges {2064384=548864,2613248=696320,3309568=540672,3850240=368640} 2024-09-14T03:55:52.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:52 smithi094 ceph-mon[86597]: osdmap e522: 8 total, 8 up, 8 in 2024-09-14T03:55:52.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:52 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:52.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:52 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-14T03:55:52.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:52 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:55:52.277 INFO:tasks.rados.rados.0.smithi070.stdout:2207: writing smithi07085154-36 from 2064384 to 2613248 tid 1 2024-09-14T03:55:52.280 INFO:tasks.rados.rados.0.smithi070.stdout:2207: writing smithi07085154-36 from 2613248 to 3309568 tid 2 2024-09-14T03:55:52.284 INFO:tasks.rados.rados.0.smithi070.stdout:2207: writing smithi07085154-36 from 3309568 to 3850240 tid 3 2024-09-14T03:55:52.286 INFO:tasks.rados.rados.0.smithi070.stdout:2207: writing smithi07085154-36 from 3850240 to 4218880 tid 4 2024-09-14T03:55:52.286 INFO:tasks.rados.rados.0.smithi070.stdout:2208: read oid 14 snap -1 2024-09-14T03:55:52.286 INFO:tasks.rados.rados.0.smithi070.stdout:2208: expect deleted 2024-09-14T03:55:52.286 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 25 current snap is 236 2024-09-14T03:55:52.286 INFO:tasks.rados.rados.0.smithi070.stdout:2209: seq_num 702 ranges {17473536=565248,18038784=581632,18620416=745472,19365888=278528} 2024-09-14T03:55:52.380 INFO:tasks.rados.rados.0.smithi070.stdout:2209: writing smithi07085154-25 from 17473536 to 18038784 tid 1 2024-09-14T03:55:52.383 INFO:tasks.rados.rados.0.smithi070.stdout:2209: writing smithi07085154-25 from 18038784 to 18620416 tid 2 2024-09-14T03:55:52.387 INFO:tasks.rados.rados.0.smithi070.stdout:2209: writing smithi07085154-25 from 18620416 to 19365888 tid 3 2024-09-14T03:55:52.389 INFO:tasks.rados.rados.0.smithi070.stdout:2209: writing smithi07085154-25 from 19365888 to 19644416 tid 4 2024-09-14T03:55:52.390 INFO:tasks.rados.rados.0.smithi070.stdout:2210: copy_from oid 47 from oid 39 current snap is 236 2024-09-14T03:55:52.390 INFO:tasks.rados.rados.0.smithi070.stdout:2204: expect (ObjNum 403 snap 140 seq_num 403) 2024-09-14T03:55:52.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:52 smithi070 ceph-mon[96704]: osdmap e522: 8 total, 8 up, 8 in 2024-09-14T03:55:52.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:52 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:52.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:52 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-14T03:55:52.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:52 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:55:52.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:52 smithi070 ceph-mon[98640]: osdmap e522: 8 total, 8 up, 8 in 2024-09-14T03:55:52.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:52 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:55:52.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:52 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-14T03:55:52.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:52 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:55:52.956 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 2040 (ObjNum 671 snap 228 seq_num 671) dirty exists 2024-09-14T03:55:52.956 INFO:tasks.rados.rados.0.smithi070.stdout:2206: finishing write tid 1 to smithi07085154-20 2024-09-14T03:55:52.956 INFO:tasks.rados.rados.0.smithi070.stdout:2206: finishing write tid 2 to smithi07085154-20 2024-09-14T03:55:52.956 INFO:tasks.rados.rados.0.smithi070.stdout:2206: finishing write tid 3 to smithi07085154-20 2024-09-14T03:55:52.956 INFO:tasks.rados.rados.0.smithi070.stdout:2206: finishing write tid 4 to smithi07085154-20 2024-09-14T03:55:52.956 INFO:tasks.rados.rados.0.smithi070.stdout:2206: finishing write tid 5 to smithi07085154-20 2024-09-14T03:55:52.956 INFO:tasks.rados.rados.0.smithi070.stdout:2206: finishing write tid 6 to smithi07085154-20 2024-09-14T03:55:52.956 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 20 v 1952 (ObjNum 700 snap 236 seq_num 700) dirty exists 2024-09-14T03:55:52.956 INFO:tasks.rados.rados.0.smithi070.stdout:2206: left oid 20 (ObjNum 700 snap 236 seq_num 700) 2024-09-14T03:55:52.956 INFO:tasks.rados.rados.0.smithi070.stdout:2207: finishing write tid 1 to smithi07085154-36 2024-09-14T03:55:52.957 INFO:tasks.rados.rados.0.smithi070.stdout:2204: done (6 left) 2024-09-14T03:55:52.957 INFO:tasks.rados.rados.0.smithi070.stdout:2205: done (5 left) 2024-09-14T03:55:52.957 INFO:tasks.rados.rados.0.smithi070.stdout:2206: done (4 left) 2024-09-14T03:55:52.957 INFO:tasks.rados.rados.0.smithi070.stdout:2211: read oid 19 snap -1 2024-09-14T03:55:52.957 INFO:tasks.rados.rados.0.smithi070.stdout:2211: expect (ObjNum 659 snap 220 seq_num 659) 2024-09-14T03:55:52.957 INFO:tasks.rados.rados.0.smithi070.stdout:2212: read oid 48 snap -1 2024-09-14T03:55:52.957 INFO:tasks.rados.rados.0.smithi070.stdout:2212: expect (ObjNum 682 snap 230 seq_num 682) 2024-09-14T03:55:52.957 INFO:tasks.rados.rados.0.smithi070.stdout:2213: rollback oid 18 current snap is 236 2024-09-14T03:55:52.957 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 18 to 234 2024-09-14T03:55:52.957 INFO:tasks.rados.rados.0.smithi070.stdout:2207: finishing write tid 2 to smithi07085154-36 2024-09-14T03:55:52.957 INFO:tasks.rados.rados.0.smithi070.stdout:2207: finishing write tid 3 to smithi07085154-36 2024-09-14T03:55:52.958 INFO:tasks.rados.rados.0.smithi070.stdout:2207: finishing write tid 4 to smithi07085154-36 2024-09-14T03:55:52.958 INFO:tasks.rados.rados.0.smithi070.stdout:2208: done (6 left) 2024-09-14T03:55:52.958 INFO:tasks.rados.rados.0.smithi070.stdout:2214: rollback oid 30 current snap is 236 2024-09-14T03:55:52.958 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 30 to 195 2024-09-14T03:55:52.958 INFO:tasks.rados.rados.0.smithi070.stdout:2207: finishing write tid 5 to smithi07085154-36 2024-09-14T03:55:52.958 INFO:tasks.rados.rados.0.smithi070.stdout:2215: read oid 5 snap -1 2024-09-14T03:55:52.958 INFO:tasks.rados.rados.0.smithi070.stdout:2215: expect (ObjNum 686 snap 231 seq_num 686) 2024-09-14T03:55:52.958 INFO:tasks.rados.rados.0.smithi070.stdout:2216: snap_remove snap 230 2024-09-14T03:55:53.393 INFO:tasks.rados.rados.0.smithi070.stdout:2216: done (8 left) 2024-09-14T03:55:53.393 INFO:tasks.rados.rados.0.smithi070.stdout:2217: snap_create 2024-09-14T03:55:53.393 INFO:tasks.rados.rados.0.smithi070.stdout:2207: finishing write tid 6 to smithi07085154-36 2024-09-14T03:55:53.393 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 36 v 2044 (ObjNum 701 snap 236 seq_num 701) dirty exists 2024-09-14T03:55:53.393 INFO:tasks.rados.rados.0.smithi070.stdout:2207: left oid 36 (ObjNum 701 snap 236 seq_num 701) 2024-09-14T03:55:53.393 INFO:tasks.rados.rados.0.smithi070.stdout:2209: finishing write tid 1 to smithi07085154-25 2024-09-14T03:55:53.393 INFO:tasks.rados.rados.0.smithi070.stdout:2209: finishing write tid 2 to smithi07085154-25 2024-09-14T03:55:53.393 INFO:tasks.rados.rados.0.smithi070.stdout:2209: finishing write tid 3 to smithi07085154-25 2024-09-14T03:55:53.394 INFO:tasks.rados.rados.0.smithi070.stdout:2209: finishing write tid 4 to smithi07085154-25 2024-09-14T03:55:53.394 INFO:tasks.rados.rados.0.smithi070.stdout:2209: finishing write tid 5 to smithi07085154-25 2024-09-14T03:55:53.394 INFO:tasks.rados.rados.0.smithi070.stdout:2209: finishing write tid 6 to smithi07085154-25 2024-09-14T03:55:53.394 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 2208 (ObjNum 702 snap 236 seq_num 702) dirty exists 2024-09-14T03:55:53.394 INFO:tasks.rados.rados.0.smithi070.stdout:2209: left oid 25 (ObjNum 702 snap 236 seq_num 702) 2024-09-14T03:55:53.394 INFO:tasks.rados.rados.0.smithi070.stdout:2210: finishing copy_from racing read to smithi07085154-47 2024-09-14T03:55:53.394 INFO:tasks.rados.rados.0.smithi070.stdout:2210: finishing copy_from to smithi07085154-47 2024-09-14T03:55:53.394 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 47 v 1705 (ObjNum 460 snap 155 seq_num 460) dirty exists 2024-09-14T03:55:53.394 INFO:tasks.rados.rados.0.smithi070.stdout:2214: finishing rollback tid 1 to smithi07085154-30 2024-09-14T03:55:53.394 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 30 v 1695 (ObjNum 49 snap 0 seq_num 4211436775) dirty dne 2024-09-14T03:55:53.394 INFO:tasks.rados.rados.0.smithi070.stdout:2213: finishing rollback tid 0 to smithi07085154-18 2024-09-14T03:55:53.394 INFO:tasks.rados.rados.0.smithi070.stdout:2213: finishing rollback tid 1 to smithi07085154-18 2024-09-14T03:55:53.394 INFO:tasks.rados.rados.0.smithi070.stdout:2213: finishing rollback tid 2 to smithi07085154-18 2024-09-14T03:55:53.394 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 2043 (ObjNum 671 snap 228 seq_num 671) dirty exists 2024-09-14T03:55:53.395 INFO:tasks.rados.rados.0.smithi070.stdout:2215: expect (ObjNum 686 snap 231 seq_num 686) 2024-09-14T03:55:53.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:53 smithi070 ceph-mon[98640]: Upgrade: Updating osd.6 2024-09-14T03:55:53.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:53 smithi070 ceph-mon[98640]: Deploying daemon osd.6 on smithi094 2024-09-14T03:55:53.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:53 smithi070 ceph-mon[98640]: pgmap v379: 121 pgs: 121 active+clean; 1009 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 3.0 MiB/s wr, 14 op/s 2024-09-14T03:55:53.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:53 smithi070 ceph-mon[98640]: osdmap e523: 8 total, 8 up, 8 in 2024-09-14T03:55:53.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:53 smithi070 ceph-mon[96704]: Upgrade: Updating osd.6 2024-09-14T03:55:53.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:53 smithi070 ceph-mon[96704]: Deploying daemon osd.6 on smithi094 2024-09-14T03:55:53.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:53 smithi070 ceph-mon[96704]: pgmap v379: 121 pgs: 121 active+clean; 1009 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 3.0 MiB/s wr, 14 op/s 2024-09-14T03:55:53.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:53 smithi070 ceph-mon[96704]: osdmap e523: 8 total, 8 up, 8 in 2024-09-14T03:55:53.445 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:53 smithi094 ceph-mon[86597]: Upgrade: Updating osd.6 2024-09-14T03:55:53.445 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:53 smithi094 ceph-mon[86597]: Deploying daemon osd.6 on smithi094 2024-09-14T03:55:53.445 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:53 smithi094 ceph-mon[86597]: pgmap v379: 121 pgs: 121 active+clean; 1009 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 3.0 MiB/s wr, 14 op/s 2024-09-14T03:55:53.446 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:53 smithi094 ceph-mon[86597]: osdmap e523: 8 total, 8 up, 8 in 2024-09-14T03:55:53.448 INFO:tasks.rados.rados.0.smithi070.stdout:2212: expect (ObjNum 682 snap 230 seq_num 682) 2024-09-14T03:55:53.585 INFO:tasks.rados.rados.0.smithi070.stdout:2211: expect (ObjNum 659 snap 220 seq_num 659) 2024-09-14T03:55:53.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:55:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:55:53] "GET /metrics HTTP/1.1" 200 36307 "" "Prometheus/2.51.0" 2024-09-14T03:55:54.230 INFO:tasks.rados.rados.0.smithi070.stdout:2207: done (8 left) 2024-09-14T03:55:54.230 INFO:tasks.rados.rados.0.smithi070.stdout:2209: done (7 left) 2024-09-14T03:55:54.230 INFO:tasks.rados.rados.0.smithi070.stdout:2210: done (6 left) 2024-09-14T03:55:54.230 INFO:tasks.rados.rados.0.smithi070.stdout:2211: done (5 left) 2024-09-14T03:55:54.230 INFO:tasks.rados.rados.0.smithi070.stdout:2212: done (4 left) 2024-09-14T03:55:54.230 INFO:tasks.rados.rados.0.smithi070.stdout:2213: done (3 left) 2024-09-14T03:55:54.230 INFO:tasks.rados.rados.0.smithi070.stdout:2214: done (2 left) 2024-09-14T03:55:54.230 INFO:tasks.rados.rados.0.smithi070.stdout:2215: done (1 left) 2024-09-14T03:55:54.231 INFO:tasks.rados.rados.0.smithi070.stdout:2217: done (0 left) 2024-09-14T03:55:54.231 INFO:tasks.rados.rados.0.smithi070.stdout:2218: snap_create 2024-09-14T03:55:54.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:54 smithi094 ceph-mon[86597]: osdmap e524: 8 total, 8 up, 8 in 2024-09-14T03:55:54.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:54 smithi094 ceph-mon[86597]: osdmap e525: 8 total, 8 up, 8 in 2024-09-14T03:55:54.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:54 smithi094 ceph-mon[86597]: pgmap v383: 121 pgs: 121 active+clean; 1.0 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 8.0 MiB/s wr, 21 op/s 2024-09-14T03:55:54.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:54 smithi070 ceph-mon[96704]: osdmap e524: 8 total, 8 up, 8 in 2024-09-14T03:55:54.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:54 smithi070 ceph-mon[96704]: osdmap e525: 8 total, 8 up, 8 in 2024-09-14T03:55:54.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:54 smithi070 ceph-mon[96704]: pgmap v383: 121 pgs: 121 active+clean; 1.0 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 8.0 MiB/s wr, 21 op/s 2024-09-14T03:55:54.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:54 smithi070 ceph-mon[98640]: osdmap e524: 8 total, 8 up, 8 in 2024-09-14T03:55:54.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:54 smithi070 ceph-mon[98640]: osdmap e525: 8 total, 8 up, 8 in 2024-09-14T03:55:54.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:54 smithi070 ceph-mon[98640]: pgmap v383: 121 pgs: 121 active+clean; 1.0 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 8.0 MiB/s wr, 21 op/s 2024-09-14T03:55:54.815 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:54 smithi094 systemd[1]: Stopping Ceph osd.6 for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:55:55.060 INFO:tasks.rados.rados.0.smithi070.stdout:2218: done (0 left) 2024-09-14T03:55:55.060 INFO:tasks.rados.rados.0.smithi070.stdout:2219: setattr oid 22 current snap is 238 2024-09-14T03:55:55.061 INFO:tasks.rados.rados.0.smithi070.stdout:2220: snap_remove snap 144 2024-09-14T03:55:55.277 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:54 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[48695]: 2024-09-14T03:55:54.813+0000 7f5f94b8d640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:55:55.277 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:54 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[48695]: 2024-09-14T03:55:54.813+0000 7f5f94b8d640 -1 osd.6 525 *** Got signal Terminated *** 2024-09-14T03:55:55.277 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:54 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[48695]: 2024-09-14T03:55:54.813+0000 7f5f94b8d640 -1 osd.6 525 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T03:55:55.777 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:55 smithi094 podman[101039]: 2024-09-14 03:55:55.611769873 +0000 UTC m=+0.908324108 container died e66b592ac940822cebc18075043a8e549cc6a5bf338de01a0ca5547612c72e85 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-eb16b97, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-14T03:55:55.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:55 smithi094 ceph-mon[86597]: osd.6 marked itself down and dead 2024-09-14T03:55:55.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:55 smithi094 ceph-mon[86597]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:55:55.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:55 smithi094 ceph-mon[86597]: osdmap e526: 8 total, 7 up, 8 in 2024-09-14T03:55:55.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:55 smithi070 ceph-mon[96704]: osd.6 marked itself down and dead 2024-09-14T03:55:55.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:55 smithi070 ceph-mon[96704]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:55:55.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:55 smithi070 ceph-mon[96704]: osdmap e526: 8 total, 7 up, 8 in 2024-09-14T03:55:55.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:55 smithi070 ceph-mon[98640]: osd.6 marked itself down and dead 2024-09-14T03:55:55.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:55 smithi070 ceph-mon[98640]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:55:55.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:55 smithi070 ceph-mon[98640]: osdmap e526: 8 total, 7 up, 8 in 2024-09-14T03:55:56.062 INFO:tasks.rados.rados.0.smithi070.stdout:2220: done (1 left) 2024-09-14T03:55:56.063 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 11 current snap is 238 2024-09-14T03:55:56.063 INFO:tasks.rados.rados.0.smithi070.stdout:2221: seq_num 703 ranges {9109504=581632,9691136=458752,10149888=557056,10706944=417792,11124736=172032} 2024-09-14T03:55:56.118 INFO:tasks.rados.rados.0.smithi070.stdout:2221: writing smithi07085154-11 from 9109504 to 9691136 tid 1 2024-09-14T03:55:56.120 INFO:tasks.rados.rados.0.smithi070.stdout:2221: writing smithi07085154-11 from 9691136 to 10149888 tid 2 2024-09-14T03:55:56.124 INFO:tasks.rados.rados.0.smithi070.stdout:2221: writing smithi07085154-11 from 10149888 to 10706944 tid 3 2024-09-14T03:55:56.126 INFO:tasks.rados.rados.0.smithi070.stdout:2221: writing smithi07085154-11 from 10706944 to 11124736 tid 4 2024-09-14T03:55:56.128 INFO:tasks.rados.rados.0.smithi070.stdout:2221: writing smithi07085154-11 from 11124736 to 11296768 tid 5 2024-09-14T03:55:56.128 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 26 current snap is 238 2024-09-14T03:55:56.128 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 1708 (ObjNum 663 snap 220 seq_num 663) dirty exists 2024-09-14T03:55:56.128 INFO:tasks.rados.rados.0.smithi070.stdout:2221: finishing write tid 1 to smithi07085154-11 2024-09-14T03:55:56.128 INFO:tasks.rados.rados.0.smithi070.stdout:2221: finishing write tid 2 to smithi07085154-11 2024-09-14T03:55:56.128 INFO:tasks.rados.rados.0.smithi070.stdout:2222: seq_num 704 ranges {1679360=598016,2277376=745472,3022848=712704,3735552=147456} 2024-09-14T03:55:56.140 INFO:tasks.rados.rados.0.smithi070.stdout:2222: writing smithi07085154-26 from 1679360 to 2277376 tid 1 2024-09-14T03:55:56.145 INFO:tasks.rados.rados.0.smithi070.stdout:2222: writing smithi07085154-26 from 2277376 to 3022848 tid 2 2024-09-14T03:55:56.149 INFO:tasks.rados.rados.0.smithi070.stdout:2222: writing smithi07085154-26 from 3022848 to 3735552 tid 3 2024-09-14T03:55:56.150 INFO:tasks.rados.rados.0.smithi070.stdout:2222: writing smithi07085154-26 from 3735552 to 3883008 tid 4 2024-09-14T03:55:56.150 INFO:tasks.rados.rados.0.smithi070.stdout:2219: done (2 left) 2024-09-14T03:55:56.150 INFO:tasks.rados.rados.0.smithi070.stdout:2223: setattr oid 23 current snap is 238 2024-09-14T03:55:56.150 INFO:tasks.rados.rados.0.smithi070.stdout:2221: finishing write tid 3 to smithi07085154-11 2024-09-14T03:55:56.150 INFO:tasks.rados.rados.0.smithi070.stdout:2221: finishing write tid 4 to smithi07085154-11 2024-09-14T03:55:56.150 INFO:tasks.rados.rados.0.smithi070.stdout:2221: finishing write tid 5 to smithi07085154-11 2024-09-14T03:55:56.150 INFO:tasks.rados.rados.0.smithi070.stdout:2221: finishing write tid 6 to smithi07085154-11 2024-09-14T03:55:56.151 INFO:tasks.rados.rados.0.smithi070.stdout:2221: finishing write tid 7 to smithi07085154-11 2024-09-14T03:55:56.151 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 11 v 2045 (ObjNum 703 snap 238 seq_num 703) dirty exists 2024-09-14T03:55:56.151 INFO:tasks.rados.rados.0.smithi070.stdout:2221: left oid 11 (ObjNum 703 snap 238 seq_num 703) 2024-09-14T03:55:56.151 INFO:tasks.rados.rados.0.smithi070.stdout:2222: finishing write tid 1 to smithi07085154-26 2024-09-14T03:55:56.151 INFO:tasks.rados.rados.0.smithi070.stdout:2222: finishing write tid 2 to smithi07085154-26 2024-09-14T03:55:56.151 INFO:tasks.rados.rados.0.smithi070.stdout:2221: done (2 left) 2024-09-14T03:55:56.151 INFO:tasks.rados.rados.0.smithi070.stdout:2224: snap_remove snap 237 2024-09-14T03:55:56.277 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:55 smithi094 podman[101039]: 2024-09-14 03:55:55.980014289 +0000 UTC m=+1.276568524 container cleanup e66b592ac940822cebc18075043a8e549cc6a5bf338de01a0ca5547612c72e85 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=quincy-eb16b97, org.label-schema.build-date=20240731) 2024-09-14T03:55:56.277 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:55 smithi094 bash[101039]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6 2024-09-14T03:55:56.703 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:56 smithi094 podman[101060]: 2024-09-14 03:55:56.390608602 +0000 UTC m=+0.774879298 container remove e66b592ac940822cebc18075043a8e549cc6a5bf338de01a0ca5547612c72e85 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-eb16b97, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-14T03:55:56.962 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:56 smithi094 podman[101152]: 2024-09-14 03:55:56.605106377 +0000 UTC m=+0.015973353 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:55:56.962 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:56 smithi094 podman[101152]: 2024-09-14 03:55:56.708326632 +0000 UTC m=+0.119193654 container create 216d7dc4c4e01b589f7dcdace959a5882cebb5c6e20f5a999954391c5496743c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=main-a278231) 2024-09-14T03:55:57.074 INFO:tasks.rados.rados.0.smithi070.stdout:2224: done (2 left) 2024-09-14T03:55:57.074 INFO:tasks.rados.rados.0.smithi070.stdout:2225: read oid 25 snap -1 2024-09-14T03:55:57.075 INFO:tasks.rados.rados.0.smithi070.stdout:2225: expect (ObjNum 702 snap 236 seq_num 702) 2024-09-14T03:55:57.075 INFO:tasks.rados.rados.0.smithi070.stdout:2226: rollback oid 30 current snap is 238 2024-09-14T03:55:57.075 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 30 to 197 2024-09-14T03:55:57.075 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 1701 (ObjNum 289 snap 0 seq_num 0) dirty exists 2024-09-14T03:55:57.075 INFO:tasks.rados.rados.0.smithi070.stdout:2222: finishing write tid 3 to smithi07085154-26 2024-09-14T03:55:57.075 INFO:tasks.rados.rados.0.smithi070.stdout:2222: finishing write tid 4 to smithi07085154-26 2024-09-14T03:55:57.075 INFO:tasks.rados.rados.0.smithi070.stdout:2223: done (3 left) 2024-09-14T03:55:57.075 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 2 current snap is 238 2024-09-14T03:55:57.075 INFO:tasks.rados.rados.0.smithi070.stdout:2227: seq_num 705 ranges {0=614400,614400=630784,1245184=524288,1769472=450560} 2024-09-14T03:55:57.077 INFO:tasks.rados.rados.0.smithi070.stdout:2227: writing smithi07085154-2 from 0 to 614400 tid 1 2024-09-14T03:55:57.081 INFO:tasks.rados.rados.0.smithi070.stdout:2227: writing smithi07085154-2 from 614400 to 1245184 tid 2 2024-09-14T03:55:57.084 INFO:tasks.rados.rados.0.smithi070.stdout:2227: writing smithi07085154-2 from 1245184 to 1769472 tid 3 2024-09-14T03:55:57.087 INFO:tasks.rados.rados.0.smithi070.stdout:2227: writing smithi07085154-2 from 1769472 to 2220032 tid 4 2024-09-14T03:55:57.087 INFO:tasks.rados.rados.0.smithi070.stdout:2228: rollback oid 29 current snap is 238 2024-09-14T03:55:57.087 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 29 to 234 2024-09-14T03:55:57.087 INFO:tasks.rados.rados.0.smithi070.stdout:2222: finishing write tid 5 to smithi07085154-26 2024-09-14T03:55:57.087 INFO:tasks.rados.rados.0.smithi070.stdout:2222: finishing write tid 6 to smithi07085154-26 2024-09-14T03:55:57.087 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 2411 (ObjNum 704 snap 238 seq_num 704) dirty exists 2024-09-14T03:55:57.087 INFO:tasks.rados.rados.0.smithi070.stdout:2222: left oid 26 (ObjNum 704 snap 238 seq_num 704) 2024-09-14T03:55:57.088 INFO:tasks.rados.rados.0.smithi070.stdout:2226: finishing rollback tid 1 to smithi07085154-30 2024-09-14T03:55:57.088 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 30 v 1695 (ObjNum 129 snap 0 seq_num 2931765312) dirty dne 2024-09-14T03:55:57.088 INFO:tasks.rados.rados.0.smithi070.stdout:2227: finishing write tid 1 to smithi07085154-2 2024-09-14T03:55:57.088 INFO:tasks.rados.rados.0.smithi070.stdout:2222: done (4 left) 2024-09-14T03:55:57.088 INFO:tasks.rados.rados.0.smithi070.stdout:2226: done (3 left) 2024-09-14T03:55:57.088 INFO:tasks.rados.rados.0.smithi070.stdout:2229: setattr oid 34 current snap is 238 2024-09-14T03:55:57.088 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 13 current snap is 238 2024-09-14T03:55:57.088 INFO:tasks.rados.rados.0.smithi070.stdout:2230: seq_num 706 ranges {0=630784,630784=516096,1146880=729088,1875968=360448} 2024-09-14T03:55:57.092 INFO:tasks.rados.rados.0.smithi070.stdout:2230: writing smithi07085154-13 from 0 to 630784 tid 1 2024-09-14T03:55:57.095 INFO:tasks.rados.rados.0.smithi070.stdout:2230: writing smithi07085154-13 from 630784 to 1146880 tid 2 2024-09-14T03:55:57.100 INFO:tasks.rados.rados.0.smithi070.stdout:2230: writing smithi07085154-13 from 1146880 to 1875968 tid 3 2024-09-14T03:55:57.103 INFO:tasks.rados.rados.0.smithi070.stdout:2230: writing smithi07085154-13 from 1875968 to 2236416 tid 4 2024-09-14T03:55:57.103 INFO:tasks.rados.rados.0.smithi070.stdout:2231: copy_from oid 17 from oid 31 current snap is 238 2024-09-14T03:55:57.103 INFO:tasks.rados.rados.0.smithi070.stdout:2227: finishing write tid 2 to smithi07085154-2 2024-09-14T03:55:57.103 INFO:tasks.rados.rados.0.smithi070.stdout:2227: finishing write tid 3 to smithi07085154-2 2024-09-14T03:55:57.103 INFO:tasks.rados.rados.0.smithi070.stdout:2227: finishing write tid 4 to smithi07085154-2 2024-09-14T03:55:57.103 INFO:tasks.rados.rados.0.smithi070.stdout:2227: finishing write tid 5 to smithi07085154-2 2024-09-14T03:55:57.103 INFO:tasks.rados.rados.0.smithi070.stdout:2228: finishing rollback tid 0 to smithi07085154-29 2024-09-14T03:55:57.103 INFO:tasks.rados.rados.0.smithi070.stdout:2228: finishing rollback tid 1 to smithi07085154-29 2024-09-14T03:55:57.103 INFO:tasks.rados.rados.0.smithi070.stdout:2228: finishing rollback tid 2 to smithi07085154-29 2024-09-14T03:55:57.103 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 1711 (ObjNum 458 snap 155 seq_num 458) dirty exists 2024-09-14T03:55:57.103 INFO:tasks.rados.rados.0.smithi070.stdout:2227: finishing write tid 6 to smithi07085154-2 2024-09-14T03:55:57.104 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 2 v 2418 (ObjNum 705 snap 238 seq_num 705) dirty exists 2024-09-14T03:55:57.104 INFO:tasks.rados.rados.0.smithi070.stdout:2227: left oid 2 (ObjNum 705 snap 238 seq_num 705) 2024-09-14T03:55:57.104 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 1712 (ObjNum 273 snap 0 seq_num 0) dirty exists 2024-09-14T03:55:57.104 INFO:tasks.rados.rados.0.smithi070.stdout:2230: finishing write tid 1 to smithi07085154-13 2024-09-14T03:55:57.104 INFO:tasks.rados.rados.0.smithi070.stdout:2230: finishing write tid 2 to smithi07085154-13 2024-09-14T03:55:57.104 INFO:tasks.rados.rados.0.smithi070.stdout:2227: done (5 left) 2024-09-14T03:55:57.104 INFO:tasks.rados.rados.0.smithi070.stdout:2228: done (4 left) 2024-09-14T03:55:57.104 INFO:tasks.rados.rados.0.smithi070.stdout:2229: done (3 left) 2024-09-14T03:55:57.104 INFO:tasks.rados.rados.0.smithi070.stdout:2232: snap_remove snap 197 2024-09-14T03:55:57.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:57 smithi094 ceph-mon[86597]: osdmap e527: 8 total, 7 up, 8 in 2024-09-14T03:55:57.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:57 smithi094 ceph-mon[86597]: pgmap v386: 121 pgs: 7 peering, 12 stale+active+clean, 102 active+clean; 1.0 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 5.9 MiB/s wr, 14 op/s 2024-09-14T03:55:57.277 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:56 smithi094 podman[101152]: 2024-09-14 03:55:56.961362359 +0000 UTC m=+0.372229335 container init 216d7dc4c4e01b589f7dcdace959a5882cebb5c6e20f5a999954391c5496743c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=main-a278231) 2024-09-14T03:55:57.277 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:56 smithi094 podman[101152]: 2024-09-14 03:55:56.966375076 +0000 UTC m=+0.377242059 container start 216d7dc4c4e01b589f7dcdace959a5882cebb5c6e20f5a999954391c5496743c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2) 2024-09-14T03:55:57.277 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:57 smithi094 podman[101152]: 2024-09-14 03:55:57.052920368 +0000 UTC m=+0.463787348 container attach 216d7dc4c4e01b589f7dcdace959a5882cebb5c6e20f5a999954391c5496743c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6-deactivate, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-14T03:55:57.277 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:57 smithi094 podman[101152]: 2024-09-14 03:55:57.174787753 +0000 UTC m=+0.585654730 container died 216d7dc4c4e01b589f7dcdace959a5882cebb5c6e20f5a999954391c5496743c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:55:57.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:57 smithi070 ceph-mon[96704]: osdmap e527: 8 total, 7 up, 8 in 2024-09-14T03:55:57.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:57 smithi070 ceph-mon[96704]: pgmap v386: 121 pgs: 7 peering, 12 stale+active+clean, 102 active+clean; 1.0 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 5.9 MiB/s wr, 14 op/s 2024-09-14T03:55:57.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:57 smithi070 ceph-mon[98640]: osdmap e527: 8 total, 7 up, 8 in 2024-09-14T03:55:57.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:57 smithi070 ceph-mon[98640]: pgmap v386: 121 pgs: 7 peering, 12 stale+active+clean, 102 active+clean; 1.0 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 5.9 MiB/s wr, 14 op/s 2024-09-14T03:55:58.060 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:57 smithi094 podman[101195]: 2024-09-14 03:55:57.801937517 +0000 UTC m=+0.618466981 container remove 216d7dc4c4e01b589f7dcdace959a5882cebb5c6e20f5a999954391c5496743c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6-deactivate, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-14T03:55:58.060 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:57 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.6.service: Deactivated successfully. 2024-09-14T03:55:58.060 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:57 smithi094 systemd[1]: Stopped Ceph osd.6 for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:55:58.060 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:57 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.6.service: Consumed 13.413s CPU time. 2024-09-14T03:55:58.080 INFO:tasks.rados.rados.0.smithi070.stdout:2230: finishing write tid 3 to smithi07085154-13 2024-09-14T03:55:58.080 INFO:tasks.rados.rados.0.smithi070.stdout:2230: finishing write tid 4 to smithi07085154-13 2024-09-14T03:55:58.080 INFO:tasks.rados.rados.0.smithi070.stdout:2230: finishing write tid 5 to smithi07085154-13 2024-09-14T03:55:58.080 INFO:tasks.rados.rados.0.smithi070.stdout:2230: finishing write tid 6 to smithi07085154-13 2024-09-14T03:55:58.080 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 2050 (ObjNum 706 snap 238 seq_num 706) dirty exists 2024-09-14T03:55:58.080 INFO:tasks.rados.rados.0.smithi070.stdout:2230: left oid 13 (ObjNum 706 snap 238 seq_num 706) 2024-09-14T03:55:58.080 INFO:tasks.rados.rados.0.smithi070.stdout:2225: expect (ObjNum 702 snap 236 seq_num 702) 2024-09-14T03:55:58.432 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:58 smithi094 systemd[1]: Starting Ceph osd.6 for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:55:58.433 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:58 smithi094 ceph-mon[86597]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-14T03:55:58.433 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:58 smithi094 ceph-mon[86597]: osdmap e528: 8 total, 7 up, 8 in 2024-09-14T03:55:58.502 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-09-14T03:55:58.502 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:58 smithi070 ceph-mon[96704]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-14T03:55:58.502 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:58 smithi070 ceph-mon[96704]: osdmap e528: 8 total, 7 up, 8 in 2024-09-14T03:55:58.502 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:58 smithi070 ceph-mon[98640]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-14T03:55:58.502 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:58 smithi070 ceph-mon[98640]: osdmap e528: 8 total, 7 up, 8 in 2024-09-14T03:55:58.684 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:58 smithi094 podman[101299]: 2024-09-14 03:55:58.334583146 +0000 UTC m=+0.015998553 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:55:58.685 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:58 smithi094 podman[101299]: 2024-09-14 03:55:58.433723038 +0000 UTC m=+0.115138436 container create 70a08a1c3f6fee8fc86c4b4ba5a0750f5d379d1f09ead85013deef0c8e7e350e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, GIT_BRANCH=HEAD) 2024-09-14T03:55:58.685 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:58 smithi094 podman[101299]: 2024-09-14 03:55:58.678750061 +0000 UTC m=+0.360165459 container init 70a08a1c3f6fee8fc86c4b4ba5a0750f5d379d1f09ead85013deef0c8e7e350e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6-activate, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=main-a278231, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-14T03:55:58.685 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:58 smithi094 podman[101299]: 2024-09-14 03:55:58.683606176 +0000 UTC m=+0.365021580 container start 70a08a1c3f6fee8fc86c4b4ba5a0750f5d379d1f09ead85013deef0c8e7e350e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=main-a278231, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True) 2024-09-14T03:55:59.027 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:58 smithi094 podman[101299]: 2024-09-14 03:55:58.761988156 +0000 UTC m=+0.443403561 container attach 70a08a1c3f6fee8fc86c4b4ba5a0750f5d379d1f09ead85013deef0c8e7e350e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6-activate, RELEASE=main-a278231, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-14T03:55:59.027 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:58 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6-activate[101314]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:55:59.027 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:58 smithi094 bash[101299]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:55:59.027 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:58 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6-activate[101314]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:55:59.027 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:58 smithi094 bash[101299]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:55:59.065 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:55:59.065 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (5m) 96s ago 10m 21.2M - 0.27.0 11f11916f8cd 45e79e51074c 2024-09-14T03:55:59.065 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi094 *:3000 running (5m) 27s ago 10m 84.6M - 9.4.7 954c08fa6188 2240810d0368 2024-09-14T03:55:59.065 INFO:teuthology.orchestra.run.smithi070.stdout:mds.a smithi070 running (12m) 96s ago 12m 21.4M - 17.2.7-1629-geb16b97b ab51d1bebd12 52e0dbb7be50 2024-09-14T03:55:59.065 INFO:teuthology.orchestra.run.smithi070.stdout:mds.b smithi094 running (12m) 27s ago 12m 16.1M - 17.2.7-1629-geb16b97b ab51d1bebd12 e3b5e3e00189 2024-09-14T03:55:59.065 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi094 *:8443,9283,8765 running (4m) 27s ago 15m 455M - 19.3.0-4907-ga2782315 8801d2529e43 569d9c7e0f06 2024-09-14T03:55:59.065 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (5m) 96s ago 17m 521M - 19.3.0-4907-ga2782315 8801d2529e43 0ef4d8e0cdbe 2024-09-14T03:55:59.065 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (4m) 96s ago 17m 57.7M 2048M 19.3.0-4907-ga2782315 8801d2529e43 96542de31cac 2024-09-14T03:55:59.065 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi094 running (4m) 27s ago 15m 44.8M 2048M 19.3.0-4907-ga2782315 8801d2529e43 86827be98659 2024-09-14T03:55:59.065 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (4m) 96s ago 15m 51.9M 2048M 19.3.0-4907-ga2782315 8801d2529e43 5741e4fcf4b6 2024-09-14T03:55:59.065 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (5m) 96s ago 10m 12.2M - 1.7.0 72c9c2088986 e316760a30b3 2024-09-14T03:55:59.066 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi094 *:9100 running (5m) 27s ago 10m 12.0M - 1.7.0 72c9c2088986 767f8a093175 2024-09-14T03:55:59.066 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (3m) 96s ago 15m 180M 2480M 19.3.0-4907-ga2782315 8801d2529e43 2345ccac2085 2024-09-14T03:55:59.066 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (2m) 96s ago 14m 177M 2480M 19.3.0-4907-ga2782315 8801d2529e43 53868856512a 2024-09-14T03:55:59.066 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (2m) 96s ago 14m 104M 2480M 19.3.0-4907-ga2782315 8801d2529e43 39ef876d2fff 2024-09-14T03:55:59.066 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (100s) 96s ago 14m 14.5M 2480M 19.3.0-4907-ga2782315 8801d2529e43 8efc112edbe9 2024-09-14T03:55:59.066 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi094 running (67s) 27s ago 13m 162M 2480M 19.3.0-4907-ga2782315 8801d2529e43 09f583ed6c77 2024-09-14T03:55:59.066 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi094 running (32s) 27s ago 13m 12.2M 2480M 19.3.0-4907-ga2782315 8801d2529e43 162396f2e7b6 2024-09-14T03:55:59.066 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi094 running (12m) 27s ago 12m 198M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 e66b592ac940 2024-09-14T03:55:59.066 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi094 running (12m) 27s ago 12m 274M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 037c6208e9ce 2024-09-14T03:55:59.066 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi094 *:9095 running (4m) 27s ago 10m 54.0M - 2.51.0 1d3b7f56885b ce1b353828dc 2024-09-14T03:55:59.425 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:55:59.425 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-09-14T03:55:59.425 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:55:59.426 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:55:59.426 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-09-14T03:55:59.426 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:55:59.426 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:55:59.426 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-09-14T03:55:59.426 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 1, 2024-09-14T03:55:59.426 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 6 2024-09-14T03:55:59.426 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:55:59.426 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": { 2024-09-14T03:55:59.426 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:55:59.426 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:55:59.426 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-09-14T03:55:59.426 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 3, 2024-09-14T03:55:59.426 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 11 2024-09-14T03:55:59.426 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-09-14T03:55:59.427 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:55:59.427 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:59 smithi070 ceph-mon[96704]: osdmap e529: 8 total, 7 up, 8 in 2024-09-14T03:55:59.427 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:55:59 smithi070 ceph-mon[96704]: pgmap v389: 121 pgs: 11 active+undersized, 7 peering, 6 stale+active+clean, 5 active+undersized+degraded, 92 active+clean; 1.0 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.5 MiB/s wr, 19 op/s; 80/1302 objects degraded (6.144%) 2024-09-14T03:55:59.427 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:59 smithi070 ceph-mon[98640]: osdmap e529: 8 total, 7 up, 8 in 2024-09-14T03:55:59.427 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:55:59 smithi070 ceph-mon[98640]: pgmap v389: 121 pgs: 11 active+undersized, 7 peering, 6 stale+active+clean, 5 active+undersized+degraded, 92 active+clean; 1.0 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.5 MiB/s wr, 19 op/s; 80/1302 objects degraded (6.144%) 2024-09-14T03:55:59.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:59 smithi094 ceph-mon[86597]: osdmap e529: 8 total, 7 up, 8 in 2024-09-14T03:55:59.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:55:59 smithi094 ceph-mon[86597]: pgmap v389: 121 pgs: 11 active+undersized, 7 peering, 6 stale+active+clean, 5 active+undersized+degraded, 92 active+clean; 1.0 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.5 MiB/s wr, 19 op/s; 80/1302 objects degraded (6.144%) 2024-09-14T03:55:59.527 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:59 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6-activate[101314]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-14T03:55:59.527 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:59 smithi094 bash[101299]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-14T03:55:59.527 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:59 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6-activate[101314]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-14T03:55:59.527 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:59 smithi094 bash[101299]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-14T03:55:59.712 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:55:59.712 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:55:59.712 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-09-14T03:55:59.712 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:55:59.712 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [ 2024-09-14T03:55:59.712 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr", 2024-09-14T03:55:59.712 INFO:teuthology.orchestra.run.smithi070.stdout: "mon" 2024-09-14T03:55:59.712 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2024-09-14T03:55:59.712 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "11/20 daemons upgraded", 2024-09-14T03:55:59.713 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "Currently upgrading osd daemons", 2024-09-14T03:55:59.713 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-09-14T03:55:59.713 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:55:59.884 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:59 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6-activate[101314]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-14T03:55:59.884 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:59 smithi094 bash[101299]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-14T03:55:59.884 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:59 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6-activate[101314]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-14T03:55:59.885 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:59 smithi094 bash[101299]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-14T03:55:59.885 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:59 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6-activate[101314]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-14T03:55:59.885 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:59 smithi094 bash[101299]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-14T03:55:59.885 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:59 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6-activate[101314]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-14T03:55:59.885 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:59 smithi094 bash[101299]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-14T03:55:59.885 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:59 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6-activate[101314]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-14T03:55:59.885 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:59 smithi094 bash[101299]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-14T03:55:59.885 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:55:59 smithi094 podman[101299]: 2024-09-14 03:55:59.592280632 +0000 UTC m=+1.273696038 container died 70a08a1c3f6fee8fc86c4b4ba5a0750f5d379d1f09ead85013deef0c8e7e350e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6-activate, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=main-a278231, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:56:00.167 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:00 smithi094 ceph-mon[86597]: from='client.44342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:00.168 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:00 smithi094 ceph-mon[86597]: from='client.44348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:00.168 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:00 smithi094 ceph-mon[86597]: from='client.44354 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:00.168 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:00 smithi094 ceph-mon[86597]: Health check failed: Degraded data redundancy: 80/1302 objects degraded (6.144%), 5 pgs degraded (PG_DEGRADED) 2024-09-14T03:56:00.168 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:00 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/766143191' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:00.522 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:56:00 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:56:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:56:00.522 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:56:00 smithi094 podman[101469]: 2024-09-14 03:56:00.16688545 +0000 UTC m=+0.565416560 container remove 70a08a1c3f6fee8fc86c4b4ba5a0750f5d379d1f09ead85013deef0c8e7e350e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6-activate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=main-a278231, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-14T03:56:00.522 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:56:00 smithi094 podman[101536]: 2024-09-14 03:56:00.27635429 +0000 UTC m=+0.015855706 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:56:00.522 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:56:00 smithi094 podman[101536]: 2024-09-14 03:56:00.40977604 +0000 UTC m=+0.149277449 container create cbdc02cdb8e444c0e3bdaac1b66cca04fbfa0b1685c13caaebaf63506e2dc412 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-14T03:56:00.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:00 smithi070 ceph-mon[98640]: from='client.44342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:00.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:00 smithi070 ceph-mon[98640]: from='client.44348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:00.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:00 smithi070 ceph-mon[98640]: from='client.44354 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:00.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:00 smithi070 ceph-mon[98640]: Health check failed: Degraded data redundancy: 80/1302 objects degraded (6.144%), 5 pgs degraded (PG_DEGRADED) 2024-09-14T03:56:00.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:00 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/766143191' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:00.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:00 smithi070 ceph-mon[96704]: from='client.44342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:00.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:00 smithi070 ceph-mon[96704]: from='client.44348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:00.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:00 smithi070 ceph-mon[96704]: from='client.44354 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:00.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:00 smithi070 ceph-mon[96704]: Health check failed: Degraded data redundancy: 80/1302 objects degraded (6.144%), 5 pgs degraded (PG_DEGRADED) 2024-09-14T03:56:00.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:00 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/766143191' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:00.658 INFO:tasks.rados.rados.0.smithi070.stdout:2231: finishing copy_from racing read to smithi07085154-17 2024-09-14T03:56:00.659 INFO:tasks.rados.rados.0.smithi070.stdout:2231: finishing copy_from to smithi07085154-17 2024-09-14T03:56:00.659 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 2194 (ObjNum 670 snap 228 seq_num 670) dirty exists 2024-09-14T03:56:00.659 INFO:tasks.rados.rados.0.smithi070.stdout:2225: done (3 left) 2024-09-14T03:56:00.659 INFO:tasks.rados.rados.0.smithi070.stdout:2230: done (2 left) 2024-09-14T03:56:00.659 INFO:tasks.rados.rados.0.smithi070.stdout:2231: done (1 left) 2024-09-14T03:56:00.659 INFO:tasks.rados.rados.0.smithi070.stdout:2232: done (0 left) 2024-09-14T03:56:00.659 INFO:tasks.rados.rados.0.smithi070.stdout:2233: snap_create 2024-09-14T03:56:01.026 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:56:00 smithi094 podman[101536]: 2024-09-14 03:56:00.595920726 +0000 UTC m=+0.335422135 container init cbdc02cdb8e444c0e3bdaac1b66cca04fbfa0b1685c13caaebaf63506e2dc412 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=main-a278231, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0) 2024-09-14T03:56:01.027 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:56:00 smithi094 podman[101536]: 2024-09-14 03:56:00.60092683 +0000 UTC m=+0.340428239 container start cbdc02cdb8e444c0e3bdaac1b66cca04fbfa0b1685c13caaebaf63506e2dc412 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-14T03:56:01.027 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:56:00 smithi094 ceph-osd[101574]: -- 172.21.15.94:0/3047673807 <== mon.1 v2:172.21.15.94:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5647f665da40 con 0x5647f742c400 2024-09-14T03:56:01.027 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:56:00 smithi094 bash[101536]: cbdc02cdb8e444c0e3bdaac1b66cca04fbfa0b1685c13caaebaf63506e2dc412 2024-09-14T03:56:01.027 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:56:00 smithi094 systemd[1]: Started Ceph osd.6 for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:56:01.082 INFO:tasks.rados.rados.0.smithi070.stdout:2233: done (0 left) 2024-09-14T03:56:01.082 INFO:tasks.rados.rados.0.smithi070.stdout:2234: rollback oid 29 current snap is 239 2024-09-14T03:56:01.082 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 29 to 234 2024-09-14T03:56:01.082 INFO:tasks.rados.rados.0.smithi070.stdout:2235: snap_remove snap 235 2024-09-14T03:56:01.341 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:01 smithi094 ceph-mon[86597]: from='client.44366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:01.341 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:01 smithi094 ceph-mon[86597]: pgmap v390: 121 pgs: 34 active+undersized, 8 active+undersized+degraded, 79 active+clean; 1021 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 1.5 MiB/s wr, 15 op/s; 92/1278 objects degraded (7.199%) 2024-09-14T03:56:01.342 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:01 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:01.342 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:01 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:01.342 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:01 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:56:01.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:01 smithi070 ceph-mon[96704]: from='client.44366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:01.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:01 smithi070 ceph-mon[96704]: pgmap v390: 121 pgs: 34 active+undersized, 8 active+undersized+degraded, 79 active+clean; 1021 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 1.5 MiB/s wr, 15 op/s; 92/1278 objects degraded (7.199%) 2024-09-14T03:56:01.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:01 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:01.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:01 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:01.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:01 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:56:01.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:01 smithi070 ceph-mon[98640]: from='client.44366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:01.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:01 smithi070 ceph-mon[98640]: pgmap v390: 121 pgs: 34 active+undersized, 8 active+undersized+degraded, 79 active+clean; 1021 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 1.5 MiB/s wr, 15 op/s; 92/1278 objects degraded (7.199%) 2024-09-14T03:56:01.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:01 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:01.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:01 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:01.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:01 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:56:02.085 INFO:tasks.rados.rados.0.smithi070.stdout:2235: done (1 left) 2024-09-14T03:56:02.086 INFO:tasks.rados.rados.0.smithi070.stdout:2236: rollback oid 42 current snap is 239 2024-09-14T03:56:02.086 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 42 to 179 2024-09-14T03:56:02.086 INFO:tasks.rados.rados.0.smithi070.stdout:2234: finishing rollback tid 0 to smithi07085154-29 2024-09-14T03:56:02.086 INFO:tasks.rados.rados.0.smithi070.stdout:2234: finishing rollback tid 1 to smithi07085154-29 2024-09-14T03:56:02.086 INFO:tasks.rados.rados.0.smithi070.stdout:2234: finishing rollback tid 2 to smithi07085154-29 2024-09-14T03:56:02.086 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 1715 (ObjNum 458 snap 155 seq_num 458) dirty exists 2024-09-14T03:56:02.086 INFO:tasks.rados.rados.0.smithi070.stdout:2234: done (1 left) 2024-09-14T03:56:02.086 INFO:tasks.rados.rados.0.smithi070.stdout:2237: delete oid 11 current snap is 239 2024-09-14T03:56:02.097 INFO:tasks.rados.rados.0.smithi070.stdout:2237: done (1 left) 2024-09-14T03:56:02.098 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 49 current snap is 239 2024-09-14T03:56:02.098 INFO:tasks.rados.rados.0.smithi070.stdout:2238: seq_num 708 ranges {8232960=663552,8896512=737280,9633792=696320,10330112=172032} 2024-09-14T03:56:02.145 INFO:tasks.rados.rados.0.smithi070.stdout:2238: writing smithi07085154-49 from 8232960 to 8896512 tid 1 2024-09-14T03:56:02.149 INFO:tasks.rados.rados.0.smithi070.stdout:2238: writing smithi07085154-49 from 8896512 to 9633792 tid 2 2024-09-14T03:56:02.153 INFO:tasks.rados.rados.0.smithi070.stdout:2238: writing smithi07085154-49 from 9633792 to 10330112 tid 3 2024-09-14T03:56:02.154 INFO:tasks.rados.rados.0.smithi070.stdout:2238: writing smithi07085154-49 from 10330112 to 10502144 tid 4 2024-09-14T03:56:02.155 INFO:tasks.rados.rados.0.smithi070.stdout:2239: rmattr oid 12 current snap is 239 2024-09-14T03:56:02.155 INFO:tasks.rados.rados.0.smithi070.stdout:2239: done (2 left) 2024-09-14T03:56:02.155 INFO:tasks.rados.rados.0.smithi070.stdout:2240: rmattr oid 10 current snap is 239 2024-09-14T03:56:02.155 INFO:tasks.rados.rados.0.smithi070.stdout:2240: done (2 left) 2024-09-14T03:56:02.155 INFO:tasks.rados.rados.0.smithi070.stdout:2241: copy_from oid 23 from oid 36 current snap is 239 2024-09-14T03:56:02.155 INFO:tasks.rados.rados.0.smithi070.stdout:2236: finishing rollback tid 0 to smithi07085154-42 2024-09-14T03:56:02.155 INFO:tasks.rados.rados.0.smithi070.stdout:2236: finishing rollback tid 1 to smithi07085154-42 2024-09-14T03:56:02.155 INFO:tasks.rados.rados.0.smithi070.stdout:2236: finishing rollback tid 2 to smithi07085154-42 2024-09-14T03:56:02.155 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 2436 (ObjNum 507 snap 166 seq_num 507) dirty exists 2024-09-14T03:56:02.155 INFO:tasks.rados.rados.0.smithi070.stdout:2238: finishing write tid 1 to smithi07085154-49 2024-09-14T03:56:02.155 INFO:tasks.rados.rados.0.smithi070.stdout:2238: finishing write tid 2 to smithi07085154-49 2024-09-14T03:56:02.155 INFO:tasks.rados.rados.0.smithi070.stdout:2236: done (2 left) 2024-09-14T03:56:02.155 INFO:tasks.rados.rados.0.smithi070.stdout:2242: copy_from oid 42 from oid 41 current snap is 239 2024-09-14T03:56:02.156 INFO:tasks.rados.rados.0.smithi070.stdout:2243: snap_remove snap 229 2024-09-14T03:56:02.345 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:02 smithi094 ceph-mon[86597]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-14T03:56:02.345 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:02 smithi094 ceph-mon[86597]: osdmap e530: 8 total, 7 up, 8 in 2024-09-14T03:56:02.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:02 smithi070 ceph-mon[96704]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-14T03:56:02.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:02 smithi070 ceph-mon[96704]: osdmap e530: 8 total, 7 up, 8 in 2024-09-14T03:56:02.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:02 smithi070 ceph-mon[98640]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-14T03:56:02.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:02 smithi070 ceph-mon[98640]: osdmap e530: 8 total, 7 up, 8 in 2024-09-14T03:56:02.776 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:56:02 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T03:56:02.484+0000 7f90d0c80640 -1 Falling back to public interface 2024-09-14T03:56:03.098 INFO:tasks.rados.rados.0.smithi070.stdout:2243: done (3 left) 2024-09-14T03:56:03.098 INFO:tasks.rados.rados.0.smithi070.stdout:2244: rollback oid 27 current snap is 239 2024-09-14T03:56:03.098 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 27 to 199 2024-09-14T03:56:03.098 INFO:tasks.rados.rados.0.smithi070.stdout:2245: rollback oid 26 current snap is 239 2024-09-14T03:56:03.098 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 26 to 217 2024-09-14T03:56:03.098 INFO:tasks.rados.rados.0.smithi070.stdout:2242: finishing copy_from to smithi07085154-42 2024-09-14T03:56:03.098 INFO:tasks.rados.rados.0.smithi070.stdout:2242: got expected ENOENT (src dne) 2024-09-14T03:56:03.098 INFO:tasks.rados.rados.0.smithi070.stdout:2242: finishing copy_from racing read to smithi07085154-42 2024-09-14T03:56:03.098 INFO:tasks.rados.rados.0.smithi070.stdout:2238: finishing write tid 3 to smithi07085154-49 2024-09-14T03:56:03.098 INFO:tasks.rados.rados.0.smithi070.stdout:2238: finishing write tid 4 to smithi07085154-49 2024-09-14T03:56:03.098 INFO:tasks.rados.rados.0.smithi070.stdout:2238: finishing write tid 5 to smithi07085154-49 2024-09-14T03:56:03.098 INFO:tasks.rados.rados.0.smithi070.stdout:2238: finishing write tid 6 to smithi07085154-49 2024-09-14T03:56:03.099 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 49 v 1955 (ObjNum 708 snap 239 seq_num 708) dirty exists 2024-09-14T03:56:03.099 INFO:tasks.rados.rados.0.smithi070.stdout:2238: left oid 49 (ObjNum 708 snap 239 seq_num 708) 2024-09-14T03:56:03.099 INFO:tasks.rados.rados.0.smithi070.stdout:2241: finishing copy_from racing read to smithi07085154-23 2024-09-14T03:56:03.099 INFO:tasks.rados.rados.0.smithi070.stdout:2241: finishing copy_from to smithi07085154-23 2024-09-14T03:56:03.099 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 2044 (ObjNum 701 snap 236 seq_num 701) dirty exists 2024-09-14T03:56:03.099 INFO:tasks.rados.rados.0.smithi070.stdout:2238: done (4 left) 2024-09-14T03:56:03.099 INFO:tasks.rados.rados.0.smithi070.stdout:2241: done (3 left) 2024-09-14T03:56:03.099 INFO:tasks.rados.rados.0.smithi070.stdout:2242: done (2 left) 2024-09-14T03:56:03.099 INFO:tasks.rados.rados.0.smithi070.stdout:2246: read oid 49 snap -1 2024-09-14T03:56:03.099 INFO:tasks.rados.rados.0.smithi070.stdout:2246: expect (ObjNum 708 snap 239 seq_num 708) 2024-09-14T03:56:03.099 INFO:tasks.rados.rados.0.smithi070.stdout:2247: rollback oid 48 current snap is 239 2024-09-14T03:56:03.099 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 48 to 208 2024-09-14T03:56:03.099 INFO:tasks.rados.rados.0.smithi070.stdout:2248: snap_remove snap 126 2024-09-14T03:56:03.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:03 smithi094 ceph-mon[86597]: osdmap e531: 8 total, 7 up, 8 in 2024-09-14T03:56:03.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:03 smithi094 ceph-mon[86597]: pgmap v393: 121 pgs: 34 active+undersized, 8 active+undersized+degraded, 79 active+clean; 1.0 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.9 MiB/s wr, 17 op/s; 92/1281 objects degraded (7.182%) 2024-09-14T03:56:03.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:03 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:03.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:03 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:56:03.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:03 smithi070 ceph-mon[98640]: osdmap e531: 8 total, 7 up, 8 in 2024-09-14T03:56:03.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:03 smithi070 ceph-mon[98640]: pgmap v393: 121 pgs: 34 active+undersized, 8 active+undersized+degraded, 79 active+clean; 1.0 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.9 MiB/s wr, 17 op/s; 92/1281 objects degraded (7.182%) 2024-09-14T03:56:03.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:03 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:03.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:03 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:56:03.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:03 smithi070 ceph-mon[96704]: osdmap e531: 8 total, 7 up, 8 in 2024-09-14T03:56:03.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:03 smithi070 ceph-mon[96704]: pgmap v393: 121 pgs: 34 active+undersized, 8 active+undersized+degraded, 79 active+clean; 1.0 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.9 MiB/s wr, 17 op/s; 92/1281 objects degraded (7.182%) 2024-09-14T03:56:03.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:03 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:03.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:03 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:56:03.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:56:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:56:03] "GET /metrics HTTP/1.1" 200 36307 "" "Prometheus/2.51.0" 2024-09-14T03:56:04.063 INFO:tasks.rados.rados.0.smithi070.stdout:2247: finishing rollback tid 0 to smithi07085154-48 2024-09-14T03:56:04.063 INFO:tasks.rados.rados.0.smithi070.stdout:2247: finishing rollback tid 1 to smithi07085154-48 2024-09-14T03:56:04.063 INFO:tasks.rados.rados.0.smithi070.stdout:2247: finishing rollback tid 2 to smithi07085154-48 2024-09-14T03:56:04.063 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 2197 (ObjNum 313 snap 105 seq_num 313) dirty exists 2024-09-14T03:56:04.063 INFO:tasks.rados.rados.0.smithi070.stdout:2244: finishing rollback tid 0 to smithi07085154-27 2024-09-14T03:56:04.063 INFO:tasks.rados.rados.0.smithi070.stdout:2244: finishing rollback tid 1 to smithi07085154-27 2024-09-14T03:56:04.063 INFO:tasks.rados.rados.0.smithi070.stdout:2244: finishing rollback tid 2 to smithi07085154-27 2024-09-14T03:56:04.063 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 2442 (ObjNum 590 snap 195 seq_num 590) dirty exists 2024-09-14T03:56:04.064 INFO:tasks.rados.rados.0.smithi070.stdout:2245: finishing rollback tid 0 to smithi07085154-26 2024-09-14T03:56:04.064 INFO:tasks.rados.rados.0.smithi070.stdout:2245: finishing rollback tid 1 to smithi07085154-26 2024-09-14T03:56:04.064 INFO:tasks.rados.rados.0.smithi070.stdout:2245: finishing rollback tid 2 to smithi07085154-26 2024-09-14T03:56:04.064 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 2446 (ObjNum 629 snap 211 seq_num 629) dirty exists 2024-09-14T03:56:04.064 INFO:tasks.rados.rados.0.smithi070.stdout:2246: expect (ObjNum 708 snap 239 seq_num 708) 2024-09-14T03:56:04.484 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:04 smithi094 ceph-mon[86597]: osdmap e532: 8 total, 7 up, 8 in 2024-09-14T03:56:04.484 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:04 smithi094 ceph-mon[86597]: osdmap e533: 8 total, 7 up, 8 in 2024-09-14T03:56:04.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:04 smithi070 ceph-mon[96704]: osdmap e532: 8 total, 7 up, 8 in 2024-09-14T03:56:04.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:04 smithi070 ceph-mon[96704]: osdmap e533: 8 total, 7 up, 8 in 2024-09-14T03:56:04.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:04 smithi070 ceph-mon[98640]: osdmap e532: 8 total, 7 up, 8 in 2024-09-14T03:56:04.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:04 smithi070 ceph-mon[98640]: osdmap e533: 8 total, 7 up, 8 in 2024-09-14T03:56:04.784 INFO:tasks.rados.rados.0.smithi070.stdout:2244: done (4 left) 2024-09-14T03:56:04.784 INFO:tasks.rados.rados.0.smithi070.stdout:2245: done (3 left) 2024-09-14T03:56:04.784 INFO:tasks.rados.rados.0.smithi070.stdout:2246: done (2 left) 2024-09-14T03:56:04.784 INFO:tasks.rados.rados.0.smithi070.stdout:2247: done (1 left) 2024-09-14T03:56:04.785 INFO:tasks.rados.rados.0.smithi070.stdout:2248: done (0 left) 2024-09-14T03:56:04.785 INFO:tasks.rados.rados.0.smithi070.stdout:2249: rollback oid 10 current snap is 239 2024-09-14T03:56:04.785 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 10 to 217 2024-09-14T03:56:04.785 INFO:tasks.rados.rados.0.smithi070.stdout:2250: delete oid 37 current snap is 239 2024-09-14T03:56:04.787 INFO:tasks.rados.rados.0.smithi070.stdout:2250: done (1 left) 2024-09-14T03:56:04.787 INFO:tasks.rados.rados.0.smithi070.stdout:2251: rollback oid 35 current snap is 239 2024-09-14T03:56:04.787 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 35 to 215 2024-09-14T03:56:04.787 INFO:tasks.rados.rados.0.smithi070.stdout:2252: snap_create 2024-09-14T03:56:04.788 INFO:tasks.rados.rados.0.smithi070.stdout:2249: finishing rollback tid 0 to smithi07085154-10 2024-09-14T03:56:04.789 INFO:tasks.rados.rados.0.smithi070.stdout:2249: finishing rollback tid 1 to smithi07085154-10 2024-09-14T03:56:04.789 INFO:tasks.rados.rados.0.smithi070.stdout:2249: finishing rollback tid 2 to smithi07085154-10 2024-09-14T03:56:04.789 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 2474 (ObjNum 508 snap 166 seq_num 508) dirty exists 2024-09-14T03:56:04.789 INFO:tasks.rados.rados.0.smithi070.stdout:2251: finishing rollback tid 0 to smithi07085154-35 2024-09-14T03:56:04.790 INFO:tasks.rados.rados.0.smithi070.stdout:2251: finishing rollback tid 1 to smithi07085154-35 2024-09-14T03:56:04.790 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 35 v 1687 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-14T03:56:05.098 INFO:tasks.rados.rados.0.smithi070.stdout:2249: done (2 left) 2024-09-14T03:56:05.099 INFO:tasks.rados.rados.0.smithi070.stdout:2251: done (1 left) 2024-09-14T03:56:05.099 INFO:tasks.rados.rados.0.smithi070.stdout:2252: done (0 left) 2024-09-14T03:56:05.099 INFO:tasks.rados.rados.0.smithi070.stdout:2253: snap_create 2024-09-14T03:56:05.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:05 smithi094 ceph-mon[86597]: pgmap v396: 121 pgs: 34 active+undersized, 8 active+undersized+degraded, 79 active+clean; 1.0 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 1.6 MiB/s wr, 10 op/s; 93/1287 objects degraded (7.226%) 2024-09-14T03:56:05.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:05 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:05.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:05 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:05.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:05 smithi094 ceph-mon[86597]: Health check update: Degraded data redundancy: 93/1287 objects degraded (7.226%), 8 pgs degraded (PG_DEGRADED) 2024-09-14T03:56:05.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:05 smithi070 ceph-mon[96704]: pgmap v396: 121 pgs: 34 active+undersized, 8 active+undersized+degraded, 79 active+clean; 1.0 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 1.6 MiB/s wr, 10 op/s; 93/1287 objects degraded (7.226%) 2024-09-14T03:56:05.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:05 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:05.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:05 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:05.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:05 smithi070 ceph-mon[96704]: Health check update: Degraded data redundancy: 93/1287 objects degraded (7.226%), 8 pgs degraded (PG_DEGRADED) 2024-09-14T03:56:05.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:05 smithi070 ceph-mon[98640]: pgmap v396: 121 pgs: 34 active+undersized, 8 active+undersized+degraded, 79 active+clean; 1.0 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 1.6 MiB/s wr, 10 op/s; 93/1287 objects degraded (7.226%) 2024-09-14T03:56:05.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:05 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:05.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:05 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:05.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:05 smithi070 ceph-mon[98640]: Health check update: Degraded data redundancy: 93/1287 objects degraded (7.226%), 8 pgs degraded (PG_DEGRADED) 2024-09-14T03:56:06.082 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:56:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T03:56:05.954+0000 7f90d0c80640 -1 osd.6 525 log_to_monitors true 2024-09-14T03:56:06.101 INFO:tasks.rados.rados.0.smithi070.stdout:2253: done (0 left) 2024-09-14T03:56:06.101 INFO:tasks.rados.rados.0.smithi070.stdout:2254: snap_create 2024-09-14T03:56:06.526 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 03:56:06 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T03:56:06.124+0000 7f90c8a2a640 -1 osd.6 525 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:56:06.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:06 smithi094 ceph-mon[86597]: osdmap e534: 8 total, 7 up, 8 in 2024-09-14T03:56:06.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:06 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:06.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:06 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:06.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:06 smithi094 ceph-mon[86597]: from='osd.6 [v2:172.21.15.94:6816/1180082390,v1:172.21.15.94:6817/1180082390]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-14T03:56:06.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:06 smithi094 ceph-mon[86597]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-14T03:56:06.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:06 smithi070 ceph-mon[98640]: osdmap e534: 8 total, 7 up, 8 in 2024-09-14T03:56:06.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:06 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:06.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:06 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:06.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:06 smithi070 ceph-mon[98640]: from='osd.6 [v2:172.21.15.94:6816/1180082390,v1:172.21.15.94:6817/1180082390]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-14T03:56:06.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:06 smithi070 ceph-mon[98640]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-14T03:56:06.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:06 smithi070 ceph-mon[96704]: osdmap e534: 8 total, 7 up, 8 in 2024-09-14T03:56:06.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:06 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:06.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:06 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:06.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:06 smithi070 ceph-mon[96704]: from='osd.6 [v2:172.21.15.94:6816/1180082390,v1:172.21.15.94:6817/1180082390]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-14T03:56:06.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:06 smithi070 ceph-mon[96704]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-14T03:56:07.113 INFO:tasks.rados.rados.0.smithi070.stdout:2254: done (0 left) 2024-09-14T03:56:07.113 INFO:tasks.rados.rados.0.smithi070.stdout:2255: read oid 21 snap -1 2024-09-14T03:56:07.113 INFO:tasks.rados.rados.0.smithi070.stdout:2255: expect deleted 2024-09-14T03:56:07.113 INFO:tasks.rados.rados.0.smithi070.stdout:2256: delete oid 10 current snap is 242 2024-09-14T03:56:07.125 INFO:tasks.rados.rados.0.smithi070.stdout:2255: done (1 left) 2024-09-14T03:56:07.125 INFO:tasks.rados.rados.0.smithi070.stdout:2256: done (0 left) 2024-09-14T03:56:07.125 INFO:tasks.rados.rados.0.smithi070.stdout:2257: rollback oid 38 current snap is 242 2024-09-14T03:56:07.125 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 38 to 220 2024-09-14T03:56:07.125 INFO:tasks.rados.rados.0.smithi070.stdout:2258: snap_remove snap 179 2024-09-14T03:56:07.180 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:07 smithi094 ceph-mon[86597]: pgmap v398: 121 pgs: 34 active+undersized, 8 active+undersized+degraded, 79 active+clean; 1019 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 553 KiB/s wr, 3 op/s; 93/1278 objects degraded (7.277%) 2024-09-14T03:56:07.180 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:07 smithi094 ceph-mon[86597]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-14T03:56:07.180 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:07 smithi094 ceph-mon[86597]: osdmap e535: 8 total, 7 up, 8 in 2024-09-14T03:56:07.180 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:07 smithi094 ceph-mon[86597]: from='osd.6 [v2:172.21.15.94:6816/1180082390,v1:172.21.15.94:6817/1180082390]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi094", "root=default"]} : dispatch 2024-09-14T03:56:07.180 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:07 smithi094 ceph-mon[86597]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi094", "root=default"]} : dispatch 2024-09-14T03:56:07.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:07 smithi070 ceph-mon[96704]: pgmap v398: 121 pgs: 34 active+undersized, 8 active+undersized+degraded, 79 active+clean; 1019 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 553 KiB/s wr, 3 op/s; 93/1278 objects degraded (7.277%) 2024-09-14T03:56:07.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:07 smithi070 ceph-mon[96704]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-14T03:56:07.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:07 smithi070 ceph-mon[96704]: osdmap e535: 8 total, 7 up, 8 in 2024-09-14T03:56:07.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:07 smithi070 ceph-mon[96704]: from='osd.6 [v2:172.21.15.94:6816/1180082390,v1:172.21.15.94:6817/1180082390]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi094", "root=default"]} : dispatch 2024-09-14T03:56:07.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:07 smithi070 ceph-mon[96704]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi094", "root=default"]} : dispatch 2024-09-14T03:56:07.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:07 smithi070 ceph-mon[98640]: pgmap v398: 121 pgs: 34 active+undersized, 8 active+undersized+degraded, 79 active+clean; 1019 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 553 KiB/s wr, 3 op/s; 93/1278 objects degraded (7.277%) 2024-09-14T03:56:07.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:07 smithi070 ceph-mon[98640]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-14T03:56:07.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:07 smithi070 ceph-mon[98640]: osdmap e535: 8 total, 7 up, 8 in 2024-09-14T03:56:07.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:07 smithi070 ceph-mon[98640]: from='osd.6 [v2:172.21.15.94:6816/1180082390,v1:172.21.15.94:6817/1180082390]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi094", "root=default"]} : dispatch 2024-09-14T03:56:07.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:07 smithi070 ceph-mon[98640]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi094", "root=default"]} : dispatch 2024-09-14T03:56:08.116 INFO:tasks.rados.rados.0.smithi070.stdout:2258: done (1 left) 2024-09-14T03:56:08.117 INFO:tasks.rados.rados.0.smithi070.stdout:2259: snap_create 2024-09-14T03:56:08.117 INFO:tasks.rados.rados.0.smithi070.stdout:2257: finishing rollback tid 0 to smithi07085154-38 2024-09-14T03:56:08.117 INFO:tasks.rados.rados.0.smithi070.stdout:2257: finishing rollback tid 1 to smithi07085154-38 2024-09-14T03:56:08.117 INFO:tasks.rados.rados.0.smithi070.stdout:2257: finishing rollback tid 2 to smithi07085154-38 2024-09-14T03:56:08.117 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 38 v 1708 (ObjNum 645 snap 216 seq_num 645) dirty exists 2024-09-14T03:56:08.196 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:08 smithi094 ceph-mon[86597]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:56:08.196 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:08 smithi094 ceph-mon[86597]: osd.6 [v2:172.21.15.94:6816/1180082390,v1:172.21.15.94:6817/1180082390] boot 2024-09-14T03:56:08.196 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:08 smithi094 ceph-mon[86597]: osdmap e536: 8 total, 8 up, 8 in 2024-09-14T03:56:08.196 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:08 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-14T03:56:08.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:08 smithi070 ceph-mon[96704]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:56:08.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:08 smithi070 ceph-mon[96704]: osd.6 [v2:172.21.15.94:6816/1180082390,v1:172.21.15.94:6817/1180082390] boot 2024-09-14T03:56:08.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:08 smithi070 ceph-mon[96704]: osdmap e536: 8 total, 8 up, 8 in 2024-09-14T03:56:08.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:08 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-14T03:56:08.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:08 smithi070 ceph-mon[98640]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:56:08.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:08 smithi070 ceph-mon[98640]: osd.6 [v2:172.21.15.94:6816/1180082390,v1:172.21.15.94:6817/1180082390] boot 2024-09-14T03:56:08.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:08 smithi070 ceph-mon[98640]: osdmap e536: 8 total, 8 up, 8 in 2024-09-14T03:56:08.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:08 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-14T03:56:09.119 INFO:tasks.rados.rados.0.smithi070.stdout:2257: done (1 left) 2024-09-14T03:56:09.119 INFO:tasks.rados.rados.0.smithi070.stdout:2259: done (0 left) 2024-09-14T03:56:09.120 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 40 current snap is 243 2024-09-14T03:56:09.120 INFO:tasks.rados.rados.0.smithi070.stdout:2260: seq_num 711 ranges {1769472=712704,2482176=786432,3268608=516096,3784704=303104} 2024-09-14T03:56:09.132 INFO:tasks.rados.rados.0.smithi070.stdout:2260: writing smithi07085154-40 from 1769472 to 2482176 tid 1 2024-09-14T03:56:09.137 INFO:tasks.rados.rados.0.smithi070.stdout:2260: writing smithi07085154-40 from 2482176 to 3268608 tid 2 2024-09-14T03:56:09.140 INFO:tasks.rados.rados.0.smithi070.stdout:2260: writing smithi07085154-40 from 3268608 to 3784704 tid 3 2024-09-14T03:56:09.142 INFO:tasks.rados.rados.0.smithi070.stdout:2260: writing smithi07085154-40 from 3784704 to 4087808 tid 4 2024-09-14T03:56:09.142 INFO:tasks.rados.rados.0.smithi070.stdout:2261: setattr oid 18 current snap is 243 2024-09-14T03:56:09.142 INFO:tasks.rados.rados.0.smithi070.stdout:2260: finishing write tid 1 to smithi07085154-40 2024-09-14T03:56:09.142 INFO:tasks.rados.rados.0.smithi070.stdout:2260: finishing write tid 2 to smithi07085154-40 2024-09-14T03:56:09.144 INFO:tasks.rados.rados.0.smithi070.stdout:2262: setattr oid 7 current snap is 243 2024-09-14T03:56:09.145 INFO:tasks.rados.rados.0.smithi070.stdout:2260: finishing write tid 3 to smithi07085154-40 2024-09-14T03:56:09.145 INFO:tasks.rados.rados.0.smithi070.stdout:2260: finishing write tid 4 to smithi07085154-40 2024-09-14T03:56:09.145 INFO:tasks.rados.rados.0.smithi070.stdout:2260: finishing write tid 5 to smithi07085154-40 2024-09-14T03:56:09.146 INFO:tasks.rados.rados.0.smithi070.stdout:2260: finishing write tid 6 to smithi07085154-40 2024-09-14T03:56:09.146 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 2049 (ObjNum 711 snap 243 seq_num 711) dirty exists 2024-09-14T03:56:09.146 INFO:tasks.rados.rados.0.smithi070.stdout:2260: left oid 40 (ObjNum 711 snap 243 seq_num 711) 2024-09-14T03:56:09.147 INFO:tasks.rados.rados.0.smithi070.stdout:2260: done (2 left) 2024-09-14T03:56:09.147 INFO:tasks.rados.rados.0.smithi070.stdout:2263: rollback oid 19 current snap is 243 2024-09-14T03:56:09.147 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 19 to 162 2024-09-14T03:56:09.147 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 1 current snap is 243 2024-09-14T03:56:09.147 INFO:tasks.rados.rados.0.smithi070.stdout:2264: seq_num 712 ranges {0=729088,729088=671744,1400832=671744,2072576=262144} 2024-09-14T03:56:09.152 INFO:tasks.rados.rados.0.smithi070.stdout:2264: writing smithi07085154-1 from 0 to 729088 tid 1 2024-09-14T03:56:09.156 INFO:tasks.rados.rados.0.smithi070.stdout:2264: writing smithi07085154-1 from 729088 to 1400832 tid 2 2024-09-14T03:56:09.159 INFO:tasks.rados.rados.0.smithi070.stdout:2264: writing smithi07085154-1 from 1400832 to 2072576 tid 3 2024-09-14T03:56:09.161 INFO:tasks.rados.rados.0.smithi070.stdout:2264: writing smithi07085154-1 from 2072576 to 2334720 tid 4 2024-09-14T03:56:09.162 INFO:tasks.rados.rados.0.smithi070.stdout:2265: read oid 26 snap 222 2024-09-14T03:56:09.162 INFO:tasks.rados.rados.0.smithi070.stdout:2265: expect deleted 2024-09-14T03:56:09.162 INFO:tasks.rados.rados.0.smithi070.stdout:2266: copy_from oid 23 from oid 25 current snap is 243 2024-09-14T03:56:09.162 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 2045 (ObjNum 671 snap 228 seq_num 671) dirty exists 2024-09-14T03:56:09.162 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 1953 (ObjNum 454 snap 154 seq_num 454) dirty exists 2024-09-14T03:56:09.162 INFO:tasks.rados.rados.0.smithi070.stdout:2263: finishing rollback tid 0 to smithi07085154-19 2024-09-14T03:56:09.162 INFO:tasks.rados.rados.0.smithi070.stdout:2263: finishing rollback tid 1 to smithi07085154-19 2024-09-14T03:56:09.162 INFO:tasks.rados.rados.0.smithi070.stdout:2263: finishing rollback tid 2 to smithi07085154-19 2024-09-14T03:56:09.162 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 19 v 1718 (ObjNum 466 snap 158 seq_num 466) dirty exists 2024-09-14T03:56:09.162 INFO:tasks.rados.rados.0.smithi070.stdout:2264: finishing write tid 1 to smithi07085154-1 2024-09-14T03:56:09.162 INFO:tasks.rados.rados.0.smithi070.stdout:2264: finishing write tid 2 to smithi07085154-1 2024-09-14T03:56:09.162 INFO:tasks.rados.rados.0.smithi070.stdout:2261: done (5 left) 2024-09-14T03:56:09.163 INFO:tasks.rados.rados.0.smithi070.stdout:2262: done (4 left) 2024-09-14T03:56:09.163 INFO:tasks.rados.rados.0.smithi070.stdout:2263: done (3 left) 2024-09-14T03:56:09.163 INFO:tasks.rados.rados.0.smithi070.stdout:2267: read oid 4 snap 162 2024-09-14T03:56:09.163 INFO:tasks.rados.rados.0.smithi070.stdout:2267: expect (ObjNum 289 snap 91 seq_num 289) 2024-09-14T03:56:09.163 INFO:tasks.rados.rados.0.smithi070.stdout:2268: snap_create 2024-09-14T03:56:09.163 INFO:tasks.rados.rados.0.smithi070.stdout:2267: expect (ObjNum 289 snap 91 seq_num 289) 2024-09-14T03:56:09.164 INFO:tasks.rados.rados.0.smithi070.stdout:2264: finishing write tid 3 to smithi07085154-1 2024-09-14T03:56:09.165 INFO:tasks.rados.rados.0.smithi070.stdout:2264: finishing write tid 4 to smithi07085154-1 2024-09-14T03:56:09.167 INFO:tasks.rados.rados.0.smithi070.stdout:2264: finishing write tid 5 to smithi07085154-1 2024-09-14T03:56:09.168 INFO:tasks.rados.rados.0.smithi070.stdout:2264: finishing write tid 6 to smithi07085154-1 2024-09-14T03:56:09.168 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 1 v 2054 (ObjNum 712 snap 243 seq_num 712) dirty exists 2024-09-14T03:56:09.168 INFO:tasks.rados.rados.0.smithi070.stdout:2264: left oid 1 (ObjNum 712 snap 243 seq_num 712) 2024-09-14T03:56:09.227 INFO:tasks.rados.rados.0.smithi070.stdout:2266: finishing copy_from racing read to smithi07085154-23 2024-09-14T03:56:09.229 INFO:tasks.rados.rados.0.smithi070.stdout:2266: finishing copy_from to smithi07085154-23 2024-09-14T03:56:09.229 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 2208 (ObjNum 702 snap 236 seq_num 702) dirty exists 2024-09-14T03:56:09.285 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:56:08 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T03:56:08.900+0000 7fa4af5ba640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (15 PGs are or would become offline) 2024-09-14T03:56:09.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:09 smithi070 ceph-mon[96704]: pgmap v401: 121 pgs: 7 peering, 30 active+undersized, 5 active+undersized+degraded, 79 active+clean; 1.0 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s rd, 1.5 MiB/s wr, 6 op/s; 47/1248 objects degraded (3.766%) 2024-09-14T03:56:09.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:09 smithi070 ceph-mon[96704]: osdmap e537: 8 total, 8 up, 8 in 2024-09-14T03:56:09.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:09 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:09.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:09 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:09.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:09 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:56:09.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:09 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:56:09.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:09 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:09.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:09 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:56:09.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:09 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:09.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:09 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:09.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:09 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:09.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:09 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-14T03:56:09.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:09 smithi070 ceph-mon[98640]: pgmap v401: 121 pgs: 7 peering, 30 active+undersized, 5 active+undersized+degraded, 79 active+clean; 1.0 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s rd, 1.5 MiB/s wr, 6 op/s; 47/1248 objects degraded (3.766%) 2024-09-14T03:56:09.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:09 smithi070 ceph-mon[98640]: osdmap e537: 8 total, 8 up, 8 in 2024-09-14T03:56:09.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:09 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:09.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:09 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:09.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:09 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:56:09.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:09 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:56:09.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:09 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:09.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:09 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:56:09.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:09 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:09.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:09 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:09.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:09 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:09.288 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:09 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-14T03:56:09.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:09 smithi094 ceph-mon[86597]: pgmap v401: 121 pgs: 7 peering, 30 active+undersized, 5 active+undersized+degraded, 79 active+clean; 1.0 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s rd, 1.5 MiB/s wr, 6 op/s; 47/1248 objects degraded (3.766%) 2024-09-14T03:56:09.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:09 smithi094 ceph-mon[86597]: osdmap e537: 8 total, 8 up, 8 in 2024-09-14T03:56:09.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:09 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:09.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:09 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:09.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:09 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:56:09.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:09 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:56:09.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:09 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:09.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:09 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:56:09.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:09 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:09.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:09 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:09.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:09 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:09.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:09 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-14T03:56:10.125 INFO:tasks.rados.rados.0.smithi070.stdout:2264: done (4 left) 2024-09-14T03:56:10.125 INFO:tasks.rados.rados.0.smithi070.stdout:2265: done (3 left) 2024-09-14T03:56:10.125 INFO:tasks.rados.rados.0.smithi070.stdout:2266: done (2 left) 2024-09-14T03:56:10.125 INFO:tasks.rados.rados.0.smithi070.stdout:2267: done (1 left) 2024-09-14T03:56:10.125 INFO:tasks.rados.rados.0.smithi070.stdout:2268: done (0 left) 2024-09-14T03:56:10.125 INFO:tasks.rados.rados.0.smithi070.stdout:2269: copy_from oid 23 from oid 16 current snap is 244 2024-09-14T03:56:10.126 INFO:tasks.rados.rados.0.smithi070.stdout:2270: setattr oid 43 current snap is 244 2024-09-14T03:56:10.126 INFO:tasks.rados.rados.0.smithi070.stdout:2271: snap_create 2024-09-14T03:56:10.132 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 2046 (ObjNum 681 snap 230 seq_num 681) dirty exists 2024-09-14T03:56:10.153 INFO:tasks.rados.rados.0.smithi070.stdout:2269: finishing copy_from racing read to smithi07085154-23 2024-09-14T03:56:10.154 INFO:tasks.rados.rados.0.smithi070.stdout:2269: finishing copy_from to smithi07085154-23 2024-09-14T03:56:10.155 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 2247 (ObjNum 114 snap 24 seq_num 114) dirty exists 2024-09-14T03:56:10.429 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:10 smithi094 ceph-mon[86597]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-14T03:56:10.429 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:10 smithi094 ceph-mon[86597]: Upgrade: unsafe to stop osd(s) at this time (15 PGs are or would become offline) 2024-09-14T03:56:10.429 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:10 smithi094 ceph-mon[86597]: osdmap e538: 8 total, 8 up, 8 in 2024-09-14T03:56:10.429 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:56:10 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:56:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:56:10.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:10 smithi070 ceph-mon[96704]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-14T03:56:10.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:10 smithi070 ceph-mon[96704]: Upgrade: unsafe to stop osd(s) at this time (15 PGs are or would become offline) 2024-09-14T03:56:10.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:10 smithi070 ceph-mon[96704]: osdmap e538: 8 total, 8 up, 8 in 2024-09-14T03:56:10.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:10 smithi070 ceph-mon[98640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-14T03:56:10.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:10 smithi070 ceph-mon[98640]: Upgrade: unsafe to stop osd(s) at this time (15 PGs are or would become offline) 2024-09-14T03:56:10.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:10 smithi070 ceph-mon[98640]: osdmap e538: 8 total, 8 up, 8 in 2024-09-14T03:56:11.127 INFO:tasks.rados.rados.0.smithi070.stdout:2269: done (2 left) 2024-09-14T03:56:11.127 INFO:tasks.rados.rados.0.smithi070.stdout:2270: done (1 left) 2024-09-14T03:56:11.127 INFO:tasks.rados.rados.0.smithi070.stdout:2271: done (0 left) 2024-09-14T03:56:11.127 INFO:tasks.rados.rados.0.smithi070.stdout:2272: copy_from oid 9 from oid 43 current snap is 245 2024-09-14T03:56:11.127 INFO:tasks.rados.rados.0.smithi070.stdout:2273: copy_from oid 20 from oid 18 current snap is 245 2024-09-14T03:56:11.127 INFO:tasks.rados.rados.0.smithi070.stdout:2274: read oid 15 snap 201 2024-09-14T03:56:11.128 INFO:tasks.rados.rados.0.smithi070.stdout:2274: expect deleted 2024-09-14T03:56:11.128 INFO:tasks.rados.rados.0.smithi070.stdout:2275: read oid 44 snap -1 2024-09-14T03:56:11.128 INFO:tasks.rados.rados.0.smithi070.stdout:2275: expect deleted 2024-09-14T03:56:11.128 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 22 current snap is 245 2024-09-14T03:56:11.128 INFO:tasks.rados.rados.0.smithi070.stdout:2276: seq_num 713 ranges {2809856=745472,3555328=557056,4112384=483328,4595712=565248} 2024-09-14T03:56:11.146 INFO:tasks.rados.rados.0.smithi070.stdout:2276: writing smithi07085154-22 from 2809856 to 3555328 tid 1 2024-09-14T03:56:11.149 INFO:tasks.rados.rados.0.smithi070.stdout:2276: writing smithi07085154-22 from 3555328 to 4112384 tid 2 2024-09-14T03:56:11.152 INFO:tasks.rados.rados.0.smithi070.stdout:2276: writing smithi07085154-22 from 4112384 to 4595712 tid 3 2024-09-14T03:56:11.156 INFO:tasks.rados.rados.0.smithi070.stdout:2276: writing smithi07085154-22 from 4595712 to 5160960 tid 4 2024-09-14T03:56:11.156 INFO:tasks.rados.rados.0.smithi070.stdout:2277: read oid 39 snap -1 2024-09-14T03:56:11.156 INFO:tasks.rados.rados.0.smithi070.stdout:2277: expect (ObjNum 460 snap 155 seq_num 460) 2024-09-14T03:56:11.156 INFO:tasks.rados.rados.0.smithi070.stdout:2278: read oid 17 snap 240 2024-09-14T03:56:11.156 INFO:tasks.rados.rados.0.smithi070.stdout:2278: expect (ObjNum 670 snap 228 seq_num 670) 2024-09-14T03:56:11.156 INFO:tasks.rados.rados.0.smithi070.stdout:2279: snap_create 2024-09-14T03:56:11.157 INFO:tasks.rados.rados.0.smithi070.stdout:2276: finishing write tid 1 to smithi07085154-22 2024-09-14T03:56:11.157 INFO:tasks.rados.rados.0.smithi070.stdout:2276: finishing write tid 2 to smithi07085154-22 2024-09-14T03:56:11.157 INFO:tasks.rados.rados.0.smithi070.stdout:2276: finishing write tid 3 to smithi07085154-22 2024-09-14T03:56:11.163 INFO:tasks.rados.rados.0.smithi070.stdout:2276: finishing write tid 4 to smithi07085154-22 2024-09-14T03:56:11.165 INFO:tasks.rados.rados.0.smithi070.stdout:2276: finishing write tid 5 to smithi07085154-22 2024-09-14T03:56:11.167 INFO:tasks.rados.rados.0.smithi070.stdout:2272: finishing copy_from racing read to smithi07085154-9 2024-09-14T03:56:11.169 INFO:tasks.rados.rados.0.smithi070.stdout:2276: finishing write tid 6 to smithi07085154-22 2024-09-14T03:56:11.169 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 1723 (ObjNum 713 snap 245 seq_num 713) dirty exists 2024-09-14T03:56:11.169 INFO:tasks.rados.rados.0.smithi070.stdout:2276: left oid 22 (ObjNum 713 snap 245 seq_num 713) 2024-09-14T03:56:11.171 INFO:tasks.rados.rados.0.smithi070.stdout:2273: finishing copy_from racing read to smithi07085154-20 2024-09-14T03:56:11.172 INFO:tasks.rados.rados.0.smithi070.stdout:2272: finishing copy_from to smithi07085154-9 2024-09-14T03:56:11.172 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 2046 (ObjNum 681 snap 230 seq_num 681) dirty exists 2024-09-14T03:56:11.172 INFO:tasks.rados.rados.0.smithi070.stdout:2273: finishing copy_from to smithi07085154-20 2024-09-14T03:56:11.172 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 20 v 2045 (ObjNum 671 snap 228 seq_num 671) dirty exists 2024-09-14T03:56:11.175 INFO:tasks.rados.rados.0.smithi070.stdout:2278: expect (ObjNum 670 snap 228 seq_num 670) 2024-09-14T03:56:11.293 INFO:tasks.rados.rados.0.smithi070.stdout:2277: expect (ObjNum 460 snap 155 seq_num 460) 2024-09-14T03:56:11.444 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:11 smithi094 ceph-mon[86597]: pgmap v404: 121 pgs: 7 peering, 13 active+undersized, 3 active+undersized+degraded, 98 active+clean; 1022 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 1.0 MiB/s wr, 8 op/s; 39/1221 objects degraded (3.194%) 2024-09-14T03:56:11.445 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:11 smithi094 ceph-mon[86597]: Health check update: Degraded data redundancy: 39/1221 objects degraded (3.194%), 3 pgs degraded (PG_DEGRADED) 2024-09-14T03:56:11.445 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:11 smithi094 ceph-mon[86597]: osdmap e539: 8 total, 8 up, 8 in 2024-09-14T03:56:11.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:11 smithi070 ceph-mon[98640]: pgmap v404: 121 pgs: 7 peering, 13 active+undersized, 3 active+undersized+degraded, 98 active+clean; 1022 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 1.0 MiB/s wr, 8 op/s; 39/1221 objects degraded (3.194%) 2024-09-14T03:56:11.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:11 smithi070 ceph-mon[98640]: Health check update: Degraded data redundancy: 39/1221 objects degraded (3.194%), 3 pgs degraded (PG_DEGRADED) 2024-09-14T03:56:11.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:11 smithi070 ceph-mon[98640]: osdmap e539: 8 total, 8 up, 8 in 2024-09-14T03:56:11.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:11 smithi070 ceph-mon[96704]: pgmap v404: 121 pgs: 7 peering, 13 active+undersized, 3 active+undersized+degraded, 98 active+clean; 1022 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 1.0 MiB/s wr, 8 op/s; 39/1221 objects degraded (3.194%) 2024-09-14T03:56:11.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:11 smithi070 ceph-mon[96704]: Health check update: Degraded data redundancy: 39/1221 objects degraded (3.194%), 3 pgs degraded (PG_DEGRADED) 2024-09-14T03:56:11.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:11 smithi070 ceph-mon[96704]: osdmap e539: 8 total, 8 up, 8 in 2024-09-14T03:56:12.136 INFO:tasks.rados.rados.0.smithi070.stdout:2272: done (7 left) 2024-09-14T03:56:12.136 INFO:tasks.rados.rados.0.smithi070.stdout:2273: done (6 left) 2024-09-14T03:56:12.137 INFO:tasks.rados.rados.0.smithi070.stdout:2274: done (5 left) 2024-09-14T03:56:12.137 INFO:tasks.rados.rados.0.smithi070.stdout:2275: done (4 left) 2024-09-14T03:56:12.137 INFO:tasks.rados.rados.0.smithi070.stdout:2276: done (3 left) 2024-09-14T03:56:12.137 INFO:tasks.rados.rados.0.smithi070.stdout:2277: done (2 left) 2024-09-14T03:56:12.137 INFO:tasks.rados.rados.0.smithi070.stdout:2278: done (1 left) 2024-09-14T03:56:12.137 INFO:tasks.rados.rados.0.smithi070.stdout:2279: done (0 left) 2024-09-14T03:56:12.137 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 27 current snap is 246 2024-09-14T03:56:12.137 INFO:tasks.rados.rados.0.smithi070.stdout:2280: seq_num 714 ranges {16457728=761856,17219584=442368,17661952=688128,18350080=475136} 2024-09-14T03:56:12.226 INFO:tasks.rados.rados.0.smithi070.stdout:2280: writing smithi07085154-27 from 16457728 to 17219584 tid 1 2024-09-14T03:56:12.229 INFO:tasks.rados.rados.0.smithi070.stdout:2280: writing smithi07085154-27 from 17219584 to 17661952 tid 2 2024-09-14T03:56:12.233 INFO:tasks.rados.rados.0.smithi070.stdout:2280: writing smithi07085154-27 from 17661952 to 18350080 tid 3 2024-09-14T03:56:12.236 INFO:tasks.rados.rados.0.smithi070.stdout:2280: writing smithi07085154-27 from 18350080 to 18825216 tid 4 2024-09-14T03:56:12.236 INFO:tasks.rados.rados.0.smithi070.stdout:2281: setattr oid 31 current snap is 246 2024-09-14T03:56:12.236 INFO:tasks.rados.rados.0.smithi070.stdout:2280: finishing write tid 1 to smithi07085154-27 2024-09-14T03:56:12.236 INFO:tasks.rados.rados.0.smithi070.stdout:2280: finishing write tid 2 to smithi07085154-27 2024-09-14T03:56:12.237 INFO:tasks.rados.rados.0.smithi070.stdout:2282: read oid 26 snap -1 2024-09-14T03:56:12.237 INFO:tasks.rados.rados.0.smithi070.stdout:2282: expect (ObjNum 629 snap 211 seq_num 629) 2024-09-14T03:56:12.237 INFO:tasks.rados.rados.0.smithi070.stdout:2283: rollback oid 36 current snap is 246 2024-09-14T03:56:12.237 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 36 to 226 2024-09-14T03:56:12.237 INFO:tasks.rados.rados.0.smithi070.stdout:2284: rmattr oid 2 current snap is 246 2024-09-14T03:56:12.238 INFO:tasks.rados.rados.0.smithi070.stdout:2284: done (4 left) 2024-09-14T03:56:12.238 INFO:tasks.rados.rados.0.smithi070.stdout:2285: delete oid 34 current snap is 246 2024-09-14T03:56:12.238 INFO:tasks.rados.rados.0.smithi070.stdout:2280: finishing write tid 3 to smithi07085154-27 2024-09-14T03:56:12.241 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 2209 (ObjNum 670 snap 228 seq_num 670) dirty exists 2024-09-14T03:56:12.242 INFO:tasks.rados.rados.0.smithi070.stdout:2281: done (4 left) 2024-09-14T03:56:12.242 INFO:tasks.rados.rados.0.smithi070.stdout:2285: done (3 left) 2024-09-14T03:56:12.242 INFO:tasks.rados.rados.0.smithi070.stdout:2286: delete oid 4 current snap is 246 2024-09-14T03:56:12.243 INFO:tasks.rados.rados.0.smithi070.stdout:2283: finishing rollback tid 0 to smithi07085154-36 2024-09-14T03:56:12.243 INFO:tasks.rados.rados.0.smithi070.stdout:2283: finishing rollback tid 1 to smithi07085154-36 2024-09-14T03:56:12.243 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 36 v 2056 (ObjNum 81 snap 0 seq_num 2931183792) dirty dne 2024-09-14T03:56:12.244 INFO:tasks.rados.rados.0.smithi070.stdout:2280: finishing write tid 4 to smithi07085154-27 2024-09-14T03:56:12.244 INFO:tasks.rados.rados.0.smithi070.stdout:2280: finishing write tid 5 to smithi07085154-27 2024-09-14T03:56:12.245 INFO:tasks.rados.rados.0.smithi070.stdout:2283: done (3 left) 2024-09-14T03:56:12.245 INFO:tasks.rados.rados.0.smithi070.stdout:2286: done (2 left) 2024-09-14T03:56:12.245 INFO:tasks.rados.rados.0.smithi070.stdout:2287: snap_create 2024-09-14T03:56:12.246 INFO:tasks.rados.rados.0.smithi070.stdout:2282: expect (ObjNum 629 snap 211 seq_num 629) 2024-09-14T03:56:12.270 INFO:tasks.rados.rados.0.smithi070.stdout:2280: finishing write tid 6 to smithi07085154-27 2024-09-14T03:56:12.270 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 2498 (ObjNum 714 snap 246 seq_num 714) dirty exists 2024-09-14T03:56:12.270 INFO:tasks.rados.rados.0.smithi070.stdout:2280: left oid 27 (ObjNum 714 snap 246 seq_num 714) 2024-09-14T03:56:12.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:12 smithi094 ceph-mon[86597]: osdmap e540: 8 total, 8 up, 8 in 2024-09-14T03:56:12.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:12 smithi094 ceph-mon[86597]: pgmap v407: 121 pgs: 7 peering, 114 active+clean; 1.0 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 8.5 MiB/s wr, 13 op/s; 1.0 MiB/s, 0 objects/s recovering 2024-09-14T03:56:12.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:12 smithi070 ceph-mon[96704]: osdmap e540: 8 total, 8 up, 8 in 2024-09-14T03:56:12.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:12 smithi070 ceph-mon[96704]: pgmap v407: 121 pgs: 7 peering, 114 active+clean; 1.0 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 8.5 MiB/s wr, 13 op/s; 1.0 MiB/s, 0 objects/s recovering 2024-09-14T03:56:12.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:12 smithi070 ceph-mon[98640]: osdmap e540: 8 total, 8 up, 8 in 2024-09-14T03:56:12.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:12 smithi070 ceph-mon[98640]: pgmap v407: 121 pgs: 7 peering, 114 active+clean; 1.0 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 8.5 MiB/s wr, 13 op/s; 1.0 MiB/s, 0 objects/s recovering 2024-09-14T03:56:13.140 INFO:tasks.rados.rados.0.smithi070.stdout:2280: done (2 left) 2024-09-14T03:56:13.140 INFO:tasks.rados.rados.0.smithi070.stdout:2282: done (1 left) 2024-09-14T03:56:13.140 INFO:tasks.rados.rados.0.smithi070.stdout:2287: done (0 left) 2024-09-14T03:56:13.140 INFO:tasks.rados.rados.0.smithi070.stdout:2288: rmattr oid 43 current snap is 247 2024-09-14T03:56:13.140 INFO:tasks.rados.rados.0.smithi070.stdout:2289: read oid 29 snap -1 2024-09-14T03:56:13.141 INFO:tasks.rados.rados.0.smithi070.stdout:2289: expect (ObjNum 458 snap 155 seq_num 458) 2024-09-14T03:56:13.141 INFO:tasks.rados.rados.0.smithi070.stdout:2290: read oid 15 snap 242 2024-09-14T03:56:13.141 INFO:tasks.rados.rados.0.smithi070.stdout:2290: expect deleted 2024-09-14T03:56:13.141 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 7 current snap is 247 2024-09-14T03:56:13.141 INFO:tasks.rados.rados.0.smithi070.stdout:2291: seq_num 717 ranges {5513216=16384} 2024-09-14T03:56:13.170 INFO:tasks.rados.rados.0.smithi070.stdout:2291: writing smithi07085154-7 from 5513216 to 5529600 tid 1 2024-09-14T03:56:13.170 INFO:tasks.rados.rados.0.smithi070.stdout:2292: snap_create 2024-09-14T03:56:13.170 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 2047 (ObjNum 681 snap 230 seq_num 681) dirty exists 2024-09-14T03:56:13.170 INFO:tasks.rados.rados.0.smithi070.stdout:2289: expect (ObjNum 458 snap 155 seq_num 458) 2024-09-14T03:56:13.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:13 smithi070 ceph-mon[96704]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 39/1221 objects degraded (3.194%), 3 pgs degraded) 2024-09-14T03:56:13.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:13 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T03:56:13.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:13 smithi070 ceph-mon[96704]: osdmap e541: 8 total, 8 up, 8 in 2024-09-14T03:56:13.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:13 smithi070 ceph-mon[98640]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 39/1221 objects degraded (3.194%), 3 pgs degraded) 2024-09-14T03:56:13.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:13 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T03:56:13.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:13 smithi070 ceph-mon[98640]: osdmap e541: 8 total, 8 up, 8 in 2024-09-14T03:56:13.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:13 smithi094 ceph-mon[86597]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 39/1221 objects degraded (3.194%), 3 pgs degraded) 2024-09-14T03:56:13.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:13 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T03:56:13.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:13 smithi094 ceph-mon[86597]: osdmap e541: 8 total, 8 up, 8 in 2024-09-14T03:56:13.712 INFO:tasks.rados.rados.0.smithi070.stdout:2291: finishing write tid 1 to smithi07085154-7 2024-09-14T03:56:13.713 INFO:tasks.rados.rados.0.smithi070.stdout:2291: finishing write tid 2 to smithi07085154-7 2024-09-14T03:56:13.713 INFO:tasks.rados.rados.0.smithi070.stdout:2291: finishing write tid 3 to smithi07085154-7 2024-09-14T03:56:13.713 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 2048 (ObjNum 717 snap 247 seq_num 717) dirty exists 2024-09-14T03:56:13.713 INFO:tasks.rados.rados.0.smithi070.stdout:2291: left oid 7 (ObjNum 717 snap 247 seq_num 717) 2024-09-14T03:56:13.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:56:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:56:13] "GET /metrics HTTP/1.1" 200 36307 "" "Prometheus/2.51.0" 2024-09-14T03:56:14.146 INFO:tasks.rados.rados.0.smithi070.stdout:2288: done (4 left) 2024-09-14T03:56:14.146 INFO:tasks.rados.rados.0.smithi070.stdout:2289: done (3 left) 2024-09-14T03:56:14.146 INFO:tasks.rados.rados.0.smithi070.stdout:2290: done (2 left) 2024-09-14T03:56:14.146 INFO:tasks.rados.rados.0.smithi070.stdout:2291: done (1 left) 2024-09-14T03:56:14.146 INFO:tasks.rados.rados.0.smithi070.stdout:2292: done (0 left) 2024-09-14T03:56:14.146 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 32 current snap is 248 2024-09-14T03:56:14.146 INFO:tasks.rados.rados.0.smithi070.stdout:2293: seq_num 718 ranges {11223040=32768} 2024-09-14T03:56:14.205 INFO:tasks.rados.rados.0.smithi070.stdout:2293: writing smithi07085154-32 from 11223040 to 11255808 tid 1 2024-09-14T03:56:14.205 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 12 current snap is 248 2024-09-14T03:56:14.205 INFO:tasks.rados.rados.0.smithi070.stdout:2294: seq_num 719 ranges {20447232=49152} 2024-09-14T03:56:14.312 INFO:tasks.rados.rados.0.smithi070.stdout:2294: writing smithi07085154-12 from 20447232 to 20496384 tid 1 2024-09-14T03:56:14.312 INFO:tasks.rados.rados.0.smithi070.stdout:2295: rmattr oid 2 current snap is 248 2024-09-14T03:56:14.312 INFO:tasks.rados.rados.0.smithi070.stdout:2295: done (2 left) 2024-09-14T03:56:14.312 INFO:tasks.rados.rados.0.smithi070.stdout:2296: rollback oid 46 current snap is 248 2024-09-14T03:56:14.312 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 46 to 214 2024-09-14T03:56:14.312 INFO:tasks.rados.rados.0.smithi070.stdout:2293: finishing write tid 1 to smithi07085154-32 2024-09-14T03:56:14.312 INFO:tasks.rados.rados.0.smithi070.stdout:2293: finishing write tid 2 to smithi07085154-32 2024-09-14T03:56:14.313 INFO:tasks.rados.rados.0.smithi070.stdout:2293: finishing write tid 3 to smithi07085154-32 2024-09-14T03:56:14.313 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 2053 (ObjNum 718 snap 248 seq_num 718) dirty exists 2024-09-14T03:56:14.313 INFO:tasks.rados.rados.0.smithi070.stdout:2293: left oid 32 (ObjNum 718 snap 248 seq_num 718) 2024-09-14T03:56:14.313 INFO:tasks.rados.rados.0.smithi070.stdout:2293: done (2 left) 2024-09-14T03:56:14.313 INFO:tasks.rados.rados.0.smithi070.stdout:2297: delete oid 14 current snap is 248 2024-09-14T03:56:14.316 INFO:tasks.rados.rados.0.smithi070.stdout:2294: finishing write tid 1 to smithi07085154-12 2024-09-14T03:56:14.316 INFO:tasks.rados.rados.0.smithi070.stdout:2294: finishing write tid 2 to smithi07085154-12 2024-09-14T03:56:14.316 INFO:tasks.rados.rados.0.smithi070.stdout:2296: finishing rollback tid 0 to smithi07085154-46 2024-09-14T03:56:14.317 INFO:tasks.rados.rados.0.smithi070.stdout:2297: done (2 left) 2024-09-14T03:56:14.317 INFO:tasks.rados.rados.0.smithi070.stdout:2298: setattr oid 10 current snap is 248 2024-09-14T03:56:14.318 INFO:tasks.rados.rados.0.smithi070.stdout:2294: finishing write tid 3 to smithi07085154-12 2024-09-14T03:56:14.318 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 2199 (ObjNum 719 snap 248 seq_num 719) dirty exists 2024-09-14T03:56:14.318 INFO:tasks.rados.rados.0.smithi070.stdout:2294: left oid 12 (ObjNum 719 snap 248 seq_num 719) 2024-09-14T03:56:14.318 INFO:tasks.rados.rados.0.smithi070.stdout:2296: finishing rollback tid 1 to smithi07085154-46 2024-09-14T03:56:14.318 INFO:tasks.rados.rados.0.smithi070.stdout:2296: finishing rollback tid 2 to smithi07085154-46 2024-09-14T03:56:14.318 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 46 v 2502 (ObjNum 603 snap 201 seq_num 603) dirty exists 2024-09-14T03:56:14.319 INFO:tasks.rados.rados.0.smithi070.stdout:2294: done (2 left) 2024-09-14T03:56:14.319 INFO:tasks.rados.rados.0.smithi070.stdout:2296: done (1 left) 2024-09-14T03:56:14.320 INFO:tasks.rados.rados.0.smithi070.stdout:2299: rmattr oid 50 current snap is 248 2024-09-14T03:56:14.320 INFO:tasks.rados.rados.0.smithi070.stdout:2299: done (1 left) 2024-09-14T03:56:14.320 INFO:tasks.rados.rados.0.smithi070.stdout:2300: read oid 44 snap -1 2024-09-14T03:56:14.320 INFO:tasks.rados.rados.0.smithi070.stdout:2300: expect deleted 2024-09-14T03:56:14.320 INFO:tasks.rados.rados.0.smithi070.stdout:2301: copy_from oid 46 from oid 35 current snap is 248 2024-09-14T03:56:14.320 INFO:tasks.rados.rados.0.smithi070.stdout:2302: setattr oid 26 current snap is 248 2024-09-14T03:56:14.321 INFO:tasks.rados.rados.0.smithi070.stdout:2301: finishing copy_from to smithi07085154-46 2024-09-14T03:56:14.322 INFO:tasks.rados.rados.0.smithi070.stdout:2301: got expected ENOENT (src dne) 2024-09-14T03:56:14.322 INFO:tasks.rados.rados.0.smithi070.stdout:2301: finishing copy_from racing read to smithi07085154-46 2024-09-14T03:56:14.322 INFO:tasks.rados.rados.0.smithi070.stdout:2300: done (3 left) 2024-09-14T03:56:14.322 INFO:tasks.rados.rados.0.smithi070.stdout:2301: done (2 left) 2024-09-14T03:56:14.322 INFO:tasks.rados.rados.0.smithi070.stdout:2303: rollback oid 12 current snap is 248 2024-09-14T03:56:14.322 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 12 to 234 2024-09-14T03:56:14.322 INFO:tasks.rados.rados.0.smithi070.stdout:2304: rollback oid 15 current snap is 248 2024-09-14T03:56:14.322 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 15 to 162 2024-09-14T03:56:14.322 INFO:tasks.rados.rados.0.smithi070.stdout:2305: copy_from oid 16 from oid 30 current snap is 248 2024-09-14T03:56:14.322 INFO:tasks.rados.rados.0.smithi070.stdout:2306: read oid 45 snap 245 2024-09-14T03:56:14.322 INFO:tasks.rados.rados.0.smithi070.stdout:2306: expect deleted 2024-09-14T03:56:14.322 INFO:tasks.rados.rados.0.smithi070.stdout:2307: delete oid 22 current snap is 248 2024-09-14T03:56:14.323 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 2503 (ObjNum 97 snap 0 seq_num 2933384544) dirty exists 2024-09-14T03:56:14.325 INFO:tasks.rados.rados.0.smithi070.stdout:2303: finishing rollback tid 0 to smithi07085154-12 2024-09-14T03:56:14.325 INFO:tasks.rados.rados.0.smithi070.stdout:2298: done (6 left) 2024-09-14T03:56:14.325 INFO:tasks.rados.rados.0.smithi070.stdout:2306: done (5 left) 2024-09-14T03:56:14.325 INFO:tasks.rados.rados.0.smithi070.stdout:2307: done (4 left) 2024-09-14T03:56:14.326 INFO:tasks.rados.rados.0.smithi070.stdout:2308: rmattr oid 45 current snap is 248 2024-09-14T03:56:14.326 INFO:tasks.rados.rados.0.smithi070.stdout:2308: done (4 left) 2024-09-14T03:56:14.326 INFO:tasks.rados.rados.0.smithi070.stdout:2309: delete oid 41 current snap is 248 2024-09-14T03:56:14.330 INFO:tasks.rados.rados.0.smithi070.stdout:2303: finishing rollback tid 1 to smithi07085154-12 2024-09-14T03:56:14.330 INFO:tasks.rados.rados.0.smithi070.stdout:2305: finishing copy_from to smithi07085154-16 2024-09-14T03:56:14.330 INFO:tasks.rados.rados.0.smithi070.stdout:2305: got expected ENOENT (src dne) 2024-09-14T03:56:14.330 INFO:tasks.rados.rados.0.smithi070.stdout:2305: finishing copy_from racing read to smithi07085154-16 2024-09-14T03:56:14.330 INFO:tasks.rados.rados.0.smithi070.stdout:2303: finishing rollback tid 2 to smithi07085154-12 2024-09-14T03:56:14.330 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 2202 (ObjNum 656 snap 218 seq_num 656) dirty exists 2024-09-14T03:56:14.330 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 2505 (ObjNum 629 snap 211 seq_num 629) dirty exists 2024-09-14T03:56:14.330 INFO:tasks.rados.rados.0.smithi070.stdout:2304: finishing rollback tid 1 to smithi07085154-15 2024-09-14T03:56:14.330 INFO:tasks.rados.rados.0.smithi070.stdout:2304: finishing rollback tid 2 to smithi07085154-15 2024-09-14T03:56:14.330 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 15 v 2049 (ObjNum 469 snap 160 seq_num 469) dirty exists 2024-09-14T03:56:14.330 INFO:tasks.rados.rados.0.smithi070.stdout:2302: done (4 left) 2024-09-14T03:56:14.330 INFO:tasks.rados.rados.0.smithi070.stdout:2303: done (3 left) 2024-09-14T03:56:14.331 INFO:tasks.rados.rados.0.smithi070.stdout:2304: done (2 left) 2024-09-14T03:56:14.331 INFO:tasks.rados.rados.0.smithi070.stdout:2305: done (1 left) 2024-09-14T03:56:14.331 INFO:tasks.rados.rados.0.smithi070.stdout:2309: done (0 left) 2024-09-14T03:56:14.331 INFO:tasks.rados.rados.0.smithi070.stdout:2310: copy_from oid 11 from oid 18 current snap is 248 2024-09-14T03:56:14.331 INFO:tasks.rados.rados.0.smithi070.stdout:2311: read oid 33 snap 194 2024-09-14T03:56:14.331 INFO:tasks.rados.rados.0.smithi070.stdout:2311: expect (ObjNum 467 snap 160 seq_num 467) 2024-09-14T03:56:14.331 INFO:tasks.rados.rados.0.smithi070.stdout:2312: snap_remove snap 240 2024-09-14T03:56:14.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:14 smithi094 ceph-mon[86597]: osdmap e542: 8 total, 8 up, 8 in 2024-09-14T03:56:14.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:14 smithi094 ceph-mon[86597]: pgmap v410: 121 pgs: 121 active+clean; 1.0 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 10 MiB/s wr, 21 op/s; 7.0 MiB/s, 1 objects/s recovering 2024-09-14T03:56:14.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:14 smithi070 ceph-mon[96704]: osdmap e542: 8 total, 8 up, 8 in 2024-09-14T03:56:14.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:14 smithi070 ceph-mon[96704]: pgmap v410: 121 pgs: 121 active+clean; 1.0 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 10 MiB/s wr, 21 op/s; 7.0 MiB/s, 1 objects/s recovering 2024-09-14T03:56:14.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:14 smithi070 ceph-mon[98640]: osdmap e542: 8 total, 8 up, 8 in 2024-09-14T03:56:14.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:14 smithi070 ceph-mon[98640]: pgmap v410: 121 pgs: 121 active+clean; 1.0 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 10 MiB/s wr, 21 op/s; 7.0 MiB/s, 1 objects/s recovering 2024-09-14T03:56:15.150 INFO:tasks.rados.rados.0.smithi070.stdout:2312: done (2 left) 2024-09-14T03:56:15.150 INFO:tasks.rados.rados.0.smithi070.stdout:2313: copy_from oid 10 from oid 1 current snap is 248 2024-09-14T03:56:15.150 INFO:tasks.rados.rados.0.smithi070.stdout:2311: expect (ObjNum 467 snap 160 seq_num 467) 2024-09-14T03:56:15.411 INFO:tasks.rados.rados.0.smithi070.stdout:2310: finishing copy_from racing read to smithi07085154-11 2024-09-14T03:56:15.411 INFO:tasks.rados.rados.0.smithi070.stdout:2310: finishing copy_from to smithi07085154-11 2024-09-14T03:56:15.412 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 11 v 2045 (ObjNum 671 snap 228 seq_num 671) dirty exists 2024-09-14T03:56:15.412 INFO:tasks.rados.rados.0.smithi070.stdout:2310: done (2 left) 2024-09-14T03:56:15.412 INFO:tasks.rados.rados.0.smithi070.stdout:2311: done (1 left) 2024-09-14T03:56:15.412 INFO:tasks.rados.rados.0.smithi070.stdout:2314: snap_remove snap 214 2024-09-14T03:56:15.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:15 smithi094 ceph-mon[86597]: osdmap e543: 8 total, 8 up, 8 in 2024-09-14T03:56:15.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:15 smithi070 ceph-mon[98640]: osdmap e543: 8 total, 8 up, 8 in 2024-09-14T03:56:15.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:15 smithi070 ceph-mon[96704]: osdmap e543: 8 total, 8 up, 8 in 2024-09-14T03:56:16.157 INFO:tasks.rados.rados.0.smithi070.stdout:2314: done (1 left) 2024-09-14T03:56:16.157 INFO:tasks.rados.rados.0.smithi070.stdout:2315: read oid 47 snap -1 2024-09-14T03:56:16.157 INFO:tasks.rados.rados.0.smithi070.stdout:2315: expect (ObjNum 460 snap 155 seq_num 460) 2024-09-14T03:56:16.157 INFO:tasks.rados.rados.0.smithi070.stdout:2316: rollback oid 44 current snap is 248 2024-09-14T03:56:16.157 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 44 to 220 2024-09-14T03:56:16.157 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 11 current snap is 248 2024-09-14T03:56:16.157 INFO:tasks.rados.rados.0.smithi070.stdout:2317: seq_num 723 ranges {9338880=114688} 2024-09-14T03:56:16.211 INFO:tasks.rados.rados.0.smithi070.stdout:2317: writing smithi07085154-11 from 9338880 to 9453568 tid 1 2024-09-14T03:56:16.211 INFO:tasks.rados.rados.0.smithi070.stdout:2318: read oid 8 snap 208 2024-09-14T03:56:16.211 INFO:tasks.rados.rados.0.smithi070.stdout:2318: expect (ObjNum 617 snap 207 seq_num 617) 2024-09-14T03:56:16.211 INFO:tasks.rados.rados.0.smithi070.stdout:2319: rmattr oid 6 current snap is 248 2024-09-14T03:56:16.211 INFO:tasks.rados.rados.0.smithi070.stdout:2313: finishing copy_from racing read to smithi07085154-10 2024-09-14T03:56:16.211 INFO:tasks.rados.rados.0.smithi070.stdout:2313: finishing copy_from to smithi07085154-10 2024-09-14T03:56:16.211 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 2054 (ObjNum 712 snap 243 seq_num 712) dirty exists 2024-09-14T03:56:16.211 INFO:tasks.rados.rados.0.smithi070.stdout:2316: finishing rollback tid 1 to smithi07085154-44 2024-09-14T03:56:16.211 INFO:tasks.rados.rados.0.smithi070.stdout:2316: finishing rollback tid 2 to smithi07085154-44 2024-09-14T03:56:16.211 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 2058 (ObjNum 536 snap 176 seq_num 536) dirty exists 2024-09-14T03:56:16.211 INFO:tasks.rados.rados.0.smithi070.stdout:2315: expect (ObjNum 460 snap 155 seq_num 460) 2024-09-14T03:56:16.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:16 smithi094 ceph-mon[86597]: osdmap e544: 8 total, 8 up, 8 in 2024-09-14T03:56:16.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:16 smithi094 ceph-mon[86597]: pgmap v413: 121 pgs: 121 active+clean; 1.0 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 1.7 MiB/s wr, 20 op/s; 6.0 MiB/s, 1 objects/s recovering 2024-09-14T03:56:16.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:16 smithi070 ceph-mon[96704]: osdmap e544: 8 total, 8 up, 8 in 2024-09-14T03:56:16.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:16 smithi070 ceph-mon[96704]: pgmap v413: 121 pgs: 121 active+clean; 1.0 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 1.7 MiB/s wr, 20 op/s; 6.0 MiB/s, 1 objects/s recovering 2024-09-14T03:56:16.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:16 smithi070 ceph-mon[98640]: osdmap e544: 8 total, 8 up, 8 in 2024-09-14T03:56:16.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:16 smithi070 ceph-mon[98640]: pgmap v413: 121 pgs: 121 active+clean; 1.0 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s rd, 1.7 MiB/s wr, 20 op/s; 6.0 MiB/s, 1 objects/s recovering 2024-09-14T03:56:16.605 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 2210 (ObjNum 671 snap 228 seq_num 671) dirty exists 2024-09-14T03:56:16.605 INFO:tasks.rados.rados.0.smithi070.stdout:2313: done (5 left) 2024-09-14T03:56:16.605 INFO:tasks.rados.rados.0.smithi070.stdout:2315: done (4 left) 2024-09-14T03:56:16.605 INFO:tasks.rados.rados.0.smithi070.stdout:2316: done (3 left) 2024-09-14T03:56:16.605 INFO:tasks.rados.rados.0.smithi070.stdout:2319: done (2 left) 2024-09-14T03:56:16.605 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 41 current snap is 248 2024-09-14T03:56:16.605 INFO:tasks.rados.rados.0.smithi070.stdout:2317: finishing write tid 1 to smithi07085154-11 2024-09-14T03:56:16.605 INFO:tasks.rados.rados.0.smithi070.stdout:2317: finishing write tid 2 to smithi07085154-11 2024-09-14T03:56:16.605 INFO:tasks.rados.rados.0.smithi070.stdout:2318: expect (ObjNum 617 snap 207 seq_num 617) 2024-09-14T03:56:16.648 INFO:tasks.rados.rados.0.smithi070.stdout:2317: finishing write tid 3 to smithi07085154-11 2024-09-14T03:56:16.649 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 11 v 2055 (ObjNum 723 snap 248 seq_num 723) dirty exists 2024-09-14T03:56:16.649 INFO:tasks.rados.rados.0.smithi070.stdout:2317: left oid 11 (ObjNum 723 snap 248 seq_num 723) 2024-09-14T03:56:16.649 INFO:tasks.rados.rados.0.smithi070.stdout:2320: seq_num 724 ranges {0=131072} 2024-09-14T03:56:16.649 INFO:tasks.rados.rados.0.smithi070.stdout:2320: writing smithi07085154-41 from 0 to 131072 tid 1 2024-09-14T03:56:16.650 INFO:tasks.rados.rados.0.smithi070.stdout:2317: done (2 left) 2024-09-14T03:56:16.650 INFO:tasks.rados.rados.0.smithi070.stdout:2318: done (1 left) 2024-09-14T03:56:16.650 INFO:tasks.rados.rados.0.smithi070.stdout:2321: snap_create 2024-09-14T03:56:16.652 INFO:tasks.rados.rados.0.smithi070.stdout:2320: finishing write tid 1 to smithi07085154-41 2024-09-14T03:56:16.652 INFO:tasks.rados.rados.0.smithi070.stdout:2320: finishing write tid 2 to smithi07085154-41 2024-09-14T03:56:16.653 INFO:tasks.rados.rados.0.smithi070.stdout:2320: finishing write tid 3 to smithi07085154-41 2024-09-14T03:56:16.653 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 2051 (ObjNum 724 snap 248 seq_num 724) dirty exists 2024-09-14T03:56:16.653 INFO:tasks.rados.rados.0.smithi070.stdout:2320: left oid 41 (ObjNum 724 snap 248 seq_num 724) 2024-09-14T03:56:17.160 INFO:tasks.rados.rados.0.smithi070.stdout:2320: done (1 left) 2024-09-14T03:56:17.160 INFO:tasks.rados.rados.0.smithi070.stdout:2321: done (0 left) 2024-09-14T03:56:17.160 INFO:tasks.rados.rados.0.smithi070.stdout:2322: read oid 13 snap 239 2024-09-14T03:56:17.161 INFO:tasks.rados.rados.0.smithi070.stdout:2322: expect (ObjNum 706 snap 238 seq_num 706) 2024-09-14T03:56:17.161 INFO:tasks.rados.rados.0.smithi070.stdout:2323: read oid 44 snap 181 2024-09-14T03:56:17.161 INFO:tasks.rados.rados.0.smithi070.stdout:2323: expect (ObjNum 536 snap 176 seq_num 536) 2024-09-14T03:56:17.161 INFO:tasks.rados.rados.0.smithi070.stdout:2324: rmattr oid 35 current snap is 249 2024-09-14T03:56:17.161 INFO:tasks.rados.rados.0.smithi070.stdout:2324: done (2 left) 2024-09-14T03:56:17.161 INFO:tasks.rados.rados.0.smithi070.stdout:2325: read oid 31 snap -1 2024-09-14T03:56:17.161 INFO:tasks.rados.rados.0.smithi070.stdout:2325: expect (ObjNum 670 snap 228 seq_num 670) 2024-09-14T03:56:17.161 INFO:tasks.rados.rados.0.smithi070.stdout:2326: read oid 42 snap -1 2024-09-14T03:56:17.161 INFO:tasks.rados.rados.0.smithi070.stdout:2326: expect (ObjNum 507 snap 166 seq_num 507) 2024-09-14T03:56:17.161 INFO:tasks.rados.rados.0.smithi070.stdout:2327: rmattr oid 17 current snap is 249 2024-09-14T03:56:17.161 INFO:tasks.rados.rados.0.smithi070.stdout:2327: done (4 left) 2024-09-14T03:56:17.161 INFO:tasks.rados.rados.0.smithi070.stdout:2328: read oid 39 snap -1 2024-09-14T03:56:17.161 INFO:tasks.rados.rados.0.smithi070.stdout:2328: expect (ObjNum 460 snap 155 seq_num 460) 2024-09-14T03:56:17.162 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 45 current snap is 249 2024-09-14T03:56:17.162 INFO:tasks.rados.rados.0.smithi070.stdout:2329: seq_num 725 ranges {0=147456} 2024-09-14T03:56:17.162 INFO:tasks.rados.rados.0.smithi070.stdout:2329: writing smithi07085154-45 from 0 to 147456 tid 1 2024-09-14T03:56:17.162 INFO:tasks.rados.rados.0.smithi070.stdout:2330: snap_remove snap 227 2024-09-14T03:56:17.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:17 smithi094 ceph-mon[86597]: osdmap e545: 8 total, 8 up, 8 in 2024-09-14T03:56:17.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:17 smithi070 ceph-mon[96704]: osdmap e545: 8 total, 8 up, 8 in 2024-09-14T03:56:17.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:17 smithi070 ceph-mon[98640]: osdmap e545: 8 total, 8 up, 8 in 2024-09-14T03:56:18.166 INFO:tasks.rados.rados.0.smithi070.stdout:2330: done (6 left) 2024-09-14T03:56:18.166 INFO:tasks.rados.rados.0.smithi070.stdout:2331: copy_from oid 6 from oid 29 current snap is 249 2024-09-14T03:56:18.166 INFO:tasks.rados.rados.0.smithi070.stdout:2322: expect (ObjNum 706 snap 238 seq_num 706) 2024-09-14T03:56:18.236 INFO:tasks.rados.rados.0.smithi070.stdout:2326: expect (ObjNum 507 snap 166 seq_num 507) 2024-09-14T03:56:18.268 INFO:tasks.rados.rados.0.smithi070.stdout:2329: finishing write tid 1 to smithi07085154-45 2024-09-14T03:56:18.269 INFO:tasks.rados.rados.0.smithi070.stdout:2329: finishing write tid 2 to smithi07085154-45 2024-09-14T03:56:18.269 INFO:tasks.rados.rados.0.smithi070.stdout:2329: finishing write tid 3 to smithi07085154-45 2024-09-14T03:56:18.269 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 45 v 1957 (ObjNum 725 snap 249 seq_num 725) dirty exists 2024-09-14T03:56:18.269 INFO:tasks.rados.rados.0.smithi070.stdout:2329: left oid 45 (ObjNum 725 snap 249 seq_num 725) 2024-09-14T03:56:18.269 INFO:tasks.rados.rados.0.smithi070.stdout:2323: expect (ObjNum 536 snap 176 seq_num 536) 2024-09-14T03:56:18.349 INFO:tasks.rados.rados.0.smithi070.stdout:2325: expect (ObjNum 670 snap 228 seq_num 670) 2024-09-14T03:56:18.469 INFO:tasks.rados.rados.0.smithi070.stdout:2328: expect (ObjNum 460 snap 155 seq_num 460) 2024-09-14T03:56:18.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:18 smithi094 ceph-mon[86597]: osdmap e546: 8 total, 8 up, 8 in 2024-09-14T03:56:18.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:18 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:18.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:18 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:56:18.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:18 smithi094 ceph-mon[86597]: pgmap v416: 121 pgs: 121 active+clean; 1.0 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 3.4 MiB/s wr, 26 op/s 2024-09-14T03:56:18.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:18 smithi070 ceph-mon[96704]: osdmap e546: 8 total, 8 up, 8 in 2024-09-14T03:56:18.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:18 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:18.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:18 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:56:18.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:18 smithi070 ceph-mon[96704]: pgmap v416: 121 pgs: 121 active+clean; 1.0 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 3.4 MiB/s wr, 26 op/s 2024-09-14T03:56:18.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:18 smithi070 ceph-mon[98640]: osdmap e546: 8 total, 8 up, 8 in 2024-09-14T03:56:18.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:18 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:18.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:18 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:56:18.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:18 smithi070 ceph-mon[98640]: pgmap v416: 121 pgs: 121 active+clean; 1.0 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 3.4 MiB/s wr, 26 op/s 2024-09-14T03:56:18.860 INFO:tasks.rados.rados.0.smithi070.stdout:2331: finishing copy_from racing read to smithi07085154-6 2024-09-14T03:56:18.861 INFO:tasks.rados.rados.0.smithi070.stdout:2331: finishing copy_from to smithi07085154-6 2024-09-14T03:56:18.861 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 1715 (ObjNum 458 snap 155 seq_num 458) dirty exists 2024-09-14T03:56:18.861 INFO:tasks.rados.rados.0.smithi070.stdout:2322: done (6 left) 2024-09-14T03:56:18.861 INFO:tasks.rados.rados.0.smithi070.stdout:2323: done (5 left) 2024-09-14T03:56:18.861 INFO:tasks.rados.rados.0.smithi070.stdout:2325: done (4 left) 2024-09-14T03:56:18.861 INFO:tasks.rados.rados.0.smithi070.stdout:2326: done (3 left) 2024-09-14T03:56:18.862 INFO:tasks.rados.rados.0.smithi070.stdout:2328: done (2 left) 2024-09-14T03:56:18.862 INFO:tasks.rados.rados.0.smithi070.stdout:2329: done (1 left) 2024-09-14T03:56:18.862 INFO:tasks.rados.rados.0.smithi070.stdout:2331: done (0 left) 2024-09-14T03:56:18.862 INFO:tasks.rados.rados.0.smithi070.stdout:2332: copy_from oid 11 from oid 38 current snap is 249 2024-09-14T03:56:18.862 INFO:tasks.rados.rados.0.smithi070.stdout:2333: delete oid 36 current snap is 249 2024-09-14T03:56:18.863 INFO:tasks.rados.rados.0.smithi070.stdout:2333: done (1 left) 2024-09-14T03:56:18.863 INFO:tasks.rados.rados.0.smithi070.stdout:2334: read oid 9 snap -1 2024-09-14T03:56:18.863 INFO:tasks.rados.rados.0.smithi070.stdout:2334: expect (ObjNum 681 snap 230 seq_num 681) 2024-09-14T03:56:18.864 INFO:tasks.rados.rados.0.smithi070.stdout:2335: snap_remove snap 208 2024-09-14T03:56:19.059 INFO:tasks.rados.rados.0.smithi070.stdout:2335: done (2 left) 2024-09-14T03:56:19.059 INFO:tasks.rados.rados.0.smithi070.stdout:2336: snap_create 2024-09-14T03:56:19.059 INFO:tasks.rados.rados.0.smithi070.stdout:2334: expect (ObjNum 681 snap 230 seq_num 681) 2024-09-14T03:56:19.122 INFO:tasks.rados.rados.0.smithi070.stdout:2332: finishing copy_from racing read to smithi07085154-11 2024-09-14T03:56:19.122 INFO:tasks.rados.rados.0.smithi070.stdout:2332: finishing copy_from to smithi07085154-11 2024-09-14T03:56:19.122 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 11 v 1708 (ObjNum 645 snap 216 seq_num 645) dirty exists 2024-09-14T03:56:19.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:19 smithi094 ceph-mon[86597]: osdmap e547: 8 total, 8 up, 8 in 2024-09-14T03:56:19.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:19 smithi094 ceph-mon[86597]: osdmap e548: 8 total, 8 up, 8 in 2024-09-14T03:56:19.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:19 smithi070 ceph-mon[96704]: osdmap e547: 8 total, 8 up, 8 in 2024-09-14T03:56:19.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:19 smithi070 ceph-mon[96704]: osdmap e548: 8 total, 8 up, 8 in 2024-09-14T03:56:19.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:19 smithi070 ceph-mon[98640]: osdmap e547: 8 total, 8 up, 8 in 2024-09-14T03:56:19.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:19 smithi070 ceph-mon[98640]: osdmap e548: 8 total, 8 up, 8 in 2024-09-14T03:56:20.061 INFO:tasks.rados.rados.0.smithi070.stdout:2332: done (2 left) 2024-09-14T03:56:20.062 INFO:tasks.rados.rados.0.smithi070.stdout:2334: done (1 left) 2024-09-14T03:56:20.062 INFO:tasks.rados.rados.0.smithi070.stdout:2336: done (0 left) 2024-09-14T03:56:20.062 INFO:tasks.rados.rados.0.smithi070.stdout:2337: snap_remove snap 195 2024-09-14T03:56:20.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:56:20 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:56:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:56:21.067 INFO:tasks.rados.rados.0.smithi070.stdout:2337: done (0 left) 2024-09-14T03:56:21.068 INFO:tasks.rados.rados.0.smithi070.stdout:2338: copy_from oid 33 from oid 14 current snap is 250 2024-09-14T03:56:21.068 INFO:tasks.rados.rados.0.smithi070.stdout:2339: rmattr oid 20 current snap is 250 2024-09-14T03:56:21.068 INFO:tasks.rados.rados.0.smithi070.stdout:2340: snap_remove snap 241 2024-09-14T03:56:21.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:21 smithi094 ceph-mon[86597]: osdmap e549: 8 total, 8 up, 8 in 2024-09-14T03:56:21.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:21 smithi094 ceph-mon[86597]: pgmap v420: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 1.2 MiB/s wr, 26 op/s 2024-09-14T03:56:21.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:21 smithi070 ceph-mon[96704]: osdmap e549: 8 total, 8 up, 8 in 2024-09-14T03:56:21.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:21 smithi070 ceph-mon[96704]: pgmap v420: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 1.2 MiB/s wr, 26 op/s 2024-09-14T03:56:21.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:21 smithi070 ceph-mon[98640]: osdmap e549: 8 total, 8 up, 8 in 2024-09-14T03:56:21.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:21 smithi070 ceph-mon[98640]: pgmap v420: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 1.2 MiB/s wr, 26 op/s 2024-09-14T03:56:22.071 INFO:tasks.rados.rados.0.smithi070.stdout:2340: done (2 left) 2024-09-14T03:56:22.071 INFO:tasks.rados.rados.0.smithi070.stdout:2341: snap_create 2024-09-14T03:56:22.072 INFO:tasks.rados.rados.0.smithi070.stdout:2338: finishing copy_from to smithi07085154-33 2024-09-14T03:56:22.072 INFO:tasks.rados.rados.0.smithi070.stdout:2338: got expected ENOENT (src dne) 2024-09-14T03:56:22.072 INFO:tasks.rados.rados.0.smithi070.stdout:2338: finishing copy_from racing read to smithi07085154-33 2024-09-14T03:56:22.072 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 20 v 2049 (ObjNum 671 snap 228 seq_num 671) dirty exists 2024-09-14T03:56:22.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:22 smithi094 ceph-mon[86597]: osdmap e550: 8 total, 8 up, 8 in 2024-09-14T03:56:22.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:22 smithi070 ceph-mon[96704]: osdmap e550: 8 total, 8 up, 8 in 2024-09-14T03:56:22.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:22 smithi070 ceph-mon[98640]: osdmap e550: 8 total, 8 up, 8 in 2024-09-14T03:56:23.077 INFO:tasks.rados.rados.0.smithi070.stdout:2338: done (2 left) 2024-09-14T03:56:23.077 INFO:tasks.rados.rados.0.smithi070.stdout:2339: done (1 left) 2024-09-14T03:56:23.077 INFO:tasks.rados.rados.0.smithi070.stdout:2341: done (0 left) 2024-09-14T03:56:23.077 INFO:tasks.rados.rados.0.smithi070.stdout:2342: read oid 47 snap -1 2024-09-14T03:56:23.077 INFO:tasks.rados.rados.0.smithi070.stdout:2342: expect (ObjNum 460 snap 155 seq_num 460) 2024-09-14T03:56:23.077 INFO:tasks.rados.rados.0.smithi070.stdout:2343: read oid 33 snap 199 2024-09-14T03:56:23.077 INFO:tasks.rados.rados.0.smithi070.stdout:2343: expect (ObjNum 597 snap 199 seq_num 597) 2024-09-14T03:56:23.077 INFO:tasks.rados.rados.0.smithi070.stdout:2344: read oid 11 snap -1 2024-09-14T03:56:23.078 INFO:tasks.rados.rados.0.smithi070.stdout:2344: expect (ObjNum 645 snap 216 seq_num 645) 2024-09-14T03:56:23.078 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 17 current snap is 251 2024-09-14T03:56:23.078 INFO:tasks.rados.rados.0.smithi070.stdout:2345: seq_num 727 ranges {3481600=180224} 2024-09-14T03:56:23.097 INFO:tasks.rados.rados.0.smithi070.stdout:2345: writing smithi07085154-17 from 3481600 to 3661824 tid 1 2024-09-14T03:56:23.097 INFO:tasks.rados.rados.0.smithi070.stdout:2346: snap_create 2024-09-14T03:56:23.101 INFO:tasks.rados.rados.0.smithi070.stdout:2344: expect (ObjNum 645 snap 216 seq_num 645) 2024-09-14T03:56:23.370 INFO:tasks.rados.rados.0.smithi070.stdout:2345: finishing write tid 1 to smithi07085154-17 2024-09-14T03:56:23.371 INFO:tasks.rados.rados.0.smithi070.stdout:2345: finishing write tid 2 to smithi07085154-17 2024-09-14T03:56:23.371 INFO:tasks.rados.rados.0.smithi070.stdout:2345: finishing write tid 3 to smithi07085154-17 2024-09-14T03:56:23.371 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 2212 (ObjNum 727 snap 251 seq_num 727) dirty exists 2024-09-14T03:56:23.371 INFO:tasks.rados.rados.0.smithi070.stdout:2345: left oid 17 (ObjNum 727 snap 251 seq_num 727) 2024-09-14T03:56:23.371 INFO:tasks.rados.rados.0.smithi070.stdout:2343: expect (ObjNum 597 snap 199 seq_num 597) 2024-09-14T03:56:23.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:23 smithi070 ceph-mon[96704]: osdmap e551: 8 total, 8 up, 8 in 2024-09-14T03:56:23.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:23 smithi070 ceph-mon[96704]: pgmap v423: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.0 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-09-14T03:56:23.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:23 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:23.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:23 smithi070 ceph-mon[98640]: osdmap e551: 8 total, 8 up, 8 in 2024-09-14T03:56:23.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:23 smithi070 ceph-mon[98640]: pgmap v423: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.0 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-09-14T03:56:23.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:23 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:23.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:23 smithi094 ceph-mon[86597]: osdmap e551: 8 total, 8 up, 8 in 2024-09-14T03:56:23.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:23 smithi094 ceph-mon[86597]: pgmap v423: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.0 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-09-14T03:56:23.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:23 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:23.700 INFO:tasks.rados.rados.0.smithi070.stdout:2342: expect (ObjNum 460 snap 155 seq_num 460) 2024-09-14T03:56:23.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:56:23 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:56:23] "GET /metrics HTTP/1.1" 200 36301 "" "Prometheus/2.51.0" 2024-09-14T03:56:24.092 INFO:tasks.rados.rados.0.smithi070.stdout:2342: done (4 left) 2024-09-14T03:56:24.092 INFO:tasks.rados.rados.0.smithi070.stdout:2343: done (3 left) 2024-09-14T03:56:24.092 INFO:tasks.rados.rados.0.smithi070.stdout:2344: done (2 left) 2024-09-14T03:56:24.093 INFO:tasks.rados.rados.0.smithi070.stdout:2345: done (1 left) 2024-09-14T03:56:24.093 INFO:tasks.rados.rados.0.smithi070.stdout:2346: done (0 left) 2024-09-14T03:56:24.093 INFO:tasks.rados.rados.0.smithi070.stdout:2347: read oid 8 snap -1 2024-09-14T03:56:24.093 INFO:tasks.rados.rados.0.smithi070.stdout:2347: expect (ObjNum 668 snap 228 seq_num 668) 2024-09-14T03:56:24.093 INFO:tasks.rados.rados.0.smithi070.stdout:2348: read oid 29 snap 213 2024-09-14T03:56:24.093 INFO:tasks.rados.rados.0.smithi070.stdout:2348: expect (ObjNum 623 snap 209 seq_num 623) 2024-09-14T03:56:24.093 INFO:tasks.rados.rados.0.smithi070.stdout:2349: snap_remove snap 194 2024-09-14T03:56:24.413 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:24 smithi094 ceph-mon[86597]: osdmap e552: 8 total, 8 up, 8 in 2024-09-14T03:56:24.413 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:24 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-14T03:56:24.413 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:24 smithi094 ceph-mon[86597]: osdmap e553: 8 total, 8 up, 8 in 2024-09-14T03:56:24.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:24 smithi070 ceph-mon[98640]: osdmap e552: 8 total, 8 up, 8 in 2024-09-14T03:56:24.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:24 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-14T03:56:24.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:24 smithi070 ceph-mon[98640]: osdmap e553: 8 total, 8 up, 8 in 2024-09-14T03:56:24.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:24 smithi070 ceph-mon[96704]: osdmap e552: 8 total, 8 up, 8 in 2024-09-14T03:56:24.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:24 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-14T03:56:24.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:24 smithi070 ceph-mon[96704]: osdmap e553: 8 total, 8 up, 8 in 2024-09-14T03:56:25.085 INFO:tasks.rados.rados.0.smithi070.stdout:2349: done (2 left) 2024-09-14T03:56:25.085 INFO:tasks.rados.rados.0.smithi070.stdout:2350: rmattr oid 39 current snap is 252 2024-09-14T03:56:25.085 INFO:tasks.rados.rados.0.smithi070.stdout:2348: expect (ObjNum 623 snap 209 seq_num 623) 2024-09-14T03:56:25.135 INFO:tasks.rados.rados.0.smithi070.stdout:2347: expect (ObjNum 668 snap 228 seq_num 668) 2024-09-14T03:56:25.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:25 smithi094 ceph-mon[86597]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-14T03:56:25.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:25 smithi094 ceph-mon[86597]: Upgrade: osd.7 is safe to restart 2024-09-14T03:56:25.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:25 smithi094 ceph-mon[86597]: pgmap v426: 121 pgs: 121 active+clean; 1.0 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 2.8 MiB/s wr, 8 op/s 2024-09-14T03:56:25.303 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 39 v 1706 (ObjNum 460 snap 155 seq_num 460) dirty exists 2024-09-14T03:56:25.303 INFO:tasks.rados.rados.0.smithi070.stdout:2347: done (2 left) 2024-09-14T03:56:25.303 INFO:tasks.rados.rados.0.smithi070.stdout:2348: done (1 left) 2024-09-14T03:56:25.303 INFO:tasks.rados.rados.0.smithi070.stdout:2350: done (0 left) 2024-09-14T03:56:25.303 INFO:tasks.rados.rados.0.smithi070.stdout:2351: delete oid 3 current snap is 252 2024-09-14T03:56:25.306 INFO:tasks.rados.rados.0.smithi070.stdout:2351: done (0 left) 2024-09-14T03:56:25.306 INFO:tasks.rados.rados.0.smithi070.stdout:2352: rollback oid 38 current snap is 252 2024-09-14T03:56:25.306 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 38 to 193 2024-09-14T03:56:25.306 INFO:tasks.rados.rados.0.smithi070.stdout:2353: rollback oid 39 current snap is 252 2024-09-14T03:56:25.306 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 39 to 219 2024-09-14T03:56:25.306 INFO:tasks.rados.rados.0.smithi070.stdout:2354: read oid 2 snap 212 2024-09-14T03:56:25.306 INFO:tasks.rados.rados.0.smithi070.stdout:2354: expect deleted 2024-09-14T03:56:25.306 INFO:tasks.rados.rados.0.smithi070.stdout:2355: read oid 37 snap 249 2024-09-14T03:56:25.306 INFO:tasks.rados.rados.0.smithi070.stdout:2355: expect deleted 2024-09-14T03:56:25.307 INFO:tasks.rados.rados.0.smithi070.stdout:2356: rollback oid 4 current snap is 252 2024-09-14T03:56:25.307 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 4 to 251 2024-09-14T03:56:25.307 INFO:tasks.rados.rados.0.smithi070.stdout:2357: rollback oid 50 current snap is 252 2024-09-14T03:56:25.307 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 50 to 238 2024-09-14T03:56:25.307 INFO:tasks.rados.rados.0.smithi070.stdout:2358: snap_create 2024-09-14T03:56:25.307 INFO:tasks.rados.rados.0.smithi070.stdout:2356: finishing rollback tid 1 to smithi07085154-4 2024-09-14T03:56:25.307 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 2046 (ObjNum 449 snap 0 seq_num 1436216095) dirty dne 2024-09-14T03:56:25.310 INFO:tasks.rados.rados.0.smithi070.stdout:2353: finishing rollback tid 0 to smithi07085154-39 2024-09-14T03:56:25.310 INFO:tasks.rados.rados.0.smithi070.stdout:2353: finishing rollback tid 1 to smithi07085154-39 2024-09-14T03:56:25.311 INFO:tasks.rados.rados.0.smithi070.stdout:2357: finishing rollback tid 0 to smithi07085154-50 2024-09-14T03:56:25.311 INFO:tasks.rados.rados.0.smithi070.stdout:2353: finishing rollback tid 2 to smithi07085154-39 2024-09-14T03:56:25.311 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 39 v 1709 (ObjNum 641 snap 215 seq_num 641) dirty exists 2024-09-14T03:56:25.311 INFO:tasks.rados.rados.0.smithi070.stdout:2357: finishing rollback tid 1 to smithi07085154-50 2024-09-14T03:56:25.311 INFO:tasks.rados.rados.0.smithi070.stdout:2357: finishing rollback tid 2 to smithi07085154-50 2024-09-14T03:56:25.311 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 1728 (ObjNum 657 snap 220 seq_num 657) dirty exists 2024-09-14T03:56:25.312 INFO:tasks.rados.rados.0.smithi070.stdout:2352: finishing rollback tid 0 to smithi07085154-38 2024-09-14T03:56:25.313 INFO:tasks.rados.rados.0.smithi070.stdout:2352: finishing rollback tid 1 to smithi07085154-38 2024-09-14T03:56:25.313 INFO:tasks.rados.rados.0.smithi070.stdout:2352: finishing rollback tid 2 to smithi07085154-38 2024-09-14T03:56:25.313 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 38 v 1712 (ObjNum 540 snap 176 seq_num 540) dirty exists 2024-09-14T03:56:25.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:25 smithi070 ceph-mon[96704]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-14T03:56:25.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:25 smithi070 ceph-mon[96704]: Upgrade: osd.7 is safe to restart 2024-09-14T03:56:25.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:25 smithi070 ceph-mon[96704]: pgmap v426: 121 pgs: 121 active+clean; 1.0 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 2.8 MiB/s wr, 8 op/s 2024-09-14T03:56:25.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:25 smithi070 ceph-mon[98640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-14T03:56:25.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:25 smithi070 ceph-mon[98640]: Upgrade: osd.7 is safe to restart 2024-09-14T03:56:25.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:25 smithi070 ceph-mon[98640]: pgmap v426: 121 pgs: 121 active+clean; 1.0 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 2.8 MiB/s wr, 8 op/s 2024-09-14T03:56:26.082 INFO:tasks.rados.rados.0.smithi070.stdout:2352: done (6 left) 2024-09-14T03:56:26.083 INFO:tasks.rados.rados.0.smithi070.stdout:2353: done (5 left) 2024-09-14T03:56:26.083 INFO:tasks.rados.rados.0.smithi070.stdout:2354: done (4 left) 2024-09-14T03:56:26.083 INFO:tasks.rados.rados.0.smithi070.stdout:2355: done (3 left) 2024-09-14T03:56:26.083 INFO:tasks.rados.rados.0.smithi070.stdout:2356: done (2 left) 2024-09-14T03:56:26.083 INFO:tasks.rados.rados.0.smithi070.stdout:2357: done (1 left) 2024-09-14T03:56:26.083 INFO:tasks.rados.rados.0.smithi070.stdout:2358: done (0 left) 2024-09-14T03:56:26.083 INFO:tasks.rados.rados.0.smithi070.stdout:2359: copy_from oid 12 from oid 43 current snap is 253 2024-09-14T03:56:26.083 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 49 current snap is 253 2024-09-14T03:56:26.083 INFO:tasks.rados.rados.0.smithi070.stdout:2360: seq_num 729 ranges {10502144=212992} 2024-09-14T03:56:26.140 INFO:tasks.rados.rados.0.smithi070.stdout:2360: writing smithi07085154-49 from 10502144 to 10715136 tid 1 2024-09-14T03:56:26.140 INFO:tasks.rados.rados.0.smithi070.stdout:2361: copy_from oid 6 from oid 21 current snap is 253 2024-09-14T03:56:26.140 INFO:tasks.rados.rados.0.smithi070.stdout:2359: finishing copy_from racing read to smithi07085154-12 2024-09-14T03:56:26.140 INFO:tasks.rados.rados.0.smithi070.stdout:2359: finishing copy_from to smithi07085154-12 2024-09-14T03:56:26.141 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 2047 (ObjNum 681 snap 230 seq_num 681) dirty exists 2024-09-14T03:56:26.141 INFO:tasks.rados.rados.0.smithi070.stdout:2359: done (2 left) 2024-09-14T03:56:26.141 INFO:tasks.rados.rados.0.smithi070.stdout:2362: snap_remove snap 219 2024-09-14T03:56:26.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:26 smithi094 ceph-mon[86597]: osdmap e554: 8 total, 8 up, 8 in 2024-09-14T03:56:26.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:26 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:26.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:26 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-14T03:56:26.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:26 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:56:26.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:26 smithi070 ceph-mon[96704]: osdmap e554: 8 total, 8 up, 8 in 2024-09-14T03:56:26.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:26 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:26.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:26 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-14T03:56:26.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:26 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:56:26.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:26 smithi070 ceph-mon[98640]: osdmap e554: 8 total, 8 up, 8 in 2024-09-14T03:56:26.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:26 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:26.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:26 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-14T03:56:26.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:26 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:56:27.093 INFO:tasks.rados.rados.0.smithi070.stdout:2362: done (2 left) 2024-09-14T03:56:27.093 INFO:tasks.rados.rados.0.smithi070.stdout:2363: snap_remove snap 239 2024-09-14T03:56:27.189 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:27 smithi094 ceph-mon[86597]: Upgrade: Updating osd.7 2024-09-14T03:56:27.189 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:27 smithi094 ceph-mon[86597]: Deploying daemon osd.7 on smithi094 2024-09-14T03:56:27.190 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:27 smithi094 ceph-mon[86597]: osdmap e555: 8 total, 8 up, 8 in 2024-09-14T03:56:27.190 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:27 smithi094 ceph-mon[86597]: pgmap v429: 121 pgs: 121 active+clean; 1.0 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 1.6 MiB/s wr, 3 op/s 2024-09-14T03:56:27.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:27 smithi070 ceph-mon[96704]: Upgrade: Updating osd.7 2024-09-14T03:56:27.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:27 smithi070 ceph-mon[96704]: Deploying daemon osd.7 on smithi094 2024-09-14T03:56:27.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:27 smithi070 ceph-mon[96704]: osdmap e555: 8 total, 8 up, 8 in 2024-09-14T03:56:27.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:27 smithi070 ceph-mon[96704]: pgmap v429: 121 pgs: 121 active+clean; 1.0 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 1.6 MiB/s wr, 3 op/s 2024-09-14T03:56:27.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:27 smithi070 ceph-mon[98640]: Upgrade: Updating osd.7 2024-09-14T03:56:27.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:27 smithi070 ceph-mon[98640]: Deploying daemon osd.7 on smithi094 2024-09-14T03:56:27.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:27 smithi070 ceph-mon[98640]: osdmap e555: 8 total, 8 up, 8 in 2024-09-14T03:56:27.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:27 smithi070 ceph-mon[98640]: pgmap v429: 121 pgs: 121 active+clean; 1.0 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 1.6 MiB/s wr, 3 op/s 2024-09-14T03:56:28.094 INFO:tasks.rados.rados.0.smithi070.stdout:2363: done (2 left) 2024-09-14T03:56:28.094 INFO:tasks.rados.rados.0.smithi070.stdout:2364: read oid 13 snap 243 2024-09-14T03:56:28.094 INFO:tasks.rados.rados.0.smithi070.stdout:2364: expect (ObjNum 706 snap 238 seq_num 706) 2024-09-14T03:56:28.094 INFO:tasks.rados.rados.0.smithi070.stdout:2365: snap_remove snap 248 2024-09-14T03:56:28.451 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:28 smithi094 ceph-mon[86597]: osdmap e556: 8 total, 8 up, 8 in 2024-09-14T03:56:28.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:28 smithi070 ceph-mon[96704]: osdmap e556: 8 total, 8 up, 8 in 2024-09-14T03:56:28.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:28 smithi070 ceph-mon[98640]: osdmap e556: 8 total, 8 up, 8 in 2024-09-14T03:56:28.777 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:28 smithi094 systemd[1]: Stopping Ceph osd.7 for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:56:28.777 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:28 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[55202]: 2024-09-14T03:56:28.694+0000 7f30ea147640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:56:28.777 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:28 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[55202]: 2024-09-14T03:56:28.694+0000 7f30ea147640 -1 osd.7 557 *** Got signal Terminated *** 2024-09-14T03:56:28.777 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:28 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[55202]: 2024-09-14T03:56:28.694+0000 7f30ea147640 -1 osd.7 557 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T03:56:29.064 INFO:tasks.rados.rados.0.smithi070.stdout:2365: done (3 left) 2024-09-14T03:56:29.064 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 43 current snap is 253 2024-09-14T03:56:29.064 INFO:tasks.rados.rados.0.smithi070.stdout:2366: seq_num 730 ranges {1818624=229376} 2024-09-14T03:56:29.076 INFO:tasks.rados.rados.0.smithi070.stdout:2366: writing smithi07085154-43 from 1818624 to 2048000 tid 1 2024-09-14T03:56:29.077 INFO:tasks.rados.rados.0.smithi070.stdout:2360: finishing write tid 1 to smithi07085154-49 2024-09-14T03:56:29.077 INFO:tasks.rados.rados.0.smithi070.stdout:2360: finishing write tid 2 to smithi07085154-49 2024-09-14T03:56:29.077 INFO:tasks.rados.rados.0.smithi070.stdout:2360: finishing write tid 3 to smithi07085154-49 2024-09-14T03:56:29.077 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 49 v 1959 (ObjNum 729 snap 253 seq_num 729) dirty exists 2024-09-14T03:56:29.077 INFO:tasks.rados.rados.0.smithi070.stdout:2360: left oid 49 (ObjNum 729 snap 253 seq_num 729) 2024-09-14T03:56:29.077 INFO:tasks.rados.rados.0.smithi070.stdout:2361: finishing copy_from to smithi07085154-6 2024-09-14T03:56:29.077 INFO:tasks.rados.rados.0.smithi070.stdout:2361: got expected ENOENT (src dne) 2024-09-14T03:56:29.077 INFO:tasks.rados.rados.0.smithi070.stdout:2361: finishing copy_from racing read to smithi07085154-6 2024-09-14T03:56:29.077 INFO:tasks.rados.rados.0.smithi070.stdout:2364: expect (ObjNum 706 snap 238 seq_num 706) 2024-09-14T03:56:29.141 INFO:tasks.rados.rados.0.smithi070.stdout:2366: finishing write tid 1 to smithi07085154-43 2024-09-14T03:56:29.141 INFO:tasks.rados.rados.0.smithi070.stdout:2366: finishing write tid 2 to smithi07085154-43 2024-09-14T03:56:29.141 INFO:tasks.rados.rados.0.smithi070.stdout:2366: finishing write tid 3 to smithi07085154-43 2024-09-14T03:56:29.141 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 2053 (ObjNum 730 snap 253 seq_num 730) dirty exists 2024-09-14T03:56:29.141 INFO:tasks.rados.rados.0.smithi070.stdout:2366: left oid 43 (ObjNum 730 snap 253 seq_num 730) 2024-09-14T03:56:29.141 INFO:tasks.rados.rados.0.smithi070.stdout:2360: done (3 left) 2024-09-14T03:56:29.141 INFO:tasks.rados.rados.0.smithi070.stdout:2361: done (2 left) 2024-09-14T03:56:29.141 INFO:tasks.rados.rados.0.smithi070.stdout:2364: done (1 left) 2024-09-14T03:56:29.141 INFO:tasks.rados.rados.0.smithi070.stdout:2366: done (0 left) 2024-09-14T03:56:29.141 INFO:tasks.rados.rados.0.smithi070.stdout:2367: rollback oid 6 current snap is 253 2024-09-14T03:56:29.141 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 6 to 238 2024-09-14T03:56:29.142 INFO:tasks.rados.rados.0.smithi070.stdout:2368: read oid 37 snap -1 2024-09-14T03:56:29.142 INFO:tasks.rados.rados.0.smithi070.stdout:2368: expect deleted 2024-09-14T03:56:29.142 INFO:tasks.rados.rados.0.smithi070.stdout:2369: snap_create 2024-09-14T03:56:29.145 INFO:tasks.rados.rados.0.smithi070.stdout:2367: finishing rollback tid 0 to smithi07085154-6 2024-09-14T03:56:29.145 INFO:tasks.rados.rados.0.smithi070.stdout:2367: finishing rollback tid 1 to smithi07085154-6 2024-09-14T03:56:29.145 INFO:tasks.rados.rados.0.smithi070.stdout:2367: finishing rollback tid 2 to smithi07085154-6 2024-09-14T03:56:29.145 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 2215 (ObjNum 671 snap 228 seq_num 671) dirty exists 2024-09-14T03:56:29.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:29 smithi094 ceph-mon[86597]: pgmap v432: 121 pgs: 121 active+clean; 1.0 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 0 B/s wr, 6 op/s 2024-09-14T03:56:29.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:29 smithi094 ceph-mon[86597]: osdmap e557: 8 total, 8 up, 8 in 2024-09-14T03:56:29.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:29 smithi094 ceph-mon[86597]: osd.7 marked itself down and dead 2024-09-14T03:56:29.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:29 smithi094 ceph-mon[86597]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:56:29.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:29 smithi094 ceph-mon[86597]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-14T03:56:29.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:29 smithi094 ceph-mon[86597]: osdmap e558: 8 total, 7 up, 8 in 2024-09-14T03:56:29.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:29 smithi070 ceph-mon[96704]: pgmap v432: 121 pgs: 121 active+clean; 1.0 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 0 B/s wr, 6 op/s 2024-09-14T03:56:29.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:29 smithi070 ceph-mon[96704]: osdmap e557: 8 total, 8 up, 8 in 2024-09-14T03:56:29.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:29 smithi070 ceph-mon[96704]: osd.7 marked itself down and dead 2024-09-14T03:56:29.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:29 smithi070 ceph-mon[96704]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:56:29.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:29 smithi070 ceph-mon[96704]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-14T03:56:29.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:29 smithi070 ceph-mon[96704]: osdmap e558: 8 total, 7 up, 8 in 2024-09-14T03:56:29.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:29 smithi070 ceph-mon[98640]: pgmap v432: 121 pgs: 121 active+clean; 1.0 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 0 B/s wr, 6 op/s 2024-09-14T03:56:29.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:29 smithi070 ceph-mon[98640]: osdmap e557: 8 total, 8 up, 8 in 2024-09-14T03:56:29.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:29 smithi070 ceph-mon[98640]: osd.7 marked itself down and dead 2024-09-14T03:56:29.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:29 smithi070 ceph-mon[98640]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T03:56:29.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:29 smithi070 ceph-mon[98640]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-14T03:56:29.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:29 smithi070 ceph-mon[98640]: osdmap e558: 8 total, 7 up, 8 in 2024-09-14T03:56:29.923 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:29 smithi094 podman[105860]: 2024-09-14 03:56:29.636417058 +0000 UTC m=+1.042755253 container died 037c6208e9ce82df2077f89728cf3d19bfb5ded8e7e577dcc2d9dc813d7cfe74 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=quincy-eb16b97, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS) 2024-09-14T03:56:30.014 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-09-14T03:56:30.066 INFO:tasks.rados.rados.0.smithi070.stdout:2367: done (2 left) 2024-09-14T03:56:30.066 INFO:tasks.rados.rados.0.smithi070.stdout:2368: done (1 left) 2024-09-14T03:56:30.066 INFO:tasks.rados.rados.0.smithi070.stdout:2369: done (0 left) 2024-09-14T03:56:30.066 INFO:tasks.rados.rados.0.smithi070.stdout:2370: delete oid 35 current snap is 254 2024-09-14T03:56:30.074 INFO:tasks.rados.rados.0.smithi070.stdout:2370: done (0 left) 2024-09-14T03:56:30.074 INFO:tasks.rados.rados.0.smithi070.stdout:2371: snap_create 2024-09-14T03:56:30.214 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:30 smithi094 podman[105860]: 2024-09-14 03:56:30.007063054 +0000 UTC m=+1.413401238 container cleanup 037c6208e9ce82df2077f89728cf3d19bfb5ded8e7e577dcc2d9dc813d7cfe74 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=quincy-eb16b97, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-14T03:56:30.214 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:30 smithi094 bash[105860]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7 2024-09-14T03:56:30.527 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:30 smithi094 podman[105882]: 2024-09-14 03:56:30.267098004 +0000 UTC m=+0.629741788 container remove 037c6208e9ce82df2077f89728cf3d19bfb5ded8e7e577dcc2d9dc813d7cfe74 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=quincy-eb16b97, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-14T03:56:30.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:56:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:56:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:56:30.579 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:56:30.579 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (6m) 2m ago 11m 21.2M - 0.27.0 11f11916f8cd 45e79e51074c 2024-09-14T03:56:30.580 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi094 *:3000 running (6m) 25s ago 10m 85.0M - 9.4.7 954c08fa6188 2240810d0368 2024-09-14T03:56:30.580 INFO:teuthology.orchestra.run.smithi070.stdout:mds.a smithi070 running (12m) 2m ago 12m 21.4M - 17.2.7-1629-geb16b97b ab51d1bebd12 52e0dbb7be50 2024-09-14T03:56:30.580 INFO:teuthology.orchestra.run.smithi070.stdout:mds.b smithi094 running (12m) 25s ago 12m 16.2M - 17.2.7-1629-geb16b97b ab51d1bebd12 e3b5e3e00189 2024-09-14T03:56:30.580 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi094 *:8443,9283,8765 running (5m) 25s ago 16m 455M - 19.3.0-4907-ga2782315 8801d2529e43 569d9c7e0f06 2024-09-14T03:56:30.580 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (5m) 2m ago 17m 521M - 19.3.0-4907-ga2782315 8801d2529e43 0ef4d8e0cdbe 2024-09-14T03:56:30.580 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (5m) 2m ago 18m 57.7M 2048M 19.3.0-4907-ga2782315 8801d2529e43 96542de31cac 2024-09-14T03:56:30.580 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi094 running (4m) 25s ago 16m 49.4M 2048M 19.3.0-4907-ga2782315 8801d2529e43 86827be98659 2024-09-14T03:56:30.580 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (4m) 2m ago 16m 51.9M 2048M 19.3.0-4907-ga2782315 8801d2529e43 5741e4fcf4b6 2024-09-14T03:56:30.580 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (6m) 2m ago 10m 12.2M - 1.7.0 72c9c2088986 e316760a30b3 2024-09-14T03:56:30.580 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi094 *:9100 running (5m) 25s ago 10m 11.9M - 1.7.0 72c9c2088986 767f8a093175 2024-09-14T03:56:30.581 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (3m) 2m ago 15m 180M 2480M 19.3.0-4907-ga2782315 8801d2529e43 2345ccac2085 2024-09-14T03:56:30.581 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (3m) 2m ago 15m 177M 2480M 19.3.0-4907-ga2782315 8801d2529e43 53868856512a 2024-09-14T03:56:30.581 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (2m) 2m ago 14m 104M 2480M 19.3.0-4907-ga2782315 8801d2529e43 39ef876d2fff 2024-09-14T03:56:30.581 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (2m) 2m ago 14m 14.5M 2480M 19.3.0-4907-ga2782315 8801d2529e43 8efc112edbe9 2024-09-14T03:56:30.581 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi094 running (98s) 25s ago 14m 200M 2480M 19.3.0-4907-ga2782315 8801d2529e43 09f583ed6c77 2024-09-14T03:56:30.581 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi094 running (64s) 25s ago 13m 157M 2480M 19.3.0-4907-ga2782315 8801d2529e43 162396f2e7b6 2024-09-14T03:56:30.581 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi094 running (30s) 25s ago 13m 11.9M 2480M 19.3.0-4907-ga2782315 8801d2529e43 cbdc02cdb8e4 2024-09-14T03:56:30.581 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi094 running (13m) 25s ago 13m 297M 2480M 17.2.7-1629-geb16b97b ab51d1bebd12 037c6208e9ce 2024-09-14T03:56:30.581 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi094 *:9095 running (5m) 25s ago 11m 54.0M - 2.51.0 1d3b7f56885b ce1b353828dc 2024-09-14T03:56:30.889 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:30 smithi094 podman[105977]: 2024-09-14 03:56:30.486112732 +0000 UTC m=+0.016081150 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:56:30.889 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:30 smithi094 podman[105977]: 2024-09-14 03:56:30.626787164 +0000 UTC m=+0.156755578 container create e2c2d437afb773ec56616abe985b87cf8e46c779f2af7865d451a084cadaadf8 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T03:56:30.889 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:30 smithi094 podman[105977]: 2024-09-14 03:56:30.888382527 +0000 UTC m=+0.418350947 container init e2c2d437afb773ec56616abe985b87cf8e46c779f2af7865d451a084cadaadf8 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=main-a278231, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-14T03:56:30.907 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:56:30.907 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-09-14T03:56:30.907 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:56:30.907 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:56:30.908 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-09-14T03:56:30.908 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:56:30.908 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:56:30.908 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-09-14T03:56:30.908 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 7 2024-09-14T03:56:30.908 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:56:30.908 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": { 2024-09-14T03:56:30.908 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2 2024-09-14T03:56:30.908 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:56:30.908 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-09-14T03:56:30.908 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 2, 2024-09-14T03:56:30.908 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 12 2024-09-14T03:56:30.908 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-09-14T03:56:30.909 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:56:31.069 INFO:tasks.rados.rados.0.smithi070.stdout:2371: done (0 left) 2024-09-14T03:56:31.069 INFO:tasks.rados.rados.0.smithi070.stdout:2372: copy_from oid 2 from oid 45 current snap is 255 2024-09-14T03:56:31.069 INFO:tasks.rados.rados.0.smithi070.stdout:2373: rmattr oid 23 current snap is 255 2024-09-14T03:56:31.069 INFO:tasks.rados.rados.0.smithi070.stdout:2373: done (1 left) 2024-09-14T03:56:31.069 INFO:tasks.rados.rados.0.smithi070.stdout:2374: rmattr oid 14 current snap is 255 2024-09-14T03:56:31.069 INFO:tasks.rados.rados.0.smithi070.stdout:2374: done (1 left) 2024-09-14T03:56:31.069 INFO:tasks.rados.rados.0.smithi070.stdout:2375: delete oid 18 current snap is 255 2024-09-14T03:56:31.075 INFO:tasks.rados.rados.0.smithi070.stdout:2375: done (1 left) 2024-09-14T03:56:31.075 INFO:tasks.rados.rados.0.smithi070.stdout:2376: snap_remove snap 181 2024-09-14T03:56:31.199 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:56:31.199 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:56:31.199 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-09-14T03:56:31.199 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:56:31.199 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [ 2024-09-14T03:56:31.199 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr", 2024-09-14T03:56:31.199 INFO:teuthology.orchestra.run.smithi070.stdout: "mon" 2024-09-14T03:56:31.200 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2024-09-14T03:56:31.200 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "12/20 daemons upgraded", 2024-09-14T03:56:31.200 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "Currently upgrading osd daemons", 2024-09-14T03:56:31.200 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-09-14T03:56:31.200 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:56:31.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:31 smithi070 ceph-mon[96704]: from='client.34464 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:31.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:31 smithi070 ceph-mon[96704]: osdmap e559: 8 total, 7 up, 8 in 2024-09-14T03:56:31.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:31 smithi070 ceph-mon[96704]: pgmap v435: 121 pgs: 5 active+undersized, 4 peering, 18 stale+active+clean, 94 active+clean; 1.0 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 552 KiB/s wr, 9 op/s 2024-09-14T03:56:31.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:31 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/1395822006' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:31.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:31 smithi070 ceph-mon[98640]: from='client.34464 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:31.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:31 smithi070 ceph-mon[98640]: osdmap e559: 8 total, 7 up, 8 in 2024-09-14T03:56:31.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:31 smithi070 ceph-mon[98640]: pgmap v435: 121 pgs: 5 active+undersized, 4 peering, 18 stale+active+clean, 94 active+clean; 1.0 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 552 KiB/s wr, 9 op/s 2024-09-14T03:56:31.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:31 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/1395822006' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:31.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:31 smithi094 ceph-mon[86597]: from='client.34464 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:31.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:31 smithi094 ceph-mon[86597]: osdmap e559: 8 total, 7 up, 8 in 2024-09-14T03:56:31.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:31 smithi094 ceph-mon[86597]: pgmap v435: 121 pgs: 5 active+undersized, 4 peering, 18 stale+active+clean, 94 active+clean; 1.0 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 552 KiB/s wr, 9 op/s 2024-09-14T03:56:31.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:31 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/1395822006' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:31.277 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:30 smithi094 podman[105977]: 2024-09-14 03:56:30.893338666 +0000 UTC m=+0.423307080 container start e2c2d437afb773ec56616abe985b87cf8e46c779f2af7865d451a084cadaadf8 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T03:56:31.278 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:30 smithi094 podman[105977]: 2024-09-14 03:56:30.971439247 +0000 UTC m=+0.501407680 container attach e2c2d437afb773ec56616abe985b87cf8e46c779f2af7865d451a084cadaadf8 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7-deactivate, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=main-a278231, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-14T03:56:31.278 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:31 smithi094 podman[105977]: 2024-09-14 03:56:31.081651596 +0000 UTC m=+0.611620005 container died e2c2d437afb773ec56616abe985b87cf8e46c779f2af7865d451a084cadaadf8 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, RELEASE=main-a278231, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-14T03:56:32.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:31 smithi094 podman[106020]: 2024-09-14 03:56:31.676044326 +0000 UTC m=+0.588085078 container remove e2c2d437afb773ec56616abe985b87cf8e46c779f2af7865d451a084cadaadf8 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7-deactivate, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=main-a278231, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-14T03:56:32.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:31 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.7.service: Deactivated successfully. 2024-09-14T03:56:32.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:31 smithi094 systemd[1]: Stopped Ceph osd.7 for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:56:32.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:31 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.7.service: Consumed 21.698s CPU time. 2024-09-14T03:56:32.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:31 smithi094 systemd[1]: Starting Ceph osd.7 for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:56:32.076 INFO:tasks.rados.rados.0.smithi070.stdout:2376: done (1 left) 2024-09-14T03:56:32.076 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 22 current snap is 255 2024-09-14T03:56:32.076 INFO:tasks.rados.rados.0.smithi070.stdout:2377: seq_num 733 ranges {0=278528} 2024-09-14T03:56:32.078 INFO:tasks.rados.rados.0.smithi070.stdout:2377: writing smithi07085154-22 from 0 to 278528 tid 1 2024-09-14T03:56:32.078 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 39 current snap is 255 2024-09-14T03:56:32.078 INFO:tasks.rados.rados.0.smithi070.stdout:2378: seq_num 734 ranges {6316032=294912} 2024-09-14T03:56:32.114 INFO:tasks.rados.rados.0.smithi070.stdout:2378: writing smithi07085154-39 from 6316032 to 6610944 tid 1 2024-09-14T03:56:32.115 INFO:tasks.rados.rados.0.smithi070.stdout:2379: snap_remove snap 253 2024-09-14T03:56:32.356 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:32 smithi094 ceph-mon[86597]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:32.356 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:32 smithi094 ceph-mon[86597]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:32.356 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:32 smithi094 ceph-mon[86597]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T03:56:32.356 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:32 smithi094 ceph-mon[86597]: osdmap e560: 8 total, 7 up, 8 in 2024-09-14T03:56:32.356 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:32 smithi094 podman[106124]: 2024-09-14 03:56:32.005734975 +0000 UTC m=+0.015905021 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:56:32.357 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:32 smithi094 podman[106124]: 2024-09-14 03:56:32.136241384 +0000 UTC m=+0.146411424 container create 91167540c55570974a6f94a9723e28d6b06eb2994a6b274e75a97c17b85685c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=main-a278231, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-14T03:56:32.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:32 smithi070 ceph-mon[96704]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:32.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:32 smithi070 ceph-mon[96704]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:32.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:32 smithi070 ceph-mon[96704]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T03:56:32.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:32 smithi070 ceph-mon[96704]: osdmap e560: 8 total, 7 up, 8 in 2024-09-14T03:56:32.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:32 smithi070 ceph-mon[98640]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:32.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:32 smithi070 ceph-mon[98640]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:32.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:32 smithi070 ceph-mon[98640]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T03:56:32.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:32 smithi070 ceph-mon[98640]: osdmap e560: 8 total, 7 up, 8 in 2024-09-14T03:56:32.777 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:32 smithi094 podman[106124]: 2024-09-14 03:56:32.355653396 +0000 UTC m=+0.365823436 container init 91167540c55570974a6f94a9723e28d6b06eb2994a6b274e75a97c17b85685c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=main-a278231) 2024-09-14T03:56:32.777 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:32 smithi094 podman[106124]: 2024-09-14 03:56:32.360834069 +0000 UTC m=+0.371004112 container start 91167540c55570974a6f94a9723e28d6b06eb2994a6b274e75a97c17b85685c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7-activate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-09-14T03:56:32.777 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:32 smithi094 podman[106124]: 2024-09-14 03:56:32.464222903 +0000 UTC m=+0.474392948 container attach 91167540c55570974a6f94a9723e28d6b06eb2994a6b274e75a97c17b85685c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7-activate, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True) 2024-09-14T03:56:32.777 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:32 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7-activate[106139]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:56:32.777 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:32 smithi094 bash[106124]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:56:32.777 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:32 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7-activate[106139]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:56:32.777 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:32 smithi094 bash[106124]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T03:56:33.083 INFO:tasks.rados.rados.0.smithi070.stdout:2379: done (3 left) 2024-09-14T03:56:33.083 INFO:tasks.rados.rados.0.smithi070.stdout:2380: setattr oid 10 current snap is 255 2024-09-14T03:56:33.083 INFO:tasks.rados.rados.0.smithi070.stdout:2372: finishing copy_from racing read to smithi07085154-2 2024-09-14T03:56:33.083 INFO:tasks.rados.rados.0.smithi070.stdout:2372: finishing copy_from to smithi07085154-2 2024-09-14T03:56:33.083 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 2 v 1957 (ObjNum 725 snap 249 seq_num 725) dirty exists 2024-09-14T03:56:33.083 INFO:tasks.rados.rados.0.smithi070.stdout:2377: finishing write tid 1 to smithi07085154-22 2024-09-14T03:56:33.083 INFO:tasks.rados.rados.0.smithi070.stdout:2377: finishing write tid 2 to smithi07085154-22 2024-09-14T03:56:33.084 INFO:tasks.rados.rados.0.smithi070.stdout:2377: finishing write tid 3 to smithi07085154-22 2024-09-14T03:56:33.084 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 1730 (ObjNum 733 snap 255 seq_num 733) dirty exists 2024-09-14T03:56:33.084 INFO:tasks.rados.rados.0.smithi070.stdout:2377: left oid 22 (ObjNum 733 snap 255 seq_num 733) 2024-09-14T03:56:33.084 INFO:tasks.rados.rados.0.smithi070.stdout:2378: finishing write tid 1 to smithi07085154-39 2024-09-14T03:56:33.084 INFO:tasks.rados.rados.0.smithi070.stdout:2378: finishing write tid 2 to smithi07085154-39 2024-09-14T03:56:33.084 INFO:tasks.rados.rados.0.smithi070.stdout:2378: finishing write tid 3 to smithi07085154-39 2024-09-14T03:56:33.084 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 39 v 1711 (ObjNum 734 snap 255 seq_num 734) dirty exists 2024-09-14T03:56:33.084 INFO:tasks.rados.rados.0.smithi070.stdout:2378: left oid 39 (ObjNum 734 snap 255 seq_num 734) 2024-09-14T03:56:33.085 INFO:tasks.rados.rados.0.smithi070.stdout:2372: done (3 left) 2024-09-14T03:56:33.085 INFO:tasks.rados.rados.0.smithi070.stdout:2377: done (2 left) 2024-09-14T03:56:33.085 INFO:tasks.rados.rados.0.smithi070.stdout:2378: done (1 left) 2024-09-14T03:56:33.085 INFO:tasks.rados.rados.0.smithi070.stdout:2381: rmattr oid 39 current snap is 255 2024-09-14T03:56:33.086 INFO:tasks.rados.rados.0.smithi070.stdout:2382: rollback oid 7 current snap is 255 2024-09-14T03:56:33.086 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 7 to 174 2024-09-14T03:56:33.086 INFO:tasks.rados.rados.0.smithi070.stdout:2383: snap_remove snap 222 2024-09-14T03:56:33.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:33 smithi094 ceph-mon[86597]: from='client.44402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:33.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:33 smithi094 ceph-mon[86597]: osdmap e561: 8 total, 7 up, 8 in 2024-09-14T03:56:33.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:33 smithi094 ceph-mon[86597]: pgmap v438: 121 pgs: 18 active+undersized, 4 peering, 8 stale+active+clean, 9 active+undersized+degraded, 82 active+clean; 1.0 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 588 KiB/s wr, 4 op/s; 132/1209 objects degraded (10.918%) 2024-09-14T03:56:33.243 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:33 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:33.244 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:33 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:56:33.244 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:32 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7-activate[106139]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-14T03:56:33.244 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:32 smithi094 bash[106124]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-14T03:56:33.244 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:32 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7-activate[106139]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-14T03:56:33.244 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:32 smithi094 bash[106124]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-14T03:56:33.244 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7-activate[106139]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-14T03:56:33.244 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:33 smithi094 bash[106124]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-14T03:56:33.406 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:33 smithi070 ceph-mon[96704]: from='client.44402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:33.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:33 smithi070 ceph-mon[96704]: osdmap e561: 8 total, 7 up, 8 in 2024-09-14T03:56:33.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:33 smithi070 ceph-mon[96704]: pgmap v438: 121 pgs: 18 active+undersized, 4 peering, 8 stale+active+clean, 9 active+undersized+degraded, 82 active+clean; 1.0 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 588 KiB/s wr, 4 op/s; 132/1209 objects degraded (10.918%) 2024-09-14T03:56:33.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:33 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:33.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:33 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:56:33.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:33 smithi070 ceph-mon[98640]: from='client.44402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:33.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:33 smithi070 ceph-mon[98640]: osdmap e561: 8 total, 7 up, 8 in 2024-09-14T03:56:33.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:33 smithi070 ceph-mon[98640]: pgmap v438: 121 pgs: 18 active+undersized, 4 peering, 8 stale+active+clean, 9 active+undersized+degraded, 82 active+clean; 1.0 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 588 KiB/s wr, 4 op/s; 132/1209 objects degraded (10.918%) 2024-09-14T03:56:33.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:33 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:33.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:33 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:56:33.527 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7-activate[106139]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-14T03:56:33.527 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:33 smithi094 bash[106124]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-14T03:56:33.527 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7-activate[106139]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-14T03:56:33.527 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:33 smithi094 bash[106124]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-14T03:56:33.527 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7-activate[106139]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-14T03:56:33.527 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:33 smithi094 bash[106124]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-14T03:56:33.527 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7-activate[106139]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-14T03:56:33.527 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:33 smithi094 bash[106124]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-14T03:56:33.527 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:33 smithi094 podman[106124]: 2024-09-14 03:56:33.27021523 +0000 UTC m=+1.280385271 container died 91167540c55570974a6f94a9723e28d6b06eb2994a6b274e75a97c17b85685c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=main-a278231, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:56:33.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:56:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:56:33] "GET /metrics HTTP/1.1" 200 36447 "" "Prometheus/2.51.0" 2024-09-14T03:56:34.064 INFO:tasks.rados.rados.0.smithi070.stdout:2383: done (3 left) 2024-09-14T03:56:34.065 INFO:tasks.rados.rados.0.smithi070.stdout:2384: delete oid 43 current snap is 255 2024-09-14T03:56:34.065 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 39 v 1712 (ObjNum 734 snap 255 seq_num 734) dirty exists 2024-09-14T03:56:34.065 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 2608 (ObjNum 712 snap 243 seq_num 712) dirty exists 2024-09-14T03:56:34.065 INFO:tasks.rados.rados.0.smithi070.stdout:2382: finishing rollback tid 0 to smithi07085154-7 2024-09-14T03:56:34.065 INFO:tasks.rados.rados.0.smithi070.stdout:2382: finishing rollback tid 1 to smithi07085154-7 2024-09-14T03:56:34.065 INFO:tasks.rados.rados.0.smithi070.stdout:2382: finishing rollback tid 2 to smithi07085154-7 2024-09-14T03:56:34.065 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 2052 (ObjNum 525 snap 174 seq_num 525) dirty exists 2024-09-14T03:56:34.076 INFO:tasks.rados.rados.0.smithi070.stdout:2380: done (3 left) 2024-09-14T03:56:34.076 INFO:tasks.rados.rados.0.smithi070.stdout:2381: done (2 left) 2024-09-14T03:56:34.076 INFO:tasks.rados.rados.0.smithi070.stdout:2382: done (1 left) 2024-09-14T03:56:34.076 INFO:tasks.rados.rados.0.smithi070.stdout:2384: done (0 left) 2024-09-14T03:56:34.076 INFO:tasks.rados.rados.0.smithi070.stdout:2385: read oid 50 snap -1 2024-09-14T03:56:34.076 INFO:tasks.rados.rados.0.smithi070.stdout:2385: expect (ObjNum 657 snap 220 seq_num 657) 2024-09-14T03:56:34.076 INFO:tasks.rados.rados.0.smithi070.stdout:2386: rollback oid 21 current snap is 255 2024-09-14T03:56:34.076 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 21 to 236 2024-09-14T03:56:34.076 INFO:tasks.rados.rados.0.smithi070.stdout:2387: delete oid 35 current snap is 255 2024-09-14T03:56:34.085 INFO:tasks.rados.rados.0.smithi070.stdout:2386: finishing rollback tid 1 to smithi07085154-21 2024-09-14T03:56:34.085 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 1701 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-14T03:56:34.086 INFO:tasks.rados.rados.0.smithi070.stdout:2386: done (2 left) 2024-09-14T03:56:34.086 INFO:tasks.rados.rados.0.smithi070.stdout:2387: done (1 left) 2024-09-14T03:56:34.086 INFO:tasks.rados.rados.0.smithi070.stdout:2388: delete oid 5 current snap is 255 2024-09-14T03:56:34.090 INFO:tasks.rados.rados.0.smithi070.stdout:2388: done (1 left) 2024-09-14T03:56:34.090 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 6 current snap is 255 2024-09-14T03:56:34.091 INFO:tasks.rados.rados.0.smithi070.stdout:2389: seq_num 738 ranges {9338880=368640} 2024-09-14T03:56:34.142 INFO:tasks.rados.rados.0.smithi070.stdout:2389: writing smithi07085154-6 from 9338880 to 9707520 tid 1 2024-09-14T03:56:34.143 INFO:tasks.rados.rados.0.smithi070.stdout:2390: delete oid 20 current snap is 255 2024-09-14T03:56:34.143 INFO:tasks.rados.rados.0.smithi070.stdout:2385: expect (ObjNum 657 snap 220 seq_num 657) 2024-09-14T03:56:34.171 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:34 smithi094 ceph-mon[86597]: Health check failed: Degraded data redundancy: 132/1209 objects degraded (10.918%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T03:56:34.171 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:34 smithi094 ceph-mon[86597]: osdmap e562: 8 total, 7 up, 8 in 2024-09-14T03:56:34.171 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:34 smithi094 ceph-mon[86597]: osdmap e563: 8 total, 7 up, 8 in 2024-09-14T03:56:34.171 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:33 smithi094 podman[106283]: 2024-09-14 03:56:33.9105043 +0000 UTC m=+0.632788468 container remove 91167540c55570974a6f94a9723e28d6b06eb2994a6b274e75a97c17b85685c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7-activate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-14T03:56:34.171 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:34 smithi094 podman[106346]: 2024-09-14 03:56:34.017828294 +0000 UTC m=+0.016053693 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:56:34.226 INFO:tasks.rados.rados.0.smithi070.stdout:2389: finishing write tid 1 to smithi07085154-6 2024-09-14T03:56:34.226 INFO:tasks.rados.rados.0.smithi070.stdout:2389: finishing write tid 2 to smithi07085154-6 2024-09-14T03:56:34.226 INFO:tasks.rados.rados.0.smithi070.stdout:2389: finishing write tid 3 to smithi07085154-6 2024-09-14T03:56:34.227 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 2217 (ObjNum 738 snap 255 seq_num 738) dirty exists 2024-09-14T03:56:34.227 INFO:tasks.rados.rados.0.smithi070.stdout:2389: left oid 6 (ObjNum 738 snap 255 seq_num 738) 2024-09-14T03:56:34.227 INFO:tasks.rados.rados.0.smithi070.stdout:2385: done (2 left) 2024-09-14T03:56:34.227 INFO:tasks.rados.rados.0.smithi070.stdout:2389: done (1 left) 2024-09-14T03:56:34.227 INFO:tasks.rados.rados.0.smithi070.stdout:2390: done (0 left) 2024-09-14T03:56:34.227 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 15 current snap is 255 2024-09-14T03:56:34.227 INFO:tasks.rados.rados.0.smithi070.stdout:2391: seq_num 740 ranges {4341760=401408} 2024-09-14T03:56:34.251 INFO:tasks.rados.rados.0.smithi070.stdout:2391: writing smithi07085154-15 from 4341760 to 4743168 tid 1 2024-09-14T03:56:34.251 INFO:tasks.rados.rados.0.smithi070.stdout:2392: snap_create 2024-09-14T03:56:34.255 INFO:tasks.rados.rados.0.smithi070.stdout:2391: finishing write tid 1 to smithi07085154-15 2024-09-14T03:56:34.255 INFO:tasks.rados.rados.0.smithi070.stdout:2391: finishing write tid 2 to smithi07085154-15 2024-09-14T03:56:34.256 INFO:tasks.rados.rados.0.smithi070.stdout:2391: finishing write tid 3 to smithi07085154-15 2024-09-14T03:56:34.256 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 15 v 2057 (ObjNum 740 snap 255 seq_num 740) dirty exists 2024-09-14T03:56:34.256 INFO:tasks.rados.rados.0.smithi070.stdout:2391: left oid 15 (ObjNum 740 snap 255 seq_num 740) 2024-09-14T03:56:34.457 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:34 smithi094 podman[106346]: 2024-09-14 03:56:34.170342135 +0000 UTC m=+0.168567517 container create ea9896fb7dbc3942967a81594535cb6cb3f19e645e4c71b9a49485678879a51d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=main-a278231, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T03:56:34.457 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:34 smithi094 podman[106346]: 2024-09-14 03:56:34.356397897 +0000 UTC m=+0.354623282 container init ea9896fb7dbc3942967a81594535cb6cb3f19e645e4c71b9a49485678879a51d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7, GIT_CLEAN=True, RELEASE=main-a278231, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-14T03:56:34.457 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:34 smithi094 podman[106346]: 2024-09-14 03:56:34.361249528 +0000 UTC m=+0.359474911 container start ea9896fb7dbc3942967a81594535cb6cb3f19e645e4c71b9a49485678879a51d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=main-a278231, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-14T03:56:34.457 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:34 smithi094 ceph-osd[106385]: -- 172.21.15.94:0/3033899920 <== mon.1 v2:172.21.15.94:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5569a0831a40 con 0x5569a1600000 2024-09-14T03:56:34.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:34 smithi070 ceph-mon[96704]: Health check failed: Degraded data redundancy: 132/1209 objects degraded (10.918%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T03:56:34.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:34 smithi070 ceph-mon[96704]: osdmap e562: 8 total, 7 up, 8 in 2024-09-14T03:56:34.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:34 smithi070 ceph-mon[96704]: osdmap e563: 8 total, 7 up, 8 in 2024-09-14T03:56:34.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:34 smithi070 ceph-mon[98640]: Health check failed: Degraded data redundancy: 132/1209 objects degraded (10.918%), 9 pgs degraded (PG_DEGRADED) 2024-09-14T03:56:34.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:34 smithi070 ceph-mon[98640]: osdmap e562: 8 total, 7 up, 8 in 2024-09-14T03:56:34.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:34 smithi070 ceph-mon[98640]: osdmap e563: 8 total, 7 up, 8 in 2024-09-14T03:56:34.776 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:34 smithi094 bash[106346]: ea9896fb7dbc3942967a81594535cb6cb3f19e645e4c71b9a49485678879a51d 2024-09-14T03:56:34.776 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:34 smithi094 systemd[1]: Started Ceph osd.7 for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:56:35.086 INFO:tasks.rados.rados.0.smithi070.stdout:2391: done (1 left) 2024-09-14T03:56:35.086 INFO:tasks.rados.rados.0.smithi070.stdout:2392: done (0 left) 2024-09-14T03:56:35.086 INFO:tasks.rados.rados.0.smithi070.stdout:2393: delete oid 47 current snap is 256 2024-09-14T03:56:35.089 INFO:tasks.rados.rados.0.smithi070.stdout:2393: done (0 left) 2024-09-14T03:56:35.089 INFO:tasks.rados.rados.0.smithi070.stdout:2394: copy_from oid 37 from oid 34 current snap is 256 2024-09-14T03:56:35.089 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 9 current snap is 256 2024-09-14T03:56:35.089 INFO:tasks.rados.rados.0.smithi070.stdout:2395: seq_num 742 ranges {1818624=434176} 2024-09-14T03:56:35.102 INFO:tasks.rados.rados.0.smithi070.stdout:2395: writing smithi07085154-9 from 1818624 to 2252800 tid 1 2024-09-14T03:56:35.102 INFO:tasks.rados.rados.0.smithi070.stdout:2396: rollback oid 27 current snap is 256 2024-09-14T03:56:35.102 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 27 to 174 2024-09-14T03:56:35.102 INFO:tasks.rados.rados.0.smithi070.stdout:2394: finishing copy_from to smithi07085154-37 2024-09-14T03:56:35.102 INFO:tasks.rados.rados.0.smithi070.stdout:2394: got expected ENOENT (src dne) 2024-09-14T03:56:35.102 INFO:tasks.rados.rados.0.smithi070.stdout:2394: finishing copy_from racing read to smithi07085154-37 2024-09-14T03:56:35.103 INFO:tasks.rados.rados.0.smithi070.stdout:2394: done (2 left) 2024-09-14T03:56:35.103 INFO:tasks.rados.rados.0.smithi070.stdout:2397: delete oid 48 current snap is 256 2024-09-14T03:56:35.106 INFO:tasks.rados.rados.0.smithi070.stdout:2397: done (2 left) 2024-09-14T03:56:35.106 INFO:tasks.rados.rados.0.smithi070.stdout:2398: delete oid 20 current snap is 256 2024-09-14T03:56:35.107 INFO:tasks.rados.rados.0.smithi070.stdout:2395: finishing write tid 1 to smithi07085154-9 2024-09-14T03:56:35.108 INFO:tasks.rados.rados.0.smithi070.stdout:2398: done (2 left) 2024-09-14T03:56:35.109 INFO:tasks.rados.rados.0.smithi070.stdout:2399: snap_create 2024-09-14T03:56:35.110 INFO:tasks.rados.rados.0.smithi070.stdout:2395: finishing write tid 2 to smithi07085154-9 2024-09-14T03:56:35.112 INFO:tasks.rados.rados.0.smithi070.stdout:2396: finishing rollback tid 0 to smithi07085154-27 2024-09-14T03:56:35.112 INFO:tasks.rados.rados.0.smithi070.stdout:2396: finishing rollback tid 1 to smithi07085154-27 2024-09-14T03:56:35.112 INFO:tasks.rados.rados.0.smithi070.stdout:2396: finishing rollback tid 2 to smithi07085154-27 2024-09-14T03:56:35.112 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 2615 (ObjNum 526 snap 174 seq_num 526) dirty exists 2024-09-14T03:56:35.113 INFO:tasks.rados.rados.0.smithi070.stdout:2395: finishing write tid 3 to smithi07085154-9 2024-09-14T03:56:35.113 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 2611 (ObjNum 742 snap 256 seq_num 742) dirty exists 2024-09-14T03:56:35.113 INFO:tasks.rados.rados.0.smithi070.stdout:2395: left oid 9 (ObjNum 742 snap 256 seq_num 742) 2024-09-14T03:56:35.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:35 smithi094 ceph-mon[86597]: pgmap v441: 121 pgs: 33 active+undersized, 4 peering, 13 active+undersized+degraded, 71 active+clean; 1.0 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 108 KiB/s wr, 1 op/s; 166/1212 objects degraded (13.696%) 2024-09-14T03:56:35.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:35 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:35.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:35 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:35.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:35 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:56:35.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:35 smithi070 ceph-mon[98640]: pgmap v441: 121 pgs: 33 active+undersized, 4 peering, 13 active+undersized+degraded, 71 active+clean; 1.0 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 108 KiB/s wr, 1 op/s; 166/1212 objects degraded (13.696%) 2024-09-14T03:56:35.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:35 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:35.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:35 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:35.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:35 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:56:35.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:35 smithi070 ceph-mon[96704]: pgmap v441: 121 pgs: 33 active+undersized, 4 peering, 13 active+undersized+degraded, 71 active+clean; 1.0 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 108 KiB/s wr, 1 op/s; 166/1212 objects degraded (13.696%) 2024-09-14T03:56:35.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:35 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:35.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:35 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:35.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:35 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:56:36.088 INFO:tasks.rados.rados.0.smithi070.stdout:2395: done (2 left) 2024-09-14T03:56:36.088 INFO:tasks.rados.rados.0.smithi070.stdout:2396: done (1 left) 2024-09-14T03:56:36.088 INFO:tasks.rados.rados.0.smithi070.stdout:2399: done (0 left) 2024-09-14T03:56:36.088 INFO:tasks.rados.rados.0.smithi070.stdout:2400: snap_remove snap 223 2024-09-14T03:56:36.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:36 smithi094 ceph-mon[86597]: osdmap e564: 8 total, 7 up, 8 in 2024-09-14T03:56:36.277 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:36 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T03:56:35.999+0000 7f684bbd2640 -1 Falling back to public interface 2024-09-14T03:56:36.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:36 smithi070 ceph-mon[96704]: osdmap e564: 8 total, 7 up, 8 in 2024-09-14T03:56:36.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:36 smithi070 ceph-mon[98640]: osdmap e564: 8 total, 7 up, 8 in 2024-09-14T03:56:37.099 INFO:tasks.rados.rados.0.smithi070.stdout:2400: done (0 left) 2024-09-14T03:56:37.099 INFO:tasks.rados.rados.0.smithi070.stdout:2401: snap_create 2024-09-14T03:56:37.405 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:37 smithi094 ceph-mon[86597]: osdmap e565: 8 total, 7 up, 8 in 2024-09-14T03:56:37.405 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:37 smithi094 ceph-mon[86597]: pgmap v444: 121 pgs: 37 active+undersized, 13 active+undersized+degraded, 71 active+clean; 1.0 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 170 KiB/s wr, 3 op/s; 166/1221 objects degraded (13.595%) 2024-09-14T03:56:37.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:37 smithi070 ceph-mon[96704]: osdmap e565: 8 total, 7 up, 8 in 2024-09-14T03:56:37.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:37 smithi070 ceph-mon[96704]: pgmap v444: 121 pgs: 37 active+undersized, 13 active+undersized+degraded, 71 active+clean; 1.0 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 170 KiB/s wr, 3 op/s; 166/1221 objects degraded (13.595%) 2024-09-14T03:56:37.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:37 smithi070 ceph-mon[98640]: osdmap e565: 8 total, 7 up, 8 in 2024-09-14T03:56:37.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:37 smithi070 ceph-mon[98640]: pgmap v444: 121 pgs: 37 active+undersized, 13 active+undersized+degraded, 71 active+clean; 1.0 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 170 KiB/s wr, 3 op/s; 166/1221 objects degraded (13.595%) 2024-09-14T03:56:38.099 INFO:tasks.rados.rados.0.smithi070.stdout:2401: done (0 left) 2024-09-14T03:56:38.099 INFO:tasks.rados.rados.0.smithi070.stdout:2402: snap_remove snap 251 2024-09-14T03:56:38.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:38 smithi094 ceph-mon[86597]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-14T03:56:38.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:38 smithi094 ceph-mon[86597]: osdmap e566: 8 total, 7 up, 8 in 2024-09-14T03:56:38.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:38 smithi070 ceph-mon[96704]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-14T03:56:38.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:38 smithi070 ceph-mon[96704]: osdmap e566: 8 total, 7 up, 8 in 2024-09-14T03:56:38.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:38 smithi070 ceph-mon[98640]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-14T03:56:38.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:38 smithi070 ceph-mon[98640]: osdmap e566: 8 total, 7 up, 8 in 2024-09-14T03:56:39.106 INFO:tasks.rados.rados.0.smithi070.stdout:2402: done (0 left) 2024-09-14T03:56:39.106 INFO:tasks.rados.rados.0.smithi070.stdout:2403: setattr oid 43 current snap is 258 2024-09-14T03:56:39.107 INFO:tasks.rados.rados.0.smithi070.stdout:2404: setattr oid 29 current snap is 258 2024-09-14T03:56:39.110 INFO:tasks.rados.rados.0.smithi070.stdout:2405: rmattr oid 12 current snap is 258 2024-09-14T03:56:39.110 INFO:tasks.rados.rados.0.smithi070.stdout:2406: read oid 6 snap -1 2024-09-14T03:56:39.110 INFO:tasks.rados.rados.0.smithi070.stdout:2406: expect (ObjNum 738 snap 255 seq_num 738) 2024-09-14T03:56:39.110 INFO:tasks.rados.rados.0.smithi070.stdout:2407: snap_remove snap 231 2024-09-14T03:56:39.523 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:39 smithi094 ceph-mon[86597]: pgmap v446: 121 pgs: 37 active+undersized, 13 active+undersized+degraded, 71 active+clean; 1.1 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 270 KiB/s wr, 11 op/s; 170/1242 objects degraded (13.688%) 2024-09-14T03:56:39.524 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:39 smithi094 ceph-mon[86597]: osdmap e567: 8 total, 7 up, 8 in 2024-09-14T03:56:39.524 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:39 smithi094 ceph-mon[86597]: Health check update: Degraded data redundancy: 170/1242 objects degraded (13.688%), 13 pgs degraded (PG_DEGRADED) 2024-09-14T03:56:39.524 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:39 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:39.524 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:39 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:39.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:39 smithi070 ceph-mon[96704]: pgmap v446: 121 pgs: 37 active+undersized, 13 active+undersized+degraded, 71 active+clean; 1.1 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 270 KiB/s wr, 11 op/s; 170/1242 objects degraded (13.688%) 2024-09-14T03:56:39.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:39 smithi070 ceph-mon[96704]: osdmap e567: 8 total, 7 up, 8 in 2024-09-14T03:56:39.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:39 smithi070 ceph-mon[96704]: Health check update: Degraded data redundancy: 170/1242 objects degraded (13.688%), 13 pgs degraded (PG_DEGRADED) 2024-09-14T03:56:39.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:39 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:39.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:39 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:39.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:39 smithi070 ceph-mon[98640]: pgmap v446: 121 pgs: 37 active+undersized, 13 active+undersized+degraded, 71 active+clean; 1.1 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 270 KiB/s wr, 11 op/s; 170/1242 objects degraded (13.688%) 2024-09-14T03:56:39.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:39 smithi070 ceph-mon[98640]: osdmap e567: 8 total, 7 up, 8 in 2024-09-14T03:56:39.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:39 smithi070 ceph-mon[98640]: Health check update: Degraded data redundancy: 170/1242 objects degraded (13.688%), 13 pgs degraded (PG_DEGRADED) 2024-09-14T03:56:39.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:39 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:39.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:39 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:40.123 INFO:tasks.rados.rados.0.smithi070.stdout:2407: done (4 left) 2024-09-14T03:56:40.123 INFO:tasks.rados.rados.0.smithi070.stdout:2408: snap_remove snap 212 2024-09-14T03:56:40.398 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:40 smithi094 ceph-mon[86597]: osdmap e568: 8 total, 7 up, 8 in 2024-09-14T03:56:40.398 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:40 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:40.398 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:40 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:40.398 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:40 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T03:56:40.162+0000 7f684bbd2640 -1 osd.7 557 log_to_monitors true 2024-09-14T03:56:40.399 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:56:40 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:56:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:56:40.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:40 smithi070 ceph-mon[96704]: osdmap e568: 8 total, 7 up, 8 in 2024-09-14T03:56:40.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:40 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:40.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:40 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:40.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:40 smithi070 ceph-mon[98640]: osdmap e568: 8 total, 7 up, 8 in 2024-09-14T03:56:40.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:40 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:40.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:40 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:41.131 INFO:tasks.rados.rados.0.smithi070.stdout:2408: done (4 left) 2024-09-14T03:56:41.132 INFO:tasks.rados.rados.0.smithi070.stdout:2409: rmattr oid 11 current snap is 258 2024-09-14T03:56:41.134 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 2204 (ObjNum 681 snap 230 seq_num 681) dirty exists 2024-09-14T03:56:41.135 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 1731 (ObjNum 458 snap 155 seq_num 458) dirty exists 2024-09-14T03:56:41.135 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 2058 (ObjNum 161 snap 0 seq_num 0) dirty exists 2024-09-14T03:56:41.135 INFO:tasks.rados.rados.0.smithi070.stdout:2406: expect (ObjNum 738 snap 255 seq_num 738) 2024-09-14T03:56:41.391 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:41 smithi094 ceph-mon[86597]: pgmap v449: 121 pgs: 1 active+clean+snaptrim, 37 active+undersized, 13 active+undersized+degraded, 70 active+clean; 1.1 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 263 KiB/s wr, 15 op/s; 170/1245 objects degraded (13.655%) 2024-09-14T03:56:41.391 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:41 smithi094 ceph-mon[86597]: osdmap e569: 8 total, 7 up, 8 in 2024-09-14T03:56:41.392 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:41 smithi094 ceph-mon[86597]: from='osd.7 [v2:172.21.15.94:6824/1152477702,v1:172.21.15.94:6825/1152477702]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-14T03:56:41.392 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:41 smithi094 ceph-mon[86597]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-14T03:56:41.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:41 smithi070 ceph-mon[98640]: pgmap v449: 121 pgs: 1 active+clean+snaptrim, 37 active+undersized, 13 active+undersized+degraded, 70 active+clean; 1.1 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 263 KiB/s wr, 15 op/s; 170/1245 objects degraded (13.655%) 2024-09-14T03:56:41.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:41 smithi070 ceph-mon[98640]: osdmap e569: 8 total, 7 up, 8 in 2024-09-14T03:56:41.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:41 smithi070 ceph-mon[98640]: from='osd.7 [v2:172.21.15.94:6824/1152477702,v1:172.21.15.94:6825/1152477702]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-14T03:56:41.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:41 smithi070 ceph-mon[98640]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-14T03:56:41.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:41 smithi070 ceph-mon[96704]: pgmap v449: 121 pgs: 1 active+clean+snaptrim, 37 active+undersized, 13 active+undersized+degraded, 70 active+clean; 1.1 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 263 KiB/s wr, 15 op/s; 170/1245 objects degraded (13.655%) 2024-09-14T03:56:41.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:41 smithi070 ceph-mon[96704]: osdmap e569: 8 total, 7 up, 8 in 2024-09-14T03:56:41.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:41 smithi070 ceph-mon[96704]: from='osd.7 [v2:172.21.15.94:6824/1152477702,v1:172.21.15.94:6825/1152477702]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-14T03:56:41.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:41 smithi070 ceph-mon[96704]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-14T03:56:41.655 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 03:56:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T03:56:41.443+0000 7f683f974640 -1 osd.7 557 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:56:41.978 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 11 v 2056 (ObjNum 645 snap 216 seq_num 645) dirty exists 2024-09-14T03:56:41.979 INFO:tasks.rados.rados.0.smithi070.stdout:2403: done (4 left) 2024-09-14T03:56:41.979 INFO:tasks.rados.rados.0.smithi070.stdout:2404: done (3 left) 2024-09-14T03:56:41.979 INFO:tasks.rados.rados.0.smithi070.stdout:2405: done (2 left) 2024-09-14T03:56:41.979 INFO:tasks.rados.rados.0.smithi070.stdout:2406: done (1 left) 2024-09-14T03:56:41.979 INFO:tasks.rados.rados.0.smithi070.stdout:2409: done (0 left) 2024-09-14T03:56:41.979 INFO:tasks.rados.rados.0.smithi070.stdout:2410: copy_from oid 10 from oid 38 current snap is 258 2024-09-14T03:56:41.979 INFO:tasks.rados.rados.0.smithi070.stdout:2411: read oid 37 snap -1 2024-09-14T03:56:41.979 INFO:tasks.rados.rados.0.smithi070.stdout:2411: expect deleted 2024-09-14T03:56:41.979 INFO:tasks.rados.rados.0.smithi070.stdout:2412: read oid 19 snap -1 2024-09-14T03:56:41.980 INFO:tasks.rados.rados.0.smithi070.stdout:2412: expect (ObjNum 466 snap 158 seq_num 466) 2024-09-14T03:56:41.980 INFO:tasks.rados.rados.0.smithi070.stdout:2413: setattr oid 30 current snap is 258 2024-09-14T03:56:41.981 INFO:tasks.rados.rados.0.smithi070.stdout:2411: done (3 left) 2024-09-14T03:56:41.981 INFO:tasks.rados.rados.0.smithi070.stdout:2414: read oid 43 snap 256 2024-09-14T03:56:41.981 INFO:tasks.rados.rados.0.smithi070.stdout:2414: expect deleted 2024-09-14T03:56:41.981 INFO:tasks.rados.rados.0.smithi070.stdout:2415: copy_from oid 40 from oid 49 current snap is 258 2024-09-14T03:56:41.981 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 26 current snap is 258 2024-09-14T03:56:41.981 INFO:tasks.rados.rados.0.smithi070.stdout:2416: seq_num 745 ranges {942080=483328} 2024-09-14T03:56:41.987 INFO:tasks.rados.rados.0.smithi070.stdout:2416: writing smithi07085154-26 from 942080 to 1425408 tid 1 2024-09-14T03:56:41.987 INFO:tasks.rados.rados.0.smithi070.stdout:2417: snap_remove snap 199 2024-09-14T03:56:42.450 INFO:tasks.rados.rados.0.smithi070.stdout:2417: done (6 left) 2024-09-14T03:56:42.451 INFO:tasks.rados.rados.0.smithi070.stdout:2418: copy_from oid 6 from oid 16 current snap is 258 2024-09-14T03:56:42.451 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 30 v 1714 (ObjNum 177 snap 0 seq_num 1) dirty exists 2024-09-14T03:56:42.451 INFO:tasks.rados.rados.0.smithi070.stdout:2412: expect (ObjNum 466 snap 158 seq_num 466) 2024-09-14T03:56:42.496 INFO:tasks.rados.rados.0.smithi070.stdout:2410: finishing copy_from racing read to smithi07085154-10 2024-09-14T03:56:42.497 INFO:tasks.rados.rados.0.smithi070.stdout:2410: finishing copy_from to smithi07085154-10 2024-09-14T03:56:42.497 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 1712 (ObjNum 540 snap 176 seq_num 540) dirty exists 2024-09-14T03:56:42.497 INFO:tasks.rados.rados.0.smithi070.stdout:2416: finishing write tid 1 to smithi07085154-26 2024-09-14T03:56:42.497 INFO:tasks.rados.rados.0.smithi070.stdout:2416: finishing write tid 2 to smithi07085154-26 2024-09-14T03:56:42.497 INFO:tasks.rados.rados.0.smithi070.stdout:2416: finishing write tid 3 to smithi07085154-26 2024-09-14T03:56:42.497 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 2620 (ObjNum 745 snap 258 seq_num 745) dirty exists 2024-09-14T03:56:42.497 INFO:tasks.rados.rados.0.smithi070.stdout:2416: left oid 26 (ObjNum 745 snap 258 seq_num 745) 2024-09-14T03:56:42.497 INFO:tasks.rados.rados.0.smithi070.stdout:2415: finishing copy_from racing read to smithi07085154-40 2024-09-14T03:56:42.497 INFO:tasks.rados.rados.0.smithi070.stdout:2415: finishing copy_from to smithi07085154-40 2024-09-14T03:56:42.497 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 1959 (ObjNum 729 snap 253 seq_num 729) dirty exists 2024-09-14T03:56:42.498 INFO:tasks.rados.rados.0.smithi070.stdout:2410: done (6 left) 2024-09-14T03:56:42.498 INFO:tasks.rados.rados.0.smithi070.stdout:2412: done (5 left) 2024-09-14T03:56:42.498 INFO:tasks.rados.rados.0.smithi070.stdout:2413: done (4 left) 2024-09-14T03:56:42.498 INFO:tasks.rados.rados.0.smithi070.stdout:2414: done (3 left) 2024-09-14T03:56:42.498 INFO:tasks.rados.rados.0.smithi070.stdout:2415: done (2 left) 2024-09-14T03:56:42.498 INFO:tasks.rados.rados.0.smithi070.stdout:2416: done (1 left) 2024-09-14T03:56:42.498 INFO:tasks.rados.rados.0.smithi070.stdout:2419: rmattr oid 32 current snap is 258 2024-09-14T03:56:42.498 INFO:tasks.rados.rados.0.smithi070.stdout:2419: done (1 left) 2024-09-14T03:56:42.498 INFO:tasks.rados.rados.0.smithi070.stdout:2420: snap_create 2024-09-14T03:56:42.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:42 smithi094 ceph-mon[86597]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-14T03:56:42.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:42 smithi094 ceph-mon[86597]: from='osd.7 [v2:172.21.15.94:6824/1152477702,v1:172.21.15.94:6825/1152477702]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi094", "root=default"]} : dispatch 2024-09-14T03:56:42.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:42 smithi094 ceph-mon[86597]: osdmap e570: 8 total, 7 up, 8 in 2024-09-14T03:56:42.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:42 smithi094 ceph-mon[86597]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi094", "root=default"]} : dispatch 2024-09-14T03:56:42.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:42 smithi094 ceph-mon[86597]: from='osd.7 ' entity='osd.7' 2024-09-14T03:56:42.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:42 smithi070 ceph-mon[96704]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-14T03:56:42.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:42 smithi070 ceph-mon[96704]: from='osd.7 [v2:172.21.15.94:6824/1152477702,v1:172.21.15.94:6825/1152477702]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi094", "root=default"]} : dispatch 2024-09-14T03:56:42.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:42 smithi070 ceph-mon[96704]: osdmap e570: 8 total, 7 up, 8 in 2024-09-14T03:56:42.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:42 smithi070 ceph-mon[96704]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi094", "root=default"]} : dispatch 2024-09-14T03:56:42.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:42 smithi070 ceph-mon[96704]: from='osd.7 ' entity='osd.7' 2024-09-14T03:56:42.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:42 smithi070 ceph-mon[98640]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-14T03:56:42.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:42 smithi070 ceph-mon[98640]: from='osd.7 [v2:172.21.15.94:6824/1152477702,v1:172.21.15.94:6825/1152477702]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi094", "root=default"]} : dispatch 2024-09-14T03:56:42.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:42 smithi070 ceph-mon[98640]: osdmap e570: 8 total, 7 up, 8 in 2024-09-14T03:56:42.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:42 smithi070 ceph-mon[98640]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi094", "root=default"]} : dispatch 2024-09-14T03:56:42.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:42 smithi070 ceph-mon[98640]: from='osd.7 ' entity='osd.7' 2024-09-14T03:56:43.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:43 smithi094 ceph-mon[86597]: pgmap v452: 121 pgs: 1 active+clean+snaptrim, 37 active+undersized, 13 active+undersized+degraded, 70 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 660 KiB/s wr, 17 op/s; 173/1251 objects degraded (13.829%) 2024-09-14T03:56:43.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:43 smithi094 ceph-mon[86597]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:56:43.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:43 smithi094 ceph-mon[86597]: osd.7 [v2:172.21.15.94:6824/1152477702,v1:172.21.15.94:6825/1152477702] boot 2024-09-14T03:56:43.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:43 smithi094 ceph-mon[86597]: osdmap e571: 8 total, 8 up, 8 in 2024-09-14T03:56:43.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:43 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-14T03:56:43.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:43 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:43.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:43 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:43.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:43 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:56:43.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:43 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:56:43.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:43 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:43.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:43 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:56:43.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:43 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:43.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:43 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:43.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:43 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:43.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:43 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:43.278 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:43 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:43.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:43 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-14T03:56:43.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:43 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-14T03:56:43.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:43 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-14T03:56:43.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:43 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-14T03:56:43.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:43 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-14T03:56:43.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:43 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-14T03:56:43.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:43 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-14T03:56:43.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:43 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-14T03:56:43.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:43 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-14T03:56:43.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:43 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-14T03:56:43.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:43 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-14T03:56:43.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:43 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-14T03:56:43.279 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:43 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-14T03:56:43.280 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:43 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-14T03:56:43.280 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:43 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-14T03:56:43.280 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:43 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-14T03:56:43.280 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:43 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-14T03:56:43.406 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[96704]: pgmap v452: 121 pgs: 1 active+clean+snaptrim, 37 active+undersized, 13 active+undersized+degraded, 70 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 660 KiB/s wr, 17 op/s; 173/1251 objects degraded (13.829%) 2024-09-14T03:56:43.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[96704]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:56:43.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[96704]: osd.7 [v2:172.21.15.94:6824/1152477702,v1:172.21.15.94:6825/1152477702] boot 2024-09-14T03:56:43.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[96704]: osdmap e571: 8 total, 8 up, 8 in 2024-09-14T03:56:43.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-14T03:56:43.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:43.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:43.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:56:43.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:56:43.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:43.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:56:43.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:43.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:43.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:43.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:43.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:43.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-14T03:56:43.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-14T03:56:43.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-14T03:56:43.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-14T03:56:43.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-14T03:56:43.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-14T03:56:43.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-14T03:56:43.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-14T03:56:43.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-14T03:56:43.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-14T03:56:43.409 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[98640]: pgmap v452: 121 pgs: 1 active+clean+snaptrim, 37 active+undersized, 13 active+undersized+degraded, 70 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 660 KiB/s wr, 17 op/s; 173/1251 objects degraded (13.829%) 2024-09-14T03:56:43.409 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[98640]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T03:56:43.409 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[98640]: osd.7 [v2:172.21.15.94:6824/1152477702,v1:172.21.15.94:6825/1152477702] boot 2024-09-14T03:56:43.409 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[98640]: osdmap e571: 8 total, 8 up, 8 in 2024-09-14T03:56:43.409 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-14T03:56:43.409 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:43.409 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:43.409 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:56:43.409 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:56:43.410 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:43.410 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:56:43.410 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:43.410 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:43.410 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:43.410 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:56:43.410 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:43.410 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-14T03:56:43.410 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-14T03:56:43.410 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-14T03:56:43.410 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-14T03:56:43.410 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-14T03:56:43.410 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-14T03:56:43.410 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-14T03:56:43.411 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-14T03:56:43.411 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-14T03:56:43.411 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-14T03:56:43.411 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-14T03:56:43.411 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-14T03:56:43.411 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-14T03:56:43.411 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-14T03:56:43.411 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-14T03:56:43.411 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-14T03:56:43.411 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-14T03:56:43.411 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-14T03:56:43.411 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-14T03:56:43.412 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-14T03:56:43.412 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-14T03:56:43.412 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-14T03:56:43.412 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-14T03:56:43.412 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-14T03:56:43.441 INFO:tasks.rados.rados.0.smithi070.stdout:2420: done (1 left) 2024-09-14T03:56:43.441 INFO:tasks.rados.rados.0.smithi070.stdout:2421: rollback oid 46 current snap is 259 2024-09-14T03:56:43.441 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 46 to 193 2024-09-14T03:56:43.442 INFO:tasks.rados.rados.0.smithi070.stdout:2422: read oid 37 snap -1 2024-09-14T03:56:43.442 INFO:tasks.rados.rados.0.smithi070.stdout:2422: expect deleted 2024-09-14T03:56:43.442 INFO:tasks.rados.rados.0.smithi070.stdout:2423: setattr oid 29 current snap is 259 2024-09-14T03:56:43.445 INFO:tasks.rados.rados.0.smithi070.stdout:2424: read oid 17 snap 186 2024-09-14T03:56:43.445 INFO:tasks.rados.rados.0.smithi070.stdout:2424: expect (ObjNum 544 snap 181 seq_num 544) 2024-09-14T03:56:43.445 INFO:tasks.rados.rados.0.smithi070.stdout:2425: delete oid 45 current snap is 259 2024-09-14T03:56:43.450 INFO:tasks.rados.rados.0.smithi070.stdout:2422: done (5 left) 2024-09-14T03:56:43.451 INFO:tasks.rados.rados.0.smithi070.stdout:2425: done (4 left) 2024-09-14T03:56:43.451 INFO:tasks.rados.rados.0.smithi070.stdout:2426: setattr oid 34 current snap is 259 2024-09-14T03:56:43.452 INFO:tasks.rados.rados.0.smithi070.stdout:2427: read oid 21 snap -1 2024-09-14T03:56:43.452 INFO:tasks.rados.rados.0.smithi070.stdout:2427: expect deleted 2024-09-14T03:56:43.452 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 19 current snap is 259 2024-09-14T03:56:43.452 INFO:tasks.rados.rados.0.smithi070.stdout:2428: seq_num 747 ranges {1155072=516096} 2024-09-14T03:56:43.462 INFO:tasks.rados.rados.0.smithi070.stdout:2428: writing smithi07085154-19 from 1155072 to 1671168 tid 1 2024-09-14T03:56:43.462 INFO:tasks.rados.rados.0.smithi070.stdout:2427: done (6 left) 2024-09-14T03:56:43.462 INFO:tasks.rados.rados.0.smithi070.stdout:2429: delete oid 25 current snap is 259 2024-09-14T03:56:43.474 INFO:tasks.rados.rados.0.smithi070.stdout:2424: expect (ObjNum 544 snap 181 seq_num 544) 2024-09-14T03:56:43.689 INFO:tasks.rados.rados.0.smithi070.stdout:2424: done (6 left) 2024-09-14T03:56:43.689 INFO:tasks.rados.rados.0.smithi070.stdout:2429: done (5 left) 2024-09-14T03:56:43.689 INFO:tasks.rados.rados.0.smithi070.stdout:2430: read oid 47 snap -1 2024-09-14T03:56:43.689 INFO:tasks.rados.rados.0.smithi070.stdout:2430: expect deleted 2024-09-14T03:56:43.689 INFO:tasks.rados.rados.0.smithi070.stdout:2431: rollback oid 44 current snap is 259 2024-09-14T03:56:43.689 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 44 to 247 2024-09-14T03:56:43.689 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 1732 (ObjNum 145 snap 0 seq_num 2930265776) dirty exists 2024-09-14T03:56:43.689 INFO:tasks.rados.rados.0.smithi070.stdout:2428: finishing write tid 1 to smithi07085154-19 2024-09-14T03:56:43.689 INFO:tasks.rados.rados.0.smithi070.stdout:2428: finishing write tid 2 to smithi07085154-19 2024-09-14T03:56:43.689 INFO:tasks.rados.rados.0.smithi070.stdout:2428: finishing write tid 3 to smithi07085154-19 2024-09-14T03:56:43.690 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 19 v 1734 (ObjNum 747 snap 259 seq_num 747) dirty exists 2024-09-14T03:56:43.690 INFO:tasks.rados.rados.0.smithi070.stdout:2428: left oid 19 (ObjNum 747 snap 259 seq_num 747) 2024-09-14T03:56:43.690 INFO:tasks.rados.rados.0.smithi070.stdout:2421: finishing rollback tid 0 to smithi07085154-46 2024-09-14T03:56:43.690 INFO:tasks.rados.rados.0.smithi070.stdout:2421: finishing rollback tid 1 to smithi07085154-46 2024-09-14T03:56:43.690 INFO:tasks.rados.rados.0.smithi070.stdout:2421: finishing rollback tid 2 to smithi07085154-46 2024-09-14T03:56:43.690 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 46 v 2624 (ObjNum 576 snap 193 seq_num 576) dirty exists 2024-09-14T03:56:43.690 INFO:tasks.rados.rados.0.smithi070.stdout:2418: finishing copy_from racing read to smithi07085154-6 2024-09-14T03:56:43.690 INFO:tasks.rados.rados.0.smithi070.stdout:2418: finishing copy_from to smithi07085154-6 2024-09-14T03:56:43.690 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 2247 (ObjNum 114 snap 24 seq_num 114) dirty exists 2024-09-14T03:56:43.690 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 1735 (ObjNum 458 snap 155 seq_num 458) dirty exists 2024-09-14T03:56:43.690 INFO:tasks.rados.rados.0.smithi070.stdout:2418: done (6 left) 2024-09-14T03:56:43.690 INFO:tasks.rados.rados.0.smithi070.stdout:2421: done (5 left) 2024-09-14T03:56:43.690 INFO:tasks.rados.rados.0.smithi070.stdout:2423: done (4 left) 2024-09-14T03:56:43.690 INFO:tasks.rados.rados.0.smithi070.stdout:2426: done (3 left) 2024-09-14T03:56:43.690 INFO:tasks.rados.rados.0.smithi070.stdout:2428: done (2 left) 2024-09-14T03:56:43.691 INFO:tasks.rados.rados.0.smithi070.stdout:2432: rmattr oid 6 current snap is 259 2024-09-14T03:56:43.691 INFO:tasks.rados.rados.0.smithi070.stdout:2432: done (2 left) 2024-09-14T03:56:43.691 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 9 current snap is 259 2024-09-14T03:56:43.691 INFO:tasks.rados.rados.0.smithi070.stdout:2433: seq_num 749 ranges {2252800=548864} 2024-09-14T03:56:43.704 INFO:tasks.rados.rados.0.smithi070.stdout:2433: writing smithi07085154-9 from 2252800 to 2801664 tid 1 2024-09-14T03:56:43.704 INFO:tasks.rados.rados.0.smithi070.stdout:2434: read oid 27 snap -1 2024-09-14T03:56:43.704 INFO:tasks.rados.rados.0.smithi070.stdout:2434: expect (ObjNum 526 snap 174 seq_num 526) 2024-09-14T03:56:43.704 INFO:tasks.rados.rados.0.smithi070.stdout:2435: read oid 12 snap -1 2024-09-14T03:56:43.704 INFO:tasks.rados.rados.0.smithi070.stdout:2435: expect (ObjNum 681 snap 230 seq_num 681) 2024-09-14T03:56:43.704 INFO:tasks.rados.rados.0.smithi070.stdout:2436: snap_create 2024-09-14T03:56:43.704 INFO:tasks.rados.rados.0.smithi070.stdout:2431: finishing rollback tid 0 to smithi07085154-44 2024-09-14T03:56:43.704 INFO:tasks.rados.rados.0.smithi070.stdout:2431: finishing rollback tid 1 to smithi07085154-44 2024-09-14T03:56:43.704 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 2060 (ObjNum 33 snap 0 seq_num 2931204560) dirty dne 2024-09-14T03:56:43.714 INFO:tasks.rados.rados.0.smithi070.stdout:2435: expect (ObjNum 681 snap 230 seq_num 681) 2024-09-14T03:56:43.772 INFO:tasks.rados.rados.0.smithi070.stdout:2433: finishing write tid 1 to smithi07085154-9 2024-09-14T03:56:43.772 INFO:tasks.rados.rados.0.smithi070.stdout:2433: finishing write tid 2 to smithi07085154-9 2024-09-14T03:56:43.772 INFO:tasks.rados.rados.0.smithi070.stdout:2434: expect (ObjNum 526 snap 174 seq_num 526) 2024-09-14T03:56:43.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:56:43 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:56:43] "GET /metrics HTTP/1.1" 200 36447 "" "Prometheus/2.51.0" 2024-09-14T03:56:44.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:44 smithi094 ceph-mon[86597]: Upgrade: Setting container_image for all osd 2024-09-14T03:56:44.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:44 smithi094 ceph-mon[86597]: Upgrade: Setting require_osd_release to 19 squid 2024-09-14T03:56:44.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:44 smithi094 ceph-mon[86597]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-14T03:56:44.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:44 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-14T03:56:44.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:44 smithi094 ceph-mon[86597]: osdmap e572: 8 total, 8 up, 8 in 2024-09-14T03:56:44.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:44 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mds ok-to-stop", "ids": ["a"]} : dispatch 2024-09-14T03:56:44.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:44 smithi094 ceph-mon[86597]: Upgrade: It appears safe to stop mds.a 2024-09-14T03:56:44.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:44 smithi094 ceph-mon[86597]: Health check update: Degraded data redundancy: 173/1251 objects degraded (13.829%), 13 pgs degraded (PG_DEGRADED) 2024-09-14T03:56:44.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:44 smithi094 ceph-mon[86597]: pgmap v455: 121 pgs: 11 peering, 1 active+clean+snaptrim, 28 active+undersized, 11 active+undersized+degraded, 70 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 3.1 MiB/s wr, 13 op/s; 141/1257 objects degraded (11.217%) 2024-09-14T03:56:44.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:44 smithi070 ceph-mon[98640]: Upgrade: Setting container_image for all osd 2024-09-14T03:56:44.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:44 smithi070 ceph-mon[98640]: Upgrade: Setting require_osd_release to 19 squid 2024-09-14T03:56:44.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:44 smithi070 ceph-mon[98640]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-14T03:56:44.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:44 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-14T03:56:44.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:44 smithi070 ceph-mon[98640]: osdmap e572: 8 total, 8 up, 8 in 2024-09-14T03:56:44.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:44 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mds ok-to-stop", "ids": ["a"]} : dispatch 2024-09-14T03:56:44.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:44 smithi070 ceph-mon[98640]: Upgrade: It appears safe to stop mds.a 2024-09-14T03:56:44.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:44 smithi070 ceph-mon[98640]: Health check update: Degraded data redundancy: 173/1251 objects degraded (13.829%), 13 pgs degraded (PG_DEGRADED) 2024-09-14T03:56:44.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:44 smithi070 ceph-mon[98640]: pgmap v455: 121 pgs: 11 peering, 1 active+clean+snaptrim, 28 active+undersized, 11 active+undersized+degraded, 70 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 3.1 MiB/s wr, 13 op/s; 141/1257 objects degraded (11.217%) 2024-09-14T03:56:44.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:44 smithi070 ceph-mon[96704]: Upgrade: Setting container_image for all osd 2024-09-14T03:56:44.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:44 smithi070 ceph-mon[96704]: Upgrade: Setting require_osd_release to 19 squid 2024-09-14T03:56:44.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:44 smithi070 ceph-mon[96704]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-14T03:56:44.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:44 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-14T03:56:44.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:44 smithi070 ceph-mon[96704]: osdmap e572: 8 total, 8 up, 8 in 2024-09-14T03:56:44.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:44 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mds ok-to-stop", "ids": ["a"]} : dispatch 2024-09-14T03:56:44.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:44 smithi070 ceph-mon[96704]: Upgrade: It appears safe to stop mds.a 2024-09-14T03:56:44.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:44 smithi070 ceph-mon[96704]: Health check update: Degraded data redundancy: 173/1251 objects degraded (13.829%), 13 pgs degraded (PG_DEGRADED) 2024-09-14T03:56:44.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:44 smithi070 ceph-mon[96704]: pgmap v455: 121 pgs: 11 peering, 1 active+clean+snaptrim, 28 active+undersized, 11 active+undersized+degraded, 70 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 3.1 MiB/s wr, 13 op/s; 141/1257 objects degraded (11.217%) 2024-09-14T03:56:45.767 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:45 smithi070 ceph-mon[98640]: osdmap e573: 8 total, 8 up, 8 in 2024-09-14T03:56:45.767 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:45 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:45.768 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:45 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:56:45.768 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:45 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:56:45.768 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:45 smithi070 ceph-mon[96704]: osdmap e573: 8 total, 8 up, 8 in 2024-09-14T03:56:45.768 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:45 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:45.768 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:45 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:56:45.768 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:45 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:56:45.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:45 smithi094 ceph-mon[86597]: osdmap e573: 8 total, 8 up, 8 in 2024-09-14T03:56:45.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:45 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:45.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:45 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:56:45.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:45 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:56:45.777 INFO:tasks.rados.rados.0.smithi070.stdout:2433: finishing write tid 3 to smithi07085154-9 2024-09-14T03:56:45.778 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 2627 (ObjNum 749 snap 259 seq_num 749) dirty exists 2024-09-14T03:56:45.778 INFO:tasks.rados.rados.0.smithi070.stdout:2433: left oid 9 (ObjNum 749 snap 259 seq_num 749) 2024-09-14T03:56:45.778 INFO:tasks.rados.rados.0.smithi070.stdout:2430: done (5 left) 2024-09-14T03:56:45.778 INFO:tasks.rados.rados.0.smithi070.stdout:2431: done (4 left) 2024-09-14T03:56:45.778 INFO:tasks.rados.rados.0.smithi070.stdout:2433: done (3 left) 2024-09-14T03:56:45.778 INFO:tasks.rados.rados.0.smithi070.stdout:2434: done (2 left) 2024-09-14T03:56:45.778 INFO:tasks.rados.rados.0.smithi070.stdout:2435: done (1 left) 2024-09-14T03:56:45.778 INFO:tasks.rados.rados.0.smithi070.stdout:2436: done (0 left) 2024-09-14T03:56:45.778 INFO:tasks.rados.rados.0.smithi070.stdout:2437: copy_from oid 16 from oid 7 current snap is 260 2024-09-14T03:56:45.778 INFO:tasks.rados.rados.0.smithi070.stdout:2438: read oid 14 snap -1 2024-09-14T03:56:45.778 INFO:tasks.rados.rados.0.smithi070.stdout:2438: expect deleted 2024-09-14T03:56:45.778 INFO:tasks.rados.rados.0.smithi070.stdout:2439: delete oid 40 current snap is 260 2024-09-14T03:56:45.780 INFO:tasks.rados.rados.0.smithi070.stdout:2438: done (2 left) 2024-09-14T03:56:45.780 INFO:tasks.rados.rados.0.smithi070.stdout:2439: done (1 left) 2024-09-14T03:56:45.780 INFO:tasks.rados.rados.0.smithi070.stdout:2440: snap_remove snap 203 2024-09-14T03:56:46.450 INFO:tasks.rados.rados.0.smithi070.stdout:2440: done (1 left) 2024-09-14T03:56:46.450 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 6 current snap is 260 2024-09-14T03:56:46.450 INFO:tasks.rados.rados.0.smithi070.stdout:2441: seq_num 751 ranges {4784128=581632} 2024-09-14T03:56:46.481 INFO:tasks.rados.rados.0.smithi070.stdout:2441: writing smithi07085154-6 from 4784128 to 5365760 tid 1 2024-09-14T03:56:46.481 INFO:tasks.rados.rados.0.smithi070.stdout:2442: snap_remove snap 221 2024-09-14T03:56:46.496 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:46 smithi070 ceph-mon[96704]: Upgrade: Updating mds.a 2024-09-14T03:56:46.496 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:46 smithi070 ceph-mon[96704]: Deploying daemon mds.a on smithi070 2024-09-14T03:56:46.496 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:46 smithi070 ceph-mon[96704]: pgmap v457: 121 pgs: 11 peering, 21 active+undersized, 11 active+undersized+degraded, 78 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 2.5 MiB/s wr, 7 op/s; 141/1257 objects degraded (11.217%) 2024-09-14T03:56:46.497 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:46 smithi070 ceph-mon[98640]: Upgrade: Updating mds.a 2024-09-14T03:56:46.497 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:46 smithi070 ceph-mon[98640]: Deploying daemon mds.a on smithi070 2024-09-14T03:56:46.497 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:46 smithi070 ceph-mon[98640]: pgmap v457: 121 pgs: 11 peering, 21 active+undersized, 11 active+undersized+degraded, 78 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 2.5 MiB/s wr, 7 op/s; 141/1257 objects degraded (11.217%) 2024-09-14T03:56:46.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:46 smithi094 ceph-mon[86597]: Upgrade: Updating mds.a 2024-09-14T03:56:46.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:46 smithi094 ceph-mon[86597]: Deploying daemon mds.a on smithi070 2024-09-14T03:56:46.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:46 smithi094 ceph-mon[86597]: pgmap v457: 121 pgs: 11 peering, 21 active+undersized, 11 active+undersized+degraded, 78 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 2.5 MiB/s wr, 7 op/s; 141/1257 objects degraded (11.217%) 2024-09-14T03:56:47.028 INFO:journalctl@ceph.mds.a.smithi070.stdout:Sep 14 03:56:46 smithi070 systemd[1]: Stopping Ceph mds.a for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:56:47.034 INFO:tasks.rados.rados.0.smithi070.stdout:2442: done (2 left) 2024-09-14T03:56:47.034 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 26 current snap is 260 2024-09-14T03:56:47.035 INFO:tasks.rados.rados.0.smithi070.stdout:2437: finishing copy_from racing read to smithi07085154-16 2024-09-14T03:56:47.035 INFO:tasks.rados.rados.0.smithi070.stdout:2437: finishing copy_from to smithi07085154-16 2024-09-14T03:56:47.035 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 2052 (ObjNum 525 snap 174 seq_num 525) dirty exists 2024-09-14T03:56:47.035 INFO:tasks.rados.rados.0.smithi070.stdout:2441: finishing write tid 1 to smithi07085154-6 2024-09-14T03:56:47.035 INFO:tasks.rados.rados.0.smithi070.stdout:2441: finishing write tid 2 to smithi07085154-6 2024-09-14T03:56:47.035 INFO:tasks.rados.rados.0.smithi070.stdout:2441: finishing write tid 3 to smithi07085154-6 2024-09-14T03:56:47.035 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 2249 (ObjNum 751 snap 260 seq_num 751) dirty exists 2024-09-14T03:56:47.035 INFO:tasks.rados.rados.0.smithi070.stdout:2441: left oid 6 (ObjNum 751 snap 260 seq_num 751) 2024-09-14T03:56:47.035 INFO:tasks.rados.rados.0.smithi070.stdout:2443: seq_num 752 ranges {1425408=598016} 2024-09-14T03:56:47.047 INFO:tasks.rados.rados.0.smithi070.stdout:2443: writing smithi07085154-26 from 1425408 to 2023424 tid 1 2024-09-14T03:56:47.047 INFO:tasks.rados.rados.0.smithi070.stdout:2437: done (2 left) 2024-09-14T03:56:47.048 INFO:tasks.rados.rados.0.smithi070.stdout:2441: done (1 left) 2024-09-14T03:56:47.048 INFO:tasks.rados.rados.0.smithi070.stdout:2444: copy_from oid 25 from oid 17 current snap is 260 2024-09-14T03:56:47.048 INFO:tasks.rados.rados.0.smithi070.stdout:2445: rollback oid 48 current snap is 260 2024-09-14T03:56:47.048 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 48 to 236 2024-09-14T03:56:47.048 INFO:tasks.rados.rados.0.smithi070.stdout:2446: delete oid 23 current snap is 260 2024-09-14T03:56:47.052 INFO:tasks.rados.rados.0.smithi070.stdout:2445: finishing rollback tid 1 to smithi07085154-48 2024-09-14T03:56:47.053 INFO:tasks.rados.rados.0.smithi070.stdout:2445: finishing rollback tid 2 to smithi07085154-48 2024-09-14T03:56:47.053 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 2206 (ObjNum 682 snap 230 seq_num 682) dirty exists 2024-09-14T03:56:47.057 INFO:tasks.rados.rados.0.smithi070.stdout:2445: done (3 left) 2024-09-14T03:56:47.057 INFO:tasks.rados.rados.0.smithi070.stdout:2446: done (2 left) 2024-09-14T03:56:47.057 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 18 current snap is 260 2024-09-14T03:56:47.057 INFO:tasks.rados.rados.0.smithi070.stdout:2447: seq_num 754 ranges {0=630784} 2024-09-14T03:56:47.061 INFO:tasks.rados.rados.0.smithi070.stdout:2447: writing smithi07085154-18 from 0 to 630784 tid 1 2024-09-14T03:56:47.061 INFO:tasks.rados.rados.0.smithi070.stdout:2448: copy_from oid 43 from oid 10 current snap is 260 2024-09-14T03:56:47.061 INFO:tasks.rados.rados.0.smithi070.stdout:2449: snap_remove snap 247 2024-09-14T03:56:47.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:47 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-a[96681]: 2024-09-14T03:56:47.027+0000 7f512eabf640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-14T03:56:47.286 INFO:journalctl@ceph.mds.a.smithi070.stdout:Sep 14 03:56:47 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-a[67490]: 2024-09-14T03:56:47.026+0000 7ff8c601b640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:56:47.286 INFO:journalctl@ceph.mds.a.smithi070.stdout:Sep 14 03:56:47 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-a[67490]: 2024-09-14T03:56:47.026+0000 7ff8c601b640 -1 mds.a *** got signal Terminated *** 2024-09-14T03:56:47.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:47 smithi094 ceph-mon[86597]: osdmap e574: 8 total, 8 up, 8 in 2024-09-14T03:56:47.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:47 smithi094 ceph-mon[86597]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-14T03:56:47.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:47 smithi094 ceph-mon[86597]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-14T03:56:47.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:47 smithi094 ceph-mon[86597]: osdmap e575: 8 total, 8 up, 8 in 2024-09-14T03:56:47.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:47 smithi094 ceph-mon[86597]: Standby daemon mds.b assigned to filesystem cephfs as rank 0 2024-09-14T03:56:47.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:47 smithi094 ceph-mon[86597]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-14T03:56:47.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:47 smithi094 ceph-mon[86597]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-14T03:56:47.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:47 smithi094 ceph-mon[86597]: fsmap cephfs:1/1 {0=b=up:replay} 2024-09-14T03:56:47.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:47 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:47.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:47 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:56:47.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:47 smithi070 ceph-mon[98640]: osdmap e574: 8 total, 8 up, 8 in 2024-09-14T03:56:47.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:47 smithi070 ceph-mon[98640]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-14T03:56:47.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:47 smithi070 ceph-mon[98640]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-14T03:56:47.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:47 smithi070 ceph-mon[98640]: osdmap e575: 8 total, 8 up, 8 in 2024-09-14T03:56:47.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:47 smithi070 ceph-mon[98640]: Standby daemon mds.b assigned to filesystem cephfs as rank 0 2024-09-14T03:56:47.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:47 smithi070 ceph-mon[98640]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-14T03:56:47.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:47 smithi070 ceph-mon[98640]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-14T03:56:47.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:47 smithi070 ceph-mon[98640]: fsmap cephfs:1/1 {0=b=up:replay} 2024-09-14T03:56:47.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:47 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:47.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:47 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:56:47.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:47 smithi070 ceph-mon[96704]: osdmap e574: 8 total, 8 up, 8 in 2024-09-14T03:56:47.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:47 smithi070 ceph-mon[96704]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-14T03:56:47.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:47 smithi070 ceph-mon[96704]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-14T03:56:47.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:47 smithi070 ceph-mon[96704]: osdmap e575: 8 total, 8 up, 8 in 2024-09-14T03:56:47.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:47 smithi070 ceph-mon[96704]: Standby daemon mds.b assigned to filesystem cephfs as rank 0 2024-09-14T03:56:47.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:47 smithi070 ceph-mon[96704]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-14T03:56:47.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:47 smithi070 ceph-mon[96704]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-14T03:56:47.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:47 smithi070 ceph-mon[96704]: fsmap cephfs:1/1 {0=b=up:replay} 2024-09-14T03:56:47.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:47 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:47.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:47 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:56:48.041 INFO:tasks.rados.rados.0.smithi070.stdout:2443: finishing write tid 1 to smithi07085154-26 2024-09-14T03:56:48.041 INFO:tasks.rados.rados.0.smithi070.stdout:2443: finishing write tid 2 to smithi07085154-26 2024-09-14T03:56:48.041 INFO:tasks.rados.rados.0.smithi070.stdout:2447: finishing write tid 1 to smithi07085154-18 2024-09-14T03:56:48.041 INFO:tasks.rados.rados.0.smithi070.stdout:2443: finishing write tid 3 to smithi07085154-26 2024-09-14T03:56:48.041 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 2702 (ObjNum 752 snap 260 seq_num 752) dirty exists 2024-09-14T03:56:48.041 INFO:tasks.rados.rados.0.smithi070.stdout:2443: left oid 26 (ObjNum 752 snap 260 seq_num 752) 2024-09-14T03:56:48.041 INFO:tasks.rados.rados.0.smithi070.stdout:2447: finishing write tid 2 to smithi07085154-18 2024-09-14T03:56:48.041 INFO:tasks.rados.rados.0.smithi070.stdout:2447: finishing write tid 3 to smithi07085154-18 2024-09-14T03:56:48.042 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 2060 (ObjNum 754 snap 260 seq_num 754) dirty exists 2024-09-14T03:56:48.042 INFO:tasks.rados.rados.0.smithi070.stdout:2447: left oid 18 (ObjNum 754 snap 260 seq_num 754) 2024-09-14T03:56:48.042 INFO:tasks.rados.rados.0.smithi070.stdout:2444: finishing copy_from racing read to smithi07085154-25 2024-09-14T03:56:48.042 INFO:tasks.rados.rados.0.smithi070.stdout:2444: finishing copy_from to smithi07085154-25 2024-09-14T03:56:48.042 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 2212 (ObjNum 727 snap 251 seq_num 727) dirty exists 2024-09-14T03:56:48.042 INFO:tasks.rados.rados.0.smithi070.stdout:2448: finishing copy_from racing read to smithi07085154-43 2024-09-14T03:56:48.042 INFO:tasks.rados.rados.0.smithi070.stdout:2448: finishing copy_from to smithi07085154-43 2024-09-14T03:56:48.042 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 1712 (ObjNum 540 snap 176 seq_num 540) dirty exists 2024-09-14T03:56:48.042 INFO:tasks.rados.rados.0.smithi070.stdout:2443: done (4 left) 2024-09-14T03:56:48.042 INFO:tasks.rados.rados.0.smithi070.stdout:2444: done (3 left) 2024-09-14T03:56:48.042 INFO:tasks.rados.rados.0.smithi070.stdout:2447: done (2 left) 2024-09-14T03:56:48.042 INFO:tasks.rados.rados.0.smithi070.stdout:2448: done (1 left) 2024-09-14T03:56:48.042 INFO:tasks.rados.rados.0.smithi070.stdout:2449: done (0 left) 2024-09-14T03:56:48.042 INFO:tasks.rados.rados.0.smithi070.stdout:2450: read oid 3 snap -1 2024-09-14T03:56:48.043 INFO:tasks.rados.rados.0.smithi070.stdout:2450: expect deleted 2024-09-14T03:56:48.043 INFO:tasks.rados.rados.0.smithi070.stdout:2451: snap_remove snap 258 2024-09-14T03:56:49.056 INFO:tasks.rados.rados.0.smithi070.stdout:2450: done (1 left) 2024-09-14T03:56:49.056 INFO:tasks.rados.rados.0.smithi070.stdout:2451: done (0 left) 2024-09-14T03:56:49.056 INFO:tasks.rados.rados.0.smithi070.stdout:2452: read oid 47 snap -1 2024-09-14T03:56:49.056 INFO:tasks.rados.rados.0.smithi070.stdout:2452: expect deleted 2024-09-14T03:56:49.056 INFO:tasks.rados.rados.0.smithi070.stdout:2453: rollback oid 38 current snap is 260 2024-09-14T03:56:49.056 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 38 to 257 2024-09-14T03:56:49.057 INFO:tasks.rados.rados.0.smithi070.stdout:2454: copy_from oid 39 from oid 9 current snap is 260 2024-09-14T03:56:49.057 INFO:tasks.rados.rados.0.smithi070.stdout:2455: snap_remove snap 220 2024-09-14T03:56:49.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:49 smithi094 ceph-mon[86597]: osdmap e576: 8 total, 8 up, 8 in 2024-09-14T03:56:49.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:49 smithi094 ceph-mon[86597]: mds.? [v2:172.21.15.94:6832/3620660683,v1:172.21.15.94:6833/3620660683] up:reconnect 2024-09-14T03:56:49.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:49 smithi094 ceph-mon[86597]: fsmap cephfs:1/1 {0=b=up:reconnect} 2024-09-14T03:56:49.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-14T03:56:49.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:49 smithi094 ceph-mon[86597]: pgmap v461: 121 pgs: 1 active+clean+snaptrim, 11 peering, 109 active+clean; 1.1 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.3 MiB/s wr, 14 op/s; 17 MiB/s, 4 objects/s recovering 2024-09-14T03:56:49.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:49 smithi070 ceph-mon[96704]: osdmap e576: 8 total, 8 up, 8 in 2024-09-14T03:56:49.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:49 smithi070 ceph-mon[96704]: mds.? [v2:172.21.15.94:6832/3620660683,v1:172.21.15.94:6833/3620660683] up:reconnect 2024-09-14T03:56:49.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:49 smithi070 ceph-mon[96704]: fsmap cephfs:1/1 {0=b=up:reconnect} 2024-09-14T03:56:49.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-14T03:56:49.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:49 smithi070 ceph-mon[96704]: pgmap v461: 121 pgs: 1 active+clean+snaptrim, 11 peering, 109 active+clean; 1.1 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.3 MiB/s wr, 14 op/s; 17 MiB/s, 4 objects/s recovering 2024-09-14T03:56:49.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:49 smithi070 ceph-mon[98640]: osdmap e576: 8 total, 8 up, 8 in 2024-09-14T03:56:49.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:49 smithi070 ceph-mon[98640]: mds.? [v2:172.21.15.94:6832/3620660683,v1:172.21.15.94:6833/3620660683] up:reconnect 2024-09-14T03:56:49.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:49 smithi070 ceph-mon[98640]: fsmap cephfs:1/1 {0=b=up:reconnect} 2024-09-14T03:56:49.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-14T03:56:49.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:49 smithi070 ceph-mon[98640]: pgmap v461: 121 pgs: 1 active+clean+snaptrim, 11 peering, 109 active+clean; 1.1 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.3 MiB/s wr, 14 op/s; 17 MiB/s, 4 objects/s recovering 2024-09-14T03:56:50.057 INFO:tasks.rados.rados.0.smithi070.stdout:2455: done (3 left) 2024-09-14T03:56:50.057 INFO:tasks.rados.rados.0.smithi070.stdout:2456: rollback oid 20 current snap is 260 2024-09-14T03:56:50.057 INFO:tasks.rados.rados.0.smithi070.stdout:2453: finishing rollback tid 0 to smithi07085154-38 2024-09-14T03:56:50.057 INFO:tasks.rados.rados.0.smithi070.stdout:2453: finishing rollback tid 1 to smithi07085154-38 2024-09-14T03:56:50.058 INFO:tasks.rados.rados.0.smithi070.stdout:2453: finishing rollback tid 2 to smithi07085154-38 2024-09-14T03:56:50.058 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 38 v 1717 (ObjNum 540 snap 176 seq_num 540) dirty exists 2024-09-14T03:56:50.058 INFO:tasks.rados.rados.0.smithi070.stdout:2454: finishing copy_from racing read to smithi07085154-39 2024-09-14T03:56:50.058 INFO:tasks.rados.rados.0.smithi070.stdout:2454: finishing copy_from to smithi07085154-39 2024-09-14T03:56:50.058 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 39 v 2627 (ObjNum 749 snap 259 seq_num 749) dirty exists 2024-09-14T03:56:50.058 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 20 to 215 2024-09-14T03:56:50.058 INFO:tasks.rados.rados.0.smithi070.stdout:2452: done (3 left) 2024-09-14T03:56:50.058 INFO:tasks.rados.rados.0.smithi070.stdout:2453: done (2 left) 2024-09-14T03:56:50.058 INFO:tasks.rados.rados.0.smithi070.stdout:2454: done (1 left) 2024-09-14T03:56:50.058 INFO:tasks.rados.rados.0.smithi070.stdout:2457: read oid 5 snap 255 2024-09-14T03:56:50.058 INFO:tasks.rados.rados.0.smithi070.stdout:2457: expect deleted 2024-09-14T03:56:50.058 INFO:tasks.rados.rados.0.smithi070.stdout:2458: snap_create 2024-09-14T03:56:50.076 INFO:tasks.rados.rados.0.smithi070.stdout:2456: finishing rollback tid 1 to smithi07085154-20 2024-09-14T03:56:50.077 INFO:tasks.rados.rados.0.smithi070.stdout:2456: finishing rollback tid 2 to smithi07085154-20 2024-09-14T03:56:50.077 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 20 v 2055 (ObjNum 524 snap 174 seq_num 524) dirty exists 2024-09-14T03:56:50.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:50 smithi094 ceph-mon[86597]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 141/1257 objects degraded (11.217%), 11 pgs degraded) 2024-09-14T03:56:50.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:50 smithi094 ceph-mon[86597]: mds.? [v2:172.21.15.94:6832/3620660683,v1:172.21.15.94:6833/3620660683] up:rejoin 2024-09-14T03:56:50.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:50 smithi094 ceph-mon[86597]: fsmap cephfs:1/1 {0=b=up:rejoin} 2024-09-14T03:56:50.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:50 smithi094 ceph-mon[86597]: osdmap e577: 8 total, 8 up, 8 in 2024-09-14T03:56:50.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:50 smithi094 ceph-mon[86597]: daemon mds.b is now active in filesystem cephfs as rank 0 2024-09-14T03:56:50.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:56:50 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:56:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:56:50.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:50 smithi070 ceph-mon[96704]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 141/1257 objects degraded (11.217%), 11 pgs degraded) 2024-09-14T03:56:50.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:50 smithi070 ceph-mon[96704]: mds.? [v2:172.21.15.94:6832/3620660683,v1:172.21.15.94:6833/3620660683] up:rejoin 2024-09-14T03:56:50.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:50 smithi070 ceph-mon[96704]: fsmap cephfs:1/1 {0=b=up:rejoin} 2024-09-14T03:56:50.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:50 smithi070 ceph-mon[96704]: osdmap e577: 8 total, 8 up, 8 in 2024-09-14T03:56:50.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:50 smithi070 ceph-mon[96704]: daemon mds.b is now active in filesystem cephfs as rank 0 2024-09-14T03:56:50.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:50 smithi070 ceph-mon[98640]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 141/1257 objects degraded (11.217%), 11 pgs degraded) 2024-09-14T03:56:50.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:50 smithi070 ceph-mon[98640]: mds.? [v2:172.21.15.94:6832/3620660683,v1:172.21.15.94:6833/3620660683] up:rejoin 2024-09-14T03:56:50.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:50 smithi070 ceph-mon[98640]: fsmap cephfs:1/1 {0=b=up:rejoin} 2024-09-14T03:56:50.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:50 smithi070 ceph-mon[98640]: osdmap e577: 8 total, 8 up, 8 in 2024-09-14T03:56:50.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:50 smithi070 ceph-mon[98640]: daemon mds.b is now active in filesystem cephfs as rank 0 2024-09-14T03:56:51.060 INFO:tasks.rados.rados.0.smithi070.stdout:2456: done (2 left) 2024-09-14T03:56:51.060 INFO:tasks.rados.rados.0.smithi070.stdout:2457: done (1 left) 2024-09-14T03:56:51.060 INFO:tasks.rados.rados.0.smithi070.stdout:2458: done (0 left) 2024-09-14T03:56:51.060 INFO:tasks.rados.rados.0.smithi070.stdout:2459: setattr oid 9 current snap is 261 2024-09-14T03:56:51.061 INFO:tasks.rados.rados.0.smithi070.stdout:2460: read oid 10 snap -1 2024-09-14T03:56:51.061 INFO:tasks.rados.rados.0.smithi070.stdout:2460: expect (ObjNum 540 snap 176 seq_num 540) 2024-09-14T03:56:51.061 INFO:tasks.rados.rados.0.smithi070.stdout:2461: snap_create 2024-09-14T03:56:51.068 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 2744 (ObjNum 749 snap 259 seq_num 749) dirty exists 2024-09-14T03:56:51.073 INFO:tasks.rados.rados.0.smithi070.stdout:2460: expect (ObjNum 540 snap 176 seq_num 540) 2024-09-14T03:56:51.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:51 smithi094 ceph-mon[86597]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-14T03:56:51.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:51 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T03:56:51.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:51 smithi094 ceph-mon[86597]: mds.? [v2:172.21.15.94:6832/3620660683,v1:172.21.15.94:6833/3620660683] up:active 2024-09-14T03:56:51.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:51 smithi094 ceph-mon[86597]: fsmap cephfs:1 {0=b=up:active} 2024-09-14T03:56:51.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:51 smithi094 ceph-mon[86597]: osdmap e578: 8 total, 8 up, 8 in 2024-09-14T03:56:51.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:51 smithi094 ceph-mon[86597]: pgmap v464: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 4.3 MiB/s wr, 11 op/s; 17 MiB/s, 3 objects/s recovering 2024-09-14T03:56:51.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:51 smithi070 ceph-mon[96704]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-14T03:56:51.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:51 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T03:56:51.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:51 smithi070 ceph-mon[96704]: mds.? [v2:172.21.15.94:6832/3620660683,v1:172.21.15.94:6833/3620660683] up:active 2024-09-14T03:56:51.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:51 smithi070 ceph-mon[96704]: fsmap cephfs:1 {0=b=up:active} 2024-09-14T03:56:51.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:51 smithi070 ceph-mon[96704]: osdmap e578: 8 total, 8 up, 8 in 2024-09-14T03:56:51.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:51 smithi070 ceph-mon[96704]: pgmap v464: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 4.3 MiB/s wr, 11 op/s; 17 MiB/s, 3 objects/s recovering 2024-09-14T03:56:51.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:51 smithi070 ceph-mon[98640]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-14T03:56:51.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:51 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T03:56:51.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:51 smithi070 ceph-mon[98640]: mds.? [v2:172.21.15.94:6832/3620660683,v1:172.21.15.94:6833/3620660683] up:active 2024-09-14T03:56:51.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:51 smithi070 ceph-mon[98640]: fsmap cephfs:1 {0=b=up:active} 2024-09-14T03:56:51.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:51 smithi070 ceph-mon[98640]: osdmap e578: 8 total, 8 up, 8 in 2024-09-14T03:56:51.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:51 smithi070 ceph-mon[98640]: pgmap v464: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 4.3 MiB/s wr, 11 op/s; 17 MiB/s, 3 objects/s recovering 2024-09-14T03:56:52.066 INFO:tasks.rados.rados.0.smithi070.stdout:2459: done (2 left) 2024-09-14T03:56:52.066 INFO:tasks.rados.rados.0.smithi070.stdout:2460: done (1 left) 2024-09-14T03:56:52.066 INFO:tasks.rados.rados.0.smithi070.stdout:2461: done (0 left) 2024-09-14T03:56:52.066 INFO:tasks.rados.rados.0.smithi070.stdout:2462: delete oid 42 current snap is 262 2024-09-14T03:56:52.072 INFO:tasks.rados.rados.0.smithi070.stdout:2462: done (0 left) 2024-09-14T03:56:52.072 INFO:tasks.rados.rados.0.smithi070.stdout:2463: snap_remove snap 249 2024-09-14T03:56:52.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:52 smithi094 ceph-mon[86597]: osdmap e579: 8 total, 8 up, 8 in 2024-09-14T03:56:52.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:52 smithi070 ceph-mon[96704]: osdmap e579: 8 total, 8 up, 8 in 2024-09-14T03:56:52.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:52 smithi070 ceph-mon[98640]: osdmap e579: 8 total, 8 up, 8 in 2024-09-14T03:56:53.072 INFO:tasks.rados.rados.0.smithi070.stdout:2463: done (0 left) 2024-09-14T03:56:53.073 INFO:tasks.rados.rados.0.smithi070.stdout:2464: read oid 23 snap -1 2024-09-14T03:56:53.073 INFO:tasks.rados.rados.0.smithi070.stdout:2464: expect deleted 2024-09-14T03:56:53.073 INFO:tasks.rados.rados.0.smithi070.stdout:2465: read oid 13 snap -1 2024-09-14T03:56:53.073 INFO:tasks.rados.rados.0.smithi070.stdout:2465: expect (ObjNum 706 snap 238 seq_num 706) 2024-09-14T03:56:53.073 INFO:tasks.rados.rados.0.smithi070.stdout:2466: snap_create 2024-09-14T03:56:53.086 INFO:tasks.rados.rados.0.smithi070.stdout:2465: expect (ObjNum 706 snap 238 seq_num 706) 2024-09-14T03:56:53.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:53 smithi070 ceph-mon[96704]: osdmap e580: 8 total, 8 up, 8 in 2024-09-14T03:56:53.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:53 smithi070 ceph-mon[96704]: pgmap v467: 121 pgs: 121 active+clean; 1.1 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 2.7 MiB/s wr, 25 op/s; 3.5 MiB/s, 0 objects/s recovering 2024-09-14T03:56:53.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:53 smithi070 ceph-mon[98640]: osdmap e580: 8 total, 8 up, 8 in 2024-09-14T03:56:53.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:53 smithi070 ceph-mon[98640]: pgmap v467: 121 pgs: 121 active+clean; 1.1 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 2.7 MiB/s wr, 25 op/s; 3.5 MiB/s, 0 objects/s recovering 2024-09-14T03:56:53.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:53 smithi094 ceph-mon[86597]: osdmap e580: 8 total, 8 up, 8 in 2024-09-14T03:56:53.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:53 smithi094 ceph-mon[86597]: pgmap v467: 121 pgs: 121 active+clean; 1.1 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 2.7 MiB/s wr, 25 op/s; 3.5 MiB/s, 0 objects/s recovering 2024-09-14T03:56:53.527 INFO:journalctl@ceph.mds.b.smithi094.stdout:Sep 14 03:56:53 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-b[59374]: 2024-09-14T03:56:53.039+0000 7f5f06dc2640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-09-14T03:56:53.664 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:56:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:56:53] "GET /metrics HTTP/1.1" 200 36282 "" "Prometheus/2.51.0" 2024-09-14T03:56:54.071 INFO:tasks.rados.rados.0.smithi070.stdout:2464: done (2 left) 2024-09-14T03:56:54.071 INFO:tasks.rados.rados.0.smithi070.stdout:2465: done (1 left) 2024-09-14T03:56:54.071 INFO:tasks.rados.rados.0.smithi070.stdout:2466: done (0 left) 2024-09-14T03:56:54.071 INFO:tasks.rados.rados.0.smithi070.stdout:2467: copy_from oid 44 from oid 8 current snap is 263 2024-09-14T03:56:54.071 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 35 current snap is 263 2024-09-14T03:56:54.071 INFO:tasks.rados.rados.0.smithi070.stdout:2468: seq_num 756 ranges {0=671744} 2024-09-14T03:56:54.075 INFO:tasks.rados.rados.0.smithi070.stdout:2468: writing smithi07085154-35 from 0 to 671744 tid 1 2024-09-14T03:56:54.075 INFO:tasks.rados.rados.0.smithi070.stdout:2469: read oid 11 snap -1 2024-09-14T03:56:54.075 INFO:tasks.rados.rados.0.smithi070.stdout:2469: expect (ObjNum 645 snap 216 seq_num 645) 2024-09-14T03:56:54.076 INFO:tasks.rados.rados.0.smithi070.stdout:2470: read oid 17 snap -1 2024-09-14T03:56:54.076 INFO:tasks.rados.rados.0.smithi070.stdout:2470: expect (ObjNum 727 snap 251 seq_num 727) 2024-09-14T03:56:54.076 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 48 current snap is 263 2024-09-14T03:56:54.076 INFO:tasks.rados.rados.0.smithi070.stdout:2471: seq_num 757 ranges {3850240=688128} 2024-09-14T03:56:54.101 INFO:tasks.rados.rados.0.smithi070.stdout:2471: writing smithi07085154-48 from 3850240 to 4538368 tid 1 2024-09-14T03:56:54.101 INFO:tasks.rados.rados.0.smithi070.stdout:2472: rollback oid 14 current snap is 263 2024-09-14T03:56:54.101 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 14 to 182 2024-09-14T03:56:54.101 INFO:tasks.rados.rados.0.smithi070.stdout:2468: finishing write tid 1 to smithi07085154-35 2024-09-14T03:56:54.101 INFO:tasks.rados.rados.0.smithi070.stdout:2468: finishing write tid 2 to smithi07085154-35 2024-09-14T03:56:54.101 INFO:tasks.rados.rados.0.smithi070.stdout:2468: finishing write tid 3 to smithi07085154-35 2024-09-14T03:56:54.101 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 35 v 1689 (ObjNum 756 snap 263 seq_num 756) dirty exists 2024-09-14T03:56:54.101 INFO:tasks.rados.rados.0.smithi070.stdout:2468: left oid 35 (ObjNum 756 snap 263 seq_num 756) 2024-09-14T03:56:54.102 INFO:tasks.rados.rados.0.smithi070.stdout:2470: expect (ObjNum 727 snap 251 seq_num 727) 2024-09-14T03:56:54.283 INFO:tasks.rados.rados.0.smithi070.stdout:2469: expect (ObjNum 645 snap 216 seq_num 645) 2024-09-14T03:56:54.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:54 smithi094 ceph-mon[86597]: osdmap e581: 8 total, 8 up, 8 in 2024-09-14T03:56:54.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:54 smithi070 ceph-mon[98640]: osdmap e581: 8 total, 8 up, 8 in 2024-09-14T03:56:54.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:54 smithi070 ceph-mon[96704]: osdmap e581: 8 total, 8 up, 8 in 2024-09-14T03:56:54.554 INFO:tasks.rados.rados.0.smithi070.stdout:2467: finishing copy_from racing read to smithi07085154-44 2024-09-14T03:56:54.554 INFO:tasks.rados.rados.0.smithi070.stdout:2467: finishing copy_from to smithi07085154-44 2024-09-14T03:56:54.554 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 2029 (ObjNum 668 snap 228 seq_num 668) dirty exists 2024-09-14T03:56:54.554 INFO:tasks.rados.rados.0.smithi070.stdout:2467: done (5 left) 2024-09-14T03:56:54.554 INFO:tasks.rados.rados.0.smithi070.stdout:2468: done (4 left) 2024-09-14T03:56:54.554 INFO:tasks.rados.rados.0.smithi070.stdout:2469: done (3 left) 2024-09-14T03:56:54.555 INFO:tasks.rados.rados.0.smithi070.stdout:2470: done (2 left) 2024-09-14T03:56:54.555 INFO:tasks.rados.rados.0.smithi070.stdout:2473: snap_remove snap 259 2024-09-14T03:56:55.080 INFO:tasks.rados.rados.0.smithi070.stdout:2473: done (2 left) 2024-09-14T03:56:55.080 INFO:tasks.rados.rados.0.smithi070.stdout:2474: snap_create 2024-09-14T03:56:55.080 INFO:tasks.rados.rados.0.smithi070.stdout:2471: finishing write tid 1 to smithi07085154-48 2024-09-14T03:56:55.080 INFO:tasks.rados.rados.0.smithi070.stdout:2472: finishing rollback tid 1 to smithi07085154-14 2024-09-14T03:56:55.080 INFO:tasks.rados.rados.0.smithi070.stdout:2471: finishing write tid 2 to smithi07085154-48 2024-09-14T03:56:55.080 INFO:tasks.rados.rados.0.smithi070.stdout:2472: finishing rollback tid 2 to smithi07085154-14 2024-09-14T03:56:55.081 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 2250 (ObjNum 398 snap 138 seq_num 398) dirty exists 2024-09-14T03:56:55.081 INFO:tasks.rados.rados.0.smithi070.stdout:2471: finishing write tid 3 to smithi07085154-48 2024-09-14T03:56:55.081 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 2208 (ObjNum 757 snap 263 seq_num 757) dirty exists 2024-09-14T03:56:55.081 INFO:tasks.rados.rados.0.smithi070.stdout:2471: left oid 48 (ObjNum 757 snap 263 seq_num 757) 2024-09-14T03:56:55.443 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:55 smithi070 ceph-mon[96704]: osdmap e582: 8 total, 8 up, 8 in 2024-09-14T03:56:55.443 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:55 smithi070 ceph-mon[96704]: pgmap v470: 121 pgs: 121 active+clean; 1.1 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 684 KiB/s wr, 18 op/s 2024-09-14T03:56:55.443 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:55 smithi070 ceph-mon[98640]: osdmap e582: 8 total, 8 up, 8 in 2024-09-14T03:56:55.443 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:55 smithi070 ceph-mon[98640]: pgmap v470: 121 pgs: 121 active+clean; 1.1 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 684 KiB/s wr, 18 op/s 2024-09-14T03:56:55.443 INFO:journalctl@ceph.mds.a.smithi070.stdout:Sep 14 03:56:55 smithi070 podman[122308]: 2024-09-14 03:56:55.136607657 +0000 UTC m=+8.221172023 container died 52e0dbb7be5086597de29f732d6b365e9675acc0ccf69c4844dfb3eff8876270 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-a, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=quincy-eb16b97, maintainer=Guillaume Abrioux ) 2024-09-14T03:56:55.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:55 smithi094 ceph-mon[86597]: osdmap e582: 8 total, 8 up, 8 in 2024-09-14T03:56:55.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:55 smithi094 ceph-mon[86597]: pgmap v470: 121 pgs: 121 active+clean; 1.1 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 684 KiB/s wr, 18 op/s 2024-09-14T03:56:55.785 INFO:journalctl@ceph.mds.a.smithi070.stdout:Sep 14 03:56:55 smithi070 podman[122308]: 2024-09-14 03:56:55.632714149 +0000 UTC m=+8.717278517 container cleanup 52e0dbb7be5086597de29f732d6b365e9675acc0ccf69c4844dfb3eff8876270 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-a, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=quincy-eb16b97, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-14T03:56:55.786 INFO:journalctl@ceph.mds.a.smithi070.stdout:Sep 14 03:56:55 smithi070 bash[122308]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-a 2024-09-14T03:56:56.081 INFO:tasks.rados.rados.0.smithi070.stdout:2471: done (2 left) 2024-09-14T03:56:56.081 INFO:tasks.rados.rados.0.smithi070.stdout:2472: done (1 left) 2024-09-14T03:56:56.081 INFO:tasks.rados.rados.0.smithi070.stdout:2474: done (0 left) 2024-09-14T03:56:56.081 INFO:tasks.rados.rados.0.smithi070.stdout:2475: copy_from oid 22 from oid 29 current snap is 264 2024-09-14T03:56:56.081 INFO:tasks.rados.rados.0.smithi070.stdout:2476: read oid 10 snap -1 2024-09-14T03:56:56.081 INFO:tasks.rados.rados.0.smithi070.stdout:2476: expect (ObjNum 540 snap 176 seq_num 540) 2024-09-14T03:56:56.081 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 47 current snap is 264 2024-09-14T03:56:56.082 INFO:tasks.rados.rados.0.smithi070.stdout:2477: seq_num 758 ranges {0=704512} 2024-09-14T03:56:56.085 INFO:tasks.rados.rados.0.smithi070.stdout:2477: writing smithi07085154-47 from 0 to 704512 tid 1 2024-09-14T03:56:56.085 INFO:tasks.rados.rados.0.smithi070.stdout:2478: rollback oid 21 current snap is 264 2024-09-14T03:56:56.085 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 21 to 263 2024-09-14T03:56:56.085 INFO:tasks.rados.rados.0.smithi070.stdout:2479: setattr oid 49 current snap is 264 2024-09-14T03:56:56.086 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 2 current snap is 264 2024-09-14T03:56:56.086 INFO:tasks.rados.rados.0.smithi070.stdout:2480: seq_num 759 ranges {147456=720896} 2024-09-14T03:56:56.092 INFO:tasks.rados.rados.0.smithi070.stdout:2480: writing smithi07085154-2 from 147456 to 868352 tid 1 2024-09-14T03:56:56.092 INFO:tasks.rados.rados.0.smithi070.stdout:2481: delete oid 15 current snap is 264 2024-09-14T03:56:56.093 INFO:tasks.rados.rados.0.smithi070.stdout:2478: finishing rollback tid 1 to smithi07085154-21 2024-09-14T03:56:56.093 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 1701 (ObjNum 33 snap 0 seq_num 2931204560) dirty dne 2024-09-14T03:56:56.093 INFO:tasks.rados.rados.0.smithi070.stdout:2477: finishing write tid 1 to smithi07085154-47 2024-09-14T03:56:56.093 INFO:tasks.rados.rados.0.smithi070.stdout:2477: finishing write tid 2 to smithi07085154-47 2024-09-14T03:56:56.093 INFO:tasks.rados.rados.0.smithi070.stdout:2477: finishing write tid 3 to smithi07085154-47 2024-09-14T03:56:56.093 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 47 v 1719 (ObjNum 758 snap 264 seq_num 758) dirty exists 2024-09-14T03:56:56.094 INFO:tasks.rados.rados.0.smithi070.stdout:2477: left oid 47 (ObjNum 758 snap 264 seq_num 758) 2024-09-14T03:56:56.094 INFO:tasks.rados.rados.0.smithi070.stdout:2476: expect (ObjNum 540 snap 176 seq_num 540) 2024-09-14T03:56:56.151 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 49 v 1961 (ObjNum 729 snap 253 seq_num 729) dirty exists 2024-09-14T03:56:56.151 INFO:tasks.rados.rados.0.smithi070.stdout:2480: finishing write tid 1 to smithi07085154-2 2024-09-14T03:56:56.151 INFO:tasks.rados.rados.0.smithi070.stdout:2480: finishing write tid 2 to smithi07085154-2 2024-09-14T03:56:56.151 INFO:tasks.rados.rados.0.smithi070.stdout:2480: finishing write tid 3 to smithi07085154-2 2024-09-14T03:56:56.151 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 2 v 2773 (ObjNum 759 snap 264 seq_num 759) dirty exists 2024-09-14T03:56:56.152 INFO:tasks.rados.rados.0.smithi070.stdout:2480: left oid 2 (ObjNum 759 snap 264 seq_num 759) 2024-09-14T03:56:56.152 INFO:tasks.rados.rados.0.smithi070.stdout:2475: finishing copy_from racing read to smithi07085154-22 2024-09-14T03:56:56.152 INFO:tasks.rados.rados.0.smithi070.stdout:2475: finishing copy_from to smithi07085154-22 2024-09-14T03:56:56.152 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 1735 (ObjNum 458 snap 155 seq_num 458) dirty exists 2024-09-14T03:56:56.152 INFO:tasks.rados.rados.0.smithi070.stdout:2475: done (6 left) 2024-09-14T03:56:56.152 INFO:tasks.rados.rados.0.smithi070.stdout:2476: done (5 left) 2024-09-14T03:56:56.152 INFO:tasks.rados.rados.0.smithi070.stdout:2477: done (4 left) 2024-09-14T03:56:56.152 INFO:tasks.rados.rados.0.smithi070.stdout:2478: done (3 left) 2024-09-14T03:56:56.152 INFO:tasks.rados.rados.0.smithi070.stdout:2479: done (2 left) 2024-09-14T03:56:56.152 INFO:tasks.rados.rados.0.smithi070.stdout:2480: done (1 left) 2024-09-14T03:56:56.152 INFO:tasks.rados.rados.0.smithi070.stdout:2481: done (0 left) 2024-09-14T03:56:56.152 INFO:tasks.rados.rados.0.smithi070.stdout:2482: setattr oid 7 current snap is 264 2024-09-14T03:56:56.154 INFO:tasks.rados.rados.0.smithi070.stdout:2483: copy_from oid 43 from oid 12 current snap is 264 2024-09-14T03:56:56.154 INFO:tasks.rados.rados.0.smithi070.stdout:2484: read oid 28 snap -1 2024-09-14T03:56:56.154 INFO:tasks.rados.rados.0.smithi070.stdout:2484: expect (ObjNum 639 snap 214 seq_num 639) 2024-09-14T03:56:56.154 INFO:tasks.rados.rados.0.smithi070.stdout:2485: rollback oid 47 current snap is 264 2024-09-14T03:56:56.154 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 47 to 234 2024-09-14T03:56:56.154 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 10 current snap is 264 2024-09-14T03:56:56.154 INFO:tasks.rados.rados.0.smithi070.stdout:2486: seq_num 761 ranges {1851392=753664} 2024-09-14T03:56:56.167 INFO:tasks.rados.rados.0.smithi070.stdout:2486: writing smithi07085154-10 from 1851392 to 2605056 tid 1 2024-09-14T03:56:56.167 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 34 current snap is 264 2024-09-14T03:56:56.168 INFO:tasks.rados.rados.0.smithi070.stdout:2485: finishing rollback tid 0 to smithi07085154-47 2024-09-14T03:56:56.168 INFO:tasks.rados.rados.0.smithi070.stdout:2485: finishing rollback tid 1 to smithi07085154-47 2024-09-14T03:56:56.168 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 2056 (ObjNum 525 snap 174 seq_num 525) dirty exists 2024-09-14T03:56:56.168 INFO:tasks.rados.rados.0.smithi070.stdout:2485: finishing rollback tid 2 to smithi07085154-47 2024-09-14T03:56:56.168 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 47 v 1722 (ObjNum 514 snap 171 seq_num 514) dirty exists 2024-09-14T03:56:56.168 INFO:tasks.rados.rados.0.smithi070.stdout:2484: expect (ObjNum 639 snap 214 seq_num 639) 2024-09-14T03:56:56.197 INFO:tasks.rados.rados.0.smithi070.stdout:2483: finishing copy_from racing read to smithi07085154-43 2024-09-14T03:56:56.197 INFO:tasks.rados.rados.0.smithi070.stdout:2483: finishing copy_from to smithi07085154-43 2024-09-14T03:56:56.197 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 2204 (ObjNum 681 snap 230 seq_num 681) dirty exists 2024-09-14T03:56:56.198 INFO:tasks.rados.rados.0.smithi070.stdout:2486: finishing write tid 1 to smithi07085154-10 2024-09-14T03:56:56.198 INFO:tasks.rados.rados.0.smithi070.stdout:2486: finishing write tid 2 to smithi07085154-10 2024-09-14T03:56:56.198 INFO:tasks.rados.rados.0.smithi070.stdout:2486: finishing write tid 3 to smithi07085154-10 2024-09-14T03:56:56.198 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 2776 (ObjNum 761 snap 264 seq_num 761) dirty exists 2024-09-14T03:56:56.198 INFO:tasks.rados.rados.0.smithi070.stdout:2486: left oid 10 (ObjNum 761 snap 264 seq_num 761) 2024-09-14T03:56:56.198 INFO:tasks.rados.rados.0.smithi070.stdout:2487: seq_num 762 ranges {0=770048} 2024-09-14T03:56:56.201 INFO:tasks.rados.rados.0.smithi070.stdout:2487: writing smithi07085154-34 from 0 to 770048 tid 1 2024-09-14T03:56:56.201 INFO:tasks.rados.rados.0.smithi070.stdout:2482: done (5 left) 2024-09-14T03:56:56.201 INFO:tasks.rados.rados.0.smithi070.stdout:2483: done (4 left) 2024-09-14T03:56:56.202 INFO:tasks.rados.rados.0.smithi070.stdout:2484: done (3 left) 2024-09-14T03:56:56.202 INFO:tasks.rados.rados.0.smithi070.stdout:2485: done (2 left) 2024-09-14T03:56:56.202 INFO:tasks.rados.rados.0.smithi070.stdout:2486: done (1 left) 2024-09-14T03:56:56.202 INFO:tasks.rados.rados.0.smithi070.stdout:2488: snap_create 2024-09-14T03:56:56.205 INFO:tasks.rados.rados.0.smithi070.stdout:2487: finishing write tid 1 to smithi07085154-34 2024-09-14T03:56:56.206 INFO:tasks.rados.rados.0.smithi070.stdout:2487: finishing write tid 2 to smithi07085154-34 2024-09-14T03:56:56.207 INFO:tasks.rados.rados.0.smithi070.stdout:2487: finishing write tid 3 to smithi07085154-34 2024-09-14T03:56:56.207 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 1737 (ObjNum 762 snap 264 seq_num 762) dirty exists 2024-09-14T03:56:56.207 INFO:tasks.rados.rados.0.smithi070.stdout:2487: left oid 34 (ObjNum 762 snap 264 seq_num 762) 2024-09-14T03:56:56.239 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:56 smithi070 ceph-mon[98640]: osdmap e583: 8 total, 8 up, 8 in 2024-09-14T03:56:56.239 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:56 smithi070 ceph-mon[96704]: osdmap e583: 8 total, 8 up, 8 in 2024-09-14T03:56:56.240 INFO:journalctl@ceph.mds.a.smithi070.stdout:Sep 14 03:56:55 smithi070 podman[122338]: 2024-09-14 03:56:55.884206764 +0000 UTC m=+0.745236114 container remove 52e0dbb7be5086597de29f732d6b365e9675acc0ccf69c4844dfb3eff8876270 (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-a, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-eb16b97, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:56:56.240 INFO:journalctl@ceph.mds.a.smithi070.stdout:Sep 14 03:56:56 smithi070 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@mds.a.service: Deactivated successfully. 2024-09-14T03:56:56.240 INFO:journalctl@ceph.mds.a.smithi070.stdout:Sep 14 03:56:56 smithi070 systemd[1]: Stopped Ceph mds.a for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:56:56.240 INFO:journalctl@ceph.mds.a.smithi070.stdout:Sep 14 03:56:56 smithi070 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@mds.a.service: Consumed 1.692s CPU time. 2024-09-14T03:56:56.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:56 smithi094 ceph-mon[86597]: osdmap e583: 8 total, 8 up, 8 in 2024-09-14T03:56:56.535 INFO:journalctl@ceph.mds.a.smithi070.stdout:Sep 14 03:56:56 smithi070 systemd[1]: Starting Ceph mds.a for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:56:56.535 INFO:journalctl@ceph.mds.a.smithi070.stdout:Sep 14 03:56:56 smithi070 podman[122474]: 2024-09-14 03:56:56.443533959 +0000 UTC m=+0.111345046 container create 4fa138982c4f2ae1c100cf405b61fba395e9e5083e6cc5b3190909a8152ec8a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-a, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=main-a278231, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-14T03:56:56.536 INFO:journalctl@ceph.mds.a.smithi070.stdout:Sep 14 03:56:56 smithi070 podman[122474]: 2024-09-14 03:56:56.347992258 +0000 UTC m=+0.015803353 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:56:57.036 INFO:journalctl@ceph.mds.a.smithi070.stdout:Sep 14 03:56:56 smithi070 podman[122474]: 2024-09-14 03:56:56.76301581 +0000 UTC m=+0.430826901 container init 4fa138982c4f2ae1c100cf405b61fba395e9e5083e6cc5b3190909a8152ec8a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-a, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=main-a278231) 2024-09-14T03:56:57.036 INFO:journalctl@ceph.mds.a.smithi070.stdout:Sep 14 03:56:56 smithi070 podman[122474]: 2024-09-14 03:56:56.767818705 +0000 UTC m=+0.435629794 container start 4fa138982c4f2ae1c100cf405b61fba395e9e5083e6cc5b3190909a8152ec8a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-a, org.label-schema.schema-version=1.0, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:56:57.036 INFO:journalctl@ceph.mds.a.smithi070.stdout:Sep 14 03:56:56 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-a[122489]: starting mds.a at 2024-09-14T03:56:57.036 INFO:journalctl@ceph.mds.a.smithi070.stdout:Sep 14 03:56:56 smithi070 bash[122474]: 4fa138982c4f2ae1c100cf405b61fba395e9e5083e6cc5b3190909a8152ec8a8 2024-09-14T03:56:57.036 INFO:journalctl@ceph.mds.a.smithi070.stdout:Sep 14 03:56:56 smithi070 systemd[1]: Started Ceph mds.a for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:56:57.089 INFO:tasks.rados.rados.0.smithi070.stdout:2487: done (1 left) 2024-09-14T03:56:57.089 INFO:tasks.rados.rados.0.smithi070.stdout:2488: done (0 left) 2024-09-14T03:56:57.090 INFO:tasks.rados.rados.0.smithi070.stdout:2489: rollback oid 45 current snap is 265 2024-09-14T03:56:57.090 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 45 to 264 2024-09-14T03:56:57.090 INFO:tasks.rados.rados.0.smithi070.stdout:2490: read oid 14 snap -1 2024-09-14T03:56:57.090 INFO:tasks.rados.rados.0.smithi070.stdout:2490: expect (ObjNum 398 snap 138 seq_num 398) 2024-09-14T03:56:57.090 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 27 current snap is 265 2024-09-14T03:56:57.090 INFO:tasks.rados.rados.0.smithi070.stdout:2491: seq_num 763 ranges {17793024=786432} 2024-09-14T03:56:57.190 INFO:tasks.rados.rados.0.smithi070.stdout:2491: writing smithi07085154-27 from 17793024 to 18579456 tid 1 2024-09-14T03:56:57.190 INFO:tasks.rados.rados.0.smithi070.stdout:2492: snap_create 2024-09-14T03:56:57.190 INFO:tasks.rados.rados.0.smithi070.stdout:2489: finishing rollback tid 1 to smithi07085154-45 2024-09-14T03:56:57.190 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 45 v 1960 (ObjNum 177 snap 0 seq_num 0) dirty dne 2024-09-14T03:56:57.190 INFO:tasks.rados.rados.0.smithi070.stdout:2490: expect (ObjNum 398 snap 138 seq_num 398) 2024-09-14T03:56:57.265 INFO:tasks.rados.rados.0.smithi070.stdout:2491: finishing write tid 1 to smithi07085154-27 2024-09-14T03:56:57.265 INFO:tasks.rados.rados.0.smithi070.stdout:2491: finishing write tid 2 to smithi07085154-27 2024-09-14T03:56:57.265 INFO:tasks.rados.rados.0.smithi070.stdout:2491: finishing write tid 3 to smithi07085154-27 2024-09-14T03:56:57.265 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 2779 (ObjNum 763 snap 265 seq_num 763) dirty exists 2024-09-14T03:56:57.266 INFO:tasks.rados.rados.0.smithi070.stdout:2491: left oid 27 (ObjNum 763 snap 265 seq_num 763) 2024-09-14T03:56:57.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:57 smithi094 ceph-mon[86597]: osdmap e584: 8 total, 8 up, 8 in 2024-09-14T03:56:57.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:57 smithi094 ceph-mon[86597]: pgmap v473: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 684 KiB/s wr, 1 op/s 2024-09-14T03:56:57.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:57 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:57.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:57 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:57.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:57 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:56:57.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:57 smithi070 ceph-mon[96704]: osdmap e584: 8 total, 8 up, 8 in 2024-09-14T03:56:57.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:57 smithi070 ceph-mon[96704]: pgmap v473: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 684 KiB/s wr, 1 op/s 2024-09-14T03:56:57.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:57 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:57.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:57 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:57.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:57 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:56:57.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:57 smithi070 ceph-mon[98640]: osdmap e584: 8 total, 8 up, 8 in 2024-09-14T03:56:57.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:57 smithi070 ceph-mon[98640]: pgmap v473: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 684 KiB/s wr, 1 op/s 2024-09-14T03:56:57.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:57 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:57.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:57 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:57.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:57 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:56:58.093 INFO:tasks.rados.rados.0.smithi070.stdout:2489: done (3 left) 2024-09-14T03:56:58.093 INFO:tasks.rados.rados.0.smithi070.stdout:2490: done (2 left) 2024-09-14T03:56:58.094 INFO:tasks.rados.rados.0.smithi070.stdout:2491: done (1 left) 2024-09-14T03:56:58.094 INFO:tasks.rados.rados.0.smithi070.stdout:2492: done (0 left) 2024-09-14T03:56:58.094 INFO:tasks.rados.rados.0.smithi070.stdout:2493: read oid 2 snap -1 2024-09-14T03:56:58.094 INFO:tasks.rados.rados.0.smithi070.stdout:2493: expect (ObjNum 759 snap 264 seq_num 759) 2024-09-14T03:56:58.094 INFO:tasks.rados.rados.0.smithi070.stdout:2494: snap_create 2024-09-14T03:56:58.097 INFO:tasks.rados.rados.0.smithi070.stdout:2493: expect (ObjNum 759 snap 264 seq_num 759) 2024-09-14T03:56:58.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:58 smithi070 ceph-mon[96704]: mds.? [v2:172.21.15.70:6834/2011419410,v1:172.21.15.70:6835/2011419410] up:boot 2024-09-14T03:56:58.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:58 smithi070 ceph-mon[96704]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-14T03:56:58.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:58 smithi070 ceph-mon[96704]: osdmap e585: 8 total, 8 up, 8 in 2024-09-14T03:56:58.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:58 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-14T03:56:58.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:58 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:58.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:58 smithi070 ceph-mon[98640]: mds.? [v2:172.21.15.70:6834/2011419410,v1:172.21.15.70:6835/2011419410] up:boot 2024-09-14T03:56:58.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:58 smithi070 ceph-mon[98640]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-14T03:56:58.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:58 smithi070 ceph-mon[98640]: osdmap e585: 8 total, 8 up, 8 in 2024-09-14T03:56:58.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:58 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-14T03:56:58.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:58 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:58.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:58 smithi094 ceph-mon[86597]: mds.? [v2:172.21.15.70:6834/2011419410,v1:172.21.15.70:6835/2011419410] up:boot 2024-09-14T03:56:58.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:58 smithi094 ceph-mon[86597]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-14T03:56:58.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:58 smithi094 ceph-mon[86597]: osdmap e585: 8 total, 8 up, 8 in 2024-09-14T03:56:58.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:58 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-14T03:56:58.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:58 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:56:59.072 INFO:tasks.rados.rados.0.smithi070.stdout:2493: done (1 left) 2024-09-14T03:56:59.072 INFO:tasks.rados.rados.0.smithi070.stdout:2494: done (0 left) 2024-09-14T03:56:59.072 INFO:tasks.rados.rados.0.smithi070.stdout:2495: snap_create 2024-09-14T03:56:59.095 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:59 smithi070 ceph-mon[96704]: osdmap e586: 8 total, 8 up, 8 in 2024-09-14T03:56:59.095 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:59 smithi070 ceph-mon[96704]: pgmap v476: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 2.2 MiB/s wr, 14 op/s 2024-09-14T03:56:59.095 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:56:59 smithi070 ceph-mon[96704]: osdmap e587: 8 total, 8 up, 8 in 2024-09-14T03:56:59.515 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:59 smithi070 ceph-mon[98640]: osdmap e586: 8 total, 8 up, 8 in 2024-09-14T03:56:59.515 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:59 smithi070 ceph-mon[98640]: pgmap v476: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 2.2 MiB/s wr, 14 op/s 2024-09-14T03:56:59.515 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:56:59 smithi070 ceph-mon[98640]: osdmap e587: 8 total, 8 up, 8 in 2024-09-14T03:56:59.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:59 smithi094 ceph-mon[86597]: osdmap e586: 8 total, 8 up, 8 in 2024-09-14T03:56:59.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:59 smithi094 ceph-mon[86597]: pgmap v476: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 2.2 MiB/s wr, 14 op/s 2024-09-14T03:56:59.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:56:59 smithi094 ceph-mon[86597]: osdmap e587: 8 total, 8 up, 8 in 2024-09-14T03:57:00.078 INFO:tasks.rados.rados.0.smithi070.stdout:2495: done (0 left) 2024-09-14T03:57:00.078 INFO:tasks.rados.rados.0.smithi070.stdout:2496: copy_from oid 41 from oid 34 current snap is 268 2024-09-14T03:57:00.078 INFO:tasks.rados.rados.0.smithi070.stdout:2497: read oid 14 snap 256 2024-09-14T03:57:00.079 INFO:tasks.rados.rados.0.smithi070.stdout:2497: expect deleted 2024-09-14T03:57:00.079 INFO:tasks.rados.rados.0.smithi070.stdout:2498: delete oid 13 current snap is 268 2024-09-14T03:57:00.081 INFO:tasks.rados.rados.0.smithi070.stdout:2497: done (2 left) 2024-09-14T03:57:00.081 INFO:tasks.rados.rados.0.smithi070.stdout:2498: done (1 left) 2024-09-14T03:57:00.081 INFO:tasks.rados.rados.0.smithi070.stdout:2499: rmattr oid 40 current snap is 268 2024-09-14T03:57:00.081 INFO:tasks.rados.rados.0.smithi070.stdout:2499: done (1 left) 2024-09-14T03:57:00.082 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 5 current snap is 268 2024-09-14T03:57:00.082 INFO:tasks.rados.rados.0.smithi070.stdout:2500: seq_num 765 ranges {0=417792,417792=401408} 2024-09-14T03:57:00.084 INFO:tasks.rados.rados.0.smithi070.stdout:2500: writing smithi07085154-5 from 0 to 417792 tid 1 2024-09-14T03:57:00.086 INFO:tasks.rados.rados.0.smithi070.stdout:2500: writing smithi07085154-5 from 417792 to 819200 tid 2 2024-09-14T03:57:00.087 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 37 current snap is 268 2024-09-14T03:57:00.087 INFO:tasks.rados.rados.0.smithi070.stdout:2501: seq_num 766 ranges {0=434176,434176=401408} 2024-09-14T03:57:00.089 INFO:tasks.rados.rados.0.smithi070.stdout:2501: writing smithi07085154-37 from 0 to 434176 tid 1 2024-09-14T03:57:00.092 INFO:tasks.rados.rados.0.smithi070.stdout:2501: writing smithi07085154-37 from 434176 to 835584 tid 2 2024-09-14T03:57:00.092 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 10 current snap is 268 2024-09-14T03:57:00.092 INFO:tasks.rados.rados.0.smithi070.stdout:2502: seq_num 767 ranges {2605056=450560,3055616=401408} 2024-09-14T03:57:00.109 INFO:tasks.rados.rados.0.smithi070.stdout:2502: writing smithi07085154-10 from 2605056 to 3055616 tid 1 2024-09-14T03:57:00.112 INFO:tasks.rados.rados.0.smithi070.stdout:2502: writing smithi07085154-10 from 3055616 to 3457024 tid 2 2024-09-14T03:57:00.112 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 16 current snap is 268 2024-09-14T03:57:00.112 INFO:tasks.rados.rados.0.smithi070.stdout:2500: finishing write tid 1 to smithi07085154-5 2024-09-14T03:57:00.112 INFO:tasks.rados.rados.0.smithi070.stdout:2503: seq_num 768 ranges {3760128=466944,4227072=401408} 2024-09-14T03:57:00.135 INFO:tasks.rados.rados.0.smithi070.stdout:2503: writing smithi07085154-16 from 3760128 to 4227072 tid 1 2024-09-14T03:57:00.137 INFO:tasks.rados.rados.0.smithi070.stdout:2503: writing smithi07085154-16 from 4227072 to 4628480 tid 2 2024-09-14T03:57:00.137 INFO:tasks.rados.rados.0.smithi070.stdout:2504: read oid 30 snap 246 2024-09-14T03:57:00.137 INFO:tasks.rados.rados.0.smithi070.stdout:2504: expect deleted 2024-09-14T03:57:00.138 INFO:tasks.rados.rados.0.smithi070.stdout:2505: snap_remove snap 263 2024-09-14T03:57:00.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:57:00 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:57:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:57:01.083 INFO:tasks.rados.rados.0.smithi070.stdout:2505: done (6 left) 2024-09-14T03:57:01.083 INFO:tasks.rados.rados.0.smithi070.stdout:2506: snap_create 2024-09-14T03:57:01.083 INFO:tasks.rados.rados.0.smithi070.stdout:2500: finishing write tid 2 to smithi07085154-5 2024-09-14T03:57:01.083 INFO:tasks.rados.rados.0.smithi070.stdout:2496: finishing copy_from racing read to smithi07085154-41 2024-09-14T03:57:01.083 INFO:tasks.rados.rados.0.smithi070.stdout:2500: finishing write tid 3 to smithi07085154-5 2024-09-14T03:57:01.083 INFO:tasks.rados.rados.0.smithi070.stdout:2500: finishing write tid 4 to smithi07085154-5 2024-09-14T03:57:01.084 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 570 (ObjNum 765 snap 268 seq_num 765) dirty exists 2024-09-14T03:57:01.084 INFO:tasks.rados.rados.0.smithi070.stdout:2500: left oid 5 (ObjNum 765 snap 268 seq_num 765) 2024-09-14T03:57:01.084 INFO:tasks.rados.rados.0.smithi070.stdout:2496: finishing copy_from to smithi07085154-41 2024-09-14T03:57:01.084 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 1737 (ObjNum 762 snap 264 seq_num 762) dirty exists 2024-09-14T03:57:01.084 INFO:tasks.rados.rados.0.smithi070.stdout:2501: finishing write tid 1 to smithi07085154-37 2024-09-14T03:57:01.084 INFO:tasks.rados.rados.0.smithi070.stdout:2501: finishing write tid 2 to smithi07085154-37 2024-09-14T03:57:01.084 INFO:tasks.rados.rados.0.smithi070.stdout:2501: finishing write tid 3 to smithi07085154-37 2024-09-14T03:57:01.084 INFO:tasks.rados.rados.0.smithi070.stdout:2501: finishing write tid 4 to smithi07085154-37 2024-09-14T03:57:01.084 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 37 v 2207 (ObjNum 766 snap 268 seq_num 766) dirty exists 2024-09-14T03:57:01.084 INFO:tasks.rados.rados.0.smithi070.stdout:2501: left oid 37 (ObjNum 766 snap 268 seq_num 766) 2024-09-14T03:57:01.084 INFO:tasks.rados.rados.0.smithi070.stdout:2502: finishing write tid 1 to smithi07085154-10 2024-09-14T03:57:01.084 INFO:tasks.rados.rados.0.smithi070.stdout:2502: finishing write tid 2 to smithi07085154-10 2024-09-14T03:57:01.084 INFO:tasks.rados.rados.0.smithi070.stdout:2502: finishing write tid 3 to smithi07085154-10 2024-09-14T03:57:01.085 INFO:tasks.rados.rados.0.smithi070.stdout:2502: finishing write tid 4 to smithi07085154-10 2024-09-14T03:57:01.085 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 2783 (ObjNum 767 snap 268 seq_num 767) dirty exists 2024-09-14T03:57:01.085 INFO:tasks.rados.rados.0.smithi070.stdout:2502: left oid 10 (ObjNum 767 snap 268 seq_num 767) 2024-09-14T03:57:01.085 INFO:tasks.rados.rados.0.smithi070.stdout:2503: finishing write tid 1 to smithi07085154-16 2024-09-14T03:57:01.085 INFO:tasks.rados.rados.0.smithi070.stdout:2503: finishing write tid 2 to smithi07085154-16 2024-09-14T03:57:01.085 INFO:tasks.rados.rados.0.smithi070.stdout:2503: finishing write tid 3 to smithi07085154-16 2024-09-14T03:57:01.085 INFO:tasks.rados.rados.0.smithi070.stdout:2503: finishing write tid 4 to smithi07085154-16 2024-09-14T03:57:01.085 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 2787 (ObjNum 768 snap 268 seq_num 768) dirty exists 2024-09-14T03:57:01.085 INFO:tasks.rados.rados.0.smithi070.stdout:2503: left oid 16 (ObjNum 768 snap 268 seq_num 768) 2024-09-14T03:57:01.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:00 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:01.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:00 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:01.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:00 smithi070 ceph-mon[96704]: osdmap e588: 8 total, 8 up, 8 in 2024-09-14T03:57:01.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:00 smithi070 ceph-mon[96704]: pgmap v479: 121 pgs: 121 active+clean; 1.1 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.8 MiB/s rd, 3.8 MiB/s wr, 21 op/s 2024-09-14T03:57:01.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:00 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:01.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:00 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:01.213 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:00 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:01.213 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:00 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:01.213 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:00 smithi070 ceph-mon[98640]: osdmap e588: 8 total, 8 up, 8 in 2024-09-14T03:57:01.213 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:00 smithi070 ceph-mon[98640]: pgmap v479: 121 pgs: 121 active+clean; 1.1 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.8 MiB/s rd, 3.8 MiB/s wr, 21 op/s 2024-09-14T03:57:01.214 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:00 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:01.214 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:00 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:01.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:00 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:01.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:00 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:01.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:00 smithi094 ceph-mon[86597]: osdmap e588: 8 total, 8 up, 8 in 2024-09-14T03:57:01.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:00 smithi094 ceph-mon[86597]: pgmap v479: 121 pgs: 121 active+clean; 1.1 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.8 MiB/s rd, 3.8 MiB/s wr, 21 op/s 2024-09-14T03:57:01.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:00 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:01.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:00 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:01.504 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-09-14T03:57:02.064 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:57:02.064 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (6m) 2s ago 11m 21.2M - 0.27.0 11f11916f8cd 45e79e51074c 2024-09-14T03:57:02.064 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi094 *:3000 running (6m) 23s ago 11m 85.6M - 9.4.7 954c08fa6188 2240810d0368 2024-09-14T03:57:02.064 INFO:teuthology.orchestra.run.smithi070.stdout:mds.a smithi070 running (5s) 2s ago 13m 15.5M - 19.3.0-4907-ga2782315 8801d2529e43 4fa138982c4f 2024-09-14T03:57:02.064 INFO:teuthology.orchestra.run.smithi070.stdout:mds.b smithi094 running (13m) 23s ago 13m 16.3M - 17.2.7-1629-geb16b97b ab51d1bebd12 e3b5e3e00189 2024-09-14T03:57:02.064 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi094 *:8443,9283,8765 running (5m) 23s ago 16m 456M - 19.3.0-4907-ga2782315 8801d2529e43 569d9c7e0f06 2024-09-14T03:57:02.064 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (6m) 2s ago 18m 529M - 19.3.0-4907-ga2782315 8801d2529e43 0ef4d8e0cdbe 2024-09-14T03:57:02.064 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (5m) 2s ago 18m 69.1M 2048M 19.3.0-4907-ga2782315 8801d2529e43 96542de31cac 2024-09-14T03:57:02.064 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi094 running (5m) 23s ago 17m 51.5M 2048M 19.3.0-4907-ga2782315 8801d2529e43 86827be98659 2024-09-14T03:57:02.064 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (5m) 2s ago 16m 55.7M 2048M 19.3.0-4907-ga2782315 8801d2529e43 5741e4fcf4b6 2024-09-14T03:57:02.064 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (6m) 2s ago 11m 11.2M - 1.7.0 72c9c2088986 e316760a30b3 2024-09-14T03:57:02.065 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi094 *:9100 running (6m) 23s ago 11m 12.2M - 1.7.0 72c9c2088986 767f8a093175 2024-09-14T03:57:02.065 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (4m) 2s ago 16m 315M 2480M 19.3.0-4907-ga2782315 8801d2529e43 2345ccac2085 2024-09-14T03:57:02.065 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (3m) 2s ago 15m 341M 2480M 19.3.0-4907-ga2782315 8801d2529e43 53868856512a 2024-09-14T03:57:02.065 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (3m) 2s ago 15m 219M 2480M 19.3.0-4907-ga2782315 8801d2529e43 39ef876d2fff 2024-09-14T03:57:02.065 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (2m) 2s ago 15m 149M 2480M 19.3.0-4907-ga2782315 8801d2529e43 8efc112edbe9 2024-09-14T03:57:02.065 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi094 running (2m) 23s ago 14m 234M 2480M 19.3.0-4907-ga2782315 8801d2529e43 09f583ed6c77 2024-09-14T03:57:02.065 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi094 running (95s) 23s ago 14m 185M 2480M 19.3.0-4907-ga2782315 8801d2529e43 162396f2e7b6 2024-09-14T03:57:02.065 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi094 running (61s) 23s ago 13m 107M 2480M 19.3.0-4907-ga2782315 8801d2529e43 cbdc02cdb8e4 2024-09-14T03:57:02.065 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi094 running (28s) 23s ago 13m 12.1M 2480M 19.3.0-4907-ga2782315 8801d2529e43 ea9896fb7dbc 2024-09-14T03:57:02.065 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi094 *:9095 running (5m) 23s ago 12m 54.0M - 2.51.0 1d3b7f56885b ce1b353828dc 2024-09-14T03:57:02.087 INFO:tasks.rados.rados.0.smithi070.stdout:2496: done (6 left) 2024-09-14T03:57:02.087 INFO:tasks.rados.rados.0.smithi070.stdout:2500: done (5 left) 2024-09-14T03:57:02.087 INFO:tasks.rados.rados.0.smithi070.stdout:2501: done (4 left) 2024-09-14T03:57:02.088 INFO:tasks.rados.rados.0.smithi070.stdout:2502: done (3 left) 2024-09-14T03:57:02.088 INFO:tasks.rados.rados.0.smithi070.stdout:2503: done (2 left) 2024-09-14T03:57:02.088 INFO:tasks.rados.rados.0.smithi070.stdout:2504: done (1 left) 2024-09-14T03:57:02.088 INFO:tasks.rados.rados.0.smithi070.stdout:2506: done (0 left) 2024-09-14T03:57:02.088 INFO:tasks.rados.rados.0.smithi070.stdout:2507: snap_create 2024-09-14T03:57:02.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:02 smithi070 ceph-mon[96704]: osdmap e589: 8 total, 8 up, 8 in 2024-09-14T03:57:02.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:02 smithi070 ceph-mon[98640]: osdmap e589: 8 total, 8 up, 8 in 2024-09-14T03:57:02.413 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:57:02.413 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-09-14T03:57:02.413 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:57:02.414 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:57:02.414 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-09-14T03:57:02.414 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:57:02.414 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:57:02.414 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-09-14T03:57:02.414 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 8 2024-09-14T03:57:02.414 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:57:02.414 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": { 2024-09-14T03:57:02.414 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 1, 2024-09-14T03:57:02.414 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 1 2024-09-14T03:57:02.414 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:57:02.414 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-09-14T03:57:02.414 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.7-1629-geb16b97b (eb16b97b08b1a378bbb6cbc2dadc21f07d8ac10b) quincy (stable)": 1, 2024-09-14T03:57:02.415 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 14 2024-09-14T03:57:02.415 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-09-14T03:57:02.415 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:57:02.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:02 smithi094 ceph-mon[86597]: osdmap e589: 8 total, 8 up, 8 in 2024-09-14T03:57:02.708 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:57:02.708 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:57:02.708 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-09-14T03:57:02.708 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:57:02.708 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [ 2024-09-14T03:57:02.708 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr", 2024-09-14T03:57:02.708 INFO:teuthology.orchestra.run.smithi070.stdout: "mon", 2024-09-14T03:57:02.708 INFO:teuthology.orchestra.run.smithi070.stdout: "osd" 2024-09-14T03:57:02.708 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2024-09-14T03:57:02.708 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "14/20 daemons upgraded", 2024-09-14T03:57:02.708 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "Currently upgrading mds daemons", 2024-09-14T03:57:02.709 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-09-14T03:57:02.709 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:57:03.093 INFO:tasks.rados.rados.0.smithi070.stdout:2507: done (0 left) 2024-09-14T03:57:03.094 INFO:tasks.rados.rados.0.smithi070.stdout:2508: delete oid 40 current snap is 270 2024-09-14T03:57:03.096 INFO:tasks.rados.rados.0.smithi070.stdout:2508: done (0 left) 2024-09-14T03:57:03.096 INFO:tasks.rados.rados.0.smithi070.stdout:2509: snap_remove snap 182 2024-09-14T03:57:03.382 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:03 smithi070 ceph-mon[98640]: from='client.44420 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:03.383 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:03 smithi070 ceph-mon[98640]: from='client.44426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:03.383 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:03 smithi070 ceph-mon[98640]: from='client.34515 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:03.383 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:03 smithi070 ceph-mon[98640]: osdmap e590: 8 total, 8 up, 8 in 2024-09-14T03:57:03.383 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:03 smithi070 ceph-mon[98640]: pgmap v482: 121 pgs: 121 active+clean; 1.1 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 2.5 MiB/s wr, 14 op/s 2024-09-14T03:57:03.383 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:03 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:03.383 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:03 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:57:03.383 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:03 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/169811262' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:03.383 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:03 smithi070 ceph-mon[96704]: from='client.44420 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:03.383 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:03 smithi070 ceph-mon[96704]: from='client.44426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:03.383 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:03 smithi070 ceph-mon[96704]: from='client.34515 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:03.383 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:03 smithi070 ceph-mon[96704]: osdmap e590: 8 total, 8 up, 8 in 2024-09-14T03:57:03.384 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:03 smithi070 ceph-mon[96704]: pgmap v482: 121 pgs: 121 active+clean; 1.1 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 2.5 MiB/s wr, 14 op/s 2024-09-14T03:57:03.384 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:03 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:03.384 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:03 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:57:03.384 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:03 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/169811262' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:03.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:03 smithi094 ceph-mon[86597]: from='client.44420 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:03.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:03 smithi094 ceph-mon[86597]: from='client.44426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:03.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:03 smithi094 ceph-mon[86597]: from='client.34515 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:03.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:03 smithi094 ceph-mon[86597]: osdmap e590: 8 total, 8 up, 8 in 2024-09-14T03:57:03.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:03 smithi094 ceph-mon[86597]: pgmap v482: 121 pgs: 121 active+clean; 1.1 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 2.5 MiB/s wr, 14 op/s 2024-09-14T03:57:03.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:03 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:03.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:03 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:57:03.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:03 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/169811262' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:03.707 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:57:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:57:03] "GET /metrics HTTP/1.1" 200 36508 "" "Prometheus/2.51.0" 2024-09-14T03:57:04.074 INFO:tasks.rados.rados.0.smithi070.stdout:2509: done (0 left) 2024-09-14T03:57:04.074 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 5 current snap is 270 2024-09-14T03:57:04.074 INFO:tasks.rados.rados.0.smithi070.stdout:2510: seq_num 770 ranges {819200=499712,1318912=401408} 2024-09-14T03:57:04.082 INFO:tasks.rados.rados.0.smithi070.stdout:2510: writing smithi07085154-5 from 819200 to 1318912 tid 1 2024-09-14T03:57:04.085 INFO:tasks.rados.rados.0.smithi070.stdout:2510: writing smithi07085154-5 from 1318912 to 1720320 tid 2 2024-09-14T03:57:04.085 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 24 current snap is 270 2024-09-14T03:57:04.085 INFO:tasks.rados.rados.0.smithi070.stdout:2511: seq_num 771 ranges {0=516096,516096=401408} 2024-09-14T03:57:04.089 INFO:tasks.rados.rados.0.smithi070.stdout:2511: writing smithi07085154-24 from 0 to 516096 tid 1 2024-09-14T03:57:04.092 INFO:tasks.rados.rados.0.smithi070.stdout:2511: writing smithi07085154-24 from 516096 to 917504 tid 2 2024-09-14T03:57:04.092 INFO:tasks.rados.rados.0.smithi070.stdout:2512: rollback oid 13 current snap is 270 2024-09-14T03:57:04.092 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 13 to 226 2024-09-14T03:57:04.092 INFO:tasks.rados.rados.0.smithi070.stdout:2513: delete oid 31 current snap is 270 2024-09-14T03:57:04.093 INFO:tasks.rados.rados.0.smithi070.stdout:2510: finishing write tid 1 to smithi07085154-5 2024-09-14T03:57:04.100 INFO:tasks.rados.rados.0.smithi070.stdout:2512: finishing rollback tid 1 to smithi07085154-13 2024-09-14T03:57:04.100 INFO:tasks.rados.rados.0.smithi070.stdout:2510: finishing write tid 2 to smithi07085154-5 2024-09-14T03:57:04.100 INFO:tasks.rados.rados.0.smithi070.stdout:2510: finishing write tid 3 to smithi07085154-5 2024-09-14T03:57:04.100 INFO:tasks.rados.rados.0.smithi070.stdout:2510: finishing write tid 4 to smithi07085154-5 2024-09-14T03:57:04.100 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 573 (ObjNum 770 snap 270 seq_num 770) dirty exists 2024-09-14T03:57:04.101 INFO:tasks.rados.rados.0.smithi070.stdout:2510: left oid 5 (ObjNum 770 snap 270 seq_num 770) 2024-09-14T03:57:04.101 INFO:tasks.rados.rados.0.smithi070.stdout:2512: finishing rollback tid 2 to smithi07085154-13 2024-09-14T03:57:04.101 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 2059 (ObjNum 459 snap 155 seq_num 459) dirty exists 2024-09-14T03:57:04.101 INFO:tasks.rados.rados.0.smithi070.stdout:2510: done (3 left) 2024-09-14T03:57:04.101 INFO:tasks.rados.rados.0.smithi070.stdout:2512: done (2 left) 2024-09-14T03:57:04.101 INFO:tasks.rados.rados.0.smithi070.stdout:2513: done (1 left) 2024-09-14T03:57:04.101 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 31 current snap is 270 2024-09-14T03:57:04.101 INFO:tasks.rados.rados.0.smithi070.stdout:2514: seq_num 773 ranges {0=548864,548864=401408} 2024-09-14T03:57:04.104 INFO:tasks.rados.rados.0.smithi070.stdout:2514: writing smithi07085154-31 from 0 to 548864 tid 1 2024-09-14T03:57:04.107 INFO:tasks.rados.rados.0.smithi070.stdout:2514: writing smithi07085154-31 from 548864 to 950272 tid 2 2024-09-14T03:57:04.107 INFO:tasks.rados.rados.0.smithi070.stdout:2515: delete oid 44 current snap is 270 2024-09-14T03:57:04.107 INFO:tasks.rados.rados.0.smithi070.stdout:2511: finishing write tid 1 to smithi07085154-24 2024-09-14T03:57:04.107 INFO:tasks.rados.rados.0.smithi070.stdout:2511: finishing write tid 2 to smithi07085154-24 2024-09-14T03:57:04.107 INFO:tasks.rados.rados.0.smithi070.stdout:2511: finishing write tid 3 to smithi07085154-24 2024-09-14T03:57:04.108 INFO:tasks.rados.rados.0.smithi070.stdout:2511: finishing write tid 4 to smithi07085154-24 2024-09-14T03:57:04.108 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 2253 (ObjNum 771 snap 270 seq_num 771) dirty exists 2024-09-14T03:57:04.108 INFO:tasks.rados.rados.0.smithi070.stdout:2511: left oid 24 (ObjNum 771 snap 270 seq_num 771) 2024-09-14T03:57:04.110 INFO:tasks.rados.rados.0.smithi070.stdout:2511: done (2 left) 2024-09-14T03:57:04.111 INFO:tasks.rados.rados.0.smithi070.stdout:2515: done (1 left) 2024-09-14T03:57:04.111 INFO:tasks.rados.rados.0.smithi070.stdout:2516: snap_remove snap 269 2024-09-14T03:57:04.347 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:04 smithi094 ceph-mon[86597]: from='client.34527 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:04.348 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:04 smithi094 ceph-mon[86597]: osdmap e591: 8 total, 8 up, 8 in 2024-09-14T03:57:04.348 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:04 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:04.348 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:04 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:04.348 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:04 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:57:04.348 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:04 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:57:04.348 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:04 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:04.348 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:04 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:57:04.348 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:04 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:04.348 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:04 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:04.348 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:04 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:04.348 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:04 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:04.349 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:04 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mds ok-to-stop", "ids": ["b"]} : dispatch 2024-09-14T03:57:04.349 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:04 smithi094 ceph-mon[86597]: osdmap e592: 8 total, 8 up, 8 in 2024-09-14T03:57:04.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:04 smithi070 ceph-mon[96704]: from='client.34527 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:04.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:04 smithi070 ceph-mon[96704]: osdmap e591: 8 total, 8 up, 8 in 2024-09-14T03:57:04.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:04 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:04.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:04 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:04.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:04 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:57:04.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:04 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:57:04.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:04 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:04.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:04 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:57:04.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:04 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:04.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:04 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:04.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:04 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:04.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:04 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:04.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:04 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mds ok-to-stop", "ids": ["b"]} : dispatch 2024-09-14T03:57:04.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:04 smithi070 ceph-mon[96704]: osdmap e592: 8 total, 8 up, 8 in 2024-09-14T03:57:04.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:04 smithi070 ceph-mon[98640]: from='client.34527 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:04.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:04 smithi070 ceph-mon[98640]: osdmap e591: 8 total, 8 up, 8 in 2024-09-14T03:57:04.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:04 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:04.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:04 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:04.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:04 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:57:04.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:04 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:57:04.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:04 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:04.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:04 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:57:04.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:04 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:04.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:04 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:04.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:04 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:04.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:04 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:04.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:04 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mds ok-to-stop", "ids": ["b"]} : dispatch 2024-09-14T03:57:04.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:04 smithi070 ceph-mon[98640]: osdmap e592: 8 total, 8 up, 8 in 2024-09-14T03:57:05.099 INFO:tasks.rados.rados.0.smithi070.stdout:2516: done (1 left) 2024-09-14T03:57:05.100 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 39 current snap is 270 2024-09-14T03:57:05.100 INFO:tasks.rados.rados.0.smithi070.stdout:2517: seq_num 775 ranges {2801664=589824,3391488=401408} 2024-09-14T03:57:05.117 INFO:tasks.rados.rados.0.smithi070.stdout:2517: writing smithi07085154-39 from 2801664 to 3391488 tid 1 2024-09-14T03:57:05.119 INFO:tasks.rados.rados.0.smithi070.stdout:2517: writing smithi07085154-39 from 3391488 to 3792896 tid 2 2024-09-14T03:57:05.119 INFO:tasks.rados.rados.0.smithi070.stdout:2518: snap_remove snap 244 2024-09-14T03:57:05.208 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:05 smithi094 ceph-mon[86597]: Detected new or changed devices on smithi070 2024-09-14T03:57:05.208 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:05 smithi094 ceph-mon[86597]: Upgrade: It appears safe to stop mds.b 2024-09-14T03:57:05.208 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:05 smithi094 ceph-mon[86597]: pgmap v485: 121 pgs: 121 active+clean; 1.1 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 1000 KiB/s wr, 6 op/s 2024-09-14T03:57:05.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:05 smithi070 ceph-mon[98640]: Detected new or changed devices on smithi070 2024-09-14T03:57:05.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:05 smithi070 ceph-mon[98640]: Upgrade: It appears safe to stop mds.b 2024-09-14T03:57:05.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:05 smithi070 ceph-mon[98640]: pgmap v485: 121 pgs: 121 active+clean; 1.1 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 1000 KiB/s wr, 6 op/s 2024-09-14T03:57:05.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:05 smithi070 ceph-mon[96704]: Detected new or changed devices on smithi070 2024-09-14T03:57:05.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:05 smithi070 ceph-mon[96704]: Upgrade: It appears safe to stop mds.b 2024-09-14T03:57:05.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:05 smithi070 ceph-mon[96704]: pgmap v485: 121 pgs: 121 active+clean; 1.1 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 1000 KiB/s wr, 6 op/s 2024-09-14T03:57:06.106 INFO:tasks.rados.rados.0.smithi070.stdout:2514: finishing write tid 1 to smithi07085154-31 2024-09-14T03:57:06.106 INFO:tasks.rados.rados.0.smithi070.stdout:2514: finishing write tid 2 to smithi07085154-31 2024-09-14T03:57:06.107 INFO:tasks.rados.rados.0.smithi070.stdout:2514: finishing write tid 3 to smithi07085154-31 2024-09-14T03:57:06.107 INFO:tasks.rados.rados.0.smithi070.stdout:2514: finishing write tid 4 to smithi07085154-31 2024-09-14T03:57:06.107 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 2253 (ObjNum 773 snap 270 seq_num 773) dirty exists 2024-09-14T03:57:06.107 INFO:tasks.rados.rados.0.smithi070.stdout:2514: left oid 31 (ObjNum 773 snap 270 seq_num 773) 2024-09-14T03:57:06.107 INFO:tasks.rados.rados.0.smithi070.stdout:2517: finishing write tid 1 to smithi07085154-39 2024-09-14T03:57:06.107 INFO:tasks.rados.rados.0.smithi070.stdout:2517: finishing write tid 2 to smithi07085154-39 2024-09-14T03:57:06.107 INFO:tasks.rados.rados.0.smithi070.stdout:2517: finishing write tid 3 to smithi07085154-39 2024-09-14T03:57:06.107 INFO:tasks.rados.rados.0.smithi070.stdout:2517: finishing write tid 4 to smithi07085154-39 2024-09-14T03:57:06.107 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 39 v 2630 (ObjNum 775 snap 270 seq_num 775) dirty exists 2024-09-14T03:57:06.107 INFO:tasks.rados.rados.0.smithi070.stdout:2517: left oid 39 (ObjNum 775 snap 270 seq_num 775) 2024-09-14T03:57:06.107 INFO:tasks.rados.rados.0.smithi070.stdout:2514: done (2 left) 2024-09-14T03:57:06.107 INFO:tasks.rados.rados.0.smithi070.stdout:2517: done (1 left) 2024-09-14T03:57:06.107 INFO:tasks.rados.rados.0.smithi070.stdout:2518: done (0 left) 2024-09-14T03:57:06.108 INFO:tasks.rados.rados.0.smithi070.stdout:2519: copy_from oid 16 from oid 18 current snap is 270 2024-09-14T03:57:06.108 INFO:tasks.rados.rados.0.smithi070.stdout:2520: read oid 42 snap 246 2024-09-14T03:57:06.108 INFO:tasks.rados.rados.0.smithi070.stdout:2520: expect (ObjNum 507 snap 166 seq_num 507) 2024-09-14T03:57:06.108 INFO:tasks.rados.rados.0.smithi070.stdout:2521: copy_from oid 10 from oid 28 current snap is 270 2024-09-14T03:57:06.108 INFO:tasks.rados.rados.0.smithi070.stdout:2522: rmattr oid 37 current snap is 270 2024-09-14T03:57:06.108 INFO:tasks.rados.rados.0.smithi070.stdout:2522: done (3 left) 2024-09-14T03:57:06.108 INFO:tasks.rados.rados.0.smithi070.stdout:2523: rmattr oid 22 current snap is 270 2024-09-14T03:57:06.108 INFO:tasks.rados.rados.0.smithi070.stdout:2524: snap_create 2024-09-14T03:57:06.126 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 1738 (ObjNum 458 snap 155 seq_num 458) dirty exists 2024-09-14T03:57:06.127 INFO:tasks.rados.rados.0.smithi070.stdout:2519: finishing copy_from racing read to smithi07085154-16 2024-09-14T03:57:06.130 INFO:tasks.rados.rados.0.smithi070.stdout:2520: expect (ObjNum 507 snap 166 seq_num 507) 2024-09-14T03:57:06.165 INFO:tasks.rados.rados.0.smithi070.stdout:2519: finishing copy_from to smithi07085154-16 2024-09-14T03:57:06.166 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 2060 (ObjNum 754 snap 260 seq_num 754) dirty exists 2024-09-14T03:57:06.166 INFO:tasks.rados.rados.0.smithi070.stdout:2521: finishing copy_from racing read to smithi07085154-10 2024-09-14T03:57:06.166 INFO:tasks.rados.rados.0.smithi070.stdout:2521: finishing copy_from to smithi07085154-10 2024-09-14T03:57:06.166 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 1262 (ObjNum 639 snap 214 seq_num 639) dirty exists 2024-09-14T03:57:06.265 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:06 smithi094 ceph-mon[86597]: osdmap e593: 8 total, 8 up, 8 in 2024-09-14T03:57:06.266 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:06 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:06.266 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:06 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:57:06.266 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:06 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:57:06.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:06 smithi070 ceph-mon[96704]: osdmap e593: 8 total, 8 up, 8 in 2024-09-14T03:57:06.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:06 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:06.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:06 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:57:06.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:06 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:57:06.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:06 smithi070 ceph-mon[98640]: osdmap e593: 8 total, 8 up, 8 in 2024-09-14T03:57:06.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:06 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:06.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:06 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T03:57:06.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:06 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:57:07.105 INFO:tasks.rados.rados.0.smithi070.stdout:2519: done (4 left) 2024-09-14T03:57:07.106 INFO:tasks.rados.rados.0.smithi070.stdout:2520: done (3 left) 2024-09-14T03:57:07.106 INFO:tasks.rados.rados.0.smithi070.stdout:2521: done (2 left) 2024-09-14T03:57:07.106 INFO:tasks.rados.rados.0.smithi070.stdout:2523: done (1 left) 2024-09-14T03:57:07.106 INFO:tasks.rados.rados.0.smithi070.stdout:2524: done (0 left) 2024-09-14T03:57:07.106 INFO:tasks.rados.rados.0.smithi070.stdout:2525: read oid 19 snap -1 2024-09-14T03:57:07.106 INFO:tasks.rados.rados.0.smithi070.stdout:2525: expect (ObjNum 747 snap 259 seq_num 747) 2024-09-14T03:57:07.106 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 41 current snap is 271 2024-09-14T03:57:07.106 INFO:tasks.rados.rados.0.smithi070.stdout:2526: seq_num 776 ranges {770048=606208,1376256=401408} 2024-09-14T03:57:07.113 INFO:tasks.rados.rados.0.smithi070.stdout:2526: writing smithi07085154-41 from 770048 to 1376256 tid 1 2024-09-14T03:57:07.115 INFO:tasks.rados.rados.0.smithi070.stdout:2526: writing smithi07085154-41 from 1376256 to 1777664 tid 2 2024-09-14T03:57:07.115 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 7 current snap is 271 2024-09-14T03:57:07.116 INFO:tasks.rados.rados.0.smithi070.stdout:2527: seq_num 777 ranges {3760128=622592,4382720=401408} 2024-09-14T03:57:07.139 INFO:tasks.rados.rados.0.smithi070.stdout:2527: writing smithi07085154-7 from 3760128 to 4382720 tid 1 2024-09-14T03:57:07.142 INFO:tasks.rados.rados.0.smithi070.stdout:2527: writing smithi07085154-7 from 4382720 to 4784128 tid 2 2024-09-14T03:57:07.142 INFO:tasks.rados.rados.0.smithi070.stdout:2528: snap_create 2024-09-14T03:57:07.142 INFO:tasks.rados.rados.0.smithi070.stdout:2525: expect (ObjNum 747 snap 259 seq_num 747) 2024-09-14T03:57:07.214 INFO:tasks.rados.rados.0.smithi070.stdout:2526: finishing write tid 1 to smithi07085154-41 2024-09-14T03:57:07.214 INFO:tasks.rados.rados.0.smithi070.stdout:2526: finishing write tid 2 to smithi07085154-41 2024-09-14T03:57:07.214 INFO:tasks.rados.rados.0.smithi070.stdout:2526: finishing write tid 3 to smithi07085154-41 2024-09-14T03:57:07.214 INFO:tasks.rados.rados.0.smithi070.stdout:2526: finishing write tid 4 to smithi07085154-41 2024-09-14T03:57:07.214 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 2210 (ObjNum 776 snap 271 seq_num 776) dirty exists 2024-09-14T03:57:07.215 INFO:tasks.rados.rados.0.smithi070.stdout:2526: left oid 41 (ObjNum 776 snap 271 seq_num 776) 2024-09-14T03:57:07.215 INFO:tasks.rados.rados.0.smithi070.stdout:2527: finishing write tid 1 to smithi07085154-7 2024-09-14T03:57:07.215 INFO:tasks.rados.rados.0.smithi070.stdout:2527: finishing write tid 2 to smithi07085154-7 2024-09-14T03:57:07.215 INFO:tasks.rados.rados.0.smithi070.stdout:2527: finishing write tid 3 to smithi07085154-7 2024-09-14T03:57:07.215 INFO:tasks.rados.rados.0.smithi070.stdout:2527: finishing write tid 4 to smithi07085154-7 2024-09-14T03:57:07.215 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 2059 (ObjNum 777 snap 271 seq_num 777) dirty exists 2024-09-14T03:57:07.215 INFO:tasks.rados.rados.0.smithi070.stdout:2527: left oid 7 (ObjNum 777 snap 271 seq_num 777) 2024-09-14T03:57:07.277 INFO:tasks.rados.rados.0.smithi070.stdout:2525: done (3 left) 2024-09-14T03:57:07.277 INFO:tasks.rados.rados.0.smithi070.stdout:2526: done (2 left) 2024-09-14T03:57:07.277 INFO:tasks.rados.rados.0.smithi070.stdout:2527: done (1 left) 2024-09-14T03:57:07.277 INFO:tasks.rados.rados.0.smithi070.stdout:2528: done (0 left) 2024-09-14T03:57:07.277 INFO:tasks.rados.rados.0.smithi070.stdout:2529: read oid 29 snap -1 2024-09-14T03:57:07.277 INFO:tasks.rados.rados.0.smithi070.stdout:2529: expect (ObjNum 458 snap 155 seq_num 458) 2024-09-14T03:57:07.277 INFO:tasks.rados.rados.0.smithi070.stdout:2530: snap_create 2024-09-14T03:57:07.305 INFO:tasks.rados.rados.0.smithi070.stdout:2529: expect (ObjNum 458 snap 155 seq_num 458) 2024-09-14T03:57:07.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:07 smithi094 ceph-mon[86597]: Upgrade: Updating mds.b 2024-09-14T03:57:07.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:07 smithi094 ceph-mon[86597]: Deploying daemon mds.b on smithi094 2024-09-14T03:57:07.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:07 smithi094 ceph-mon[86597]: pgmap v487: 121 pgs: 121 active+clean; 1.1 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 391 KiB/s wr, 1 op/s 2024-09-14T03:57:07.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:07 smithi094 ceph-mon[86597]: osdmap e594: 8 total, 8 up, 8 in 2024-09-14T03:57:07.527 INFO:journalctl@ceph.mds.b.smithi094.stdout:Sep 14 03:57:07 smithi094 systemd[1]: Stopping Ceph mds.b for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:57:07.528 INFO:journalctl@ceph.mds.b.smithi094.stdout:Sep 14 03:57:07 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-b[59374]: 2024-09-14T03:57:07.270+0000 7f5f0adca640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:57:07.528 INFO:journalctl@ceph.mds.b.smithi094.stdout:Sep 14 03:57:07 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-b[59374]: 2024-09-14T03:57:07.270+0000 7f5f0adca640 -1 mds.b *** got signal Terminated *** 2024-09-14T03:57:07.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:07 smithi070 ceph-mon[98640]: Upgrade: Updating mds.b 2024-09-14T03:57:07.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:07 smithi070 ceph-mon[98640]: Deploying daemon mds.b on smithi094 2024-09-14T03:57:07.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:07 smithi070 ceph-mon[98640]: pgmap v487: 121 pgs: 121 active+clean; 1.1 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 391 KiB/s wr, 1 op/s 2024-09-14T03:57:07.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:07 smithi070 ceph-mon[98640]: osdmap e594: 8 total, 8 up, 8 in 2024-09-14T03:57:07.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:07 smithi070 ceph-mon[96704]: Upgrade: Updating mds.b 2024-09-14T03:57:07.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:07 smithi070 ceph-mon[96704]: Deploying daemon mds.b on smithi094 2024-09-14T03:57:07.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:07 smithi070 ceph-mon[96704]: pgmap v487: 121 pgs: 121 active+clean; 1.1 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 391 KiB/s wr, 1 op/s 2024-09-14T03:57:07.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:07 smithi070 ceph-mon[96704]: osdmap e594: 8 total, 8 up, 8 in 2024-09-14T03:57:07.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:07 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-a[96681]: 2024-09-14T03:57:07.271+0000 7f512eabf640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-14T03:57:08.285 INFO:tasks.rados.rados.0.smithi070.stdout:2529: done (1 left) 2024-09-14T03:57:08.286 INFO:tasks.rados.rados.0.smithi070.stdout:2530: done (0 left) 2024-09-14T03:57:08.286 INFO:tasks.rados.rados.0.smithi070.stdout:2531: rollback oid 44 current snap is 273 2024-09-14T03:57:08.286 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 44 to 266 2024-09-14T03:57:08.286 INFO:tasks.rados.rados.0.smithi070.stdout:2532: snap_create 2024-09-14T03:57:08.288 INFO:tasks.rados.rados.0.smithi070.stdout:2531: finishing rollback tid 1 to smithi07085154-44 2024-09-14T03:57:08.289 INFO:tasks.rados.rados.0.smithi070.stdout:2531: finishing rollback tid 2 to smithi07085154-44 2024-09-14T03:57:08.289 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 2064 (ObjNum 668 snap 228 seq_num 668) dirty exists 2024-09-14T03:57:08.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:08 smithi094 ceph-mon[86597]: osdmap e595: 8 total, 8 up, 8 in 2024-09-14T03:57:08.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:08 smithi094 ceph-mon[86597]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-14T03:57:08.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:08 smithi094 ceph-mon[86597]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-14T03:57:08.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:08 smithi094 ceph-mon[86597]: osdmap e596: 8 total, 8 up, 8 in 2024-09-14T03:57:08.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:08 smithi094 ceph-mon[86597]: Standby daemon mds.a assigned to filesystem cephfs as rank 0 2024-09-14T03:57:08.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:08 smithi094 ceph-mon[86597]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-14T03:57:08.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:08 smithi094 ceph-mon[86597]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-14T03:57:08.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:08 smithi094 ceph-mon[86597]: fsmap cephfs:1/1 {0=a=up:replay} 2024-09-14T03:57:08.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:08 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "b"} : dispatch 2024-09-14T03:57:08.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:08 smithi070 ceph-mon[96704]: osdmap e595: 8 total, 8 up, 8 in 2024-09-14T03:57:08.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:08 smithi070 ceph-mon[96704]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-14T03:57:08.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:08 smithi070 ceph-mon[96704]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-14T03:57:08.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:08 smithi070 ceph-mon[96704]: osdmap e596: 8 total, 8 up, 8 in 2024-09-14T03:57:08.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:08 smithi070 ceph-mon[96704]: Standby daemon mds.a assigned to filesystem cephfs as rank 0 2024-09-14T03:57:08.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:08 smithi070 ceph-mon[96704]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-14T03:57:08.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:08 smithi070 ceph-mon[96704]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-14T03:57:08.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:08 smithi070 ceph-mon[96704]: fsmap cephfs:1/1 {0=a=up:replay} 2024-09-14T03:57:08.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:08 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "b"} : dispatch 2024-09-14T03:57:08.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:08 smithi070 ceph-mon[98640]: osdmap e595: 8 total, 8 up, 8 in 2024-09-14T03:57:08.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:08 smithi070 ceph-mon[98640]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-14T03:57:08.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:08 smithi070 ceph-mon[98640]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-14T03:57:08.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:08 smithi070 ceph-mon[98640]: osdmap e596: 8 total, 8 up, 8 in 2024-09-14T03:57:08.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:08 smithi070 ceph-mon[98640]: Standby daemon mds.a assigned to filesystem cephfs as rank 0 2024-09-14T03:57:08.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:08 smithi070 ceph-mon[98640]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-14T03:57:08.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:08 smithi070 ceph-mon[98640]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-14T03:57:08.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:08 smithi070 ceph-mon[98640]: fsmap cephfs:1/1 {0=a=up:replay} 2024-09-14T03:57:08.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:08 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "b"} : dispatch 2024-09-14T03:57:09.077 INFO:tasks.rados.rados.0.smithi070.stdout:2531: done (1 left) 2024-09-14T03:57:09.077 INFO:tasks.rados.rados.0.smithi070.stdout:2532: done (0 left) 2024-09-14T03:57:09.077 INFO:tasks.rados.rados.0.smithi070.stdout:2533: snap_remove snap 252 2024-09-14T03:57:09.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:09 smithi094 ceph-mon[86597]: pgmap v491: 121 pgs: 121 active+clean; 1.1 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 955 KiB/s rd, 1.2 MiB/s wr, 15 op/s 2024-09-14T03:57:09.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:09 smithi094 ceph-mon[86597]: osdmap e597: 8 total, 8 up, 8 in 2024-09-14T03:57:09.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:09 smithi094 ceph-mon[86597]: mds.? [v2:172.21.15.70:6834/2011419410,v1:172.21.15.70:6835/2011419410] up:reconnect 2024-09-14T03:57:09.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:09 smithi094 ceph-mon[86597]: fsmap cephfs:1/1 {0=a=up:reconnect} 2024-09-14T03:57:09.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:09 smithi094 ceph-mon[86597]: osdmap e598: 8 total, 8 up, 8 in 2024-09-14T03:57:09.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:09 smithi070 ceph-mon[98640]: pgmap v491: 121 pgs: 121 active+clean; 1.1 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 955 KiB/s rd, 1.2 MiB/s wr, 15 op/s 2024-09-14T03:57:09.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:09 smithi070 ceph-mon[98640]: osdmap e597: 8 total, 8 up, 8 in 2024-09-14T03:57:09.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:09 smithi070 ceph-mon[98640]: mds.? [v2:172.21.15.70:6834/2011419410,v1:172.21.15.70:6835/2011419410] up:reconnect 2024-09-14T03:57:09.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:09 smithi070 ceph-mon[98640]: fsmap cephfs:1/1 {0=a=up:reconnect} 2024-09-14T03:57:09.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:09 smithi070 ceph-mon[98640]: osdmap e598: 8 total, 8 up, 8 in 2024-09-14T03:57:09.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:09 smithi070 ceph-mon[96704]: pgmap v491: 121 pgs: 121 active+clean; 1.1 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 955 KiB/s rd, 1.2 MiB/s wr, 15 op/s 2024-09-14T03:57:09.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:09 smithi070 ceph-mon[96704]: osdmap e597: 8 total, 8 up, 8 in 2024-09-14T03:57:09.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:09 smithi070 ceph-mon[96704]: mds.? [v2:172.21.15.70:6834/2011419410,v1:172.21.15.70:6835/2011419410] up:reconnect 2024-09-14T03:57:09.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:09 smithi070 ceph-mon[96704]: fsmap cephfs:1/1 {0=a=up:reconnect} 2024-09-14T03:57:09.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:09 smithi070 ceph-mon[96704]: osdmap e598: 8 total, 8 up, 8 in 2024-09-14T03:57:10.078 INFO:tasks.rados.rados.0.smithi070.stdout:2533: done (0 left) 2024-09-14T03:57:10.078 INFO:tasks.rados.rados.0.smithi070.stdout:2534: delete oid 47 current snap is 274 2024-09-14T03:57:10.093 INFO:tasks.rados.rados.0.smithi070.stdout:2534: done (0 left) 2024-09-14T03:57:10.093 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 27 current snap is 274 2024-09-14T03:57:10.093 INFO:tasks.rados.rados.0.smithi070.stdout:2535: seq_num 779 ranges {18579456=655360,19234816=401408} 2024-09-14T03:57:10.195 INFO:tasks.rados.rados.0.smithi070.stdout:2535: writing smithi07085154-27 from 18579456 to 19234816 tid 1 2024-09-14T03:57:10.197 INFO:tasks.rados.rados.0.smithi070.stdout:2535: writing smithi07085154-27 from 19234816 to 19636224 tid 2 2024-09-14T03:57:10.198 INFO:tasks.rados.rados.0.smithi070.stdout:2536: read oid 18 snap -1 2024-09-14T03:57:10.198 INFO:tasks.rados.rados.0.smithi070.stdout:2536: expect (ObjNum 754 snap 260 seq_num 754) 2024-09-14T03:57:10.198 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 43 current snap is 274 2024-09-14T03:57:10.198 INFO:tasks.rados.rados.0.smithi070.stdout:2537: seq_num 780 ranges {1818624=671744,2490368=401408} 2024-09-14T03:57:10.211 INFO:tasks.rados.rados.0.smithi070.stdout:2537: writing smithi07085154-43 from 1818624 to 2490368 tid 1 2024-09-14T03:57:10.214 INFO:tasks.rados.rados.0.smithi070.stdout:2537: writing smithi07085154-43 from 2490368 to 2891776 tid 2 2024-09-14T03:57:10.214 INFO:tasks.rados.rados.0.smithi070.stdout:2538: read oid 8 snap 174 2024-09-14T03:57:10.214 INFO:tasks.rados.rados.0.smithi070.stdout:2538: expect (ObjNum 488 snap 164 seq_num 488) 2024-09-14T03:57:10.214 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 34 current snap is 274 2024-09-14T03:57:10.214 INFO:tasks.rados.rados.0.smithi070.stdout:2539: seq_num 781 ranges {770048=688128,1458176=401408} 2024-09-14T03:57:10.222 INFO:tasks.rados.rados.0.smithi070.stdout:2539: writing smithi07085154-34 from 770048 to 1458176 tid 1 2024-09-14T03:57:10.225 INFO:tasks.rados.rados.0.smithi070.stdout:2539: writing smithi07085154-34 from 1458176 to 1859584 tid 2 2024-09-14T03:57:10.225 INFO:tasks.rados.rados.0.smithi070.stdout:2540: delete oid 17 current snap is 274 2024-09-14T03:57:10.225 INFO:tasks.rados.rados.0.smithi070.stdout:2536: expect (ObjNum 754 snap 260 seq_num 754) 2024-09-14T03:57:10.243 INFO:tasks.rados.rados.0.smithi070.stdout:2535: finishing write tid 1 to smithi07085154-27 2024-09-14T03:57:10.243 INFO:tasks.rados.rados.0.smithi070.stdout:2535: finishing write tid 2 to smithi07085154-27 2024-09-14T03:57:10.243 INFO:tasks.rados.rados.0.smithi070.stdout:2535: finishing write tid 3 to smithi07085154-27 2024-09-14T03:57:10.243 INFO:tasks.rados.rados.0.smithi070.stdout:2535: finishing write tid 4 to smithi07085154-27 2024-09-14T03:57:10.243 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 2847 (ObjNum 779 snap 274 seq_num 779) dirty exists 2024-09-14T03:57:10.243 INFO:tasks.rados.rados.0.smithi070.stdout:2535: left oid 27 (ObjNum 779 snap 274 seq_num 779) 2024-09-14T03:57:10.243 INFO:tasks.rados.rados.0.smithi070.stdout:2537: finishing write tid 1 to smithi07085154-43 2024-09-14T03:57:10.244 INFO:tasks.rados.rados.0.smithi070.stdout:2537: finishing write tid 2 to smithi07085154-43 2024-09-14T03:57:10.244 INFO:tasks.rados.rados.0.smithi070.stdout:2535: done (5 left) 2024-09-14T03:57:10.244 INFO:tasks.rados.rados.0.smithi070.stdout:2536: done (4 left) 2024-09-14T03:57:10.244 INFO:tasks.rados.rados.0.smithi070.stdout:2540: done (3 left) 2024-09-14T03:57:10.244 INFO:tasks.rados.rados.0.smithi070.stdout:2541: setattr oid 24 current snap is 274 2024-09-14T03:57:10.244 INFO:tasks.rados.rados.0.smithi070.stdout:2537: finishing write tid 3 to smithi07085154-43 2024-09-14T03:57:10.244 INFO:tasks.rados.rados.0.smithi070.stdout:2537: finishing write tid 4 to smithi07085154-43 2024-09-14T03:57:10.244 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 2213 (ObjNum 780 snap 274 seq_num 780) dirty exists 2024-09-14T03:57:10.244 INFO:tasks.rados.rados.0.smithi070.stdout:2537: left oid 43 (ObjNum 780 snap 274 seq_num 780) 2024-09-14T03:57:10.244 INFO:tasks.rados.rados.0.smithi070.stdout:2539: finishing write tid 1 to smithi07085154-34 2024-09-14T03:57:10.244 INFO:tasks.rados.rados.0.smithi070.stdout:2539: finishing write tid 2 to smithi07085154-34 2024-09-14T03:57:10.244 INFO:tasks.rados.rados.0.smithi070.stdout:2539: finishing write tid 3 to smithi07085154-34 2024-09-14T03:57:10.244 INFO:tasks.rados.rados.0.smithi070.stdout:2539: finishing write tid 4 to smithi07085154-34 2024-09-14T03:57:10.244 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 1741 (ObjNum 781 snap 274 seq_num 781) dirty exists 2024-09-14T03:57:10.245 INFO:tasks.rados.rados.0.smithi070.stdout:2539: left oid 34 (ObjNum 781 snap 274 seq_num 781) 2024-09-14T03:57:10.245 INFO:tasks.rados.rados.0.smithi070.stdout:2538: expect (ObjNum 488 snap 164 seq_num 488) 2024-09-14T03:57:10.463 INFO:tasks.rados.rados.0.smithi070.stdout:2537: done (3 left) 2024-09-14T03:57:10.464 INFO:tasks.rados.rados.0.smithi070.stdout:2538: done (2 left) 2024-09-14T03:57:10.464 INFO:tasks.rados.rados.0.smithi070.stdout:2539: done (1 left) 2024-09-14T03:57:10.464 INFO:tasks.rados.rados.0.smithi070.stdout:2542: read oid 10 snap -1 2024-09-14T03:57:10.464 INFO:tasks.rados.rados.0.smithi070.stdout:2542: expect (ObjNum 639 snap 214 seq_num 639) 2024-09-14T03:57:10.464 INFO:tasks.rados.rados.0.smithi070.stdout:2543: setattr oid 36 current snap is 274 2024-09-14T03:57:10.465 INFO:tasks.rados.rados.0.smithi070.stdout:2544: rmattr oid 8 current snap is 274 2024-09-14T03:57:10.465 INFO:tasks.rados.rados.0.smithi070.stdout:2544: done (3 left) 2024-09-14T03:57:10.465 INFO:tasks.rados.rados.0.smithi070.stdout:2545: read oid 15 snap -1 2024-09-14T03:57:10.465 INFO:tasks.rados.rados.0.smithi070.stdout:2545: expect deleted 2024-09-14T03:57:10.465 INFO:tasks.rados.rados.0.smithi070.stdout:2546: delete oid 25 current snap is 274 2024-09-14T03:57:10.466 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 2254 (ObjNum 771 snap 270 seq_num 771) dirty exists 2024-09-14T03:57:10.468 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 36 v 2065 (ObjNum 184 snap 0 seq_num 304) dirty exists 2024-09-14T03:57:10.468 INFO:tasks.rados.rados.0.smithi070.stdout:2541: done (4 left) 2024-09-14T03:57:10.468 INFO:tasks.rados.rados.0.smithi070.stdout:2543: done (3 left) 2024-09-14T03:57:10.468 INFO:tasks.rados.rados.0.smithi070.stdout:2545: done (2 left) 2024-09-14T03:57:10.468 INFO:tasks.rados.rados.0.smithi070.stdout:2546: done (1 left) 2024-09-14T03:57:10.468 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 36 current snap is 274 2024-09-14T03:57:10.468 INFO:tasks.rados.rados.0.smithi070.stdout:2547: seq_num 784 ranges {0=737280,737280=401408} 2024-09-14T03:57:10.472 INFO:tasks.rados.rados.0.smithi070.stdout:2547: writing smithi07085154-36 from 0 to 737280 tid 1 2024-09-14T03:57:10.474 INFO:tasks.rados.rados.0.smithi070.stdout:2547: writing smithi07085154-36 from 737280 to 1138688 tid 2 2024-09-14T03:57:10.474 INFO:tasks.rados.rados.0.smithi070.stdout:2548: setattr oid 4 current snap is 274 2024-09-14T03:57:10.474 INFO:tasks.rados.rados.0.smithi070.stdout:2542: expect (ObjNum 639 snap 214 seq_num 639) 2024-09-14T03:57:10.503 INFO:tasks.rados.rados.0.smithi070.stdout:2547: finishing write tid 1 to smithi07085154-36 2024-09-14T03:57:10.504 INFO:tasks.rados.rados.0.smithi070.stdout:2547: finishing write tid 2 to smithi07085154-36 2024-09-14T03:57:10.504 INFO:tasks.rados.rados.0.smithi070.stdout:2547: finishing write tid 3 to smithi07085154-36 2024-09-14T03:57:10.504 INFO:tasks.rados.rados.0.smithi070.stdout:2547: finishing write tid 4 to smithi07085154-36 2024-09-14T03:57:10.504 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 36 v 2068 (ObjNum 784 snap 274 seq_num 784) dirty exists 2024-09-14T03:57:10.504 INFO:tasks.rados.rados.0.smithi070.stdout:2547: left oid 36 (ObjNum 784 snap 274 seq_num 784) 2024-09-14T03:57:10.504 INFO:tasks.rados.rados.0.smithi070.stdout:2542: done (2 left) 2024-09-14T03:57:10.504 INFO:tasks.rados.rados.0.smithi070.stdout:2547: done (1 left) 2024-09-14T03:57:10.504 INFO:tasks.rados.rados.0.smithi070.stdout:2549: delete oid 29 current snap is 274 2024-09-14T03:57:10.506 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 2060 (ObjNum 184 snap 0 seq_num 304) dirty exists 2024-09-14T03:57:10.508 INFO:tasks.rados.rados.0.smithi070.stdout:2548: done (1 left) 2024-09-14T03:57:10.508 INFO:tasks.rados.rados.0.smithi070.stdout:2549: done (0 left) 2024-09-14T03:57:10.508 INFO:tasks.rados.rados.0.smithi070.stdout:2550: delete oid 38 current snap is 274 2024-09-14T03:57:10.512 INFO:tasks.rados.rados.0.smithi070.stdout:2550: done (0 left) 2024-09-14T03:57:10.512 INFO:tasks.rados.rados.0.smithi070.stdout:2551: read oid 48 snap -1 2024-09-14T03:57:10.512 INFO:tasks.rados.rados.0.smithi070.stdout:2551: expect (ObjNum 757 snap 263 seq_num 757) 2024-09-14T03:57:10.512 INFO:tasks.rados.rados.0.smithi070.stdout:2552: delete oid 31 current snap is 274 2024-09-14T03:57:10.514 INFO:tasks.rados.rados.0.smithi070.stdout:2552: done (1 left) 2024-09-14T03:57:10.515 INFO:tasks.rados.rados.0.smithi070.stdout:2553: read oid 21 snap 261 2024-09-14T03:57:10.515 INFO:tasks.rados.rados.0.smithi070.stdout:2553: expect deleted 2024-09-14T03:57:10.515 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 16 current snap is 274 2024-09-14T03:57:10.515 INFO:tasks.rados.rados.0.smithi070.stdout:2554: seq_num 788 ranges {630784=802816,1433600=401408} 2024-09-14T03:57:10.522 INFO:tasks.rados.rados.0.smithi070.stdout:2554: writing smithi07085154-16 from 630784 to 1433600 tid 1 2024-09-14T03:57:10.525 INFO:tasks.rados.rados.0.smithi070.stdout:2554: writing smithi07085154-16 from 1433600 to 1835008 tid 2 2024-09-14T03:57:10.525 INFO:tasks.rados.rados.0.smithi070.stdout:2555: read oid 10 snap -1 2024-09-14T03:57:10.525 INFO:tasks.rados.rados.0.smithi070.stdout:2555: expect (ObjNum 639 snap 214 seq_num 639) 2024-09-14T03:57:10.525 INFO:tasks.rados.rados.0.smithi070.stdout:2556: read oid 4 snap 261 2024-09-14T03:57:10.525 INFO:tasks.rados.rados.0.smithi070.stdout:2556: expect deleted 2024-09-14T03:57:10.525 INFO:tasks.rados.rados.0.smithi070.stdout:2557: delete oid 7 current snap is 274 2024-09-14T03:57:10.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:57:10 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:57:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:57:10.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:10 smithi094 ceph-mon[86597]: mds.? [v2:172.21.15.70:6834/2011419410,v1:172.21.15.70:6835/2011419410] up:rejoin 2024-09-14T03:57:10.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:10 smithi094 ceph-mon[86597]: fsmap cephfs:1/1 {0=a=up:rejoin} 2024-09-14T03:57:10.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:10 smithi094 ceph-mon[86597]: daemon mds.a is now active in filesystem cephfs as rank 0 2024-09-14T03:57:10.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:10 smithi094 ceph-mon[86597]: osdmap e599: 8 total, 8 up, 8 in 2024-09-14T03:57:10.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:10 smithi094 ceph-mon[86597]: pgmap v495: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.7 KiB/s rd, 961 KiB/s wr, 7 op/s 2024-09-14T03:57:10.529 INFO:tasks.rados.rados.0.smithi070.stdout:2553: done (5 left) 2024-09-14T03:57:10.529 INFO:tasks.rados.rados.0.smithi070.stdout:2556: done (4 left) 2024-09-14T03:57:10.529 INFO:tasks.rados.rados.0.smithi070.stdout:2557: done (3 left) 2024-09-14T03:57:10.529 INFO:tasks.rados.rados.0.smithi070.stdout:2558: snap_create 2024-09-14T03:57:10.530 INFO:tasks.rados.rados.0.smithi070.stdout:2554: finishing write tid 1 to smithi07085154-16 2024-09-14T03:57:10.530 INFO:tasks.rados.rados.0.smithi070.stdout:2554: finishing write tid 2 to smithi07085154-16 2024-09-14T03:57:10.530 INFO:tasks.rados.rados.0.smithi070.stdout:2554: finishing write tid 3 to smithi07085154-16 2024-09-14T03:57:10.533 INFO:tasks.rados.rados.0.smithi070.stdout:2551: expect (ObjNum 757 snap 263 seq_num 757) 2024-09-14T03:57:10.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:10 smithi070 ceph-mon[98640]: mds.? [v2:172.21.15.70:6834/2011419410,v1:172.21.15.70:6835/2011419410] up:rejoin 2024-09-14T03:57:10.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:10 smithi070 ceph-mon[98640]: fsmap cephfs:1/1 {0=a=up:rejoin} 2024-09-14T03:57:10.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:10 smithi070 ceph-mon[98640]: daemon mds.a is now active in filesystem cephfs as rank 0 2024-09-14T03:57:10.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:10 smithi070 ceph-mon[98640]: osdmap e599: 8 total, 8 up, 8 in 2024-09-14T03:57:10.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:10 smithi070 ceph-mon[98640]: pgmap v495: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.7 KiB/s rd, 961 KiB/s wr, 7 op/s 2024-09-14T03:57:10.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:10 smithi070 ceph-mon[96704]: mds.? [v2:172.21.15.70:6834/2011419410,v1:172.21.15.70:6835/2011419410] up:rejoin 2024-09-14T03:57:10.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:10 smithi070 ceph-mon[96704]: fsmap cephfs:1/1 {0=a=up:rejoin} 2024-09-14T03:57:10.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:10 smithi070 ceph-mon[96704]: daemon mds.a is now active in filesystem cephfs as rank 0 2024-09-14T03:57:10.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:10 smithi070 ceph-mon[96704]: osdmap e599: 8 total, 8 up, 8 in 2024-09-14T03:57:10.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:10 smithi070 ceph-mon[96704]: pgmap v495: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.7 KiB/s rd, 961 KiB/s wr, 7 op/s 2024-09-14T03:57:10.749 INFO:tasks.rados.rados.0.smithi070.stdout:2554: finishing write tid 4 to smithi07085154-16 2024-09-14T03:57:10.749 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 2851 (ObjNum 788 snap 274 seq_num 788) dirty exists 2024-09-14T03:57:10.749 INFO:tasks.rados.rados.0.smithi070.stdout:2554: left oid 16 (ObjNum 788 snap 274 seq_num 788) 2024-09-14T03:57:10.749 INFO:tasks.rados.rados.0.smithi070.stdout:2555: expect (ObjNum 639 snap 214 seq_num 639) 2024-09-14T03:57:11.293 INFO:tasks.rados.rados.0.smithi070.stdout:2551: done (3 left) 2024-09-14T03:57:11.293 INFO:tasks.rados.rados.0.smithi070.stdout:2554: done (2 left) 2024-09-14T03:57:11.293 INFO:tasks.rados.rados.0.smithi070.stdout:2555: done (1 left) 2024-09-14T03:57:11.293 INFO:tasks.rados.rados.0.smithi070.stdout:2558: done (0 left) 2024-09-14T03:57:11.293 INFO:tasks.rados.rados.0.smithi070.stdout:2559: snap_remove snap 257 2024-09-14T03:57:11.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:11 smithi070 ceph-mon[96704]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-14T03:57:11.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:11 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T03:57:11.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:11 smithi070 ceph-mon[96704]: mds.? [v2:172.21.15.70:6834/2011419410,v1:172.21.15.70:6835/2011419410] up:active 2024-09-14T03:57:11.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:11 smithi070 ceph-mon[96704]: fsmap cephfs:1 {0=a=up:active} 2024-09-14T03:57:11.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:11 smithi070 ceph-mon[98640]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-14T03:57:11.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:11 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T03:57:11.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:11 smithi070 ceph-mon[98640]: mds.? [v2:172.21.15.70:6834/2011419410,v1:172.21.15.70:6835/2011419410] up:active 2024-09-14T03:57:11.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:11 smithi070 ceph-mon[98640]: fsmap cephfs:1 {0=a=up:active} 2024-09-14T03:57:11.670 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:11 smithi094 ceph-mon[86597]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-14T03:57:11.670 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:11 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T03:57:11.670 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:11 smithi094 ceph-mon[86597]: mds.? [v2:172.21.15.70:6834/2011419410,v1:172.21.15.70:6835/2011419410] up:active 2024-09-14T03:57:11.670 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:11 smithi094 ceph-mon[86597]: fsmap cephfs:1 {0=a=up:active} 2024-09-14T03:57:12.301 INFO:tasks.rados.rados.0.smithi070.stdout:2559: done (0 left) 2024-09-14T03:57:12.301 INFO:tasks.rados.rados.0.smithi070.stdout:2560: rmattr oid 26 current snap is 275 2024-09-14T03:57:12.301 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 11 current snap is 275 2024-09-14T03:57:12.301 INFO:tasks.rados.rados.0.smithi070.stdout:2561: seq_num 790 ranges {4808704=434176,5242880=548864,5791744=253952} 2024-09-14T03:57:12.330 INFO:tasks.rados.rados.0.smithi070.stdout:2561: writing smithi07085154-11 from 4808704 to 5242880 tid 1 2024-09-14T03:57:12.333 INFO:tasks.rados.rados.0.smithi070.stdout:2561: writing smithi07085154-11 from 5242880 to 5791744 tid 2 2024-09-14T03:57:12.336 INFO:tasks.rados.rados.0.smithi070.stdout:2561: writing smithi07085154-11 from 5791744 to 6045696 tid 3 2024-09-14T03:57:12.336 INFO:tasks.rados.rados.0.smithi070.stdout:2562: copy_from oid 24 from oid 3 current snap is 275 2024-09-14T03:57:12.336 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 2853 (ObjNum 752 snap 260 seq_num 752) dirty exists 2024-09-14T03:57:12.336 INFO:tasks.rados.rados.0.smithi070.stdout:2561: finishing write tid 1 to smithi07085154-11 2024-09-14T03:57:12.336 INFO:tasks.rados.rados.0.smithi070.stdout:2560: done (2 left) 2024-09-14T03:57:12.336 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 42 current snap is 275 2024-09-14T03:57:12.336 INFO:tasks.rados.rados.0.smithi070.stdout:2563: seq_num 791 ranges {0=450560,450560=475136,925696=327680} 2024-09-14T03:57:12.338 INFO:tasks.rados.rados.0.smithi070.stdout:2563: writing smithi07085154-42 from 0 to 450560 tid 1 2024-09-14T03:57:12.341 INFO:tasks.rados.rados.0.smithi070.stdout:2563: writing smithi07085154-42 from 450560 to 925696 tid 2 2024-09-14T03:57:12.344 INFO:tasks.rados.rados.0.smithi070.stdout:2563: writing smithi07085154-42 from 925696 to 1253376 tid 3 2024-09-14T03:57:12.344 INFO:tasks.rados.rados.0.smithi070.stdout:2564: delete oid 39 current snap is 275 2024-09-14T03:57:12.344 INFO:tasks.rados.rados.0.smithi070.stdout:2562: finishing copy_from to smithi07085154-24 2024-09-14T03:57:12.344 INFO:tasks.rados.rados.0.smithi070.stdout:2562: got expected ENOENT (src dne) 2024-09-14T03:57:12.344 INFO:tasks.rados.rados.0.smithi070.stdout:2562: finishing copy_from racing read to smithi07085154-24 2024-09-14T03:57:12.344 INFO:tasks.rados.rados.0.smithi070.stdout:2561: finishing write tid 2 to smithi07085154-11 2024-09-14T03:57:12.345 INFO:tasks.rados.rados.0.smithi070.stdout:2561: finishing write tid 3 to smithi07085154-11 2024-09-14T03:57:12.345 INFO:tasks.rados.rados.0.smithi070.stdout:2561: finishing write tid 4 to smithi07085154-11 2024-09-14T03:57:12.345 INFO:tasks.rados.rados.0.smithi070.stdout:2561: finishing write tid 5 to smithi07085154-11 2024-09-14T03:57:12.345 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 11 v 2063 (ObjNum 790 snap 275 seq_num 790) dirty exists 2024-09-14T03:57:12.345 INFO:tasks.rados.rados.0.smithi070.stdout:2561: left oid 11 (ObjNum 790 snap 275 seq_num 790) 2024-09-14T03:57:12.345 INFO:tasks.rados.rados.0.smithi070.stdout:2563: finishing write tid 1 to smithi07085154-42 2024-09-14T03:57:12.346 INFO:tasks.rados.rados.0.smithi070.stdout:2561: done (3 left) 2024-09-14T03:57:12.346 INFO:tasks.rados.rados.0.smithi070.stdout:2562: done (2 left) 2024-09-14T03:57:12.346 INFO:tasks.rados.rados.0.smithi070.stdout:2564: done (1 left) 2024-09-14T03:57:12.346 INFO:tasks.rados.rados.0.smithi070.stdout:2565: rollback oid 41 current snap is 275 2024-09-14T03:57:12.347 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 41 to 260 2024-09-14T03:57:12.347 INFO:tasks.rados.rados.0.smithi070.stdout:2566: rollback oid 35 current snap is 275 2024-09-14T03:57:12.347 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 35 to 234 2024-09-14T03:57:12.347 INFO:tasks.rados.rados.0.smithi070.stdout:2567: rollback oid 19 current snap is 275 2024-09-14T03:57:12.347 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 19 to 264 2024-09-14T03:57:12.347 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 4 current snap is 275 2024-09-14T03:57:12.347 INFO:tasks.rados.rados.0.smithi070.stdout:2568: seq_num 793 ranges {0=483328,483328=647168,1130496=155648} 2024-09-14T03:57:12.349 INFO:tasks.rados.rados.0.smithi070.stdout:2568: writing smithi07085154-4 from 0 to 483328 tid 1 2024-09-14T03:57:12.354 INFO:tasks.rados.rados.0.smithi070.stdout:2568: writing smithi07085154-4 from 483328 to 1130496 tid 2 2024-09-14T03:57:12.355 INFO:tasks.rados.rados.0.smithi070.stdout:2568: writing smithi07085154-4 from 1130496 to 1286144 tid 3 2024-09-14T03:57:12.356 INFO:tasks.rados.rados.0.smithi070.stdout:2569: snap_create 2024-09-14T03:57:12.356 INFO:tasks.rados.rados.0.smithi070.stdout:2563: finishing write tid 2 to smithi07085154-42 2024-09-14T03:57:12.356 INFO:tasks.rados.rados.0.smithi070.stdout:2563: finishing write tid 3 to smithi07085154-42 2024-09-14T03:57:12.356 INFO:tasks.rados.rados.0.smithi070.stdout:2563: finishing write tid 4 to smithi07085154-42 2024-09-14T03:57:12.356 INFO:tasks.rados.rados.0.smithi070.stdout:2566: finishing rollback tid 0 to smithi07085154-35 2024-09-14T03:57:12.356 INFO:tasks.rados.rados.0.smithi070.stdout:2567: finishing rollback tid 0 to smithi07085154-19 2024-09-14T03:57:12.356 INFO:tasks.rados.rados.0.smithi070.stdout:2567: finishing rollback tid 1 to smithi07085154-19 2024-09-14T03:57:12.356 INFO:tasks.rados.rados.0.smithi070.stdout:2566: finishing rollback tid 1 to smithi07085154-35 2024-09-14T03:57:12.356 INFO:tasks.rados.rados.0.smithi070.stdout:2566: finishing rollback tid 2 to smithi07085154-35 2024-09-14T03:57:12.356 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 35 v 1692 (ObjNum 645 snap 216 seq_num 645) dirty exists 2024-09-14T03:57:12.356 INFO:tasks.rados.rados.0.smithi070.stdout:2567: finishing rollback tid 2 to smithi07085154-19 2024-09-14T03:57:12.356 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 19 v 1745 (ObjNum 747 snap 259 seq_num 747) dirty exists 2024-09-14T03:57:12.357 INFO:tasks.rados.rados.0.smithi070.stdout:2563: finishing write tid 5 to smithi07085154-42 2024-09-14T03:57:12.357 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 2873 (ObjNum 791 snap 275 seq_num 791) dirty exists 2024-09-14T03:57:12.357 INFO:tasks.rados.rados.0.smithi070.stdout:2563: left oid 42 (ObjNum 791 snap 275 seq_num 791) 2024-09-14T03:57:12.357 INFO:tasks.rados.rados.0.smithi070.stdout:2565: finishing rollback tid 0 to smithi07085154-41 2024-09-14T03:57:12.357 INFO:tasks.rados.rados.0.smithi070.stdout:2565: finishing rollback tid 1 to smithi07085154-41 2024-09-14T03:57:12.357 INFO:tasks.rados.rados.0.smithi070.stdout:2565: finishing rollback tid 2 to smithi07085154-41 2024-09-14T03:57:12.357 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 2216 (ObjNum 724 snap 248 seq_num 724) dirty exists 2024-09-14T03:57:12.357 INFO:tasks.rados.rados.0.smithi070.stdout:2568: finishing write tid 1 to smithi07085154-4 2024-09-14T03:57:12.358 INFO:tasks.rados.rados.0.smithi070.stdout:2568: finishing write tid 2 to smithi07085154-4 2024-09-14T03:57:12.360 INFO:tasks.rados.rados.0.smithi070.stdout:2568: finishing write tid 3 to smithi07085154-4 2024-09-14T03:57:12.360 INFO:tasks.rados.rados.0.smithi070.stdout:2568: finishing write tid 4 to smithi07085154-4 2024-09-14T03:57:12.361 INFO:tasks.rados.rados.0.smithi070.stdout:2568: finishing write tid 5 to smithi07085154-4 2024-09-14T03:57:12.361 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 2065 (ObjNum 793 snap 275 seq_num 793) dirty exists 2024-09-14T03:57:12.361 INFO:tasks.rados.rados.0.smithi070.stdout:2568: left oid 4 (ObjNum 793 snap 275 seq_num 793) 2024-09-14T03:57:12.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:12 smithi070 ceph-mon[96704]: osdmap e600: 8 total, 8 up, 8 in 2024-09-14T03:57:12.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:12 smithi070 ceph-mon[96704]: pgmap v497: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.6 MiB/s rd, 1.7 MiB/s wr, 41 op/s 2024-09-14T03:57:12.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:12 smithi070 ceph-mon[98640]: osdmap e600: 8 total, 8 up, 8 in 2024-09-14T03:57:12.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:12 smithi070 ceph-mon[98640]: pgmap v497: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.6 MiB/s rd, 1.7 MiB/s wr, 41 op/s 2024-09-14T03:57:12.537 INFO:journalctl@ceph.mds.a.smithi070.stdout:Sep 14 03:57:12 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-a[122489]: 2024-09-14T03:57:12.094+0000 7f7f2891e640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-09-14T03:57:12.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:12 smithi094 ceph-mon[86597]: osdmap e600: 8 total, 8 up, 8 in 2024-09-14T03:57:12.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:12 smithi094 ceph-mon[86597]: pgmap v497: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.6 MiB/s rd, 1.7 MiB/s wr, 41 op/s 2024-09-14T03:57:13.304 INFO:tasks.rados.rados.0.smithi070.stdout:2563: done (5 left) 2024-09-14T03:57:13.304 INFO:tasks.rados.rados.0.smithi070.stdout:2565: done (4 left) 2024-09-14T03:57:13.304 INFO:tasks.rados.rados.0.smithi070.stdout:2566: done (3 left) 2024-09-14T03:57:13.304 INFO:tasks.rados.rados.0.smithi070.stdout:2567: done (2 left) 2024-09-14T03:57:13.304 INFO:tasks.rados.rados.0.smithi070.stdout:2568: done (1 left) 2024-09-14T03:57:13.304 INFO:tasks.rados.rados.0.smithi070.stdout:2569: done (0 left) 2024-09-14T03:57:13.304 INFO:tasks.rados.rados.0.smithi070.stdout:2570: read oid 19 snap -1 2024-09-14T03:57:13.304 INFO:tasks.rados.rados.0.smithi070.stdout:2570: expect (ObjNum 747 snap 259 seq_num 747) 2024-09-14T03:57:13.304 INFO:tasks.rados.rados.0.smithi070.stdout:2571: snap_remove snap 162 2024-09-14T03:57:13.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:13 smithi094 ceph-mon[86597]: osdmap e601: 8 total, 8 up, 8 in 2024-09-14T03:57:13.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:13 smithi070 ceph-mon[98640]: osdmap e601: 8 total, 8 up, 8 in 2024-09-14T03:57:13.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:13 smithi070 ceph-mon[96704]: osdmap e601: 8 total, 8 up, 8 in 2024-09-14T03:57:13.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:57:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:57:13] "GET /metrics HTTP/1.1" 200 36508 "" "Prometheus/2.51.0" 2024-09-14T03:57:14.077 INFO:tasks.rados.rados.0.smithi070.stdout:2571: done (1 left) 2024-09-14T03:57:14.078 INFO:tasks.rados.rados.0.smithi070.stdout:2572: snap_create 2024-09-14T03:57:14.078 INFO:tasks.rados.rados.0.smithi070.stdout:2570: expect (ObjNum 747 snap 259 seq_num 747) 2024-09-14T03:57:14.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:14 smithi094 ceph-mon[86597]: osdmap e602: 8 total, 8 up, 8 in 2024-09-14T03:57:14.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:14 smithi094 ceph-mon[86597]: osdmap e603: 8 total, 8 up, 8 in 2024-09-14T03:57:14.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:14 smithi094 ceph-mon[86597]: pgmap v501: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.3 MiB/s wr, 44 op/s 2024-09-14T03:57:14.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:14 smithi070 ceph-mon[96704]: osdmap e602: 8 total, 8 up, 8 in 2024-09-14T03:57:14.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:14 smithi070 ceph-mon[96704]: osdmap e603: 8 total, 8 up, 8 in 2024-09-14T03:57:14.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:14 smithi070 ceph-mon[96704]: pgmap v501: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.3 MiB/s wr, 44 op/s 2024-09-14T03:57:14.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:14 smithi070 ceph-mon[98640]: osdmap e602: 8 total, 8 up, 8 in 2024-09-14T03:57:14.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:14 smithi070 ceph-mon[98640]: osdmap e603: 8 total, 8 up, 8 in 2024-09-14T03:57:14.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:14 smithi070 ceph-mon[98640]: pgmap v501: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.3 MiB/s wr, 44 op/s 2024-09-14T03:57:15.082 INFO:tasks.rados.rados.0.smithi070.stdout:2570: done (1 left) 2024-09-14T03:57:15.082 INFO:tasks.rados.rados.0.smithi070.stdout:2572: done (0 left) 2024-09-14T03:57:15.082 INFO:tasks.rados.rados.0.smithi070.stdout:2573: copy_from oid 50 from oid 43 current snap is 277 2024-09-14T03:57:15.082 INFO:tasks.rados.rados.0.smithi070.stdout:2574: snap_create 2024-09-14T03:57:15.100 INFO:tasks.rados.rados.0.smithi070.stdout:2573: finishing copy_from racing read to smithi07085154-50 2024-09-14T03:57:15.103 INFO:tasks.rados.rados.0.smithi070.stdout:2573: finishing copy_from to smithi07085154-50 2024-09-14T03:57:15.103 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 2213 (ObjNum 780 snap 274 seq_num 780) dirty exists 2024-09-14T03:57:15.588 INFO:journalctl@ceph.mds.b.smithi094.stdout:Sep 14 03:57:15 smithi094 podman[110334]: 2024-09-14 03:57:15.308424867 +0000 UTC m=+8.135264398 container died e3b5e3e0018900bd5f9f1623ccdf58cbdaf47eed42f4ad6fb3d54c1226f800ae (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-b, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=quincy-eb16b97, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-14T03:57:15.953 INFO:journalctl@ceph.mds.b.smithi094.stdout:Sep 14 03:57:15 smithi094 podman[110334]: 2024-09-14 03:57:15.685163123 +0000 UTC m=+8.512002650 container cleanup e3b5e3e0018900bd5f9f1623ccdf58cbdaf47eed42f4ad6fb3d54c1226f800ae (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-b, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=quincy-eb16b97, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:57:15.954 INFO:journalctl@ceph.mds.b.smithi094.stdout:Sep 14 03:57:15 smithi094 bash[110334]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-b 2024-09-14T03:57:16.084 INFO:tasks.rados.rados.0.smithi070.stdout:2573: done (1 left) 2024-09-14T03:57:16.084 INFO:tasks.rados.rados.0.smithi070.stdout:2574: done (0 left) 2024-09-14T03:57:16.084 INFO:tasks.rados.rados.0.smithi070.stdout:2575: rollback oid 44 current snap is 278 2024-09-14T03:57:16.084 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 44 to 243 2024-09-14T03:57:16.085 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 42 current snap is 278 2024-09-14T03:57:16.085 INFO:tasks.rados.rados.0.smithi070.stdout:2576: seq_num 794 ranges {1253376=499712,1753088=532480,2285568=270336} 2024-09-14T03:57:16.093 INFO:tasks.rados.rados.0.smithi070.stdout:2576: writing smithi07085154-42 from 1253376 to 1753088 tid 1 2024-09-14T03:57:16.097 INFO:tasks.rados.rados.0.smithi070.stdout:2576: writing smithi07085154-42 from 1753088 to 2285568 tid 2 2024-09-14T03:57:16.099 INFO:tasks.rados.rados.0.smithi070.stdout:2576: writing smithi07085154-42 from 2285568 to 2555904 tid 3 2024-09-14T03:57:16.099 INFO:tasks.rados.rados.0.smithi070.stdout:2577: rollback oid 5 current snap is 278 2024-09-14T03:57:16.099 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 5 to 260 2024-09-14T03:57:16.099 INFO:tasks.rados.rados.0.smithi070.stdout:2575: finishing rollback tid 0 to smithi07085154-44 2024-09-14T03:57:16.099 INFO:tasks.rados.rados.0.smithi070.stdout:2575: finishing rollback tid 1 to smithi07085154-44 2024-09-14T03:57:16.100 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 2070 (ObjNum 11377 snap 0 seq_num 2931951488) dirty dne 2024-09-14T03:57:16.100 INFO:tasks.rados.rados.0.smithi070.stdout:2575: done (2 left) 2024-09-14T03:57:16.100 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 37 current snap is 278 2024-09-14T03:57:16.100 INFO:tasks.rados.rados.0.smithi070.stdout:2578: seq_num 795 ranges {835584=524288,1359872=417792,1777664=385024} 2024-09-14T03:57:16.106 INFO:tasks.rados.rados.0.smithi070.stdout:2578: writing smithi07085154-37 from 835584 to 1359872 tid 1 2024-09-14T03:57:16.109 INFO:tasks.rados.rados.0.smithi070.stdout:2578: writing smithi07085154-37 from 1359872 to 1777664 tid 2 2024-09-14T03:57:16.112 INFO:tasks.rados.rados.0.smithi070.stdout:2578: writing smithi07085154-37 from 1777664 to 2162688 tid 3 2024-09-14T03:57:16.112 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 9 current snap is 278 2024-09-14T03:57:16.112 INFO:tasks.rados.rados.0.smithi070.stdout:2577: finishing rollback tid 0 to smithi07085154-5 2024-09-14T03:57:16.112 INFO:tasks.rados.rados.0.smithi070.stdout:2577: finishing rollback tid 1 to smithi07085154-5 2024-09-14T03:57:16.112 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 575 (ObjNum 129 snap 0 seq_num 2930855752) dirty dne 2024-09-14T03:57:16.112 INFO:tasks.rados.rados.0.smithi070.stdout:2576: finishing write tid 1 to smithi07085154-42 2024-09-14T03:57:16.112 INFO:tasks.rados.rados.0.smithi070.stdout:2579: seq_num 796 ranges {2801664=540672,3342336=704512,4046848=98304} 2024-09-14T03:57:16.130 INFO:tasks.rados.rados.0.smithi070.stdout:2579: writing smithi07085154-9 from 2801664 to 3342336 tid 1 2024-09-14T03:57:16.133 INFO:tasks.rados.rados.0.smithi070.stdout:2579: writing smithi07085154-9 from 3342336 to 4046848 tid 2 2024-09-14T03:57:16.135 INFO:tasks.rados.rados.0.smithi070.stdout:2579: writing smithi07085154-9 from 4046848 to 4145152 tid 3 2024-09-14T03:57:16.135 INFO:tasks.rados.rados.0.smithi070.stdout:2577: done (3 left) 2024-09-14T03:57:16.135 INFO:tasks.rados.rados.0.smithi070.stdout:2580: read oid 44 snap -1 2024-09-14T03:57:16.135 INFO:tasks.rados.rados.0.smithi070.stdout:2580: expect deleted 2024-09-14T03:57:16.135 INFO:tasks.rados.rados.0.smithi070.stdout:2581: delete oid 17 current snap is 278 2024-09-14T03:57:16.135 INFO:tasks.rados.rados.0.smithi070.stdout:2576: finishing write tid 2 to smithi07085154-42 2024-09-14T03:57:16.136 INFO:tasks.rados.rados.0.smithi070.stdout:2576: finishing write tid 3 to smithi07085154-42 2024-09-14T03:57:16.136 INFO:tasks.rados.rados.0.smithi070.stdout:2576: finishing write tid 4 to smithi07085154-42 2024-09-14T03:57:16.136 INFO:tasks.rados.rados.0.smithi070.stdout:2576: finishing write tid 5 to smithi07085154-42 2024-09-14T03:57:16.136 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 2890 (ObjNum 794 snap 278 seq_num 794) dirty exists 2024-09-14T03:57:16.136 INFO:tasks.rados.rados.0.smithi070.stdout:2576: left oid 42 (ObjNum 794 snap 278 seq_num 794) 2024-09-14T03:57:16.136 INFO:tasks.rados.rados.0.smithi070.stdout:2578: finishing write tid 1 to smithi07085154-37 2024-09-14T03:57:16.136 INFO:tasks.rados.rados.0.smithi070.stdout:2578: finishing write tid 2 to smithi07085154-37 2024-09-14T03:57:16.136 INFO:tasks.rados.rados.0.smithi070.stdout:2578: finishing write tid 3 to smithi07085154-37 2024-09-14T03:57:16.136 INFO:tasks.rados.rados.0.smithi070.stdout:2578: finishing write tid 4 to smithi07085154-37 2024-09-14T03:57:16.136 INFO:tasks.rados.rados.0.smithi070.stdout:2578: finishing write tid 5 to smithi07085154-37 2024-09-14T03:57:16.136 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 37 v 2220 (ObjNum 795 snap 278 seq_num 795) dirty exists 2024-09-14T03:57:16.137 INFO:tasks.rados.rados.0.smithi070.stdout:2578: left oid 37 (ObjNum 795 snap 278 seq_num 795) 2024-09-14T03:57:16.137 INFO:tasks.rados.rados.0.smithi070.stdout:2579: finishing write tid 1 to smithi07085154-9 2024-09-14T03:57:16.138 INFO:tasks.rados.rados.0.smithi070.stdout:2576: done (4 left) 2024-09-14T03:57:16.138 INFO:tasks.rados.rados.0.smithi070.stdout:2578: done (3 left) 2024-09-14T03:57:16.138 INFO:tasks.rados.rados.0.smithi070.stdout:2580: done (2 left) 2024-09-14T03:57:16.138 INFO:tasks.rados.rados.0.smithi070.stdout:2581: done (1 left) 2024-09-14T03:57:16.138 INFO:tasks.rados.rados.0.smithi070.stdout:2582: read oid 20 snap -1 2024-09-14T03:57:16.138 INFO:tasks.rados.rados.0.smithi070.stdout:2582: expect (ObjNum 524 snap 174 seq_num 524) 2024-09-14T03:57:16.138 INFO:tasks.rados.rados.0.smithi070.stdout:2583: delete oid 41 current snap is 278 2024-09-14T03:57:16.140 INFO:tasks.rados.rados.0.smithi070.stdout:2579: finishing write tid 2 to smithi07085154-9 2024-09-14T03:57:16.142 INFO:tasks.rados.rados.0.smithi070.stdout:2583: done (2 left) 2024-09-14T03:57:16.143 INFO:tasks.rados.rados.0.smithi070.stdout:2584: read oid 5 snap -1 2024-09-14T03:57:16.143 INFO:tasks.rados.rados.0.smithi070.stdout:2584: expect deleted 2024-09-14T03:57:16.143 INFO:tasks.rados.rados.0.smithi070.stdout:2585: read oid 6 snap -1 2024-09-14T03:57:16.143 INFO:tasks.rados.rados.0.smithi070.stdout:2585: expect (ObjNum 751 snap 260 seq_num 751) 2024-09-14T03:57:16.143 INFO:tasks.rados.rados.0.smithi070.stdout:2586: read oid 26 snap -1 2024-09-14T03:57:16.143 INFO:tasks.rados.rados.0.smithi070.stdout:2586: expect (ObjNum 752 snap 260 seq_num 752) 2024-09-14T03:57:16.143 INFO:tasks.rados.rados.0.smithi070.stdout:2587: read oid 44 snap -1 2024-09-14T03:57:16.143 INFO:tasks.rados.rados.0.smithi070.stdout:2587: expect deleted 2024-09-14T03:57:16.143 INFO:tasks.rados.rados.0.smithi070.stdout:2588: copy_from oid 11 from oid 24 current snap is 278 2024-09-14T03:57:16.143 INFO:tasks.rados.rados.0.smithi070.stdout:2589: snap_create 2024-09-14T03:57:16.143 INFO:tasks.rados.rados.0.smithi070.stdout:2579: finishing write tid 3 to smithi07085154-9 2024-09-14T03:57:16.143 INFO:tasks.rados.rados.0.smithi070.stdout:2579: finishing write tid 4 to smithi07085154-9 2024-09-14T03:57:16.151 INFO:tasks.rados.rados.0.smithi070.stdout:2579: finishing write tid 5 to smithi07085154-9 2024-09-14T03:57:16.151 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 2895 (ObjNum 796 snap 278 seq_num 796) dirty exists 2024-09-14T03:57:16.151 INFO:tasks.rados.rados.0.smithi070.stdout:2579: left oid 9 (ObjNum 796 snap 278 seq_num 796) 2024-09-14T03:57:16.156 INFO:tasks.rados.rados.0.smithi070.stdout:2588: finishing copy_from racing read to smithi07085154-11 2024-09-14T03:57:16.157 INFO:tasks.rados.rados.0.smithi070.stdout:2588: finishing copy_from to smithi07085154-11 2024-09-14T03:57:16.157 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 11 v 2254 (ObjNum 771 snap 270 seq_num 771) dirty exists 2024-09-14T03:57:16.159 INFO:tasks.rados.rados.0.smithi070.stdout:2586: expect (ObjNum 752 snap 260 seq_num 752) 2024-09-14T03:57:16.249 INFO:tasks.rados.rados.0.smithi070.stdout:2585: expect (ObjNum 751 snap 260 seq_num 751) 2024-09-14T03:57:16.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:16 smithi094 ceph-mon[86597]: osdmap e604: 8 total, 8 up, 8 in 2024-09-14T03:57:16.277 INFO:journalctl@ceph.mds.b.smithi094.stdout:Sep 14 03:57:15 smithi094 podman[110362]: 2024-09-14 03:57:15.952983497 +0000 UTC m=+0.642948624 container remove e3b5e3e0018900bd5f9f1623ccdf58cbdaf47eed42f4ad6fb3d54c1226f800ae (image=quay.ceph.io/ceph-ci/ceph@sha256:a1fd6dadb08c4973ec538e917bdb1227697e119074c9629a2e7ba5e794a9b7b6, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-b, RELEASE=quincy-eb16b97, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-14T03:57:16.277 INFO:journalctl@ceph.mds.b.smithi094.stdout:Sep 14 03:57:16 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@mds.b.service: Deactivated successfully. 2024-09-14T03:57:16.277 INFO:journalctl@ceph.mds.b.smithi094.stdout:Sep 14 03:57:16 smithi094 systemd[1]: Stopped Ceph mds.b for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:57:16.277 INFO:journalctl@ceph.mds.b.smithi094.stdout:Sep 14 03:57:16 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@mds.b.service: Consumed 1.227s CPU time. 2024-09-14T03:57:16.531 INFO:tasks.rados.rados.0.smithi070.stdout:2582: expect (ObjNum 524 snap 174 seq_num 524) 2024-09-14T03:57:16.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:16 smithi070 ceph-mon[96704]: osdmap e604: 8 total, 8 up, 8 in 2024-09-14T03:57:16.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:16 smithi070 ceph-mon[98640]: osdmap e604: 8 total, 8 up, 8 in 2024-09-14T03:57:16.690 INFO:journalctl@ceph.mds.b.smithi094.stdout:Sep 14 03:57:16 smithi094 systemd[1]: Starting Ceph mds.b for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:57:16.690 INFO:journalctl@ceph.mds.b.smithi094.stdout:Sep 14 03:57:16 smithi094 podman[110499]: 2024-09-14 03:57:16.449727754 +0000 UTC m=+0.015873615 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:57:16.690 INFO:journalctl@ceph.mds.b.smithi094.stdout:Sep 14 03:57:16 smithi094 podman[110499]: 2024-09-14 03:57:16.562644212 +0000 UTC m=+0.128790067 container create 9f7842c26184e48971bcffd58a092eee4a7015cf63edf6c5118360985054226d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-b, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:57:17.089 INFO:journalctl@ceph.mds.b.smithi094.stdout:Sep 14 03:57:16 smithi094 podman[110499]: 2024-09-14 03:57:16.790456332 +0000 UTC m=+0.356602188 container init 9f7842c26184e48971bcffd58a092eee4a7015cf63edf6c5118360985054226d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-b, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=main-a278231, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:57:17.089 INFO:journalctl@ceph.mds.b.smithi094.stdout:Sep 14 03:57:16 smithi094 podman[110499]: 2024-09-14 03:57:16.795329617 +0000 UTC m=+0.361475473 container start 9f7842c26184e48971bcffd58a092eee4a7015cf63edf6c5118360985054226d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-b, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=main-a278231, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-14T03:57:17.089 INFO:journalctl@ceph.mds.b.smithi094.stdout:Sep 14 03:57:16 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-b[110514]: starting mds.b at 2024-09-14T03:57:17.089 INFO:journalctl@ceph.mds.b.smithi094.stdout:Sep 14 03:57:16 smithi094 bash[110499]: 9f7842c26184e48971bcffd58a092eee4a7015cf63edf6c5118360985054226d 2024-09-14T03:57:17.089 INFO:journalctl@ceph.mds.b.smithi094.stdout:Sep 14 03:57:16 smithi094 systemd[1]: Started Ceph mds.b for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:57:17.092 INFO:tasks.rados.rados.0.smithi070.stdout:2579: done (7 left) 2024-09-14T03:57:17.092 INFO:tasks.rados.rados.0.smithi070.stdout:2582: done (6 left) 2024-09-14T03:57:17.092 INFO:tasks.rados.rados.0.smithi070.stdout:2584: done (5 left) 2024-09-14T03:57:17.093 INFO:tasks.rados.rados.0.smithi070.stdout:2585: done (4 left) 2024-09-14T03:57:17.093 INFO:tasks.rados.rados.0.smithi070.stdout:2586: done (3 left) 2024-09-14T03:57:17.093 INFO:tasks.rados.rados.0.smithi070.stdout:2587: done (2 left) 2024-09-14T03:57:17.093 INFO:tasks.rados.rados.0.smithi070.stdout:2588: done (1 left) 2024-09-14T03:57:17.093 INFO:tasks.rados.rados.0.smithi070.stdout:2589: done (0 left) 2024-09-14T03:57:17.093 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 50 current snap is 279 2024-09-14T03:57:17.093 INFO:tasks.rados.rados.0.smithi070.stdout:2590: seq_num 799 ranges {2891776=589824,3481600=802816} 2024-09-14T03:57:17.110 INFO:tasks.rados.rados.0.smithi070.stdout:2590: writing smithi07085154-50 from 2891776 to 3481600 tid 1 2024-09-14T03:57:17.116 INFO:tasks.rados.rados.0.smithi070.stdout:2590: writing smithi07085154-50 from 3481600 to 4284416 tid 2 2024-09-14T03:57:17.116 INFO:tasks.rados.rados.0.smithi070.stdout:2591: read oid 39 snap 255 2024-09-14T03:57:17.116 INFO:tasks.rados.rados.0.smithi070.stdout:2591: expect (ObjNum 734 snap 255 seq_num 734) 2024-09-14T03:57:17.116 INFO:tasks.rados.rados.0.smithi070.stdout:2592: read oid 24 snap -1 2024-09-14T03:57:17.116 INFO:tasks.rados.rados.0.smithi070.stdout:2592: expect (ObjNum 771 snap 270 seq_num 771) 2024-09-14T03:57:17.116 INFO:tasks.rados.rados.0.smithi070.stdout:2593: read oid 18 snap 275 2024-09-14T03:57:17.117 INFO:tasks.rados.rados.0.smithi070.stdout:2593: expect (ObjNum 754 snap 260 seq_num 754) 2024-09-14T03:57:17.117 INFO:tasks.rados.rados.0.smithi070.stdout:2594: copy_from oid 16 from oid 38 current snap is 279 2024-09-14T03:57:17.117 INFO:tasks.rados.rados.0.smithi070.stdout:2595: snap_create 2024-09-14T03:57:17.117 INFO:tasks.rados.rados.0.smithi070.stdout:2590: finishing write tid 1 to smithi07085154-50 2024-09-14T03:57:17.121 INFO:tasks.rados.rados.0.smithi070.stdout:2594: finishing copy_from to smithi07085154-16 2024-09-14T03:57:17.121 INFO:tasks.rados.rados.0.smithi070.stdout:2594: got expected ENOENT (src dne) 2024-09-14T03:57:17.121 INFO:tasks.rados.rados.0.smithi070.stdout:2594: finishing copy_from racing read to smithi07085154-16 2024-09-14T03:57:17.121 INFO:tasks.rados.rados.0.smithi070.stdout:2592: expect (ObjNum 771 snap 270 seq_num 771) 2024-09-14T03:57:17.147 INFO:tasks.rados.rados.0.smithi070.stdout:2593: expect (ObjNum 754 snap 260 seq_num 754) 2024-09-14T03:57:17.163 INFO:tasks.rados.rados.0.smithi070.stdout:2590: finishing write tid 2 to smithi07085154-50 2024-09-14T03:57:17.163 INFO:tasks.rados.rados.0.smithi070.stdout:2590: finishing write tid 3 to smithi07085154-50 2024-09-14T03:57:17.163 INFO:tasks.rados.rados.0.smithi070.stdout:2590: finishing write tid 4 to smithi07085154-50 2024-09-14T03:57:17.163 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 2216 (ObjNum 799 snap 279 seq_num 799) dirty exists 2024-09-14T03:57:17.163 INFO:tasks.rados.rados.0.smithi070.stdout:2590: left oid 50 (ObjNum 799 snap 279 seq_num 799) 2024-09-14T03:57:17.163 INFO:tasks.rados.rados.0.smithi070.stdout:2591: expect (ObjNum 734 snap 255 seq_num 734) 2024-09-14T03:57:17.501 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:17 smithi094 ceph-mon[86597]: osdmap e605: 8 total, 8 up, 8 in 2024-09-14T03:57:17.501 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:17 smithi094 ceph-mon[86597]: pgmap v504: 121 pgs: 121 active+clean; 1.1 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s rd, 567 KiB/s wr, 14 op/s 2024-09-14T03:57:17.501 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:17 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:17.502 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:17 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:17.502 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:17 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:57:17.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:17 smithi070 ceph-mon[98640]: osdmap e605: 8 total, 8 up, 8 in 2024-09-14T03:57:17.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:17 smithi070 ceph-mon[98640]: pgmap v504: 121 pgs: 121 active+clean; 1.1 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s rd, 567 KiB/s wr, 14 op/s 2024-09-14T03:57:17.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:17 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:17.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:17 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:17.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:17 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:57:17.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:17 smithi070 ceph-mon[96704]: osdmap e605: 8 total, 8 up, 8 in 2024-09-14T03:57:17.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:17 smithi070 ceph-mon[96704]: pgmap v504: 121 pgs: 121 active+clean; 1.1 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s rd, 567 KiB/s wr, 14 op/s 2024-09-14T03:57:17.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:17 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:17.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:17 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:17.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:17 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:57:18.096 INFO:tasks.rados.rados.0.smithi070.stdout:2590: done (5 left) 2024-09-14T03:57:18.097 INFO:tasks.rados.rados.0.smithi070.stdout:2591: done (4 left) 2024-09-14T03:57:18.097 INFO:tasks.rados.rados.0.smithi070.stdout:2592: done (3 left) 2024-09-14T03:57:18.097 INFO:tasks.rados.rados.0.smithi070.stdout:2593: done (2 left) 2024-09-14T03:57:18.097 INFO:tasks.rados.rados.0.smithi070.stdout:2594: done (1 left) 2024-09-14T03:57:18.097 INFO:tasks.rados.rados.0.smithi070.stdout:2595: done (0 left) 2024-09-14T03:57:18.097 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 12 current snap is 280 2024-09-14T03:57:18.097 INFO:tasks.rados.rados.0.smithi070.stdout:2596: seq_num 800 ranges {1818624=606208,2424832=688128,3112960=114688} 2024-09-14T03:57:18.109 INFO:tasks.rados.rados.0.smithi070.stdout:2596: writing smithi07085154-12 from 1818624 to 2424832 tid 1 2024-09-14T03:57:18.113 INFO:tasks.rados.rados.0.smithi070.stdout:2596: writing smithi07085154-12 from 2424832 to 3112960 tid 2 2024-09-14T03:57:18.114 INFO:tasks.rados.rados.0.smithi070.stdout:2596: writing smithi07085154-12 from 3112960 to 3227648 tid 3 2024-09-14T03:57:18.114 INFO:tasks.rados.rados.0.smithi070.stdout:2597: read oid 33 snap -1 2024-09-14T03:57:18.114 INFO:tasks.rados.rados.0.smithi070.stdout:2597: expect (ObjNum 600 snap 199 seq_num 600) 2024-09-14T03:57:18.114 INFO:tasks.rados.rados.0.smithi070.stdout:2598: read oid 50 snap 242 2024-09-14T03:57:18.114 INFO:tasks.rados.rados.0.smithi070.stdout:2598: expect (ObjNum 657 snap 220 seq_num 657) 2024-09-14T03:57:18.114 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 45 current snap is 280 2024-09-14T03:57:18.114 INFO:tasks.rados.rados.0.smithi070.stdout:2599: seq_num 801 ranges {0=622592,622592=573440,1196032=229376} 2024-09-14T03:57:18.118 INFO:tasks.rados.rados.0.smithi070.stdout:2599: writing smithi07085154-45 from 0 to 622592 tid 1 2024-09-14T03:57:18.121 INFO:tasks.rados.rados.0.smithi070.stdout:2599: writing smithi07085154-45 from 622592 to 1196032 tid 2 2024-09-14T03:57:18.124 INFO:tasks.rados.rados.0.smithi070.stdout:2599: writing smithi07085154-45 from 1196032 to 1425408 tid 3 2024-09-14T03:57:18.124 INFO:tasks.rados.rados.0.smithi070.stdout:2600: rollback oid 36 current snap is 280 2024-09-14T03:57:18.124 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 36 to 276 2024-09-14T03:57:18.124 INFO:tasks.rados.rados.0.smithi070.stdout:2596: finishing write tid 1 to smithi07085154-12 2024-09-14T03:57:18.124 INFO:tasks.rados.rados.0.smithi070.stdout:2596: finishing write tid 2 to smithi07085154-12 2024-09-14T03:57:18.124 INFO:tasks.rados.rados.0.smithi070.stdout:2596: finishing write tid 3 to smithi07085154-12 2024-09-14T03:57:18.124 INFO:tasks.rados.rados.0.smithi070.stdout:2596: finishing write tid 4 to smithi07085154-12 2024-09-14T03:57:18.124 INFO:tasks.rados.rados.0.smithi070.stdout:2596: finishing write tid 5 to smithi07085154-12 2024-09-14T03:57:18.124 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 2212 (ObjNum 800 snap 280 seq_num 800) dirty exists 2024-09-14T03:57:18.125 INFO:tasks.rados.rados.0.smithi070.stdout:2596: left oid 12 (ObjNum 800 snap 280 seq_num 800) 2024-09-14T03:57:18.125 INFO:tasks.rados.rados.0.smithi070.stdout:2596: done (4 left) 2024-09-14T03:57:18.125 INFO:tasks.rados.rados.0.smithi070.stdout:2601: snap_remove snap 193 2024-09-14T03:57:18.487 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:18 smithi094 ceph-mon[86597]: mds.? [v2:172.21.15.94:6832/1666439704,v1:172.21.15.94:6833/1666439704] up:boot 2024-09-14T03:57:18.488 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:18 smithi094 ceph-mon[86597]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:57:18.488 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:18 smithi094 ceph-mon[86597]: osdmap e606: 8 total, 8 up, 8 in 2024-09-14T03:57:18.488 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:18 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "b"} : dispatch 2024-09-14T03:57:18.488 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:18 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:57:18.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:18 smithi070 ceph-mon[96704]: mds.? [v2:172.21.15.94:6832/1666439704,v1:172.21.15.94:6833/1666439704] up:boot 2024-09-14T03:57:18.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:18 smithi070 ceph-mon[96704]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:57:18.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:18 smithi070 ceph-mon[96704]: osdmap e606: 8 total, 8 up, 8 in 2024-09-14T03:57:18.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:18 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "b"} : dispatch 2024-09-14T03:57:18.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:18 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:57:18.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:18 smithi070 ceph-mon[98640]: mds.? [v2:172.21.15.94:6832/1666439704,v1:172.21.15.94:6833/1666439704] up:boot 2024-09-14T03:57:18.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:18 smithi070 ceph-mon[98640]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-14T03:57:18.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:18 smithi070 ceph-mon[98640]: osdmap e606: 8 total, 8 up, 8 in 2024-09-14T03:57:18.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:18 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "b"} : dispatch 2024-09-14T03:57:18.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:18 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:57:19.080 INFO:tasks.rados.rados.0.smithi070.stdout:2601: done (4 left) 2024-09-14T03:57:19.080 INFO:tasks.rados.rados.0.smithi070.stdout:2602: delete oid 39 current snap is 280 2024-09-14T03:57:19.081 INFO:tasks.rados.rados.0.smithi070.stdout:2600: finishing rollback tid 0 to smithi07085154-36 2024-09-14T03:57:19.081 INFO:tasks.rados.rados.0.smithi070.stdout:2600: finishing rollback tid 1 to smithi07085154-36 2024-09-14T03:57:19.081 INFO:tasks.rados.rados.0.smithi070.stdout:2600: finishing rollback tid 2 to smithi07085154-36 2024-09-14T03:57:19.081 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 36 v 2073 (ObjNum 784 snap 274 seq_num 784) dirty exists 2024-09-14T03:57:19.081 INFO:tasks.rados.rados.0.smithi070.stdout:2598: expect (ObjNum 657 snap 220 seq_num 657) 2024-09-14T03:57:19.174 INFO:tasks.rados.rados.0.smithi070.stdout:2599: finishing write tid 1 to smithi07085154-45 2024-09-14T03:57:19.175 INFO:tasks.rados.rados.0.smithi070.stdout:2599: finishing write tid 2 to smithi07085154-45 2024-09-14T03:57:19.175 INFO:tasks.rados.rados.0.smithi070.stdout:2599: finishing write tid 3 to smithi07085154-45 2024-09-14T03:57:19.175 INFO:tasks.rados.rados.0.smithi070.stdout:2599: finishing write tid 4 to smithi07085154-45 2024-09-14T03:57:19.175 INFO:tasks.rados.rados.0.smithi070.stdout:2597: expect (ObjNum 600 snap 199 seq_num 600) 2024-09-14T03:57:19.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:19 smithi094 ceph-mon[86597]: osdmap e607: 8 total, 8 up, 8 in 2024-09-14T03:57:19.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:19 smithi094 ceph-mon[86597]: pgmap v507: 121 pgs: 121 active+clean; 1.1 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 2.5 MiB/s wr, 18 op/s 2024-09-14T03:57:19.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:19 smithi094 ceph-mon[86597]: osdmap e608: 8 total, 8 up, 8 in 2024-09-14T03:57:19.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:19 smithi070 ceph-mon[98640]: osdmap e607: 8 total, 8 up, 8 in 2024-09-14T03:57:19.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:19 smithi070 ceph-mon[98640]: pgmap v507: 121 pgs: 121 active+clean; 1.1 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 2.5 MiB/s wr, 18 op/s 2024-09-14T03:57:19.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:19 smithi070 ceph-mon[98640]: osdmap e608: 8 total, 8 up, 8 in 2024-09-14T03:57:19.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:19 smithi070 ceph-mon[96704]: osdmap e607: 8 total, 8 up, 8 in 2024-09-14T03:57:19.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:19 smithi070 ceph-mon[96704]: pgmap v507: 121 pgs: 121 active+clean; 1.1 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 2.5 MiB/s wr, 18 op/s 2024-09-14T03:57:19.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:19 smithi070 ceph-mon[96704]: osdmap e608: 8 total, 8 up, 8 in 2024-09-14T03:57:19.658 INFO:tasks.rados.rados.0.smithi070.stdout:2599: finishing write tid 5 to smithi07085154-45 2024-09-14T03:57:19.658 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 45 v 1965 (ObjNum 801 snap 280 seq_num 801) dirty exists 2024-09-14T03:57:19.658 INFO:tasks.rados.rados.0.smithi070.stdout:2599: left oid 45 (ObjNum 801 snap 280 seq_num 801) 2024-09-14T03:57:19.658 INFO:tasks.rados.rados.0.smithi070.stdout:2597: done (4 left) 2024-09-14T03:57:19.658 INFO:tasks.rados.rados.0.smithi070.stdout:2598: done (3 left) 2024-09-14T03:57:19.658 INFO:tasks.rados.rados.0.smithi070.stdout:2599: done (2 left) 2024-09-14T03:57:19.658 INFO:tasks.rados.rados.0.smithi070.stdout:2600: done (1 left) 2024-09-14T03:57:19.658 INFO:tasks.rados.rados.0.smithi070.stdout:2602: done (0 left) 2024-09-14T03:57:19.658 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 10 current snap is 280 2024-09-14T03:57:19.658 INFO:tasks.rados.rados.0.smithi070.stdout:2603: seq_num 803 ranges {1105920=655360,1761280=745472,2506752=57344} 2024-09-14T03:57:19.667 INFO:tasks.rados.rados.0.smithi070.stdout:2603: writing smithi07085154-10 from 1105920 to 1761280 tid 1 2024-09-14T03:57:19.672 INFO:tasks.rados.rados.0.smithi070.stdout:2603: writing smithi07085154-10 from 1761280 to 2506752 tid 2 2024-09-14T03:57:19.672 INFO:tasks.rados.rados.0.smithi070.stdout:2603: writing smithi07085154-10 from 2506752 to 2564096 tid 3 2024-09-14T03:57:19.673 INFO:tasks.rados.rados.0.smithi070.stdout:2604: read oid 6 snap -1 2024-09-14T03:57:19.673 INFO:tasks.rados.rados.0.smithi070.stdout:2604: expect (ObjNum 751 snap 260 seq_num 751) 2024-09-14T03:57:19.673 INFO:tasks.rados.rados.0.smithi070.stdout:2605: rollback oid 37 current snap is 280 2024-09-14T03:57:19.673 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 37 to 255 2024-09-14T03:57:19.673 INFO:tasks.rados.rados.0.smithi070.stdout:2606: read oid 17 snap -1 2024-09-14T03:57:19.673 INFO:tasks.rados.rados.0.smithi070.stdout:2606: expect deleted 2024-09-14T03:57:19.673 INFO:tasks.rados.rados.0.smithi070.stdout:2607: snap_remove snap 274 2024-09-14T03:57:20.106 INFO:tasks.rados.rados.0.smithi070.stdout:2603: finishing write tid 1 to smithi07085154-10 2024-09-14T03:57:20.106 INFO:tasks.rados.rados.0.smithi070.stdout:2605: finishing rollback tid 0 to smithi07085154-37 2024-09-14T03:57:20.106 INFO:tasks.rados.rados.0.smithi070.stdout:2605: finishing rollback tid 1 to smithi07085154-37 2024-09-14T03:57:20.106 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 37 v 2223 (ObjNum 1057 snap 0 seq_num 4194279632) dirty dne 2024-09-14T03:57:20.106 INFO:tasks.rados.rados.0.smithi070.stdout:2603: finishing write tid 2 to smithi07085154-10 2024-09-14T03:57:20.106 INFO:tasks.rados.rados.0.smithi070.stdout:2603: finishing write tid 3 to smithi07085154-10 2024-09-14T03:57:20.106 INFO:tasks.rados.rados.0.smithi070.stdout:2603: finishing write tid 4 to smithi07085154-10 2024-09-14T03:57:20.106 INFO:tasks.rados.rados.0.smithi070.stdout:2603: finishing write tid 5 to smithi07085154-10 2024-09-14T03:57:20.106 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 2914 (ObjNum 803 snap 280 seq_num 803) dirty exists 2024-09-14T03:57:20.106 INFO:tasks.rados.rados.0.smithi070.stdout:2603: left oid 10 (ObjNum 803 snap 280 seq_num 803) 2024-09-14T03:57:20.106 INFO:tasks.rados.rados.0.smithi070.stdout:2604: expect (ObjNum 751 snap 260 seq_num 751) 2024-09-14T03:57:20.276 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:57:20 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:57:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:57:20.393 INFO:tasks.rados.rados.0.smithi070.stdout:2603: done (4 left) 2024-09-14T03:57:20.394 INFO:tasks.rados.rados.0.smithi070.stdout:2604: done (3 left) 2024-09-14T03:57:20.394 INFO:tasks.rados.rados.0.smithi070.stdout:2605: done (2 left) 2024-09-14T03:57:20.394 INFO:tasks.rados.rados.0.smithi070.stdout:2606: done (1 left) 2024-09-14T03:57:20.394 INFO:tasks.rados.rados.0.smithi070.stdout:2607: done (0 left) 2024-09-14T03:57:20.394 INFO:tasks.rados.rados.0.smithi070.stdout:2608: read oid 43 snap -1 2024-09-14T03:57:20.394 INFO:tasks.rados.rados.0.smithi070.stdout:2608: expect (ObjNum 780 snap 274 seq_num 780) 2024-09-14T03:57:20.394 INFO:tasks.rados.rados.0.smithi070.stdout:2609: rollback oid 44 current snap is 280 2024-09-14T03:57:20.394 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 44 to 276 2024-09-14T03:57:20.394 INFO:tasks.rados.rados.0.smithi070.stdout:2610: rmattr oid 26 current snap is 280 2024-09-14T03:57:20.394 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 21 current snap is 280 2024-09-14T03:57:20.394 INFO:tasks.rados.rados.0.smithi070.stdout:2611: seq_num 804 ranges {0=671744,671744=622592,1294336=180224} 2024-09-14T03:57:20.397 INFO:tasks.rados.rados.0.smithi070.stdout:2611: writing smithi07085154-21 from 0 to 671744 tid 1 2024-09-14T03:57:20.401 INFO:tasks.rados.rados.0.smithi070.stdout:2611: writing smithi07085154-21 from 671744 to 1294336 tid 2 2024-09-14T03:57:20.403 INFO:tasks.rados.rados.0.smithi070.stdout:2611: writing smithi07085154-21 from 1294336 to 1474560 tid 3 2024-09-14T03:57:20.403 INFO:tasks.rados.rados.0.smithi070.stdout:2612: snap_create 2024-09-14T03:57:20.403 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 2922 (ObjNum 752 snap 260 seq_num 752) dirty exists 2024-09-14T03:57:20.403 INFO:tasks.rados.rados.0.smithi070.stdout:2609: finishing rollback tid 1 to smithi07085154-44 2024-09-14T03:57:20.403 INFO:tasks.rados.rados.0.smithi070.stdout:2609: finishing rollback tid 2 to smithi07085154-44 2024-09-14T03:57:20.403 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 2075 (ObjNum 668 snap 228 seq_num 668) dirty exists 2024-09-14T03:57:20.403 INFO:tasks.rados.rados.0.smithi070.stdout:2611: finishing write tid 1 to smithi07085154-21 2024-09-14T03:57:20.403 INFO:tasks.rados.rados.0.smithi070.stdout:2608: expect (ObjNum 780 snap 274 seq_num 780) 2024-09-14T03:57:20.508 INFO:tasks.rados.rados.0.smithi070.stdout:2611: finishing write tid 2 to smithi07085154-21 2024-09-14T03:57:20.508 INFO:tasks.rados.rados.0.smithi070.stdout:2611: finishing write tid 3 to smithi07085154-21 2024-09-14T03:57:20.508 INFO:tasks.rados.rados.0.smithi070.stdout:2611: finishing write tid 4 to smithi07085154-21 2024-09-14T03:57:20.508 INFO:tasks.rados.rados.0.smithi070.stdout:2611: finishing write tid 5 to smithi07085154-21 2024-09-14T03:57:20.508 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 1705 (ObjNum 804 snap 280 seq_num 804) dirty exists 2024-09-14T03:57:20.508 INFO:tasks.rados.rados.0.smithi070.stdout:2611: left oid 21 (ObjNum 804 snap 280 seq_num 804) 2024-09-14T03:57:21.108 INFO:tasks.rados.rados.0.smithi070.stdout:2608: done (4 left) 2024-09-14T03:57:21.109 INFO:tasks.rados.rados.0.smithi070.stdout:2609: done (3 left) 2024-09-14T03:57:21.109 INFO:tasks.rados.rados.0.smithi070.stdout:2610: done (2 left) 2024-09-14T03:57:21.109 INFO:tasks.rados.rados.0.smithi070.stdout:2611: done (1 left) 2024-09-14T03:57:21.109 INFO:tasks.rados.rados.0.smithi070.stdout:2612: done (0 left) 2024-09-14T03:57:21.109 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 15 current snap is 281 2024-09-14T03:57:21.109 INFO:tasks.rados.rados.0.smithi070.stdout:2613: seq_num 805 ranges {0=688128,688128=507904,1196032=294912} 2024-09-14T03:57:21.112 INFO:tasks.rados.rados.0.smithi070.stdout:2613: writing smithi07085154-15 from 0 to 688128 tid 1 2024-09-14T03:57:21.115 INFO:tasks.rados.rados.0.smithi070.stdout:2613: writing smithi07085154-15 from 688128 to 1196032 tid 2 2024-09-14T03:57:21.117 INFO:tasks.rados.rados.0.smithi070.stdout:2613: writing smithi07085154-15 from 1196032 to 1490944 tid 3 2024-09-14T03:57:21.117 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 21 current snap is 281 2024-09-14T03:57:21.118 INFO:tasks.rados.rados.0.smithi070.stdout:2614: seq_num 806 ranges {1474560=704512,2179072=442368,2621440=360448} 2024-09-14T03:57:21.129 INFO:tasks.rados.rados.0.smithi070.stdout:2614: writing smithi07085154-21 from 1474560 to 2179072 tid 1 2024-09-14T03:57:21.132 INFO:tasks.rados.rados.0.smithi070.stdout:2614: writing smithi07085154-21 from 2179072 to 2621440 tid 2 2024-09-14T03:57:21.134 INFO:tasks.rados.rados.0.smithi070.stdout:2614: writing smithi07085154-21 from 2621440 to 2981888 tid 3 2024-09-14T03:57:21.134 INFO:tasks.rados.rados.0.smithi070.stdout:2615: setattr oid 46 current snap is 281 2024-09-14T03:57:21.134 INFO:tasks.rados.rados.0.smithi070.stdout:2613: finishing write tid 1 to smithi07085154-15 2024-09-14T03:57:21.135 INFO:tasks.rados.rados.0.smithi070.stdout:2613: finishing write tid 2 to smithi07085154-15 2024-09-14T03:57:21.135 INFO:tasks.rados.rados.0.smithi070.stdout:2613: finishing write tid 3 to smithi07085154-15 2024-09-14T03:57:21.135 INFO:tasks.rados.rados.0.smithi070.stdout:2613: finishing write tid 4 to smithi07085154-15 2024-09-14T03:57:21.135 INFO:tasks.rados.rados.0.smithi070.stdout:2613: finishing write tid 5 to smithi07085154-15 2024-09-14T03:57:21.135 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 15 v 2227 (ObjNum 805 snap 281 seq_num 805) dirty exists 2024-09-14T03:57:21.135 INFO:tasks.rados.rados.0.smithi070.stdout:2613: left oid 15 (ObjNum 805 snap 281 seq_num 805) 2024-09-14T03:57:21.135 INFO:tasks.rados.rados.0.smithi070.stdout:2614: finishing write tid 1 to smithi07085154-21 2024-09-14T03:57:21.136 INFO:tasks.rados.rados.0.smithi070.stdout:2614: finishing write tid 2 to smithi07085154-21 2024-09-14T03:57:21.136 INFO:tasks.rados.rados.0.smithi070.stdout:2613: done (2 left) 2024-09-14T03:57:21.136 INFO:tasks.rados.rados.0.smithi070.stdout:2616: copy_from oid 14 from oid 2 current snap is 281 2024-09-14T03:57:21.136 INFO:tasks.rados.rados.0.smithi070.stdout:2617: snap_create 2024-09-14T03:57:21.136 INFO:tasks.rados.rados.0.smithi070.stdout:2614: finishing write tid 3 to smithi07085154-21 2024-09-14T03:57:21.137 INFO:tasks.rados.rados.0.smithi070.stdout:2614: finishing write tid 4 to smithi07085154-21 2024-09-14T03:57:21.137 INFO:tasks.rados.rados.0.smithi070.stdout:2614: finishing write tid 5 to smithi07085154-21 2024-09-14T03:57:21.137 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 1709 (ObjNum 806 snap 281 seq_num 806) dirty exists 2024-09-14T03:57:21.137 INFO:tasks.rados.rados.0.smithi070.stdout:2614: left oid 21 (ObjNum 806 snap 281 seq_num 806) 2024-09-14T03:57:21.142 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 46 v 2924 (ObjNum 576 snap 193 seq_num 576) dirty exists 2024-09-14T03:57:21.145 INFO:tasks.rados.rados.0.smithi070.stdout:2616: finishing copy_from racing read to smithi07085154-14 2024-09-14T03:57:21.146 INFO:tasks.rados.rados.0.smithi070.stdout:2616: finishing copy_from to smithi07085154-14 2024-09-14T03:57:21.146 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 2773 (ObjNum 759 snap 264 seq_num 759) dirty exists 2024-09-14T03:57:21.374 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:21 smithi094 ceph-mon[86597]: pgmap v510: 121 pgs: 121 active+clean; 1.1 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 3.2 MiB/s wr, 23 op/s 2024-09-14T03:57:21.374 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:21 smithi094 ceph-mon[86597]: osdmap e609: 8 total, 8 up, 8 in 2024-09-14T03:57:21.374 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:21 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:21.374 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:21 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:21.374 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:21 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:21.374 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:21 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:21.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:21 smithi070 ceph-mon[96704]: pgmap v510: 121 pgs: 121 active+clean; 1.1 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 3.2 MiB/s wr, 23 op/s 2024-09-14T03:57:21.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:21 smithi070 ceph-mon[96704]: osdmap e609: 8 total, 8 up, 8 in 2024-09-14T03:57:21.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:21 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:21.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:21 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:21.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:21 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:21.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:21 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:21.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:21 smithi070 ceph-mon[98640]: pgmap v510: 121 pgs: 121 active+clean; 1.1 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 3.2 MiB/s wr, 23 op/s 2024-09-14T03:57:21.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:21 smithi070 ceph-mon[98640]: osdmap e609: 8 total, 8 up, 8 in 2024-09-14T03:57:21.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:21 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:21.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:21 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:21.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:21 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:21.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:21 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:22.114 INFO:tasks.rados.rados.0.smithi070.stdout:2614: done (3 left) 2024-09-14T03:57:22.114 INFO:tasks.rados.rados.0.smithi070.stdout:2615: done (2 left) 2024-09-14T03:57:22.114 INFO:tasks.rados.rados.0.smithi070.stdout:2616: done (1 left) 2024-09-14T03:57:22.114 INFO:tasks.rados.rados.0.smithi070.stdout:2617: done (0 left) 2024-09-14T03:57:22.114 INFO:tasks.rados.rados.0.smithi070.stdout:2618: rollback oid 6 current snap is 282 2024-09-14T03:57:22.114 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 6 to 243 2024-09-14T03:57:22.114 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 9 current snap is 282 2024-09-14T03:57:22.114 INFO:tasks.rados.rados.0.smithi070.stdout:2619: seq_num 807 ranges {4145152=720896,4866048=729088,5595136=73728} 2024-09-14T03:57:22.140 INFO:tasks.rados.rados.0.smithi070.stdout:2619: writing smithi07085154-9 from 4145152 to 4866048 tid 1 2024-09-14T03:57:22.144 INFO:tasks.rados.rados.0.smithi070.stdout:2619: writing smithi07085154-9 from 4866048 to 5595136 tid 2 2024-09-14T03:57:22.145 INFO:tasks.rados.rados.0.smithi070.stdout:2619: writing smithi07085154-9 from 5595136 to 5668864 tid 3 2024-09-14T03:57:22.145 INFO:tasks.rados.rados.0.smithi070.stdout:2620: copy_from oid 19 from oid 31 current snap is 282 2024-09-14T03:57:22.145 INFO:tasks.rados.rados.0.smithi070.stdout:2618: finishing rollback tid 0 to smithi07085154-6 2024-09-14T03:57:22.145 INFO:tasks.rados.rados.0.smithi070.stdout:2618: finishing rollback tid 1 to smithi07085154-6 2024-09-14T03:57:22.145 INFO:tasks.rados.rados.0.smithi070.stdout:2618: finishing rollback tid 2 to smithi07085154-6 2024-09-14T03:57:22.145 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 2259 (ObjNum 671 snap 228 seq_num 671) dirty exists 2024-09-14T03:57:22.145 INFO:tasks.rados.rados.0.smithi070.stdout:2618: done (2 left) 2024-09-14T03:57:22.145 INFO:tasks.rados.rados.0.smithi070.stdout:2621: rollback oid 34 current snap is 282 2024-09-14T03:57:22.145 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 34 to 270 2024-09-14T03:57:22.146 INFO:tasks.rados.rados.0.smithi070.stdout:2622: delete oid 7 current snap is 282 2024-09-14T03:57:22.147 INFO:tasks.rados.rados.0.smithi070.stdout:2622: done (3 left) 2024-09-14T03:57:22.147 INFO:tasks.rados.rados.0.smithi070.stdout:2623: read oid 5 snap -1 2024-09-14T03:57:22.147 INFO:tasks.rados.rados.0.smithi070.stdout:2623: expect deleted 2024-09-14T03:57:22.147 INFO:tasks.rados.rados.0.smithi070.stdout:2624: read oid 23 snap -1 2024-09-14T03:57:22.147 INFO:tasks.rados.rados.0.smithi070.stdout:2624: expect deleted 2024-09-14T03:57:22.147 INFO:tasks.rados.rados.0.smithi070.stdout:2625: delete oid 24 current snap is 282 2024-09-14T03:57:22.150 INFO:tasks.rados.rados.0.smithi070.stdout:2620: finishing copy_from to smithi07085154-19 2024-09-14T03:57:22.150 INFO:tasks.rados.rados.0.smithi070.stdout:2620: got expected ENOENT (src dne) 2024-09-14T03:57:22.150 INFO:tasks.rados.rados.0.smithi070.stdout:2620: finishing copy_from racing read to smithi07085154-19 2024-09-14T03:57:22.151 INFO:tasks.rados.rados.0.smithi070.stdout:2619: finishing write tid 1 to smithi07085154-9 2024-09-14T03:57:22.153 INFO:tasks.rados.rados.0.smithi070.stdout:2620: done (5 left) 2024-09-14T03:57:22.153 INFO:tasks.rados.rados.0.smithi070.stdout:2623: done (4 left) 2024-09-14T03:57:22.153 INFO:tasks.rados.rados.0.smithi070.stdout:2624: done (3 left) 2024-09-14T03:57:22.153 INFO:tasks.rados.rados.0.smithi070.stdout:2625: done (2 left) 2024-09-14T03:57:22.153 INFO:tasks.rados.rados.0.smithi070.stdout:2626: read oid 3 snap -1 2024-09-14T03:57:22.153 INFO:tasks.rados.rados.0.smithi070.stdout:2626: expect deleted 2024-09-14T03:57:22.153 INFO:tasks.rados.rados.0.smithi070.stdout:2627: read oid 19 snap -1 2024-09-14T03:57:22.153 INFO:tasks.rados.rados.0.smithi070.stdout:2627: expect (ObjNum 747 snap 259 seq_num 747) 2024-09-14T03:57:22.153 INFO:tasks.rados.rados.0.smithi070.stdout:2628: setattr oid 32 current snap is 282 2024-09-14T03:57:22.153 INFO:tasks.rados.rados.0.smithi070.stdout:2621: finishing rollback tid 0 to smithi07085154-34 2024-09-14T03:57:22.153 INFO:tasks.rados.rados.0.smithi070.stdout:2621: finishing rollback tid 1 to smithi07085154-34 2024-09-14T03:57:22.154 INFO:tasks.rados.rados.0.smithi070.stdout:2619: finishing write tid 2 to smithi07085154-9 2024-09-14T03:57:22.154 INFO:tasks.rados.rados.0.smithi070.stdout:2621: finishing rollback tid 2 to smithi07085154-34 2024-09-14T03:57:22.154 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 2219 (ObjNum 762 snap 264 seq_num 762) dirty exists 2024-09-14T03:57:22.154 INFO:tasks.rados.rados.0.smithi070.stdout:2619: finishing write tid 3 to smithi07085154-9 2024-09-14T03:57:22.154 INFO:tasks.rados.rados.0.smithi070.stdout:2619: finishing write tid 4 to smithi07085154-9 2024-09-14T03:57:22.154 INFO:tasks.rados.rados.0.smithi070.stdout:2621: done (4 left) 2024-09-14T03:57:22.154 INFO:tasks.rados.rados.0.smithi070.stdout:2626: done (3 left) 2024-09-14T03:57:22.154 INFO:tasks.rados.rados.0.smithi070.stdout:2629: snap_create 2024-09-14T03:57:22.155 INFO:tasks.rados.rados.0.smithi070.stdout:2619: finishing write tid 5 to smithi07085154-9 2024-09-14T03:57:22.155 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 2929 (ObjNum 807 snap 282 seq_num 807) dirty exists 2024-09-14T03:57:22.155 INFO:tasks.rados.rados.0.smithi070.stdout:2619: left oid 9 (ObjNum 807 snap 282 seq_num 807) 2024-09-14T03:57:22.156 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 2255 (ObjNum 718 snap 248 seq_num 718) dirty exists 2024-09-14T03:57:22.160 INFO:tasks.rados.rados.0.smithi070.stdout:2627: expect (ObjNum 747 snap 259 seq_num 747) 2024-09-14T03:57:22.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:22 smithi094 ceph-mon[86597]: osdmap e610: 8 total, 8 up, 8 in 2024-09-14T03:57:22.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:22 smithi070 ceph-mon[96704]: osdmap e610: 8 total, 8 up, 8 in 2024-09-14T03:57:22.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:22 smithi070 ceph-mon[98640]: osdmap e610: 8 total, 8 up, 8 in 2024-09-14T03:57:23.118 INFO:tasks.rados.rados.0.smithi070.stdout:2619: done (3 left) 2024-09-14T03:57:23.118 INFO:tasks.rados.rados.0.smithi070.stdout:2627: done (2 left) 2024-09-14T03:57:23.119 INFO:tasks.rados.rados.0.smithi070.stdout:2628: done (1 left) 2024-09-14T03:57:23.119 INFO:tasks.rados.rados.0.smithi070.stdout:2629: done (0 left) 2024-09-14T03:57:23.119 INFO:tasks.rados.rados.0.smithi070.stdout:2630: snap_remove snap 262 2024-09-14T03:57:23.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:23 smithi094 ceph-mon[86597]: pgmap v512: 121 pgs: 121 active+clean; 1.1 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 2.6 MiB/s wr, 23 op/s 2024-09-14T03:57:23.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:23 smithi094 ceph-mon[86597]: osdmap e611: 8 total, 8 up, 8 in 2024-09-14T03:57:23.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:23 smithi070 ceph-mon[96704]: pgmap v512: 121 pgs: 121 active+clean; 1.1 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 2.6 MiB/s wr, 23 op/s 2024-09-14T03:57:23.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:23 smithi070 ceph-mon[96704]: osdmap e611: 8 total, 8 up, 8 in 2024-09-14T03:57:23.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:23 smithi070 ceph-mon[98640]: pgmap v512: 121 pgs: 121 active+clean; 1.1 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 2.6 MiB/s wr, 23 op/s 2024-09-14T03:57:23.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:23 smithi070 ceph-mon[98640]: osdmap e611: 8 total, 8 up, 8 in 2024-09-14T03:57:23.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:57:23 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:57:23] "GET /metrics HTTP/1.1" 200 36507 "" "Prometheus/2.51.0" 2024-09-14T03:57:24.081 INFO:tasks.rados.rados.0.smithi070.stdout:2630: done (0 left) 2024-09-14T03:57:24.081 INFO:tasks.rados.rados.0.smithi070.stdout:2631: read oid 30 snap 267 2024-09-14T03:57:24.081 INFO:tasks.rados.rados.0.smithi070.stdout:2631: expect (ObjNum -1363867712 snap 21849 seq_num 3758578112) 2024-09-14T03:57:24.081 INFO:tasks.rados.rados.0.smithi070.stdout:2632: delete oid 14 current snap is 283 2024-09-14T03:57:24.088 INFO:tasks.rados.rados.0.smithi070.stdout:2632: done (1 left) 2024-09-14T03:57:24.088 INFO:tasks.rados.rados.0.smithi070.stdout:2633: read oid 47 snap -1 2024-09-14T03:57:24.088 INFO:tasks.rados.rados.0.smithi070.stdout:2633: expect deleted 2024-09-14T03:57:24.088 INFO:tasks.rados.rados.0.smithi070.stdout:2634: copy_from oid 42 from oid 39 current snap is 283 2024-09-14T03:57:24.088 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 34 current snap is 283 2024-09-14T03:57:24.088 INFO:tasks.rados.rados.0.smithi070.stdout:2635: seq_num 811 ranges {770048=786432,1556480=663552,2220032=139264} 2024-09-14T03:57:24.097 INFO:tasks.rados.rados.0.smithi070.stdout:2635: writing smithi07085154-34 from 770048 to 1556480 tid 1 2024-09-14T03:57:24.102 INFO:tasks.rados.rados.0.smithi070.stdout:2635: writing smithi07085154-34 from 1556480 to 2220032 tid 2 2024-09-14T03:57:24.103 INFO:tasks.rados.rados.0.smithi070.stdout:2635: writing smithi07085154-34 from 2220032 to 2359296 tid 3 2024-09-14T03:57:24.103 INFO:tasks.rados.rados.0.smithi070.stdout:2636: delete oid 32 current snap is 283 2024-09-14T03:57:24.103 INFO:tasks.rados.rados.0.smithi070.stdout:2631: expect (ObjNum -1363867712 snap 21849 seq_num 3758578112) 2024-09-14T03:57:24.105 INFO:tasks.rados.rados.0.smithi070.stdout:2634: finishing copy_from to smithi07085154-42 2024-09-14T03:57:24.105 INFO:tasks.rados.rados.0.smithi070.stdout:2634: got expected ENOENT (src dne) 2024-09-14T03:57:24.105 INFO:tasks.rados.rados.0.smithi070.stdout:2634: finishing copy_from racing read to smithi07085154-42 2024-09-14T03:57:24.105 INFO:tasks.rados.rados.0.smithi070.stdout:2635: finishing write tid 1 to smithi07085154-34 2024-09-14T03:57:24.106 INFO:tasks.rados.rados.0.smithi070.stdout:2631: done (4 left) 2024-09-14T03:57:24.106 INFO:tasks.rados.rados.0.smithi070.stdout:2633: done (3 left) 2024-09-14T03:57:24.107 INFO:tasks.rados.rados.0.smithi070.stdout:2634: done (2 left) 2024-09-14T03:57:24.107 INFO:tasks.rados.rados.0.smithi070.stdout:2636: done (1 left) 2024-09-14T03:57:24.107 INFO:tasks.rados.rados.0.smithi070.stdout:2637: setattr oid 29 current snap is 283 2024-09-14T03:57:24.108 INFO:tasks.rados.rados.0.smithi070.stdout:2635: finishing write tid 2 to smithi07085154-34 2024-09-14T03:57:24.108 INFO:tasks.rados.rados.0.smithi070.stdout:2635: finishing write tid 3 to smithi07085154-34 2024-09-14T03:57:24.108 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 26 current snap is 283 2024-09-14T03:57:24.109 INFO:tasks.rados.rados.0.smithi070.stdout:2638: seq_num 813 ranges {2023424=417792,2441216=434176,2875392=442368,3317760=327680} 2024-09-14T03:57:24.122 INFO:tasks.rados.rados.0.smithi070.stdout:2638: writing smithi07085154-26 from 2023424 to 2441216 tid 1 2024-09-14T03:57:24.124 INFO:tasks.rados.rados.0.smithi070.stdout:2638: writing smithi07085154-26 from 2441216 to 2875392 tid 2 2024-09-14T03:57:24.127 INFO:tasks.rados.rados.0.smithi070.stdout:2638: writing smithi07085154-26 from 2875392 to 3317760 tid 3 2024-09-14T03:57:24.129 INFO:tasks.rados.rados.0.smithi070.stdout:2638: writing smithi07085154-26 from 3317760 to 3645440 tid 4 2024-09-14T03:57:24.129 INFO:tasks.rados.rados.0.smithi070.stdout:2639: setattr oid 6 current snap is 283 2024-09-14T03:57:24.129 INFO:tasks.rados.rados.0.smithi070.stdout:2635: finishing write tid 4 to smithi07085154-34 2024-09-14T03:57:24.130 INFO:tasks.rados.rados.0.smithi070.stdout:2635: finishing write tid 5 to smithi07085154-34 2024-09-14T03:57:24.130 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 2223 (ObjNum 811 snap 283 seq_num 811) dirty exists 2024-09-14T03:57:24.130 INFO:tasks.rados.rados.0.smithi070.stdout:2635: left oid 34 (ObjNum 811 snap 283 seq_num 811) 2024-09-14T03:57:24.130 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 2224 (ObjNum 49 snap 0 seq_num 2931790768) dirty exists 2024-09-14T03:57:24.130 INFO:tasks.rados.rados.0.smithi070.stdout:2638: finishing write tid 1 to smithi07085154-26 2024-09-14T03:57:24.130 INFO:tasks.rados.rados.0.smithi070.stdout:2638: finishing write tid 2 to smithi07085154-26 2024-09-14T03:57:24.131 INFO:tasks.rados.rados.0.smithi070.stdout:2635: done (3 left) 2024-09-14T03:57:24.131 INFO:tasks.rados.rados.0.smithi070.stdout:2637: done (2 left) 2024-09-14T03:57:24.131 INFO:tasks.rados.rados.0.smithi070.stdout:2640: rollback oid 17 current snap is 283 2024-09-14T03:57:24.131 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 17 to 243 2024-09-14T03:57:24.131 INFO:tasks.rados.rados.0.smithi070.stdout:2641: read oid 34 snap -1 2024-09-14T03:57:24.131 INFO:tasks.rados.rados.0.smithi070.stdout:2641: expect (ObjNum 811 snap 283 seq_num 811) 2024-09-14T03:57:24.132 INFO:tasks.rados.rados.0.smithi070.stdout:2642: read oid 43 snap -1 2024-09-14T03:57:24.132 INFO:tasks.rados.rados.0.smithi070.stdout:2642: expect (ObjNum 780 snap 274 seq_num 780) 2024-09-14T03:57:24.132 INFO:tasks.rados.rados.0.smithi070.stdout:2643: copy_from oid 40 from oid 20 current snap is 283 2024-09-14T03:57:24.132 INFO:tasks.rados.rados.0.smithi070.stdout:2644: copy_from oid 3 from oid 28 current snap is 283 2024-09-14T03:57:24.132 INFO:tasks.rados.rados.0.smithi070.stdout:2645: read oid 32 snap -1 2024-09-14T03:57:24.132 INFO:tasks.rados.rados.0.smithi070.stdout:2645: expect deleted 2024-09-14T03:57:24.132 INFO:tasks.rados.rados.0.smithi070.stdout:2646: snap_remove snap 238 2024-09-14T03:57:24.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:24 smithi094 ceph-mon[86597]: osdmap e612: 8 total, 8 up, 8 in 2024-09-14T03:57:24.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:24 smithi094 ceph-mon[86597]: osdmap e613: 8 total, 8 up, 8 in 2024-09-14T03:57:24.428 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:24 smithi070 ceph-mon[98640]: osdmap e612: 8 total, 8 up, 8 in 2024-09-14T03:57:24.429 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:24 smithi070 ceph-mon[98640]: osdmap e613: 8 total, 8 up, 8 in 2024-09-14T03:57:24.429 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:24 smithi070 ceph-mon[96704]: osdmap e612: 8 total, 8 up, 8 in 2024-09-14T03:57:24.429 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:24 smithi070 ceph-mon[96704]: osdmap e613: 8 total, 8 up, 8 in 2024-09-14T03:57:25.125 INFO:tasks.rados.rados.0.smithi070.stdout:2646: done (8 left) 2024-09-14T03:57:25.126 INFO:tasks.rados.rados.0.smithi070.stdout:2647: read oid 33 snap 255 2024-09-14T03:57:25.126 INFO:tasks.rados.rados.0.smithi070.stdout:2647: expect (ObjNum 600 snap 199 seq_num 600) 2024-09-14T03:57:25.126 INFO:tasks.rados.rados.0.smithi070.stdout:2648: setattr oid 45 current snap is 283 2024-09-14T03:57:25.126 INFO:tasks.rados.rados.0.smithi070.stdout:2638: finishing write tid 3 to smithi07085154-26 2024-09-14T03:57:25.126 INFO:tasks.rados.rados.0.smithi070.stdout:2638: finishing write tid 4 to smithi07085154-26 2024-09-14T03:57:25.126 INFO:tasks.rados.rados.0.smithi070.stdout:2638: finishing write tid 5 to smithi07085154-26 2024-09-14T03:57:25.126 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 2260 (ObjNum 671 snap 228 seq_num 671) dirty exists 2024-09-14T03:57:25.126 INFO:tasks.rados.rados.0.smithi070.stdout:2638: finishing write tid 6 to smithi07085154-26 2024-09-14T03:57:25.126 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 2949 (ObjNum 813 snap 283 seq_num 813) dirty exists 2024-09-14T03:57:25.126 INFO:tasks.rados.rados.0.smithi070.stdout:2638: left oid 26 (ObjNum 813 snap 283 seq_num 813) 2024-09-14T03:57:25.126 INFO:tasks.rados.rados.0.smithi070.stdout:2640: finishing rollback tid 1 to smithi07085154-17 2024-09-14T03:57:25.126 INFO:tasks.rados.rados.0.smithi070.stdout:2640: finishing rollback tid 2 to smithi07085154-17 2024-09-14T03:57:25.127 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 2262 (ObjNum 670 snap 228 seq_num 670) dirty exists 2024-09-14T03:57:25.127 INFO:tasks.rados.rados.0.smithi070.stdout:2641: expect (ObjNum 811 snap 283 seq_num 811) 2024-09-14T03:57:25.215 INFO:tasks.rados.rados.0.smithi070.stdout:2644: finishing copy_from racing read to smithi07085154-3 2024-09-14T03:57:25.215 INFO:tasks.rados.rados.0.smithi070.stdout:2644: finishing copy_from to smithi07085154-3 2024-09-14T03:57:25.215 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 3 v 1262 (ObjNum 639 snap 214 seq_num 639) dirty exists 2024-09-14T03:57:25.215 INFO:tasks.rados.rados.0.smithi070.stdout:2642: expect (ObjNum 780 snap 274 seq_num 780) 2024-09-14T03:57:25.267 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:25 smithi094 ceph-mon[86597]: pgmap v516: 121 pgs: 121 active+clean; 1.1 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 1.6 MiB/s wr, 15 op/s 2024-09-14T03:57:25.268 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:25.268 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:25.268 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:57:25.268 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:57:25.268 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:25.268 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:57:25.268 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.268 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.268 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.268 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.268 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.269 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:25.269 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.a"} : dispatch 2024-09-14T03:57:25.269 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.a"}]': finished 2024-09-14T03:57:25.269 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.b"} : dispatch 2024-09-14T03:57:25.269 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.b"}]': finished 2024-09-14T03:57:25.269 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.269 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:25.269 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.269 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:25.269 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.269 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.269 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:25.270 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.270 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:25.270 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.270 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:25.270 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.270 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:25.270 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.270 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.270 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:25 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.322 INFO:tasks.rados.rados.0.smithi070.stdout:2643: finishing copy_from racing read to smithi07085154-40 2024-09-14T03:57:25.322 INFO:tasks.rados.rados.0.smithi070.stdout:2643: finishing copy_from to smithi07085154-40 2024-09-14T03:57:25.322 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 2055 (ObjNum 524 snap 174 seq_num 524) dirty exists 2024-09-14T03:57:25.322 INFO:tasks.rados.rados.0.smithi070.stdout:2647: expect (ObjNum 600 snap 199 seq_num 600) 2024-09-14T03:57:25.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[96704]: pgmap v516: 121 pgs: 121 active+clean; 1.1 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 1.6 MiB/s wr, 15 op/s 2024-09-14T03:57:25.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:25.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:25.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:57:25.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:57:25.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:25.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:57:25.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:25.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.a"} : dispatch 2024-09-14T03:57:25.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.a"}]': finished 2024-09-14T03:57:25.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.b"} : dispatch 2024-09-14T03:57:25.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.b"}]': finished 2024-09-14T03:57:25.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:25.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:25.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:25.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:25.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:25.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:25.539 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.539 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.539 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[98640]: pgmap v516: 121 pgs: 121 active+clean; 1.1 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 1.6 MiB/s wr, 15 op/s 2024-09-14T03:57:25.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:25.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:25.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:57:25.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:57:25.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:25.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:57:25.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:25.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.a"} : dispatch 2024-09-14T03:57:25.541 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.a"}]': finished 2024-09-14T03:57:25.541 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.b"} : dispatch 2024-09-14T03:57:25.541 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.b"}]': finished 2024-09-14T03:57:25.541 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.541 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:25.541 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.541 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:25.541 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.541 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.541 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:25.541 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.541 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:25.541 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.541 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:25.542 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.542 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:25.542 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.542 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.542 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:25 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:25.831 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 45 v 1966 (ObjNum 801 snap 280 seq_num 801) dirty exists 2024-09-14T03:57:25.831 INFO:tasks.rados.rados.0.smithi070.stdout:2638: done (9 left) 2024-09-14T03:57:25.831 INFO:tasks.rados.rados.0.smithi070.stdout:2639: done (8 left) 2024-09-14T03:57:25.831 INFO:tasks.rados.rados.0.smithi070.stdout:2640: done (7 left) 2024-09-14T03:57:25.831 INFO:tasks.rados.rados.0.smithi070.stdout:2641: done (6 left) 2024-09-14T03:57:25.831 INFO:tasks.rados.rados.0.smithi070.stdout:2642: done (5 left) 2024-09-14T03:57:25.831 INFO:tasks.rados.rados.0.smithi070.stdout:2643: done (4 left) 2024-09-14T03:57:25.831 INFO:tasks.rados.rados.0.smithi070.stdout:2644: done (3 left) 2024-09-14T03:57:25.831 INFO:tasks.rados.rados.0.smithi070.stdout:2645: done (2 left) 2024-09-14T03:57:25.831 INFO:tasks.rados.rados.0.smithi070.stdout:2647: done (1 left) 2024-09-14T03:57:25.831 INFO:tasks.rados.rados.0.smithi070.stdout:2648: done (0 left) 2024-09-14T03:57:25.831 INFO:tasks.rados.rados.0.smithi070.stdout:2649: delete oid 37 current snap is 283 2024-09-14T03:57:25.832 INFO:tasks.rados.rados.0.smithi070.stdout:2649: done (0 left) 2024-09-14T03:57:25.832 INFO:tasks.rados.rados.0.smithi070.stdout:2650: delete oid 19 current snap is 283 2024-09-14T03:57:25.835 INFO:tasks.rados.rados.0.smithi070.stdout:2650: done (0 left) 2024-09-14T03:57:25.835 INFO:tasks.rados.rados.0.smithi070.stdout:2651: rmattr oid 35 current snap is 283 2024-09-14T03:57:25.835 INFO:tasks.rados.rados.0.smithi070.stdout:2652: read oid 18 snap 265 2024-09-14T03:57:25.836 INFO:tasks.rados.rados.0.smithi070.stdout:2652: expect (ObjNum 754 snap 260 seq_num 754) 2024-09-14T03:57:25.836 INFO:tasks.rados.rados.0.smithi070.stdout:2653: copy_from oid 27 from oid 15 current snap is 283 2024-09-14T03:57:25.836 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 10 current snap is 283 2024-09-14T03:57:25.836 INFO:tasks.rados.rados.0.smithi070.stdout:2654: seq_num 816 ranges {2564096=475136,3039232=540672,3579904=614400,4194304=49152} 2024-09-14T03:57:25.851 INFO:tasks.rados.rados.0.smithi070.stdout:2654: writing smithi07085154-10 from 2564096 to 3039232 tid 1 2024-09-14T03:57:25.854 INFO:tasks.rados.rados.0.smithi070.stdout:2654: writing smithi07085154-10 from 3039232 to 3579904 tid 2 2024-09-14T03:57:25.858 INFO:tasks.rados.rados.0.smithi070.stdout:2654: writing smithi07085154-10 from 3579904 to 4194304 tid 3 2024-09-14T03:57:25.859 INFO:tasks.rados.rados.0.smithi070.stdout:2654: writing smithi07085154-10 from 4194304 to 4243456 tid 4 2024-09-14T03:57:25.859 INFO:tasks.rados.rados.0.smithi070.stdout:2655: read oid 41 snap -1 2024-09-14T03:57:25.859 INFO:tasks.rados.rados.0.smithi070.stdout:2655: expect deleted 2024-09-14T03:57:25.859 INFO:tasks.rados.rados.0.smithi070.stdout:2656: snap_remove snap 233 2024-09-14T03:57:26.144 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 35 v 1693 (ObjNum 645 snap 216 seq_num 645) dirty exists 2024-09-14T03:57:26.144 INFO:tasks.rados.rados.0.smithi070.stdout:2652: expect (ObjNum 754 snap 260 seq_num 754) 2024-09-14T03:57:26.169 INFO:tasks.rados.rados.0.smithi070.stdout:2653: finishing copy_from racing read to smithi07085154-27 2024-09-14T03:57:26.169 INFO:tasks.rados.rados.0.smithi070.stdout:2653: finishing copy_from to smithi07085154-27 2024-09-14T03:57:26.170 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 2227 (ObjNum 805 snap 281 seq_num 805) dirty exists 2024-09-14T03:57:26.170 INFO:tasks.rados.rados.0.smithi070.stdout:2654: finishing write tid 1 to smithi07085154-10 2024-09-14T03:57:26.170 INFO:tasks.rados.rados.0.smithi070.stdout:2654: finishing write tid 2 to smithi07085154-10 2024-09-14T03:57:26.170 INFO:tasks.rados.rados.0.smithi070.stdout:2654: finishing write tid 3 to smithi07085154-10 2024-09-14T03:57:26.170 INFO:tasks.rados.rados.0.smithi070.stdout:2654: finishing write tid 4 to smithi07085154-10 2024-09-14T03:57:26.170 INFO:tasks.rados.rados.0.smithi070.stdout:2654: finishing write tid 5 to smithi07085154-10 2024-09-14T03:57:26.170 INFO:tasks.rados.rados.0.smithi070.stdout:2654: finishing write tid 6 to smithi07085154-10 2024-09-14T03:57:26.170 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 2973 (ObjNum 816 snap 283 seq_num 816) dirty exists 2024-09-14T03:57:26.170 INFO:tasks.rados.rados.0.smithi070.stdout:2654: left oid 10 (ObjNum 816 snap 283 seq_num 816) 2024-09-14T03:57:26.170 INFO:tasks.rados.rados.0.smithi070.stdout:2651: done (5 left) 2024-09-14T03:57:26.170 INFO:tasks.rados.rados.0.smithi070.stdout:2652: done (4 left) 2024-09-14T03:57:26.170 INFO:tasks.rados.rados.0.smithi070.stdout:2653: done (3 left) 2024-09-14T03:57:26.170 INFO:tasks.rados.rados.0.smithi070.stdout:2654: done (2 left) 2024-09-14T03:57:26.171 INFO:tasks.rados.rados.0.smithi070.stdout:2655: done (1 left) 2024-09-14T03:57:26.171 INFO:tasks.rados.rados.0.smithi070.stdout:2656: done (0 left) 2024-09-14T03:57:26.171 INFO:tasks.rados.rados.0.smithi070.stdout:2657: copy_from oid 19 from oid 31 current snap is 283 2024-09-14T03:57:26.171 INFO:tasks.rados.rados.0.smithi070.stdout:2658: read oid 40 snap 260 2024-09-14T03:57:26.171 INFO:tasks.rados.rados.0.smithi070.stdout:2658: expect deleted 2024-09-14T03:57:26.171 INFO:tasks.rados.rados.0.smithi070.stdout:2659: delete oid 25 current snap is 283 2024-09-14T03:57:26.172 INFO:tasks.rados.rados.0.smithi070.stdout:2657: finishing copy_from to smithi07085154-19 2024-09-14T03:57:26.172 INFO:tasks.rados.rados.0.smithi070.stdout:2657: got expected ENOENT (src dne) 2024-09-14T03:57:26.172 INFO:tasks.rados.rados.0.smithi070.stdout:2657: finishing copy_from racing read to smithi07085154-19 2024-09-14T03:57:26.172 INFO:tasks.rados.rados.0.smithi070.stdout:2657: done (2 left) 2024-09-14T03:57:26.172 INFO:tasks.rados.rados.0.smithi070.stdout:2658: done (1 left) 2024-09-14T03:57:26.172 INFO:tasks.rados.rados.0.smithi070.stdout:2659: done (0 left) 2024-09-14T03:57:26.172 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 39 current snap is 283 2024-09-14T03:57:26.172 INFO:tasks.rados.rados.0.smithi070.stdout:2660: seq_num 818 ranges {0=507904,507904=704512,1212416=499712} 2024-09-14T03:57:26.174 INFO:tasks.rados.rados.0.smithi070.stdout:2660: writing smithi07085154-39 from 0 to 507904 tid 1 2024-09-14T03:57:26.178 INFO:tasks.rados.rados.0.smithi070.stdout:2660: writing smithi07085154-39 from 507904 to 1212416 tid 2 2024-09-14T03:57:26.181 INFO:tasks.rados.rados.0.smithi070.stdout:2660: writing smithi07085154-39 from 1212416 to 1712128 tid 3 2024-09-14T03:57:26.181 INFO:tasks.rados.rados.0.smithi070.stdout:2661: delete oid 7 current snap is 283 2024-09-14T03:57:26.181 INFO:tasks.rados.rados.0.smithi070.stdout:2660: finishing write tid 1 to smithi07085154-39 2024-09-14T03:57:26.181 INFO:tasks.rados.rados.0.smithi070.stdout:2660: finishing write tid 2 to smithi07085154-39 2024-09-14T03:57:26.183 INFO:tasks.rados.rados.0.smithi070.stdout:2661: done (1 left) 2024-09-14T03:57:26.183 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 6 current snap is 283 2024-09-14T03:57:26.184 INFO:tasks.rados.rados.0.smithi070.stdout:2662: seq_num 820 ranges {9338880=540672,9879552=475136,10354688=598016,10952704=131072} 2024-09-14T03:57:26.236 INFO:tasks.rados.rados.0.smithi070.stdout:2662: writing smithi07085154-6 from 9338880 to 9879552 tid 1 2024-09-14T03:57:26.238 INFO:tasks.rados.rados.0.smithi070.stdout:2662: writing smithi07085154-6 from 9879552 to 10354688 tid 2 2024-09-14T03:57:26.242 INFO:tasks.rados.rados.0.smithi070.stdout:2662: writing smithi07085154-6 from 10354688 to 10952704 tid 3 2024-09-14T03:57:26.243 INFO:tasks.rados.rados.0.smithi070.stdout:2662: writing smithi07085154-6 from 10952704 to 11083776 tid 4 2024-09-14T03:57:26.243 INFO:tasks.rados.rados.0.smithi070.stdout:2663: delete oid 25 current snap is 283 2024-09-14T03:57:26.243 INFO:tasks.rados.rados.0.smithi070.stdout:2660: finishing write tid 3 to smithi07085154-39 2024-09-14T03:57:26.243 INFO:tasks.rados.rados.0.smithi070.stdout:2660: finishing write tid 4 to smithi07085154-39 2024-09-14T03:57:26.243 INFO:tasks.rados.rados.0.smithi070.stdout:2660: finishing write tid 5 to smithi07085154-39 2024-09-14T03:57:26.243 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 39 v 2635 (ObjNum 818 snap 283 seq_num 818) dirty exists 2024-09-14T03:57:26.243 INFO:tasks.rados.rados.0.smithi070.stdout:2660: left oid 39 (ObjNum 818 snap 283 seq_num 818) 2024-09-14T03:57:26.243 INFO:tasks.rados.rados.0.smithi070.stdout:2662: finishing write tid 1 to smithi07085154-6 2024-09-14T03:57:26.243 INFO:tasks.rados.rados.0.smithi070.stdout:2662: finishing write tid 2 to smithi07085154-6 2024-09-14T03:57:26.247 INFO:tasks.rados.rados.0.smithi070.stdout:2662: finishing write tid 3 to smithi07085154-6 2024-09-14T03:57:26.247 INFO:tasks.rados.rados.0.smithi070.stdout:2662: finishing write tid 4 to smithi07085154-6 2024-09-14T03:57:26.247 INFO:tasks.rados.rados.0.smithi070.stdout:2662: finishing write tid 5 to smithi07085154-6 2024-09-14T03:57:26.249 INFO:tasks.rados.rados.0.smithi070.stdout:2662: finishing write tid 6 to smithi07085154-6 2024-09-14T03:57:26.249 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 2267 (ObjNum 820 snap 283 seq_num 820) dirty exists 2024-09-14T03:57:26.249 INFO:tasks.rados.rados.0.smithi070.stdout:2662: left oid 6 (ObjNum 820 snap 283 seq_num 820) 2024-09-14T03:57:26.250 INFO:tasks.rados.rados.0.smithi070.stdout:2660: done (2 left) 2024-09-14T03:57:26.250 INFO:tasks.rados.rados.0.smithi070.stdout:2662: done (1 left) 2024-09-14T03:57:26.250 INFO:tasks.rados.rados.0.smithi070.stdout:2663: done (0 left) 2024-09-14T03:57:26.250 INFO:tasks.rados.rados.0.smithi070.stdout:2664: snap_remove snap 217 2024-09-14T03:57:26.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:26 smithi094 ceph-mon[86597]: Detected new or changed devices on smithi094 2024-09-14T03:57:26.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:26 smithi094 ceph-mon[86597]: Upgrade: Setting container_image for all mds 2024-09-14T03:57:26.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:26 smithi094 ceph-mon[86597]: Upgrade: Setting container_image for all rgw 2024-09-14T03:57:26.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:26 smithi094 ceph-mon[86597]: Upgrade: Setting container_image for all rbd-mirror 2024-09-14T03:57:26.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:26 smithi094 ceph-mon[86597]: Upgrade: Setting container_image for all ceph-exporter 2024-09-14T03:57:26.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:26 smithi094 ceph-mon[86597]: Upgrade: Setting container_image for all iscsi 2024-09-14T03:57:26.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:26 smithi094 ceph-mon[86597]: Upgrade: Setting container_image for all nfs 2024-09-14T03:57:26.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:26 smithi094 ceph-mon[86597]: Upgrade: Setting container_image for all nvmeof 2024-09-14T03:57:26.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:26 smithi094 ceph-mon[86597]: osdmap e614: 8 total, 8 up, 8 in 2024-09-14T03:57:26.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:26 smithi070 ceph-mon[96704]: Detected new or changed devices on smithi094 2024-09-14T03:57:26.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:26 smithi070 ceph-mon[96704]: Upgrade: Setting container_image for all mds 2024-09-14T03:57:26.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:26 smithi070 ceph-mon[96704]: Upgrade: Setting container_image for all rgw 2024-09-14T03:57:26.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:26 smithi070 ceph-mon[96704]: Upgrade: Setting container_image for all rbd-mirror 2024-09-14T03:57:26.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:26 smithi070 ceph-mon[96704]: Upgrade: Setting container_image for all ceph-exporter 2024-09-14T03:57:26.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:26 smithi070 ceph-mon[96704]: Upgrade: Setting container_image for all iscsi 2024-09-14T03:57:26.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:26 smithi070 ceph-mon[96704]: Upgrade: Setting container_image for all nfs 2024-09-14T03:57:26.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:26 smithi070 ceph-mon[96704]: Upgrade: Setting container_image for all nvmeof 2024-09-14T03:57:26.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:26 smithi070 ceph-mon[96704]: osdmap e614: 8 total, 8 up, 8 in 2024-09-14T03:57:26.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:26 smithi070 ceph-mon[98640]: Detected new or changed devices on smithi094 2024-09-14T03:57:26.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:26 smithi070 ceph-mon[98640]: Upgrade: Setting container_image for all mds 2024-09-14T03:57:26.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:26 smithi070 ceph-mon[98640]: Upgrade: Setting container_image for all rgw 2024-09-14T03:57:26.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:26 smithi070 ceph-mon[98640]: Upgrade: Setting container_image for all rbd-mirror 2024-09-14T03:57:26.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:26 smithi070 ceph-mon[98640]: Upgrade: Setting container_image for all ceph-exporter 2024-09-14T03:57:26.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:26 smithi070 ceph-mon[98640]: Upgrade: Setting container_image for all iscsi 2024-09-14T03:57:26.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:26 smithi070 ceph-mon[98640]: Upgrade: Setting container_image for all nfs 2024-09-14T03:57:26.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:26 smithi070 ceph-mon[98640]: Upgrade: Setting container_image for all nvmeof 2024-09-14T03:57:26.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:26 smithi070 ceph-mon[98640]: osdmap e614: 8 total, 8 up, 8 in 2024-09-14T03:57:27.143 INFO:tasks.rados.rados.0.smithi070.stdout:2664: done (0 left) 2024-09-14T03:57:27.143 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 22 current snap is 283 2024-09-14T03:57:27.143 INFO:tasks.rados.rados.0.smithi070.stdout:2665: seq_num 822 ranges {6578176=573440,7151616=696320,7847936=507904} 2024-09-14T03:57:27.183 INFO:tasks.rados.rados.0.smithi070.stdout:2665: writing smithi07085154-22 from 6578176 to 7151616 tid 1 2024-09-14T03:57:27.187 INFO:tasks.rados.rados.0.smithi070.stdout:2665: writing smithi07085154-22 from 7151616 to 7847936 tid 2 2024-09-14T03:57:27.190 INFO:tasks.rados.rados.0.smithi070.stdout:2665: writing smithi07085154-22 from 7847936 to 8355840 tid 3 2024-09-14T03:57:27.191 INFO:tasks.rados.rados.0.smithi070.stdout:2666: read oid 40 snap -1 2024-09-14T03:57:27.191 INFO:tasks.rados.rados.0.smithi070.stdout:2666: expect (ObjNum 524 snap 174 seq_num 524) 2024-09-14T03:57:27.191 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 23 current snap is 283 2024-09-14T03:57:27.191 INFO:tasks.rados.rados.0.smithi070.stdout:2665: finishing write tid 1 to smithi07085154-22 2024-09-14T03:57:27.191 INFO:tasks.rados.rados.0.smithi070.stdout:2667: seq_num 823 ranges {0=589824,589824=573440,1163264=630784} 2024-09-14T03:57:27.194 INFO:tasks.rados.rados.0.smithi070.stdout:2667: writing smithi07085154-23 from 0 to 589824 tid 1 2024-09-14T03:57:27.197 INFO:tasks.rados.rados.0.smithi070.stdout:2667: writing smithi07085154-23 from 589824 to 1163264 tid 2 2024-09-14T03:57:27.201 INFO:tasks.rados.rados.0.smithi070.stdout:2667: writing smithi07085154-23 from 1163264 to 1794048 tid 3 2024-09-14T03:57:27.201 INFO:tasks.rados.rados.0.smithi070.stdout:2668: snap_remove snap 255 2024-09-14T03:57:27.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:27 smithi094 ceph-mon[86597]: Upgrade: Updating grafana.a 2024-09-14T03:57:27.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:27 smithi094 ceph-mon[86597]: pgmap v518: 121 pgs: 121 active+clean; 1.1 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 364 KiB/s wr, 4 op/s 2024-09-14T03:57:27.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:27 smithi094 ceph-mon[86597]: osdmap e615: 8 total, 8 up, 8 in 2024-09-14T03:57:27.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:27 smithi070 ceph-mon[98640]: Upgrade: Updating grafana.a 2024-09-14T03:57:27.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:27 smithi070 ceph-mon[98640]: pgmap v518: 121 pgs: 121 active+clean; 1.1 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 364 KiB/s wr, 4 op/s 2024-09-14T03:57:27.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:27 smithi070 ceph-mon[98640]: osdmap e615: 8 total, 8 up, 8 in 2024-09-14T03:57:27.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:27 smithi070 ceph-mon[96704]: Upgrade: Updating grafana.a 2024-09-14T03:57:27.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:27 smithi070 ceph-mon[96704]: pgmap v518: 121 pgs: 121 active+clean; 1.1 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 364 KiB/s wr, 4 op/s 2024-09-14T03:57:27.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:27 smithi070 ceph-mon[96704]: osdmap e615: 8 total, 8 up, 8 in 2024-09-14T03:57:28.149 INFO:tasks.rados.rados.0.smithi070.stdout:2668: done (3 left) 2024-09-14T03:57:28.150 INFO:tasks.rados.rados.0.smithi070.stdout:2669: snap_remove snap 256 2024-09-14T03:57:28.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:28 smithi094 ceph-mon[86597]: Deploying daemon grafana.a on smithi094 2024-09-14T03:57:28.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:28 smithi094 ceph-mon[86597]: osdmap e616: 8 total, 8 up, 8 in 2024-09-14T03:57:28.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:28 smithi070 ceph-mon[98640]: Deploying daemon grafana.a on smithi094 2024-09-14T03:57:28.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:28 smithi070 ceph-mon[98640]: osdmap e616: 8 total, 8 up, 8 in 2024-09-14T03:57:28.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:28 smithi070 ceph-mon[96704]: Deploying daemon grafana.a on smithi094 2024-09-14T03:57:28.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:28 smithi070 ceph-mon[96704]: osdmap e616: 8 total, 8 up, 8 in 2024-09-14T03:57:29.085 INFO:tasks.rados.rados.0.smithi070.stdout:2669: done (3 left) 2024-09-14T03:57:29.085 INFO:tasks.rados.rados.0.smithi070.stdout:2670: delete oid 8 current snap is 283 2024-09-14T03:57:29.085 INFO:tasks.rados.rados.0.smithi070.stdout:2665: finishing write tid 2 to smithi07085154-22 2024-09-14T03:57:29.085 INFO:tasks.rados.rados.0.smithi070.stdout:2665: finishing write tid 3 to smithi07085154-22 2024-09-14T03:57:29.085 INFO:tasks.rados.rados.0.smithi070.stdout:2665: finishing write tid 4 to smithi07085154-22 2024-09-14T03:57:29.085 INFO:tasks.rados.rados.0.smithi070.stdout:2665: finishing write tid 5 to smithi07085154-22 2024-09-14T03:57:29.085 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 2229 (ObjNum 822 snap 283 seq_num 822) dirty exists 2024-09-14T03:57:29.085 INFO:tasks.rados.rados.0.smithi070.stdout:2665: left oid 22 (ObjNum 822 snap 283 seq_num 822) 2024-09-14T03:57:29.085 INFO:tasks.rados.rados.0.smithi070.stdout:2667: finishing write tid 1 to smithi07085154-23 2024-09-14T03:57:29.085 INFO:tasks.rados.rados.0.smithi070.stdout:2667: finishing write tid 2 to smithi07085154-23 2024-09-14T03:57:29.085 INFO:tasks.rados.rados.0.smithi070.stdout:2667: finishing write tid 3 to smithi07085154-23 2024-09-14T03:57:29.085 INFO:tasks.rados.rados.0.smithi070.stdout:2667: finishing write tid 4 to smithi07085154-23 2024-09-14T03:57:29.086 INFO:tasks.rados.rados.0.smithi070.stdout:2667: finishing write tid 5 to smithi07085154-23 2024-09-14T03:57:29.086 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 2779 (ObjNum 823 snap 283 seq_num 823) dirty exists 2024-09-14T03:57:29.086 INFO:tasks.rados.rados.0.smithi070.stdout:2667: left oid 23 (ObjNum 823 snap 283 seq_num 823) 2024-09-14T03:57:29.086 INFO:tasks.rados.rados.0.smithi070.stdout:2666: expect (ObjNum 524 snap 174 seq_num 524) 2024-09-14T03:57:29.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:29 smithi094 ceph-mon[86597]: pgmap v521: 121 pgs: 121 active+clean; 1.1 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 2.7 MiB/s wr, 26 op/s 2024-09-14T03:57:29.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:29 smithi094 ceph-mon[86597]: osdmap e617: 8 total, 8 up, 8 in 2024-09-14T03:57:29.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:29 smithi094 ceph-mon[86597]: osdmap e618: 8 total, 8 up, 8 in 2024-09-14T03:57:29.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:29 smithi070 ceph-mon[98640]: pgmap v521: 121 pgs: 121 active+clean; 1.1 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 2.7 MiB/s wr, 26 op/s 2024-09-14T03:57:29.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:29 smithi070 ceph-mon[98640]: osdmap e617: 8 total, 8 up, 8 in 2024-09-14T03:57:29.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:29 smithi070 ceph-mon[98640]: osdmap e618: 8 total, 8 up, 8 in 2024-09-14T03:57:29.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:29 smithi070 ceph-mon[96704]: pgmap v521: 121 pgs: 121 active+clean; 1.1 GiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 2.7 MiB/s wr, 26 op/s 2024-09-14T03:57:29.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:29 smithi070 ceph-mon[96704]: osdmap e617: 8 total, 8 up, 8 in 2024-09-14T03:57:29.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:29 smithi070 ceph-mon[96704]: osdmap e618: 8 total, 8 up, 8 in 2024-09-14T03:57:29.538 INFO:tasks.rados.rados.0.smithi070.stdout:2665: done (3 left) 2024-09-14T03:57:29.538 INFO:tasks.rados.rados.0.smithi070.stdout:2666: done (2 left) 2024-09-14T03:57:29.538 INFO:tasks.rados.rados.0.smithi070.stdout:2667: done (1 left) 2024-09-14T03:57:29.538 INFO:tasks.rados.rados.0.smithi070.stdout:2670: done (0 left) 2024-09-14T03:57:29.538 INFO:tasks.rados.rados.0.smithi070.stdout:2671: delete oid 19 current snap is 283 2024-09-14T03:57:29.541 INFO:tasks.rados.rados.0.smithi070.stdout:2671: done (0 left) 2024-09-14T03:57:29.541 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 18 current snap is 283 2024-09-14T03:57:29.541 INFO:tasks.rados.rados.0.smithi070.stdout:2672: seq_num 826 ranges {630784=638976,1269760=630784,1900544=573440} 2024-09-14T03:57:29.547 INFO:tasks.rados.rados.0.smithi070.stdout:2672: writing smithi07085154-18 from 630784 to 1269760 tid 1 2024-09-14T03:57:29.551 INFO:tasks.rados.rados.0.smithi070.stdout:2672: writing smithi07085154-18 from 1269760 to 1900544 tid 2 2024-09-14T03:57:29.554 INFO:tasks.rados.rados.0.smithi070.stdout:2672: writing smithi07085154-18 from 1900544 to 2473984 tid 3 2024-09-14T03:57:29.555 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 14 current snap is 283 2024-09-14T03:57:29.555 INFO:tasks.rados.rados.0.smithi070.stdout:2672: finishing write tid 1 to smithi07085154-18 2024-09-14T03:57:29.555 INFO:tasks.rados.rados.0.smithi070.stdout:2672: finishing write tid 2 to smithi07085154-18 2024-09-14T03:57:29.555 INFO:tasks.rados.rados.0.smithi070.stdout:2673: seq_num 827 ranges {0=655360,655360=516096,1171456=688128} 2024-09-14T03:57:29.558 INFO:tasks.rados.rados.0.smithi070.stdout:2673: writing smithi07085154-14 from 0 to 655360 tid 1 2024-09-14T03:57:29.561 INFO:tasks.rados.rados.0.smithi070.stdout:2673: writing smithi07085154-14 from 655360 to 1171456 tid 2 2024-09-14T03:57:29.565 INFO:tasks.rados.rados.0.smithi070.stdout:2673: writing smithi07085154-14 from 1171456 to 1859584 tid 3 2024-09-14T03:57:29.565 INFO:tasks.rados.rados.0.smithi070.stdout:2674: delete oid 35 current snap is 283 2024-09-14T03:57:29.565 INFO:tasks.rados.rados.0.smithi070.stdout:2672: finishing write tid 3 to smithi07085154-18 2024-09-14T03:57:29.565 INFO:tasks.rados.rados.0.smithi070.stdout:2672: finishing write tid 4 to smithi07085154-18 2024-09-14T03:57:29.566 INFO:tasks.rados.rados.0.smithi070.stdout:2672: finishing write tid 5 to smithi07085154-18 2024-09-14T03:57:29.566 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 2232 (ObjNum 826 snap 283 seq_num 826) dirty exists 2024-09-14T03:57:29.566 INFO:tasks.rados.rados.0.smithi070.stdout:2672: left oid 18 (ObjNum 826 snap 283 seq_num 826) 2024-09-14T03:57:29.566 INFO:tasks.rados.rados.0.smithi070.stdout:2673: finishing write tid 1 to smithi07085154-14 2024-09-14T03:57:29.566 INFO:tasks.rados.rados.0.smithi070.stdout:2673: finishing write tid 2 to smithi07085154-14 2024-09-14T03:57:29.566 INFO:tasks.rados.rados.0.smithi070.stdout:2672: done (2 left) 2024-09-14T03:57:29.566 INFO:tasks.rados.rados.0.smithi070.stdout:2674: done (1 left) 2024-09-14T03:57:29.566 INFO:tasks.rados.rados.0.smithi070.stdout:2675: setattr oid 35 current snap is 283 2024-09-14T03:57:29.567 INFO:tasks.rados.rados.0.smithi070.stdout:2676: copy_from oid 26 from oid 1 current snap is 283 2024-09-14T03:57:29.567 INFO:tasks.rados.rados.0.smithi070.stdout:2677: copy_from oid 17 from oid 29 current snap is 283 2024-09-14T03:57:29.567 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 27 current snap is 283 2024-09-14T03:57:29.567 INFO:tasks.rados.rados.0.smithi070.stdout:2678: seq_num 829 ranges {1490944=688128,2179072=729088,2908160=475136} 2024-09-14T03:57:29.580 INFO:tasks.rados.rados.0.smithi070.stdout:2678: writing smithi07085154-27 from 1490944 to 2179072 tid 1 2024-09-14T03:57:29.584 INFO:tasks.rados.rados.0.smithi070.stdout:2678: writing smithi07085154-27 from 2179072 to 2908160 tid 2 2024-09-14T03:57:29.587 INFO:tasks.rados.rados.0.smithi070.stdout:2678: writing smithi07085154-27 from 2908160 to 3383296 tid 3 2024-09-14T03:57:29.587 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 2 current snap is 283 2024-09-14T03:57:29.587 INFO:tasks.rados.rados.0.smithi070.stdout:2673: finishing write tid 3 to smithi07085154-14 2024-09-14T03:57:29.587 INFO:tasks.rados.rados.0.smithi070.stdout:2679: seq_num 830 ranges {868352=704512,1572864=614400,2187264=573440,2760704=16384} 2024-09-14T03:57:29.595 INFO:tasks.rados.rados.0.smithi070.stdout:2679: writing smithi07085154-2 from 868352 to 1572864 tid 1 2024-09-14T03:57:29.598 INFO:tasks.rados.rados.0.smithi070.stdout:2679: writing smithi07085154-2 from 1572864 to 2187264 tid 2 2024-09-14T03:57:29.603 INFO:tasks.rados.rados.0.smithi070.stdout:2679: writing smithi07085154-2 from 2187264 to 2760704 tid 3 2024-09-14T03:57:29.603 INFO:tasks.rados.rados.0.smithi070.stdout:2679: writing smithi07085154-2 from 2760704 to 2777088 tid 4 2024-09-14T03:57:29.603 INFO:tasks.rados.rados.0.smithi070.stdout:2680: read oid 9 snap -1 2024-09-14T03:57:29.603 INFO:tasks.rados.rados.0.smithi070.stdout:2680: expect (ObjNum 807 snap 282 seq_num 807) 2024-09-14T03:57:29.603 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 19 current snap is 283 2024-09-14T03:57:29.603 INFO:tasks.rados.rados.0.smithi070.stdout:2673: finishing write tid 4 to smithi07085154-14 2024-09-14T03:57:29.603 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 35 v 1695 (ObjNum 289 snap 0 seq_num 1) dirty exists 2024-09-14T03:57:29.603 INFO:tasks.rados.rados.0.smithi070.stdout:2673: finishing write tid 5 to smithi07085154-14 2024-09-14T03:57:29.603 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 2783 (ObjNum 827 snap 283 seq_num 827) dirty exists 2024-09-14T03:57:29.604 INFO:tasks.rados.rados.0.smithi070.stdout:2673: left oid 14 (ObjNum 827 snap 283 seq_num 827) 2024-09-14T03:57:29.604 INFO:tasks.rados.rados.0.smithi070.stdout:2677: finishing copy_from racing read to smithi07085154-17 2024-09-14T03:57:29.604 INFO:tasks.rados.rados.0.smithi070.stdout:2677: finishing copy_from to smithi07085154-17 2024-09-14T03:57:29.604 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 2224 (ObjNum 97 snap 0 seq_num 2930931040) dirty exists 2024-09-14T03:57:29.604 INFO:tasks.rados.rados.0.smithi070.stdout:2676: finishing copy_from racing read to smithi07085154-26 2024-09-14T03:57:29.604 INFO:tasks.rados.rados.0.smithi070.stdout:2676: finishing copy_from to smithi07085154-26 2024-09-14T03:57:29.604 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 2054 (ObjNum 712 snap 243 seq_num 712) dirty exists 2024-09-14T03:57:29.604 INFO:tasks.rados.rados.0.smithi070.stdout:2678: finishing write tid 1 to smithi07085154-27 2024-09-14T03:57:29.604 INFO:tasks.rados.rados.0.smithi070.stdout:2681: seq_num 831 ranges {0=720896,720896=499712,1220608=704512} 2024-09-14T03:57:29.606 INFO:tasks.rados.rados.0.smithi070.stdout:2681: writing smithi07085154-19 from 0 to 720896 tid 1 2024-09-14T03:57:29.609 INFO:tasks.rados.rados.0.smithi070.stdout:2681: writing smithi07085154-19 from 720896 to 1220608 tid 2 2024-09-14T03:57:29.613 INFO:tasks.rados.rados.0.smithi070.stdout:2681: writing smithi07085154-19 from 1220608 to 1925120 tid 3 2024-09-14T03:57:29.614 INFO:tasks.rados.rados.0.smithi070.stdout:2673: done (7 left) 2024-09-14T03:57:29.614 INFO:tasks.rados.rados.0.smithi070.stdout:2675: done (6 left) 2024-09-14T03:57:29.614 INFO:tasks.rados.rados.0.smithi070.stdout:2676: done (5 left) 2024-09-14T03:57:29.614 INFO:tasks.rados.rados.0.smithi070.stdout:2677: done (4 left) 2024-09-14T03:57:29.614 INFO:tasks.rados.rados.0.smithi070.stdout:2682: setattr oid 28 current snap is 283 2024-09-14T03:57:29.614 INFO:tasks.rados.rados.0.smithi070.stdout:2678: finishing write tid 2 to smithi07085154-27 2024-09-14T03:57:29.614 INFO:tasks.rados.rados.0.smithi070.stdout:2678: finishing write tid 3 to smithi07085154-27 2024-09-14T03:57:29.614 INFO:tasks.rados.rados.0.smithi070.stdout:2678: finishing write tid 4 to smithi07085154-27 2024-09-14T03:57:29.614 INFO:tasks.rados.rados.0.smithi070.stdout:2678: finishing write tid 5 to smithi07085154-27 2024-09-14T03:57:29.614 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 3036 (ObjNum 829 snap 283 seq_num 829) dirty exists 2024-09-14T03:57:29.614 INFO:tasks.rados.rados.0.smithi070.stdout:2678: left oid 27 (ObjNum 829 snap 283 seq_num 829) 2024-09-14T03:57:29.615 INFO:tasks.rados.rados.0.smithi070.stdout:2679: finishing write tid 1 to smithi07085154-2 2024-09-14T03:57:29.615 INFO:tasks.rados.rados.0.smithi070.stdout:2679: finishing write tid 2 to smithi07085154-2 2024-09-14T03:57:29.615 INFO:tasks.rados.rados.0.smithi070.stdout:2679: finishing write tid 3 to smithi07085154-2 2024-09-14T03:57:29.615 INFO:tasks.rados.rados.0.smithi070.stdout:2679: finishing write tid 4 to smithi07085154-2 2024-09-14T03:57:29.615 INFO:tasks.rados.rados.0.smithi070.stdout:2679: finishing write tid 5 to smithi07085154-2 2024-09-14T03:57:29.615 INFO:tasks.rados.rados.0.smithi070.stdout:2679: finishing write tid 6 to smithi07085154-2 2024-09-14T03:57:29.615 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 2 v 3042 (ObjNum 830 snap 283 seq_num 830) dirty exists 2024-09-14T03:57:29.615 INFO:tasks.rados.rados.0.smithi070.stdout:2679: left oid 2 (ObjNum 830 snap 283 seq_num 830) 2024-09-14T03:57:29.615 INFO:tasks.rados.rados.0.smithi070.stdout:2681: finishing write tid 1 to smithi07085154-19 2024-09-14T03:57:29.615 INFO:tasks.rados.rados.0.smithi070.stdout:2681: finishing write tid 2 to smithi07085154-19 2024-09-14T03:57:29.616 INFO:tasks.rados.rados.0.smithi070.stdout:2678: done (4 left) 2024-09-14T03:57:29.616 INFO:tasks.rados.rados.0.smithi070.stdout:2679: done (3 left) 2024-09-14T03:57:29.616 INFO:tasks.rados.rados.0.smithi070.stdout:2683: snap_create 2024-09-14T03:57:29.618 INFO:tasks.rados.rados.0.smithi070.stdout:2681: finishing write tid 3 to smithi07085154-19 2024-09-14T03:57:29.618 INFO:tasks.rados.rados.0.smithi070.stdout:2681: finishing write tid 4 to smithi07085154-19 2024-09-14T03:57:29.619 INFO:tasks.rados.rados.0.smithi070.stdout:2681: finishing write tid 5 to smithi07085154-19 2024-09-14T03:57:29.619 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 19 v 2233 (ObjNum 831 snap 283 seq_num 831) dirty exists 2024-09-14T03:57:29.619 INFO:tasks.rados.rados.0.smithi070.stdout:2681: left oid 19 (ObjNum 831 snap 283 seq_num 831) 2024-09-14T03:57:29.622 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 28 v 2066 (ObjNum 639 snap 214 seq_num 639) dirty exists 2024-09-14T03:57:29.628 INFO:tasks.rados.rados.0.smithi070.stdout:2680: expect (ObjNum 807 snap 282 seq_num 807) 2024-09-14T03:57:30.145 INFO:tasks.rados.rados.0.smithi070.stdout:2680: done (3 left) 2024-09-14T03:57:30.145 INFO:tasks.rados.rados.0.smithi070.stdout:2681: done (2 left) 2024-09-14T03:57:30.145 INFO:tasks.rados.rados.0.smithi070.stdout:2682: done (1 left) 2024-09-14T03:57:30.145 INFO:tasks.rados.rados.0.smithi070.stdout:2683: done (0 left) 2024-09-14T03:57:30.145 INFO:tasks.rados.rados.0.smithi070.stdout:2684: rollback oid 14 current snap is 284 2024-09-14T03:57:30.145 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 14 to 245 2024-09-14T03:57:30.146 INFO:tasks.rados.rados.0.smithi070.stdout:2685: snap_create 2024-09-14T03:57:30.147 INFO:tasks.rados.rados.0.smithi070.stdout:2684: finishing rollback tid 0 to smithi07085154-14 2024-09-14T03:57:30.148 INFO:tasks.rados.rados.0.smithi070.stdout:2684: finishing rollback tid 1 to smithi07085154-14 2024-09-14T03:57:30.149 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 2785 (ObjNum 33 snap 0 seq_num 1851349334) dirty dne 2024-09-14T03:57:30.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:30 smithi094 ceph-mon[86597]: pgmap v524: 121 pgs: 121 active+clean; 1.1 GiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 5.6 MiB/s wr, 41 op/s 2024-09-14T03:57:30.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:30 smithi094 ceph-mon[86597]: osdmap e619: 8 total, 8 up, 8 in 2024-09-14T03:57:30.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:57:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:57:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:57:30.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:30 smithi070 ceph-mon[96704]: pgmap v524: 121 pgs: 121 active+clean; 1.1 GiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 5.6 MiB/s wr, 41 op/s 2024-09-14T03:57:30.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:30 smithi070 ceph-mon[96704]: osdmap e619: 8 total, 8 up, 8 in 2024-09-14T03:57:30.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:30 smithi070 ceph-mon[98640]: pgmap v524: 121 pgs: 121 active+clean; 1.1 GiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 5.6 MiB/s wr, 41 op/s 2024-09-14T03:57:30.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:30 smithi070 ceph-mon[98640]: osdmap e619: 8 total, 8 up, 8 in 2024-09-14T03:57:31.147 INFO:tasks.rados.rados.0.smithi070.stdout:2684: done (1 left) 2024-09-14T03:57:31.147 INFO:tasks.rados.rados.0.smithi070.stdout:2685: done (0 left) 2024-09-14T03:57:31.147 INFO:tasks.rados.rados.0.smithi070.stdout:2686: read oid 49 snap -1 2024-09-14T03:57:31.147 INFO:tasks.rados.rados.0.smithi070.stdout:2686: expect (ObjNum 729 snap 253 seq_num 729) 2024-09-14T03:57:31.147 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 48 current snap is 285 2024-09-14T03:57:31.147 INFO:tasks.rados.rados.0.smithi070.stdout:2687: seq_num 832 ranges {4538368=737280,5275648=786432,6062080=417792} 2024-09-14T03:57:31.177 INFO:tasks.rados.rados.0.smithi070.stdout:2687: writing smithi07085154-48 from 4538368 to 5275648 tid 1 2024-09-14T03:57:31.182 INFO:tasks.rados.rados.0.smithi070.stdout:2687: writing smithi07085154-48 from 5275648 to 6062080 tid 2 2024-09-14T03:57:31.185 INFO:tasks.rados.rados.0.smithi070.stdout:2687: writing smithi07085154-48 from 6062080 to 6479872 tid 3 2024-09-14T03:57:31.185 INFO:tasks.rados.rados.0.smithi070.stdout:2688: rmattr oid 11 current snap is 285 2024-09-14T03:57:31.186 INFO:tasks.rados.rados.0.smithi070.stdout:2686: expect (ObjNum 729 snap 253 seq_num 729) 2024-09-14T03:57:32.068 INFO:tasks.rados.rados.0.smithi070.stdout:2687: finishing write tid 1 to smithi07085154-48 2024-09-14T03:57:32.069 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 11 v 2257 (ObjNum 771 snap 270 seq_num 771) dirty exists 2024-09-14T03:57:32.069 INFO:tasks.rados.rados.0.smithi070.stdout:2687: finishing write tid 2 to smithi07085154-48 2024-09-14T03:57:32.069 INFO:tasks.rados.rados.0.smithi070.stdout:2687: finishing write tid 3 to smithi07085154-48 2024-09-14T03:57:32.069 INFO:tasks.rados.rados.0.smithi070.stdout:2687: finishing write tid 4 to smithi07085154-48 2024-09-14T03:57:32.069 INFO:tasks.rados.rados.0.smithi070.stdout:2687: finishing write tid 5 to smithi07085154-48 2024-09-14T03:57:32.069 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 2216 (ObjNum 832 snap 285 seq_num 832) dirty exists 2024-09-14T03:57:32.069 INFO:tasks.rados.rados.0.smithi070.stdout:2687: left oid 48 (ObjNum 832 snap 285 seq_num 832) 2024-09-14T03:57:32.069 INFO:tasks.rados.rados.0.smithi070.stdout:2686: done (2 left) 2024-09-14T03:57:32.069 INFO:tasks.rados.rados.0.smithi070.stdout:2687: done (1 left) 2024-09-14T03:57:32.069 INFO:tasks.rados.rados.0.smithi070.stdout:2688: done (0 left) 2024-09-14T03:57:32.069 INFO:tasks.rados.rados.0.smithi070.stdout:2689: read oid 8 snap -1 2024-09-14T03:57:32.069 INFO:tasks.rados.rados.0.smithi070.stdout:2689: expect deleted 2024-09-14T03:57:32.070 INFO:tasks.rados.rados.0.smithi070.stdout:2690: snap_remove snap 186 2024-09-14T03:57:32.151 INFO:tasks.rados.rados.0.smithi070.stdout:2690: done (1 left) 2024-09-14T03:57:32.151 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 21 current snap is 285 2024-09-14T03:57:32.151 INFO:tasks.rados.rados.0.smithi070.stdout:2691: seq_num 833 ranges {2981888=761856,3743744=671744,4415488=532480} 2024-09-14T03:57:32.173 INFO:tasks.rados.rados.0.smithi070.stdout:2691: writing smithi07085154-21 from 2981888 to 3743744 tid 1 2024-09-14T03:57:32.178 INFO:tasks.rados.rados.0.smithi070.stdout:2691: writing smithi07085154-21 from 3743744 to 4415488 tid 2 2024-09-14T03:57:32.182 INFO:tasks.rados.rados.0.smithi070.stdout:2691: writing smithi07085154-21 from 4415488 to 4947968 tid 3 2024-09-14T03:57:32.182 INFO:tasks.rados.rados.0.smithi070.stdout:2692: rmattr oid 39 current snap is 285 2024-09-14T03:57:32.182 INFO:tasks.rados.rados.0.smithi070.stdout:2692: done (2 left) 2024-09-14T03:57:32.182 INFO:tasks.rados.rados.0.smithi070.stdout:2693: snap_remove snap 250 2024-09-14T03:57:32.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:32 smithi094 ceph-mon[86597]: osdmap e620: 8 total, 8 up, 8 in 2024-09-14T03:57:32.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:32 smithi094 ceph-mon[86597]: pgmap v527: 121 pgs: 121 active+clean; 1.1 GiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 16 MiB/s rd, 6.5 MiB/s wr, 38 op/s 2024-09-14T03:57:32.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:32 smithi070 ceph-mon[96704]: osdmap e620: 8 total, 8 up, 8 in 2024-09-14T03:57:32.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:32 smithi070 ceph-mon[96704]: pgmap v527: 121 pgs: 121 active+clean; 1.1 GiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 16 MiB/s rd, 6.5 MiB/s wr, 38 op/s 2024-09-14T03:57:32.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:32 smithi070 ceph-mon[98640]: osdmap e620: 8 total, 8 up, 8 in 2024-09-14T03:57:32.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:32 smithi070 ceph-mon[98640]: pgmap v527: 121 pgs: 121 active+clean; 1.1 GiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 16 MiB/s rd, 6.5 MiB/s wr, 38 op/s 2024-09-14T03:57:33.004 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-09-14T03:57:33.157 INFO:tasks.rados.rados.0.smithi070.stdout:2693: done (2 left) 2024-09-14T03:57:33.157 INFO:tasks.rados.rados.0.smithi070.stdout:2694: snap_remove snap 280 2024-09-14T03:57:33.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:33 smithi070 ceph-mon[96704]: osdmap e621: 8 total, 8 up, 8 in 2024-09-14T03:57:33.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:33 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:57:33.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:33 smithi070 ceph-mon[96704]: from='client.44456 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:33.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:33 smithi070 ceph-mon[98640]: osdmap e621: 8 total, 8 up, 8 in 2024-09-14T03:57:33.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:33 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:57:33.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:33 smithi070 ceph-mon[98640]: from='client.44456 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:33.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:33 smithi094 ceph-mon[86597]: osdmap e621: 8 total, 8 up, 8 in 2024-09-14T03:57:33.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:33 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:57:33.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:33 smithi094 ceph-mon[86597]: from='client.44456 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:33.573 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:57:33.573 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (7m) 33s ago 12m 21.2M - 0.27.0 11f11916f8cd 45e79e51074c 2024-09-14T03:57:33.573 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi094 *:3000 running (7m) 13s ago 12m 84.1M - 9.4.7 954c08fa6188 2240810d0368 2024-09-14T03:57:33.573 INFO:teuthology.orchestra.run.smithi070.stdout:mds.a smithi070 running (37s) 33s ago 13m 15.5M - 19.3.0-4907-ga2782315 8801d2529e43 4fa138982c4f 2024-09-14T03:57:33.573 INFO:teuthology.orchestra.run.smithi070.stdout:mds.b smithi094 running (17s) 13s ago 13m 13.8M - 19.3.0-4907-ga2782315 8801d2529e43 9f7842c26184 2024-09-14T03:57:33.573 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi094 *:8443,9283,8765 running (6m) 13s ago 17m 456M - 19.3.0-4907-ga2782315 8801d2529e43 569d9c7e0f06 2024-09-14T03:57:33.573 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (6m) 33s ago 18m 529M - 19.3.0-4907-ga2782315 8801d2529e43 0ef4d8e0cdbe 2024-09-14T03:57:33.573 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (6m) 33s ago 19m 69.1M 2048M 19.3.0-4907-ga2782315 8801d2529e43 96542de31cac 2024-09-14T03:57:33.573 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi094 running (5m) 13s ago 17m 52.7M 2048M 19.3.0-4907-ga2782315 8801d2529e43 86827be98659 2024-09-14T03:57:33.574 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (5m) 33s ago 17m 55.7M 2048M 19.3.0-4907-ga2782315 8801d2529e43 5741e4fcf4b6 2024-09-14T03:57:33.574 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (7m) 33s ago 11m 11.2M - 1.7.0 72c9c2088986 e316760a30b3 2024-09-14T03:57:33.574 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi094 *:9100 running (6m) 13s ago 11m 11.9M - 1.7.0 72c9c2088986 767f8a093175 2024-09-14T03:57:33.574 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (4m) 33s ago 16m 315M 2480M 19.3.0-4907-ga2782315 8801d2529e43 2345ccac2085 2024-09-14T03:57:33.574 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (4m) 33s ago 16m 341M 2480M 19.3.0-4907-ga2782315 8801d2529e43 53868856512a 2024-09-14T03:57:33.574 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (3m) 33s ago 16m 219M 2480M 19.3.0-4907-ga2782315 8801d2529e43 39ef876d2fff 2024-09-14T03:57:33.574 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (3m) 33s ago 15m 149M 2480M 19.3.0-4907-ga2782315 8801d2529e43 8efc112edbe9 2024-09-14T03:57:33.574 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi094 running (2m) 13s ago 15m 257M 2480M 19.3.0-4907-ga2782315 8801d2529e43 09f583ed6c77 2024-09-14T03:57:33.574 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi094 running (2m) 13s ago 14m 249M 2480M 19.3.0-4907-ga2782315 8801d2529e43 162396f2e7b6 2024-09-14T03:57:33.574 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi094 running (93s) 13s ago 14m 128M 2480M 19.3.0-4907-ga2782315 8801d2529e43 cbdc02cdb8e4 2024-09-14T03:57:33.574 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi094 running (59s) 13s ago 14m 171M 2480M 19.3.0-4907-ga2782315 8801d2529e43 ea9896fb7dbc 2024-09-14T03:57:33.574 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi094 *:9095 running (6m) 13s ago 12m 54.0M - 2.51.0 1d3b7f56885b ce1b353828dc 2024-09-14T03:57:33.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:57:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:57:33] "GET /metrics HTTP/1.1" 200 36506 "" "Prometheus/2.51.0" 2024-09-14T03:57:33.906 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:57:33.906 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-09-14T03:57:33.906 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:57:33.906 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:57:33.907 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-09-14T03:57:33.907 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:57:33.907 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:57:33.907 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-09-14T03:57:33.907 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 8 2024-09-14T03:57:33.907 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:57:33.907 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": { 2024-09-14T03:57:33.907 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:57:33.907 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:57:33.907 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-09-14T03:57:33.907 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 15 2024-09-14T03:57:33.907 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-09-14T03:57:33.907 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:57:34.084 INFO:tasks.rados.rados.0.smithi070.stdout:2694: done (2 left) 2024-09-14T03:57:34.084 INFO:tasks.rados.rados.0.smithi070.stdout:2695: rmattr oid 29 current snap is 285 2024-09-14T03:57:34.085 INFO:tasks.rados.rados.0.smithi070.stdout:2691: finishing write tid 1 to smithi07085154-21 2024-09-14T03:57:34.085 INFO:tasks.rados.rados.0.smithi070.stdout:2691: finishing write tid 2 to smithi07085154-21 2024-09-14T03:57:34.085 INFO:tasks.rados.rados.0.smithi070.stdout:2691: finishing write tid 3 to smithi07085154-21 2024-09-14T03:57:34.085 INFO:tasks.rados.rados.0.smithi070.stdout:2691: finishing write tid 4 to smithi07085154-21 2024-09-14T03:57:34.085 INFO:tasks.rados.rados.0.smithi070.stdout:2691: finishing write tid 5 to smithi07085154-21 2024-09-14T03:57:34.085 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 1713 (ObjNum 833 snap 285 seq_num 833) dirty exists 2024-09-14T03:57:34.085 INFO:tasks.rados.rados.0.smithi070.stdout:2691: left oid 21 (ObjNum 833 snap 285 seq_num 833) 2024-09-14T03:57:34.085 INFO:tasks.rados.rados.0.smithi070.stdout:2689: done (2 left) 2024-09-14T03:57:34.085 INFO:tasks.rados.rados.0.smithi070.stdout:2691: done (1 left) 2024-09-14T03:57:34.085 INFO:tasks.rados.rados.0.smithi070.stdout:2696: read oid 37 snap -1 2024-09-14T03:57:34.085 INFO:tasks.rados.rados.0.smithi070.stdout:2696: expect deleted 2024-09-14T03:57:34.085 INFO:tasks.rados.rados.0.smithi070.stdout:2697: read oid 45 snap -1 2024-09-14T03:57:34.085 INFO:tasks.rados.rados.0.smithi070.stdout:2697: expect (ObjNum 801 snap 280 seq_num 801) 2024-09-14T03:57:34.086 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 10 current snap is 285 2024-09-14T03:57:34.086 INFO:tasks.rados.rados.0.smithi070.stdout:2698: seq_num 834 ranges {4243456=778240,5021696=557056,5578752=548864,6127616=98304} 2024-09-14T03:57:34.111 INFO:tasks.rados.rados.0.smithi070.stdout:2698: writing smithi07085154-10 from 4243456 to 5021696 tid 1 2024-09-14T03:57:34.115 INFO:tasks.rados.rados.0.smithi070.stdout:2698: writing smithi07085154-10 from 5021696 to 5578752 tid 2 2024-09-14T03:57:34.119 INFO:tasks.rados.rados.0.smithi070.stdout:2698: writing smithi07085154-10 from 5578752 to 6127616 tid 3 2024-09-14T03:57:34.120 INFO:tasks.rados.rados.0.smithi070.stdout:2698: writing smithi07085154-10 from 6127616 to 6225920 tid 4 2024-09-14T03:57:34.120 INFO:tasks.rados.rados.0.smithi070.stdout:2699: delete oid 43 current snap is 285 2024-09-14T03:57:34.121 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 2234 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-14T03:57:34.121 INFO:tasks.rados.rados.0.smithi070.stdout:2697: expect (ObjNum 801 snap 280 seq_num 801) 2024-09-14T03:57:34.178 INFO:tasks.rados.rados.0.smithi070.stdout:2698: finishing write tid 1 to smithi07085154-10 2024-09-14T03:57:34.178 INFO:tasks.rados.rados.0.smithi070.stdout:2698: finishing write tid 2 to smithi07085154-10 2024-09-14T03:57:34.178 INFO:tasks.rados.rados.0.smithi070.stdout:2698: finishing write tid 3 to smithi07085154-10 2024-09-14T03:57:34.178 INFO:tasks.rados.rados.0.smithi070.stdout:2698: finishing write tid 4 to smithi07085154-10 2024-09-14T03:57:34.178 INFO:tasks.rados.rados.0.smithi070.stdout:2698: finishing write tid 5 to smithi07085154-10 2024-09-14T03:57:34.178 INFO:tasks.rados.rados.0.smithi070.stdout:2698: finishing write tid 6 to smithi07085154-10 2024-09-14T03:57:34.178 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 3074 (ObjNum 834 snap 285 seq_num 834) dirty exists 2024-09-14T03:57:34.178 INFO:tasks.rados.rados.0.smithi070.stdout:2698: left oid 10 (ObjNum 834 snap 285 seq_num 834) 2024-09-14T03:57:34.178 INFO:tasks.rados.rados.0.smithi070.stdout:2695: done (4 left) 2024-09-14T03:57:34.178 INFO:tasks.rados.rados.0.smithi070.stdout:2696: done (3 left) 2024-09-14T03:57:34.179 INFO:tasks.rados.rados.0.smithi070.stdout:2697: done (2 left) 2024-09-14T03:57:34.179 INFO:tasks.rados.rados.0.smithi070.stdout:2698: done (1 left) 2024-09-14T03:57:34.179 INFO:tasks.rados.rados.0.smithi070.stdout:2699: done (0 left) 2024-09-14T03:57:34.179 INFO:tasks.rados.rados.0.smithi070.stdout:2700: setattr oid 8 current snap is 285 2024-09-14T03:57:34.179 INFO:tasks.rados.rados.0.smithi070.stdout:2701: rmattr oid 27 current snap is 285 2024-09-14T03:57:34.179 INFO:tasks.rados.rados.0.smithi070.stdout:2701: done (1 left) 2024-09-14T03:57:34.179 INFO:tasks.rados.rados.0.smithi070.stdout:2702: rmattr oid 30 current snap is 285 2024-09-14T03:57:34.179 INFO:tasks.rados.rados.0.smithi070.stdout:2703: read oid 13 snap 275 2024-09-14T03:57:34.179 INFO:tasks.rados.rados.0.smithi070.stdout:2703: expect (ObjNum 459 snap 155 seq_num 459) 2024-09-14T03:57:34.179 INFO:tasks.rados.rados.0.smithi070.stdout:2704: read oid 2 snap -1 2024-09-14T03:57:34.179 INFO:tasks.rados.rados.0.smithi070.stdout:2704: expect (ObjNum 830 snap 283 seq_num 830) 2024-09-14T03:57:34.179 INFO:tasks.rados.rados.0.smithi070.stdout:2705: delete oid 14 current snap is 285 2024-09-14T03:57:34.180 INFO:tasks.rados.rados.0.smithi070.stdout:2705: done (4 left) 2024-09-14T03:57:34.180 INFO:tasks.rados.rados.0.smithi070.stdout:2706: snap_create 2024-09-14T03:57:34.182 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 30 v 1725 (ObjNum 353 snap 0 seq_num 4213931367) dirty exists 2024-09-14T03:57:34.183 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 8 v 2234 (ObjNum 65 snap 0 seq_num 1800953697) dirty exists 2024-09-14T03:57:34.189 INFO:tasks.rados.rados.0.smithi070.stdout:2704: expect (ObjNum 830 snap 283 seq_num 830) 2024-09-14T03:57:34.240 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:57:34.241 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:57:34.241 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-09-14T03:57:34.241 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-14T03:57:34.241 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [ 2024-09-14T03:57:34.241 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr", 2024-09-14T03:57:34.241 INFO:teuthology.orchestra.run.smithi070.stdout: "mon", 2024-09-14T03:57:34.241 INFO:teuthology.orchestra.run.smithi070.stdout: "mds", 2024-09-14T03:57:34.241 INFO:teuthology.orchestra.run.smithi070.stdout: "osd" 2024-09-14T03:57:34.241 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2024-09-14T03:57:34.241 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "15/20 daemons upgraded", 2024-09-14T03:57:34.242 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "Currently upgrading grafana daemons", 2024-09-14T03:57:34.242 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-09-14T03:57:34.242 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:57:34.242 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:34 smithi070 ceph-mon[98640]: osdmap e622: 8 total, 8 up, 8 in 2024-09-14T03:57:34.242 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:34 smithi070 ceph-mon[98640]: from='client.44462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:34.242 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:34 smithi070 ceph-mon[98640]: from='client.34551 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:34.243 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:34 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/727494956' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:34.243 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:34 smithi070 ceph-mon[98640]: osdmap e623: 8 total, 8 up, 8 in 2024-09-14T03:57:34.243 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:34 smithi070 ceph-mon[98640]: pgmap v531: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.1 MiB/s rd, 3.7 MiB/s wr, 22 op/s 2024-09-14T03:57:34.243 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:34 smithi070 ceph-mon[96704]: osdmap e622: 8 total, 8 up, 8 in 2024-09-14T03:57:34.243 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:34 smithi070 ceph-mon[96704]: from='client.44462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:34.244 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:34 smithi070 ceph-mon[96704]: from='client.34551 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:34.244 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:34 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/727494956' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:34.244 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:34 smithi070 ceph-mon[96704]: osdmap e623: 8 total, 8 up, 8 in 2024-09-14T03:57:34.244 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:34 smithi070 ceph-mon[96704]: pgmap v531: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.1 MiB/s rd, 3.7 MiB/s wr, 22 op/s 2024-09-14T03:57:34.286 INFO:tasks.rados.rados.0.smithi070.stdout:2703: expect (ObjNum 459 snap 155 seq_num 459) 2024-09-14T03:57:34.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:34 smithi094 ceph-mon[86597]: osdmap e622: 8 total, 8 up, 8 in 2024-09-14T03:57:34.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:34 smithi094 ceph-mon[86597]: from='client.44462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:34.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:34 smithi094 ceph-mon[86597]: from='client.34551 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:34.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:34 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/727494956' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:34.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:34 smithi094 ceph-mon[86597]: osdmap e623: 8 total, 8 up, 8 in 2024-09-14T03:57:34.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:34 smithi094 ceph-mon[86597]: pgmap v531: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.1 MiB/s rd, 3.7 MiB/s wr, 22 op/s 2024-09-14T03:57:35.932 INFO:tasks.rados.rados.0.smithi070.stdout:2700: done (4 left) 2024-09-14T03:57:35.932 INFO:tasks.rados.rados.0.smithi070.stdout:2702: done (3 left) 2024-09-14T03:57:35.932 INFO:tasks.rados.rados.0.smithi070.stdout:2703: done (2 left) 2024-09-14T03:57:35.932 INFO:tasks.rados.rados.0.smithi070.stdout:2704: done (1 left) 2024-09-14T03:57:35.932 INFO:tasks.rados.rados.0.smithi070.stdout:2706: done (0 left) 2024-09-14T03:57:35.932 INFO:tasks.rados.rados.0.smithi070.stdout:2707: snap_remove snap 243 2024-09-14T03:57:36.160 INFO:tasks.rados.rados.0.smithi070.stdout:2707: done (0 left) 2024-09-14T03:57:36.160 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 31 current snap is 286 2024-09-14T03:57:36.160 INFO:tasks.rados.rados.0.smithi070.stdout:2708: seq_num 837 ranges {0=425984,425984=655360,1081344=729088,1810432=221184} 2024-09-14T03:57:36.162 INFO:tasks.rados.rados.0.smithi070.stdout:2708: writing smithi07085154-31 from 0 to 425984 tid 1 2024-09-14T03:57:36.167 INFO:tasks.rados.rados.0.smithi070.stdout:2708: writing smithi07085154-31 from 425984 to 1081344 tid 2 2024-09-14T03:57:36.172 INFO:tasks.rados.rados.0.smithi070.stdout:2708: writing smithi07085154-31 from 1081344 to 1810432 tid 3 2024-09-14T03:57:36.174 INFO:tasks.rados.rados.0.smithi070.stdout:2708: writing smithi07085154-31 from 1810432 to 2031616 tid 4 2024-09-14T03:57:36.174 INFO:tasks.rados.rados.0.smithi070.stdout:2709: delete oid 49 current snap is 286 2024-09-14T03:57:36.196 INFO:tasks.rados.rados.0.smithi070.stdout:2709: done (1 left) 2024-09-14T03:57:36.196 INFO:tasks.rados.rados.0.smithi070.stdout:2710: delete oid 34 current snap is 286 2024-09-14T03:57:36.197 INFO:tasks.rados.rados.0.smithi070.stdout:2708: finishing write tid 1 to smithi07085154-31 2024-09-14T03:57:36.197 INFO:tasks.rados.rados.0.smithi070.stdout:2708: finishing write tid 2 to smithi07085154-31 2024-09-14T03:57:36.197 INFO:tasks.rados.rados.0.smithi070.stdout:2708: finishing write tid 3 to smithi07085154-31 2024-09-14T03:57:36.200 INFO:tasks.rados.rados.0.smithi070.stdout:2708: finishing write tid 4 to smithi07085154-31 2024-09-14T03:57:36.200 INFO:tasks.rados.rados.0.smithi070.stdout:2708: finishing write tid 5 to smithi07085154-31 2024-09-14T03:57:36.201 INFO:tasks.rados.rados.0.smithi070.stdout:2710: done (1 left) 2024-09-14T03:57:36.201 INFO:tasks.rados.rados.0.smithi070.stdout:2711: snap_remove snap 270 2024-09-14T03:57:36.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:36 smithi094 ceph-mon[86597]: from='client.44480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:36.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:36 smithi094 ceph-mon[86597]: osdmap e624: 8 total, 8 up, 8 in 2024-09-14T03:57:36.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:36 smithi070 ceph-mon[96704]: from='client.44480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:36.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:36 smithi070 ceph-mon[96704]: osdmap e624: 8 total, 8 up, 8 in 2024-09-14T03:57:36.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:36 smithi070 ceph-mon[98640]: from='client.44480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:36.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:36 smithi070 ceph-mon[98640]: osdmap e624: 8 total, 8 up, 8 in 2024-09-14T03:57:36.610 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:36 smithi094 systemd[1]: Stopping Ceph grafana.a for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:57:36.873 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:36 smithi094 podman[114634]: 2024-09-14 03:57:36.608170113 +0000 UTC m=+0.121772755 container died 2240810d03680ea85e39642edc3bf217af0418f8e6889c7425ca7efcd5ccf8dc (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a, build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, version=8.5, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, io.buildah.version=1.32.0, architecture=x86_64, vcs-type=git, io.openshift.tags=base rhel8) 2024-09-14T03:57:37.162 INFO:tasks.rados.rados.0.smithi070.stdout:2708: finishing write tid 6 to smithi07085154-31 2024-09-14T03:57:37.162 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 2272 (ObjNum 837 snap 286 seq_num 837) dirty exists 2024-09-14T03:57:37.162 INFO:tasks.rados.rados.0.smithi070.stdout:2708: left oid 31 (ObjNum 837 snap 286 seq_num 837) 2024-09-14T03:57:37.162 INFO:tasks.rados.rados.0.smithi070.stdout:2708: done (1 left) 2024-09-14T03:57:37.162 INFO:tasks.rados.rados.0.smithi070.stdout:2711: done (0 left) 2024-09-14T03:57:37.162 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 45 current snap is 286 2024-09-14T03:57:37.162 INFO:tasks.rados.rados.0.smithi070.stdout:2712: seq_num 840 ranges {1425408=475136,1900544=712704,2613248=499712,3112960=393216} 2024-09-14T03:57:37.173 INFO:tasks.rados.rados.0.smithi070.stdout:2712: writing smithi07085154-45 from 1425408 to 1900544 tid 1 2024-09-14T03:57:37.178 INFO:tasks.rados.rados.0.smithi070.stdout:2712: writing smithi07085154-45 from 1900544 to 2613248 tid 2 2024-09-14T03:57:37.181 INFO:tasks.rados.rados.0.smithi070.stdout:2712: writing smithi07085154-45 from 2613248 to 3112960 tid 3 2024-09-14T03:57:37.184 INFO:tasks.rados.rados.0.smithi070.stdout:2712: writing smithi07085154-45 from 3112960 to 3506176 tid 4 2024-09-14T03:57:37.184 INFO:tasks.rados.rados.0.smithi070.stdout:2713: read oid 30 snap -1 2024-09-14T03:57:37.184 INFO:tasks.rados.rados.0.smithi070.stdout:2713: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:57:37.184 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 10 current snap is 286 2024-09-14T03:57:37.184 INFO:tasks.rados.rados.0.smithi070.stdout:2712: finishing write tid 1 to smithi07085154-45 2024-09-14T03:57:37.184 INFO:tasks.rados.rados.0.smithi070.stdout:2714: seq_num 841 ranges {6225920=491520,6717440=598016,7315456=704512,8019968=303104} 2024-09-14T03:57:37.221 INFO:tasks.rados.rados.0.smithi070.stdout:2714: writing smithi07085154-10 from 6225920 to 6717440 tid 1 2024-09-14T03:57:37.224 INFO:tasks.rados.rados.0.smithi070.stdout:2714: writing smithi07085154-10 from 6717440 to 7315456 tid 2 2024-09-14T03:57:37.228 INFO:tasks.rados.rados.0.smithi070.stdout:2714: writing smithi07085154-10 from 7315456 to 8019968 tid 3 2024-09-14T03:57:37.230 INFO:tasks.rados.rados.0.smithi070.stdout:2714: writing smithi07085154-10 from 8019968 to 8323072 tid 4 2024-09-14T03:57:37.230 INFO:tasks.rados.rados.0.smithi070.stdout:2715: rollback oid 11 current snap is 286 2024-09-14T03:57:37.230 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 11 to 236 2024-09-14T03:57:37.230 INFO:tasks.rados.rados.0.smithi070.stdout:2712: finishing write tid 2 to smithi07085154-45 2024-09-14T03:57:37.230 INFO:tasks.rados.rados.0.smithi070.stdout:2713: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:57:37.231 INFO:tasks.rados.rados.0.smithi070.stdout:2712: finishing write tid 3 to smithi07085154-45 2024-09-14T03:57:37.231 INFO:tasks.rados.rados.0.smithi070.stdout:2712: finishing write tid 4 to smithi07085154-45 2024-09-14T03:57:37.232 INFO:tasks.rados.rados.0.smithi070.stdout:2712: finishing write tid 5 to smithi07085154-45 2024-09-14T03:57:37.232 INFO:tasks.rados.rados.0.smithi070.stdout:2712: finishing write tid 6 to smithi07085154-45 2024-09-14T03:57:37.232 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 45 v 1972 (ObjNum 840 snap 286 seq_num 840) dirty exists 2024-09-14T03:57:37.232 INFO:tasks.rados.rados.0.smithi070.stdout:2712: left oid 45 (ObjNum 840 snap 286 seq_num 840) 2024-09-14T03:57:37.232 INFO:tasks.rados.rados.0.smithi070.stdout:2714: finishing write tid 1 to smithi07085154-10 2024-09-14T03:57:37.232 INFO:tasks.rados.rados.0.smithi070.stdout:2714: finishing write tid 2 to smithi07085154-10 2024-09-14T03:57:37.232 INFO:tasks.rados.rados.0.smithi070.stdout:2712: done (3 left) 2024-09-14T03:57:37.232 INFO:tasks.rados.rados.0.smithi070.stdout:2713: done (2 left) 2024-09-14T03:57:37.232 INFO:tasks.rados.rados.0.smithi070.stdout:2716: snap_remove snap 272 2024-09-14T03:57:37.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:37 smithi094 ceph-mon[86597]: pgmap v533: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.7 MiB/s rd, 474 KiB/s wr, 8 op/s 2024-09-14T03:57:37.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:37 smithi094 ceph-mon[86597]: osdmap e625: 8 total, 8 up, 8 in 2024-09-14T03:57:37.277 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:36 smithi094 podman[114634]: 2024-09-14 03:57:36.9455707 +0000 UTC m=+0.459173343 container cleanup 2240810d03680ea85e39642edc3bf217af0418f8e6889c7425ca7efcd5ccf8dc (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a, description=Ceph Grafana Container, name=ubi8, com.redhat.component=ubi8-container, release=239.1651231664, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.32.0, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public) 2024-09-14T03:57:37.277 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:36 smithi094 bash[114634]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a 2024-09-14T03:57:37.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:37 smithi070 ceph-mon[96704]: pgmap v533: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.7 MiB/s rd, 474 KiB/s wr, 8 op/s 2024-09-14T03:57:37.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:37 smithi070 ceph-mon[96704]: osdmap e625: 8 total, 8 up, 8 in 2024-09-14T03:57:37.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:37 smithi070 ceph-mon[98640]: pgmap v533: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.7 MiB/s rd, 474 KiB/s wr, 8 op/s 2024-09-14T03:57:37.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:37 smithi070 ceph-mon[98640]: osdmap e625: 8 total, 8 up, 8 in 2024-09-14T03:57:38.162 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:37 smithi094 podman[114651]: 2024-09-14 03:57:37.897071082 +0000 UTC m=+1.286363081 container remove 2240810d03680ea85e39642edc3bf217af0418f8e6889c7425ca7efcd5ccf8dc (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, name=ubi8, vendor=Red Hat, Inc., description=Ceph Grafana Container, architecture=x86_64, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, vcs-type=git, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-09-14T03:57:38.163 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:37 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-09-14T03:57:38.167 INFO:tasks.rados.rados.0.smithi070.stdout:2714: finishing write tid 3 to smithi07085154-10 2024-09-14T03:57:38.167 INFO:tasks.rados.rados.0.smithi070.stdout:2714: finishing write tid 4 to smithi07085154-10 2024-09-14T03:57:38.168 INFO:tasks.rados.rados.0.smithi070.stdout:2714: finishing write tid 5 to smithi07085154-10 2024-09-14T03:57:38.168 INFO:tasks.rados.rados.0.smithi070.stdout:2715: finishing rollback tid 0 to smithi07085154-11 2024-09-14T03:57:38.168 INFO:tasks.rados.rados.0.smithi070.stdout:2715: finishing rollback tid 1 to smithi07085154-11 2024-09-14T03:57:38.168 INFO:tasks.rados.rados.0.smithi070.stdout:2715: finishing rollback tid 2 to smithi07085154-11 2024-09-14T03:57:38.168 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 11 v 2260 (ObjNum 403 snap 140 seq_num 403) dirty exists 2024-09-14T03:57:38.168 INFO:tasks.rados.rados.0.smithi070.stdout:2714: finishing write tid 6 to smithi07085154-10 2024-09-14T03:57:38.168 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 3120 (ObjNum 841 snap 286 seq_num 841) dirty exists 2024-09-14T03:57:38.168 INFO:tasks.rados.rados.0.smithi070.stdout:2714: left oid 10 (ObjNum 841 snap 286 seq_num 841) 2024-09-14T03:57:38.168 INFO:tasks.rados.rados.0.smithi070.stdout:2714: done (2 left) 2024-09-14T03:57:38.168 INFO:tasks.rados.rados.0.smithi070.stdout:2715: done (1 left) 2024-09-14T03:57:38.168 INFO:tasks.rados.rados.0.smithi070.stdout:2716: done (0 left) 2024-09-14T03:57:38.168 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 2 current snap is 286 2024-09-14T03:57:38.168 INFO:tasks.rados.rados.0.smithi070.stdout:2717: seq_num 842 ranges {2777088=507904,3284992=475136,3760128=516096,4276224=499712,4775936=114688} 2024-09-14T03:57:38.185 INFO:tasks.rados.rados.0.smithi070.stdout:2717: writing smithi07085154-2 from 2777088 to 3284992 tid 1 2024-09-14T03:57:38.188 INFO:tasks.rados.rados.0.smithi070.stdout:2717: writing smithi07085154-2 from 3284992 to 3760128 tid 2 2024-09-14T03:57:38.192 INFO:tasks.rados.rados.0.smithi070.stdout:2717: writing smithi07085154-2 from 3760128 to 4276224 tid 3 2024-09-14T03:57:38.195 INFO:tasks.rados.rados.0.smithi070.stdout:2717: writing smithi07085154-2 from 4276224 to 4775936 tid 4 2024-09-14T03:57:38.196 INFO:tasks.rados.rados.0.smithi070.stdout:2717: writing smithi07085154-2 from 4775936 to 4890624 tid 5 2024-09-14T03:57:38.196 INFO:tasks.rados.rados.0.smithi070.stdout:2718: snap_remove snap 275 2024-09-14T03:57:38.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:38 smithi094 ceph-mon[86597]: osdmap e626: 8 total, 8 up, 8 in 2024-09-14T03:57:38.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:38 smithi094 ceph-mon[86597]: pgmap v536: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 1.1 GiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 22 MiB/s rd, 958 KiB/s wr, 17 op/s 2024-09-14T03:57:38.527 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:38 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-09-14T03:57:38.527 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:38 smithi094 systemd[1]: Stopped Ceph grafana.a for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:57:38.527 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:38 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@grafana.a.service: Consumed 7.037s CPU time. 2024-09-14T03:57:38.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:38 smithi070 ceph-mon[96704]: osdmap e626: 8 total, 8 up, 8 in 2024-09-14T03:57:38.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:38 smithi070 ceph-mon[96704]: pgmap v536: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 1.1 GiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 22 MiB/s rd, 958 KiB/s wr, 17 op/s 2024-09-14T03:57:38.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:38 smithi070 ceph-mon[98640]: osdmap e626: 8 total, 8 up, 8 in 2024-09-14T03:57:38.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:38 smithi070 ceph-mon[98640]: pgmap v536: 121 pgs: 3 active+clean+snaptrim, 118 active+clean; 1.1 GiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 22 MiB/s rd, 958 KiB/s wr, 17 op/s 2024-09-14T03:57:39.026 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:38 smithi094 systemd[1]: Starting Ceph grafana.a for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T03:57:39.027 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:38 smithi094 podman[114786]: 2024-09-14 03:57:38.749393603 +0000 UTC m=+0.016415070 image pull c8b91775d855b99270fc5d22f3c6737e8cca01ef4c25c8b0362295e0746fa39b quay.io/ceph/grafana:10.4.0 2024-09-14T03:57:39.085 INFO:tasks.rados.rados.0.smithi070.stdout:2718: done (1 left) 2024-09-14T03:57:39.086 INFO:tasks.rados.rados.0.smithi070.stdout:2719: read oid 43 snap -1 2024-09-14T03:57:39.086 INFO:tasks.rados.rados.0.smithi070.stdout:2719: expect deleted 2024-09-14T03:57:39.086 INFO:tasks.rados.rados.0.smithi070.stdout:2717: finishing write tid 1 to smithi07085154-2 2024-09-14T03:57:39.086 INFO:tasks.rados.rados.0.smithi070.stdout:2717: finishing write tid 2 to smithi07085154-2 2024-09-14T03:57:39.086 INFO:tasks.rados.rados.0.smithi070.stdout:2717: finishing write tid 3 to smithi07085154-2 2024-09-14T03:57:39.086 INFO:tasks.rados.rados.0.smithi070.stdout:2717: finishing write tid 4 to smithi07085154-2 2024-09-14T03:57:39.086 INFO:tasks.rados.rados.0.smithi070.stdout:2717: finishing write tid 5 to smithi07085154-2 2024-09-14T03:57:39.086 INFO:tasks.rados.rados.0.smithi070.stdout:2717: finishing write tid 6 to smithi07085154-2 2024-09-14T03:57:39.087 INFO:tasks.rados.rados.0.smithi070.stdout:2717: finishing write tid 7 to smithi07085154-2 2024-09-14T03:57:39.087 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 2 v 3127 (ObjNum 842 snap 286 seq_num 842) dirty exists 2024-09-14T03:57:39.087 INFO:tasks.rados.rados.0.smithi070.stdout:2717: left oid 2 (ObjNum 842 snap 286 seq_num 842) 2024-09-14T03:57:39.087 INFO:tasks.rados.rados.0.smithi070.stdout:2717: done (1 left) 2024-09-14T03:57:39.087 INFO:tasks.rados.rados.0.smithi070.stdout:2720: rollback oid 11 current snap is 286 2024-09-14T03:57:39.087 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 11 to 107 2024-09-14T03:57:39.087 INFO:tasks.rados.rados.0.smithi070.stdout:2721: setattr oid 49 current snap is 286 2024-09-14T03:57:39.088 INFO:tasks.rados.rados.0.smithi070.stdout:2722: copy_from oid 50 from oid 38 current snap is 286 2024-09-14T03:57:39.088 INFO:tasks.rados.rados.0.smithi070.stdout:2723: rmattr oid 46 current snap is 286 2024-09-14T03:57:39.088 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 44 current snap is 286 2024-09-14T03:57:39.088 INFO:tasks.rados.rados.0.smithi070.stdout:2724: seq_num 843 ranges {3964928=524288,4489216=761856,5251072=671744,5922816=172032} 2024-09-14T03:57:39.113 INFO:tasks.rados.rados.0.smithi070.stdout:2724: writing smithi07085154-44 from 3964928 to 4489216 tid 1 2024-09-14T03:57:39.118 INFO:tasks.rados.rados.0.smithi070.stdout:2724: writing smithi07085154-44 from 4489216 to 5251072 tid 2 2024-09-14T03:57:39.122 INFO:tasks.rados.rados.0.smithi070.stdout:2724: writing smithi07085154-44 from 5251072 to 5922816 tid 3 2024-09-14T03:57:39.123 INFO:tasks.rados.rados.0.smithi070.stdout:2724: writing smithi07085154-44 from 5922816 to 6094848 tid 4 2024-09-14T03:57:39.123 INFO:tasks.rados.rados.0.smithi070.stdout:2725: delete oid 19 current snap is 286 2024-09-14T03:57:39.123 INFO:tasks.rados.rados.0.smithi070.stdout:2722: finishing copy_from to smithi07085154-50 2024-09-14T03:57:39.123 INFO:tasks.rados.rados.0.smithi070.stdout:2722: got expected ENOENT (src dne) 2024-09-14T03:57:39.124 INFO:tasks.rados.rados.0.smithi070.stdout:2722: finishing copy_from racing read to smithi07085154-50 2024-09-14T03:57:39.124 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 49 v 1973 (ObjNum 33 snap 0 seq_num 2930912544) dirty exists 2024-09-14T03:57:39.124 INFO:tasks.rados.rados.0.smithi070.stdout:2720: finishing rollback tid 0 to smithi07085154-11 2024-09-14T03:57:39.124 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 46 v 3147 (ObjNum 576 snap 193 seq_num 576) dirty exists 2024-09-14T03:57:39.124 INFO:tasks.rados.rados.0.smithi070.stdout:2720: finishing rollback tid 1 to smithi07085154-11 2024-09-14T03:57:39.124 INFO:tasks.rados.rados.0.smithi070.stdout:2720: finishing rollback tid 2 to smithi07085154-11 2024-09-14T03:57:39.124 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 11 v 2263 (ObjNum 293 snap 91 seq_num 293) dirty exists 2024-09-14T03:57:39.124 INFO:tasks.rados.rados.0.smithi070.stdout:2724: finishing write tid 1 to smithi07085154-44 2024-09-14T03:57:39.124 INFO:tasks.rados.rados.0.smithi070.stdout:2724: finishing write tid 2 to smithi07085154-44 2024-09-14T03:57:39.125 INFO:tasks.rados.rados.0.smithi070.stdout:2719: done (6 left) 2024-09-14T03:57:39.125 INFO:tasks.rados.rados.0.smithi070.stdout:2720: done (5 left) 2024-09-14T03:57:39.125 INFO:tasks.rados.rados.0.smithi070.stdout:2721: done (4 left) 2024-09-14T03:57:39.126 INFO:tasks.rados.rados.0.smithi070.stdout:2722: done (3 left) 2024-09-14T03:57:39.126 INFO:tasks.rados.rados.0.smithi070.stdout:2723: done (2 left) 2024-09-14T03:57:39.126 INFO:tasks.rados.rados.0.smithi070.stdout:2725: done (1 left) 2024-09-14T03:57:39.126 INFO:tasks.rados.rados.0.smithi070.stdout:2726: snap_remove snap 266 2024-09-14T03:57:39.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:39 smithi094 ceph-mon[86597]: osdmap e627: 8 total, 8 up, 8 in 2024-09-14T03:57:39.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:39 smithi094 ceph-mon[86597]: osdmap e628: 8 total, 8 up, 8 in 2024-09-14T03:57:39.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:39 smithi070 ceph-mon[96704]: osdmap e627: 8 total, 8 up, 8 in 2024-09-14T03:57:39.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:39 smithi070 ceph-mon[96704]: osdmap e628: 8 total, 8 up, 8 in 2024-09-14T03:57:39.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:39 smithi070 ceph-mon[98640]: osdmap e627: 8 total, 8 up, 8 in 2024-09-14T03:57:39.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:39 smithi070 ceph-mon[98640]: osdmap e628: 8 total, 8 up, 8 in 2024-09-14T03:57:40.087 INFO:tasks.rados.rados.0.smithi070.stdout:2726: done (1 left) 2024-09-14T03:57:40.087 INFO:tasks.rados.rados.0.smithi070.stdout:2727: read oid 39 snap -1 2024-09-14T03:57:40.088 INFO:tasks.rados.rados.0.smithi070.stdout:2727: expect (ObjNum 818 snap 283 seq_num 818) 2024-09-14T03:57:40.088 INFO:tasks.rados.rados.0.smithi070.stdout:2724: finishing write tid 3 to smithi07085154-44 2024-09-14T03:57:40.088 INFO:tasks.rados.rados.0.smithi070.stdout:2724: finishing write tid 4 to smithi07085154-44 2024-09-14T03:57:40.088 INFO:tasks.rados.rados.0.smithi070.stdout:2724: finishing write tid 5 to smithi07085154-44 2024-09-14T03:57:40.088 INFO:tasks.rados.rados.0.smithi070.stdout:2724: finishing write tid 6 to smithi07085154-44 2024-09-14T03:57:40.088 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 2080 (ObjNum 843 snap 286 seq_num 843) dirty exists 2024-09-14T03:57:40.088 INFO:tasks.rados.rados.0.smithi070.stdout:2724: left oid 44 (ObjNum 843 snap 286 seq_num 843) 2024-09-14T03:57:40.088 INFO:tasks.rados.rados.0.smithi070.stdout:2724: done (1 left) 2024-09-14T03:57:40.088 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 4 current snap is 286 2024-09-14T03:57:40.088 INFO:tasks.rados.rados.0.smithi070.stdout:2728: seq_num 845 ranges {1286144=557056,1843200=581632,2424832=688128,3112960=335872} 2024-09-14T03:57:40.098 INFO:tasks.rados.rados.0.smithi070.stdout:2728: writing smithi07085154-4 from 1286144 to 1843200 tid 1 2024-09-14T03:57:40.102 INFO:tasks.rados.rados.0.smithi070.stdout:2728: writing smithi07085154-4 from 1843200 to 2424832 tid 2 2024-09-14T03:57:40.107 INFO:tasks.rados.rados.0.smithi070.stdout:2728: writing smithi07085154-4 from 2424832 to 3112960 tid 3 2024-09-14T03:57:40.110 INFO:tasks.rados.rados.0.smithi070.stdout:2728: writing smithi07085154-4 from 3112960 to 3448832 tid 4 2024-09-14T03:57:40.110 INFO:tasks.rados.rados.0.smithi070.stdout:2729: rollback oid 27 current snap is 286 2024-09-14T03:57:40.110 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 27 to 226 2024-09-14T03:57:40.111 INFO:tasks.rados.rados.0.smithi070.stdout:2727: expect (ObjNum 818 snap 283 seq_num 818) 2024-09-14T03:57:40.160 INFO:tasks.rados.rados.0.smithi070.stdout:2728: finishing write tid 1 to smithi07085154-4 2024-09-14T03:57:40.160 INFO:tasks.rados.rados.0.smithi070.stdout:2729: finishing rollback tid 0 to smithi07085154-27 2024-09-14T03:57:40.160 INFO:tasks.rados.rados.0.smithi070.stdout:2729: finishing rollback tid 1 to smithi07085154-27 2024-09-14T03:57:40.161 INFO:tasks.rados.rados.0.smithi070.stdout:2729: finishing rollback tid 2 to smithi07085154-27 2024-09-14T03:57:40.161 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 3165 (ObjNum 656 snap 218 seq_num 656) dirty exists 2024-09-14T03:57:40.161 INFO:tasks.rados.rados.0.smithi070.stdout:2727: done (2 left) 2024-09-14T03:57:40.161 INFO:tasks.rados.rados.0.smithi070.stdout:2729: done (1 left) 2024-09-14T03:57:40.161 INFO:tasks.rados.rados.0.smithi070.stdout:2730: rmattr oid 10 current snap is 286 2024-09-14T03:57:40.161 INFO:tasks.rados.rados.0.smithi070.stdout:2730: done (1 left) 2024-09-14T03:57:40.161 INFO:tasks.rados.rados.0.smithi070.stdout:2731: read oid 39 snap 284 2024-09-14T03:57:40.161 INFO:tasks.rados.rados.0.smithi070.stdout:2731: expect (ObjNum 818 snap 283 seq_num 818) 2024-09-14T03:57:40.161 INFO:tasks.rados.rados.0.smithi070.stdout:2732: delete oid 5 current snap is 286 2024-09-14T03:57:40.161 INFO:tasks.rados.rados.0.smithi070.stdout:2728: finishing write tid 2 to smithi07085154-4 2024-09-14T03:57:40.161 INFO:tasks.rados.rados.0.smithi070.stdout:2728: finishing write tid 3 to smithi07085154-4 2024-09-14T03:57:40.162 INFO:tasks.rados.rados.0.smithi070.stdout:2728: finishing write tid 4 to smithi07085154-4 2024-09-14T03:57:40.162 INFO:tasks.rados.rados.0.smithi070.stdout:2728: finishing write tid 5 to smithi07085154-4 2024-09-14T03:57:40.162 INFO:tasks.rados.rados.0.smithi070.stdout:2728: finishing write tid 6 to smithi07085154-4 2024-09-14T03:57:40.162 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 2071 (ObjNum 845 snap 286 seq_num 845) dirty exists 2024-09-14T03:57:40.162 INFO:tasks.rados.rados.0.smithi070.stdout:2728: left oid 4 (ObjNum 845 snap 286 seq_num 845) 2024-09-14T03:57:40.163 INFO:tasks.rados.rados.0.smithi070.stdout:2728: done (2 left) 2024-09-14T03:57:40.163 INFO:tasks.rados.rados.0.smithi070.stdout:2732: done (1 left) 2024-09-14T03:57:40.163 INFO:tasks.rados.rados.0.smithi070.stdout:2733: copy_from oid 22 from oid 46 current snap is 286 2024-09-14T03:57:40.163 INFO:tasks.rados.rados.0.smithi070.stdout:2734: read oid 15 snap -1 2024-09-14T03:57:40.163 INFO:tasks.rados.rados.0.smithi070.stdout:2734: expect (ObjNum 805 snap 281 seq_num 805) 2024-09-14T03:57:40.163 INFO:tasks.rados.rados.0.smithi070.stdout:2735: read oid 10 snap -1 2024-09-14T03:57:40.163 INFO:tasks.rados.rados.0.smithi070.stdout:2735: expect (ObjNum 841 snap 286 seq_num 841) 2024-09-14T03:57:40.163 INFO:tasks.rados.rados.0.smithi070.stdout:2736: rollback oid 9 current snap is 286 2024-09-14T03:57:40.163 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 9 to 215 2024-09-14T03:57:40.164 INFO:tasks.rados.rados.0.smithi070.stdout:2737: rollback oid 1 current snap is 286 2024-09-14T03:57:40.164 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 1 to 265 2024-09-14T03:57:40.164 INFO:tasks.rados.rados.0.smithi070.stdout:2738: setattr oid 21 current snap is 286 2024-09-14T03:57:40.164 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 2 current snap is 286 2024-09-14T03:57:40.164 INFO:tasks.rados.rados.0.smithi070.stdout:2739: seq_num 847 ranges {4890624=589824,5480448=753664,6234112=655360,6889472=196608} 2024-09-14T03:57:40.194 INFO:tasks.rados.rados.0.smithi070.stdout:2739: writing smithi07085154-2 from 4890624 to 5480448 tid 1 2024-09-14T03:57:40.198 INFO:tasks.rados.rados.0.smithi070.stdout:2739: writing smithi07085154-2 from 5480448 to 6234112 tid 2 2024-09-14T03:57:40.202 INFO:tasks.rados.rados.0.smithi070.stdout:2739: writing smithi07085154-2 from 6234112 to 6889472 tid 3 2024-09-14T03:57:40.204 INFO:tasks.rados.rados.0.smithi070.stdout:2739: writing smithi07085154-2 from 6889472 to 7086080 tid 4 2024-09-14T03:57:40.204 INFO:tasks.rados.rados.0.smithi070.stdout:2740: read oid 26 snap -1 2024-09-14T03:57:40.204 INFO:tasks.rados.rados.0.smithi070.stdout:2740: expect (ObjNum 712 snap 243 seq_num 712) 2024-09-14T03:57:40.204 INFO:tasks.rados.rados.0.smithi070.stdout:2741: delete oid 40 current snap is 286 2024-09-14T03:57:40.204 INFO:tasks.rados.rados.0.smithi070.stdout:2737: finishing rollback tid 0 to smithi07085154-1 2024-09-14T03:57:40.204 INFO:tasks.rados.rados.0.smithi070.stdout:2731: expect (ObjNum 818 snap 283 seq_num 818) 2024-09-14T03:57:40.214 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 03:57:40 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[96999]: 2024-09-14T03:57:40.103+0000 7f90586de640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap 15c after trimming was started 2024-09-14T03:57:40.214 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:39 smithi094 podman[114786]: 2024-09-14 03:57:39.901506625 +0000 UTC m=+1.168528082 container create b7cb3ebd57f568c8c28045d8be60e740b566b10beda2ce7108ab317872775ebf (image=quay.io/ceph/grafana:10.4.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-14T03:57:40.249 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 1714 (ObjNum 833 snap 285 seq_num 833) dirty exists 2024-09-14T03:57:40.249 INFO:tasks.rados.rados.0.smithi070.stdout:2734: expect (ObjNum 805 snap 281 seq_num 805) 2024-09-14T03:57:40.286 INFO:tasks.rados.rados.0.smithi070.stdout:2737: finishing rollback tid 1 to smithi07085154-1 2024-09-14T03:57:40.287 INFO:tasks.rados.rados.0.smithi070.stdout:2737: finishing rollback tid 2 to smithi07085154-1 2024-09-14T03:57:40.287 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 1 v 2083 (ObjNum 712 snap 243 seq_num 712) dirty exists 2024-09-14T03:57:40.287 INFO:tasks.rados.rados.0.smithi070.stdout:2736: finishing rollback tid 0 to smithi07085154-9 2024-09-14T03:57:40.287 INFO:tasks.rados.rados.0.smithi070.stdout:2733: finishing copy_from racing read to smithi07085154-22 2024-09-14T03:57:40.287 INFO:tasks.rados.rados.0.smithi070.stdout:2735: expect (ObjNum 841 snap 286 seq_num 841) 2024-09-14T03:57:40.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:57:40 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:57:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:57:40.716 INFO:tasks.rados.rados.0.smithi070.stdout:2736: finishing rollback tid 1 to smithi07085154-9 2024-09-14T03:57:40.717 INFO:tasks.rados.rados.0.smithi070.stdout:2733: finishing copy_from to smithi07085154-22 2024-09-14T03:57:40.717 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 3147 (ObjNum 576 snap 193 seq_num 576) dirty exists 2024-09-14T03:57:40.717 INFO:tasks.rados.rados.0.smithi070.stdout:2736: finishing rollback tid 2 to smithi07085154-9 2024-09-14T03:57:40.717 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 3179 (ObjNum 459 snap 155 seq_num 459) dirty exists 2024-09-14T03:57:40.717 INFO:tasks.rados.rados.0.smithi070.stdout:2739: finishing write tid 1 to smithi07085154-2 2024-09-14T03:57:40.717 INFO:tasks.rados.rados.0.smithi070.stdout:2739: finishing write tid 2 to smithi07085154-2 2024-09-14T03:57:40.717 INFO:tasks.rados.rados.0.smithi070.stdout:2739: finishing write tid 3 to smithi07085154-2 2024-09-14T03:57:40.717 INFO:tasks.rados.rados.0.smithi070.stdout:2731: done (9 left) 2024-09-14T03:57:40.717 INFO:tasks.rados.rados.0.smithi070.stdout:2733: done (8 left) 2024-09-14T03:57:40.717 INFO:tasks.rados.rados.0.smithi070.stdout:2734: done (7 left) 2024-09-14T03:57:40.717 INFO:tasks.rados.rados.0.smithi070.stdout:2735: done (6 left) 2024-09-14T03:57:40.717 INFO:tasks.rados.rados.0.smithi070.stdout:2736: done (5 left) 2024-09-14T03:57:40.718 INFO:tasks.rados.rados.0.smithi070.stdout:2737: done (4 left) 2024-09-14T03:57:40.718 INFO:tasks.rados.rados.0.smithi070.stdout:2738: done (3 left) 2024-09-14T03:57:40.718 INFO:tasks.rados.rados.0.smithi070.stdout:2741: done (2 left) 2024-09-14T03:57:40.718 INFO:tasks.rados.rados.0.smithi070.stdout:2742: read oid 39 snap -1 2024-09-14T03:57:40.718 INFO:tasks.rados.rados.0.smithi070.stdout:2742: expect (ObjNum 818 snap 283 seq_num 818) 2024-09-14T03:57:40.718 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 32 current snap is 286 2024-09-14T03:57:40.718 INFO:tasks.rados.rados.0.smithi070.stdout:2739: finishing write tid 4 to smithi07085154-2 2024-09-14T03:57:40.718 INFO:tasks.rados.rados.0.smithi070.stdout:2739: finishing write tid 5 to smithi07085154-2 2024-09-14T03:57:40.718 INFO:tasks.rados.rados.0.smithi070.stdout:2739: finishing write tid 6 to smithi07085154-2 2024-09-14T03:57:40.718 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 2 v 3184 (ObjNum 847 snap 286 seq_num 847) dirty exists 2024-09-14T03:57:40.718 INFO:tasks.rados.rados.0.smithi070.stdout:2739: left oid 2 (ObjNum 847 snap 286 seq_num 847) 2024-09-14T03:57:40.718 INFO:tasks.rados.rados.0.smithi070.stdout:2740: expect (ObjNum 712 snap 243 seq_num 712) 2024-09-14T03:57:40.775 INFO:tasks.rados.rados.0.smithi070.stdout:2742: expect (ObjNum 818 snap 283 seq_num 818) 2024-09-14T03:57:40.818 INFO:tasks.rados.rados.0.smithi070.stdout:2743: seq_num 849 ranges {0=622592,622592=516096,1138688=663552,1802240=425984} 2024-09-14T03:57:40.821 INFO:tasks.rados.rados.0.smithi070.stdout:2743: writing smithi07085154-32 from 0 to 622592 tid 1 2024-09-14T03:57:40.824 INFO:tasks.rados.rados.0.smithi070.stdout:2743: writing smithi07085154-32 from 622592 to 1138688 tid 2 2024-09-14T03:57:40.828 INFO:tasks.rados.rados.0.smithi070.stdout:2743: writing smithi07085154-32 from 1138688 to 1802240 tid 3 2024-09-14T03:57:40.831 INFO:tasks.rados.rados.0.smithi070.stdout:2743: writing smithi07085154-32 from 1802240 to 2228224 tid 4 2024-09-14T03:57:40.831 INFO:tasks.rados.rados.0.smithi070.stdout:2739: done (3 left) 2024-09-14T03:57:40.831 INFO:tasks.rados.rados.0.smithi070.stdout:2740: done (2 left) 2024-09-14T03:57:40.831 INFO:tasks.rados.rados.0.smithi070.stdout:2742: done (1 left) 2024-09-14T03:57:40.832 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 14 current snap is 286 2024-09-14T03:57:40.832 INFO:tasks.rados.rados.0.smithi070.stdout:2743: finishing write tid 1 to smithi07085154-32 2024-09-14T03:57:40.832 INFO:tasks.rados.rados.0.smithi070.stdout:2744: seq_num 850 ranges {0=638976,638976=802816,1441792=475136,1916928=327680} 2024-09-14T03:57:40.835 INFO:tasks.rados.rados.0.smithi070.stdout:2744: writing smithi07085154-14 from 0 to 638976 tid 1 2024-09-14T03:57:40.840 INFO:tasks.rados.rados.0.smithi070.stdout:2744: writing smithi07085154-14 from 638976 to 1441792 tid 2 2024-09-14T03:57:40.842 INFO:tasks.rados.rados.0.smithi070.stdout:2744: writing smithi07085154-14 from 1441792 to 1916928 tid 3 2024-09-14T03:57:40.845 INFO:tasks.rados.rados.0.smithi070.stdout:2744: writing smithi07085154-14 from 1916928 to 2244608 tid 4 2024-09-14T03:57:40.845 INFO:tasks.rados.rados.0.smithi070.stdout:2745: rollback oid 3 current snap is 286 2024-09-14T03:57:40.845 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 3 to 242 2024-09-14T03:57:40.845 INFO:tasks.rados.rados.0.smithi070.stdout:2743: finishing write tid 2 to smithi07085154-32 2024-09-14T03:57:40.846 INFO:tasks.rados.rados.0.smithi070.stdout:2743: finishing write tid 3 to smithi07085154-32 2024-09-14T03:57:40.846 INFO:tasks.rados.rados.0.smithi070.stdout:2743: finishing write tid 4 to smithi07085154-32 2024-09-14T03:57:40.846 INFO:tasks.rados.rados.0.smithi070.stdout:2743: finishing write tid 5 to smithi07085154-32 2024-09-14T03:57:40.846 INFO:tasks.rados.rados.0.smithi070.stdout:2743: finishing write tid 6 to smithi07085154-32 2024-09-14T03:57:40.846 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 2268 (ObjNum 849 snap 286 seq_num 849) dirty exists 2024-09-14T03:57:40.846 INFO:tasks.rados.rados.0.smithi070.stdout:2743: left oid 32 (ObjNum 849 snap 286 seq_num 849) 2024-09-14T03:57:40.846 INFO:tasks.rados.rados.0.smithi070.stdout:2744: finishing write tid 1 to smithi07085154-14 2024-09-14T03:57:40.846 INFO:tasks.rados.rados.0.smithi070.stdout:2743: done (2 left) 2024-09-14T03:57:40.846 INFO:tasks.rados.rados.0.smithi070.stdout:2746: rmattr oid 50 current snap is 286 2024-09-14T03:57:40.846 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 30 current snap is 286 2024-09-14T03:57:40.846 INFO:tasks.rados.rados.0.smithi070.stdout:2747: seq_num 851 ranges {0=655360,655360=688128,1343488=679936,2023424=237568} 2024-09-14T03:57:40.848 INFO:tasks.rados.rados.0.smithi070.stdout:2747: writing smithi07085154-30 from 0 to 655360 tid 1 2024-09-14T03:57:40.852 INFO:tasks.rados.rados.0.smithi070.stdout:2747: writing smithi07085154-30 from 655360 to 1343488 tid 2 2024-09-14T03:57:40.856 INFO:tasks.rados.rados.0.smithi070.stdout:2747: writing smithi07085154-30 from 1343488 to 2023424 tid 3 2024-09-14T03:57:40.858 INFO:tasks.rados.rados.0.smithi070.stdout:2747: writing smithi07085154-30 from 2023424 to 2260992 tid 4 2024-09-14T03:57:40.858 INFO:tasks.rados.rados.0.smithi070.stdout:2748: copy_from oid 4 from oid 46 current snap is 286 2024-09-14T03:57:40.858 INFO:tasks.rados.rados.0.smithi070.stdout:2744: finishing write tid 2 to smithi07085154-14 2024-09-14T03:57:40.858 INFO:tasks.rados.rados.0.smithi070.stdout:2744: finishing write tid 3 to smithi07085154-14 2024-09-14T03:57:40.859 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 3148 (ObjNum 799 snap 279 seq_num 799) dirty exists 2024-09-14T03:57:40.859 INFO:tasks.rados.rados.0.smithi070.stdout:2744: finishing write tid 4 to smithi07085154-14 2024-09-14T03:57:40.859 INFO:tasks.rados.rados.0.smithi070.stdout:2744: finishing write tid 5 to smithi07085154-14 2024-09-14T03:57:40.859 INFO:tasks.rados.rados.0.smithi070.stdout:2745: finishing rollback tid 0 to smithi07085154-3 2024-09-14T03:57:40.859 INFO:tasks.rados.rados.0.smithi070.stdout:2746: done (4 left) 2024-09-14T03:57:40.859 INFO:tasks.rados.rados.0.smithi070.stdout:2749: read oid 38 snap -1 2024-09-14T03:57:40.859 INFO:tasks.rados.rados.0.smithi070.stdout:2749: expect deleted 2024-09-14T03:57:40.859 INFO:tasks.rados.rados.0.smithi070.stdout:2750: setattr oid 40 current snap is 286 2024-09-14T03:57:40.859 INFO:tasks.rados.rados.0.smithi070.stdout:2745: finishing rollback tid 1 to smithi07085154-3 2024-09-14T03:57:40.859 INFO:tasks.rados.rados.0.smithi070.stdout:2744: finishing write tid 6 to smithi07085154-14 2024-09-14T03:57:40.859 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 2790 (ObjNum 850 snap 286 seq_num 850) dirty exists 2024-09-14T03:57:40.859 INFO:tasks.rados.rados.0.smithi070.stdout:2744: left oid 14 (ObjNum 850 snap 286 seq_num 850) 2024-09-14T03:57:40.859 INFO:tasks.rados.rados.0.smithi070.stdout:2745: finishing rollback tid 2 to smithi07085154-3 2024-09-14T03:57:40.860 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 3 v 1728 (ObjNum 683 snap 230 seq_num 683) dirty exists 2024-09-14T03:57:40.860 INFO:tasks.rados.rados.0.smithi070.stdout:2747: finishing write tid 1 to smithi07085154-30 2024-09-14T03:57:40.860 INFO:tasks.rados.rados.0.smithi070.stdout:2747: finishing write tid 2 to smithi07085154-30 2024-09-14T03:57:40.860 INFO:tasks.rados.rados.0.smithi070.stdout:2744: done (5 left) 2024-09-14T03:57:40.860 INFO:tasks.rados.rados.0.smithi070.stdout:2745: done (4 left) 2024-09-14T03:57:40.861 INFO:tasks.rados.rados.0.smithi070.stdout:2751: rmattr oid 10 current snap is 286 2024-09-14T03:57:40.861 INFO:tasks.rados.rados.0.smithi070.stdout:2751: done (4 left) 2024-09-14T03:57:40.861 INFO:tasks.rados.rados.0.smithi070.stdout:2752: read oid 22 snap 268 2024-09-14T03:57:40.861 INFO:tasks.rados.rados.0.smithi070.stdout:2752: expect (ObjNum 458 snap 155 seq_num 458) 2024-09-14T03:57:40.861 INFO:tasks.rados.rados.0.smithi070.stdout:2753: snap_create 2024-09-14T03:57:40.861 INFO:tasks.rados.rados.0.smithi070.stdout:2748: finishing copy_from racing read to smithi07085154-4 2024-09-14T03:57:40.862 INFO:tasks.rados.rados.0.smithi070.stdout:2748: finishing copy_from to smithi07085154-4 2024-09-14T03:57:40.862 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 3147 (ObjNum 576 snap 193 seq_num 576) dirty exists 2024-09-14T03:57:40.862 INFO:tasks.rados.rados.0.smithi070.stdout:2747: finishing write tid 3 to smithi07085154-30 2024-09-14T03:57:40.862 INFO:tasks.rados.rados.0.smithi070.stdout:2747: finishing write tid 4 to smithi07085154-30 2024-09-14T03:57:40.863 INFO:tasks.rados.rados.0.smithi070.stdout:2747: finishing write tid 5 to smithi07085154-30 2024-09-14T03:57:40.866 INFO:tasks.rados.rados.0.smithi070.stdout:2747: finishing write tid 6 to smithi07085154-30 2024-09-14T03:57:40.866 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 30 v 1733 (ObjNum 851 snap 286 seq_num 851) dirty exists 2024-09-14T03:57:40.866 INFO:tasks.rados.rados.0.smithi070.stdout:2747: left oid 30 (ObjNum 851 snap 286 seq_num 851) 2024-09-14T03:57:40.866 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 2085 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-14T03:57:40.891 INFO:tasks.rados.rados.0.smithi070.stdout:2752: expect (ObjNum 458 snap 155 seq_num 458) 2024-09-14T03:57:41.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-mon[86597]: osdmap e629: 8 total, 8 up, 8 in 2024-09-14T03:57:41.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-mon[86597]: pgmap v540: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1.1 GiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.9 MiB/s wr, 15 op/s 2024-09-14T03:57:41.277 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 podman[114786]: 2024-09-14 03:57:41.171273741 +0000 UTC m=+2.438295200 container init b7cb3ebd57f568c8c28045d8be60e740b566b10beda2ce7108ab317872775ebf (image=quay.io/ceph/grafana:10.4.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-14T03:57:41.277 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 podman[114786]: 2024-09-14 03:57:41.176133715 +0000 UTC m=+2.443155171 container start b7cb3ebd57f568c8c28045d8be60e740b566b10beda2ce7108ab317872775ebf (image=quay.io/ceph/grafana:10.4.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-14T03:57:41.277 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 bash[114786]: b7cb3ebd57f568c8c28045d8be60e740b566b10beda2ce7108ab317872775ebf 2024-09-14T03:57:41.277 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 systemd[1]: Started Ceph grafana.a for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T03:57:41.435 INFO:tasks.rados.rados.0.smithi070.stdout:2747: done (5 left) 2024-09-14T03:57:41.435 INFO:tasks.rados.rados.0.smithi070.stdout:2748: done (4 left) 2024-09-14T03:57:41.435 INFO:tasks.rados.rados.0.smithi070.stdout:2749: done (3 left) 2024-09-14T03:57:41.435 INFO:tasks.rados.rados.0.smithi070.stdout:2750: done (2 left) 2024-09-14T03:57:41.436 INFO:tasks.rados.rados.0.smithi070.stdout:2752: done (1 left) 2024-09-14T03:57:41.436 INFO:tasks.rados.rados.0.smithi070.stdout:2753: done (0 left) 2024-09-14T03:57:41.436 INFO:tasks.rados.rados.0.smithi070.stdout:2754: rollback oid 19 current snap is 287 2024-09-14T03:57:41.436 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 19 to 226 2024-09-14T03:57:41.436 INFO:tasks.rados.rados.0.smithi070.stdout:2755: snap_create 2024-09-14T03:57:41.444 INFO:tasks.rados.rados.0.smithi070.stdout:2754: finishing rollback tid 1 to smithi07085154-19 2024-09-14T03:57:41.444 INFO:tasks.rados.rados.0.smithi070.stdout:2754: finishing rollback tid 2 to smithi07085154-19 2024-09-14T03:57:41.444 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 19 v 3150 (ObjNum 659 snap 220 seq_num 659) dirty exists 2024-09-14T03:57:41.529 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=settings t=2024-09-14T03:57:41.299128192Z level=info msg="Starting Grafana" version=10.4.0 commit=03f502a94d17f7dc4e6c34acdf8428aedd986e4c branch=HEAD compiled=2024-09-14T03:57:41Z 2024-09-14T03:57:41.529 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=settings t=2024-09-14T03:57:41.299354018Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-14T03:57:41.529 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=settings t=2024-09-14T03:57:41.29936201Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-14T03:57:41.529 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=settings t=2024-09-14T03:57:41.299366147Z level=info msg="Config overridden from command line" arg="default.paths.data=/var/lib/grafana" 2024-09-14T03:57:41.529 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=settings t=2024-09-14T03:57:41.299369647Z level=info msg="Config overridden from command line" arg="default.paths.logs=/var/log/grafana" 2024-09-14T03:57:41.529 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=settings t=2024-09-14T03:57:41.299372141Z level=info msg="Config overridden from command line" arg="default.paths.plugins=/var/lib/grafana/plugins" 2024-09-14T03:57:41.529 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=settings t=2024-09-14T03:57:41.299374616Z level=info msg="Config overridden from command line" arg="default.paths.provisioning=/etc/grafana/provisioning" 2024-09-14T03:57:41.529 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=settings t=2024-09-14T03:57:41.299377216Z level=info msg="Config overridden from command line" arg="default.log.mode=console" 2024-09-14T03:57:41.529 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=settings t=2024-09-14T03:57:41.299380036Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-14T03:57:41.529 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=settings t=2024-09-14T03:57:41.299382739Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-14T03:57:41.530 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=settings t=2024-09-14T03:57:41.299385347Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-14T03:57:41.530 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=settings t=2024-09-14T03:57:41.299387842Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-14T03:57:41.530 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=settings t=2024-09-14T03:57:41.299390382Z level=info msg=Target target=[all] 2024-09-14T03:57:41.530 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=settings t=2024-09-14T03:57:41.299396908Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-14T03:57:41.530 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=settings t=2024-09-14T03:57:41.299402926Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-14T03:57:41.530 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=settings t=2024-09-14T03:57:41.299405505Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-14T03:57:41.530 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=settings t=2024-09-14T03:57:41.299407836Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-14T03:57:41.530 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=settings t=2024-09-14T03:57:41.299410307Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-14T03:57:41.531 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=settings t=2024-09-14T03:57:41.299412957Z level=info msg="App mode production" 2024-09-14T03:57:41.531 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=sqlstore t=2024-09-14T03:57:41.299655304Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-14T03:57:41.531 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=sqlstore t=2024-09-14T03:57:41.299667304Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-14T03:57:41.531 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.30016058Z level=info msg="Starting DB migrations" 2024-09-14T03:57:41.531 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.311514613Z level=info msg="Executing migration" id="Add uid column to user" 2024-09-14T03:57:41.532 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.317348667Z level=info msg="Migration successfully executed" id="Add uid column to user" duration=5.825971ms 2024-09-14T03:57:41.532 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.318003025Z level=info msg="Executing migration" id="Update uid column values for users" 2024-09-14T03:57:41.532 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.318165169Z level=info msg="Migration successfully executed" id="Update uid column values for users" duration=159.789µs 2024-09-14T03:57:41.532 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.31855804Z level=info msg="Executing migration" id="Add unique index user_uid" 2024-09-14T03:57:41.532 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.319357808Z level=info msg="Migration successfully executed" id="Add unique index user_uid" duration=782.124µs 2024-09-14T03:57:41.532 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.31985867Z level=info msg="Executing migration" id="Add playlist column created_at" 2024-09-14T03:57:41.532 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.325229578Z level=info msg="Migration successfully executed" id="Add playlist column created_at" duration=5.367561ms 2024-09-14T03:57:41.532 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.325718756Z level=info msg="Executing migration" id="Add playlist column updated_at" 2024-09-14T03:57:41.532 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.33148439Z level=info msg="Migration successfully executed" id="Add playlist column updated_at" duration=5.763724ms 2024-09-14T03:57:41.533 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.331988034Z level=info msg="Executing migration" id="Add column uid in team" 2024-09-14T03:57:41.533 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.337322559Z level=info msg="Migration successfully executed" id="Add column uid in team" duration=5.332712ms 2024-09-14T03:57:41.533 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.337750837Z level=info msg="Executing migration" id="Update uid column values in team" 2024-09-14T03:57:41.533 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.337923454Z level=info msg="Migration successfully executed" id="Update uid column values in team" duration=171.885µs 2024-09-14T03:57:41.533 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.338285017Z level=info msg="Executing migration" id="Add unique index team_org_id_uid" 2024-09-14T03:57:41.533 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.339195822Z level=info msg="Migration successfully executed" id="Add unique index team_org_id_uid" duration=909.8µs 2024-09-14T03:57:41.533 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.339686196Z level=info msg="Executing migration" id="add index user_auth_token.revoked_at" 2024-09-14T03:57:41.533 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.340414017Z level=info msg="Migration successfully executed" id="add index user_auth_token.revoked_at" duration=727.043µs 2024-09-14T03:57:41.533 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.34083424Z level=info msg="Executing migration" id="add result_fingerprint column to alert_instance" 2024-09-14T03:57:41.534 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.347072486Z level=info msg="Migration successfully executed" id="add result_fingerprint column to alert_instance" duration=6.236202ms 2024-09-14T03:57:41.534 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.347524663Z level=info msg="Executing migration" id="add last_applied column to alert_configuration_history" 2024-09-14T03:57:41.534 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.352530548Z level=info msg="Migration successfully executed" id="add last_applied column to alert_configuration_history" duration=5.004866ms 2024-09-14T03:57:41.534 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.352927515Z level=info msg="Executing migration" id="alter library_element model to mediumtext" 2024-09-14T03:57:41.534 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.352979184Z level=info msg="Migration successfully executed" id="alter library_element model to mediumtext" duration=51.408µs 2024-09-14T03:57:41.534 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.353427154Z level=info msg="Executing migration" id="permission kind migration" 2024-09-14T03:57:41.534 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.358457183Z level=info msg="Migration successfully executed" id="permission kind migration" duration=5.028656ms 2024-09-14T03:57:41.534 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.358852632Z level=info msg="Executing migration" id="permission attribute migration" 2024-09-14T03:57:41.534 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.364060314Z level=info msg="Migration successfully executed" id="permission attribute migration" duration=5.206361ms 2024-09-14T03:57:41.534 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.36441883Z level=info msg="Executing migration" id="permission identifier migration" 2024-09-14T03:57:41.534 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.369877402Z level=info msg="Migration successfully executed" id="permission identifier migration" duration=5.457235ms 2024-09-14T03:57:41.534 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.370235559Z level=info msg="Executing migration" id="add permission identifier index" 2024-09-14T03:57:41.535 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.371054221Z level=info msg="Migration successfully executed" id="add permission identifier index" duration=818.431µs 2024-09-14T03:57:41.535 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.37144015Z level=info msg="Executing migration" id="add permission action scope role_id index" 2024-09-14T03:57:41.535 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.372236236Z level=info msg="Migration successfully executed" id="add permission action scope role_id index" duration=793.006µs 2024-09-14T03:57:41.535 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.372582847Z level=info msg="Executing migration" id="remove permission role_id action scope index" 2024-09-14T03:57:41.535 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.373314247Z level=info msg="Migration successfully executed" id="remove permission role_id action scope index" duration=731.264µs 2024-09-14T03:57:41.535 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.373659555Z level=info msg="Executing migration" id="rbac disabled migrator" 2024-09-14T03:57:41.535 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.373679354Z level=info msg="Migration successfully executed" id="rbac disabled migrator" duration=20.124µs 2024-09-14T03:57:41.535 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.374096452Z level=info msg="Executing migration" id="drop index IDX_correlation_uid - v1" 2024-09-14T03:57:41.535 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.374787977Z level=info msg="Migration successfully executed" id="drop index IDX_correlation_uid - v1" duration=690.827µs 2024-09-14T03:57:41.535 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.375124383Z level=info msg="Executing migration" id="drop index IDX_correlation_source_uid - v1" 2024-09-14T03:57:41.535 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.375825459Z level=info msg="Migration successfully executed" id="drop index IDX_correlation_source_uid - v1" duration=701.013µs 2024-09-14T03:57:41.536 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.376183322Z level=info msg="Executing migration" id="Rename table correlation to correlation_tmp_qwerty - v1" 2024-09-14T03:57:41.536 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.391374329Z level=info msg="Migration successfully executed" id="Rename table correlation to correlation_tmp_qwerty - v1" duration=15.187879ms 2024-09-14T03:57:41.536 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.391903868Z level=info msg="Executing migration" id="create correlation v2" 2024-09-14T03:57:41.536 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.392690742Z level=info msg="Migration successfully executed" id="create correlation v2" duration=785.052µs 2024-09-14T03:57:41.536 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.393076141Z level=info msg="Executing migration" id="create index IDX_correlation_uid - v2" 2024-09-14T03:57:41.536 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.393790023Z level=info msg="Migration successfully executed" id="create index IDX_correlation_uid - v2" duration=713.666µs 2024-09-14T03:57:41.536 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.394124887Z level=info msg="Executing migration" id="create index IDX_correlation_source_uid - v2" 2024-09-14T03:57:41.536 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.394846746Z level=info msg="Migration successfully executed" id="create index IDX_correlation_source_uid - v2" duration=721.7µs 2024-09-14T03:57:41.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:41 smithi070 ceph-mon[96704]: osdmap e629: 8 total, 8 up, 8 in 2024-09-14T03:57:41.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:41 smithi070 ceph-mon[96704]: pgmap v540: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1.1 GiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.9 MiB/s wr, 15 op/s 2024-09-14T03:57:41.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:41 smithi070 ceph-mon[98640]: osdmap e629: 8 total, 8 up, 8 in 2024-09-14T03:57:41.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:41 smithi070 ceph-mon[98640]: pgmap v540: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1.1 GiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.9 MiB/s wr, 15 op/s 2024-09-14T03:57:41.538 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.395221918Z level=info msg="Executing migration" id="create index IDX_correlation_org_id - v2" 2024-09-14T03:57:41.538 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.395931892Z level=info msg="Migration successfully executed" id="create index IDX_correlation_org_id - v2" duration=709.748µs 2024-09-14T03:57:41.538 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.396293211Z level=info msg="Executing migration" id="copy correlation v1 to v2" 2024-09-14T03:57:41.538 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.396495048Z level=info msg="Migration successfully executed" id="copy correlation v1 to v2" duration=201.604µs 2024-09-14T03:57:41.538 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.396876991Z level=info msg="Executing migration" id="drop correlation_tmp_qwerty" 2024-09-14T03:57:41.539 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.397492609Z level=info msg="Migration successfully executed" id="drop correlation_tmp_qwerty" duration=613.576µs 2024-09-14T03:57:41.539 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.397849422Z level=info msg="Executing migration" id="add provisioning column" 2024-09-14T03:57:41.539 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.403110207Z level=info msg="Migration successfully executed" id="add provisioning column" duration=5.259385ms 2024-09-14T03:57:41.539 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.403546616Z level=info msg="Executing migration" id="backfill empty share column fields with default of public" 2024-09-14T03:57:41.539 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.403717507Z level=info msg="Migration successfully executed" id="backfill empty share column fields with default of public" duration=170.345µs 2024-09-14T03:57:41.539 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.404136384Z level=info msg="Executing migration" id="migrate contents column to mediumblob for MySQL" 2024-09-14T03:57:41.539 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.404179963Z level=info msg="Migration successfully executed" id="migrate contents column to mediumblob for MySQL" duration=42.835µs 2024-09-14T03:57:41.539 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.404597389Z level=info msg="Executing migration" id="add origin column to seed_assignment" 2024-09-14T03:57:41.539 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.410280894Z level=info msg="Migration successfully executed" id="add origin column to seed_assignment" duration=5.680466ms 2024-09-14T03:57:41.539 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.410767521Z level=info msg="Executing migration" id="add origin to plugin seed_assignment" 2024-09-14T03:57:41.539 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.411001327Z level=info msg="Migration successfully executed" id="add origin to plugin seed_assignment" duration=234.751µs 2024-09-14T03:57:41.540 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.411383306Z level=info msg="Executing migration" id="prevent seeding OnCall access" 2024-09-14T03:57:41.540 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.411495144Z level=info msg="Migration successfully executed" id="prevent seeding OnCall access" duration=111.889µs 2024-09-14T03:57:41.540 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.411820068Z level=info msg="Executing migration" id="managed folder permissions library panel actions migration" 2024-09-14T03:57:41.540 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.41277304Z level=info msg="Migration successfully executed" id="managed folder permissions library panel actions migration" duration=951.299µs 2024-09-14T03:57:41.540 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.41316415Z level=info msg="Executing migration" id="Remove unique index for folder.title and folder.parent_uid" 2024-09-14T03:57:41.540 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.413974496Z level=info msg="Migration successfully executed" id="Remove unique index for folder.title and folder.parent_uid" duration=812.153µs 2024-09-14T03:57:41.540 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.414468209Z level=info msg="Executing migration" id="Add unique index for title, parent_uid, and org_id" 2024-09-14T03:57:41.540 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.415345065Z level=info msg="Migration successfully executed" id="Add unique index for title, parent_uid, and org_id" duration=876.649µs 2024-09-14T03:57:41.540 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.41570704Z level=info msg="Executing migration" id="Sync dashboard and folder table" 2024-09-14T03:57:41.540 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.416132649Z level=info msg="Migration successfully executed" id="Sync dashboard and folder table" duration=426.869µs 2024-09-14T03:57:41.540 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.416443698Z level=info msg="Executing migration" id="Remove ghost folders from the folder table" 2024-09-14T03:57:41.540 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.416637917Z level=info msg="Migration successfully executed" id="Remove ghost folders from the folder table" duration=194.071µs 2024-09-14T03:57:41.541 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.416994238Z level=info msg="Executing migration" id="Remove unique index UQE_folder_uid_org_id" 2024-09-14T03:57:41.541 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.417697605Z level=info msg="Migration successfully executed" id="Remove unique index UQE_folder_uid_org_id" duration=703.246µs 2024-09-14T03:57:41.541 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.418043901Z level=info msg="Executing migration" id="Add unique index UQE_folder_org_id_uid" 2024-09-14T03:57:41.541 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.418744309Z level=info msg="Migration successfully executed" id="Add unique index UQE_folder_org_id_uid" duration=700.141µs 2024-09-14T03:57:41.541 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.419058406Z level=info msg="Executing migration" id="Remove unique index UQE_folder_title_parent_uid_org_id" 2024-09-14T03:57:41.541 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.41973113Z level=info msg="Migration successfully executed" id="Remove unique index UQE_folder_title_parent_uid_org_id" duration=674.43µs 2024-09-14T03:57:41.541 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.420057769Z level=info msg="Executing migration" id="Add unique index UQE_folder_org_id_parent_uid_title" 2024-09-14T03:57:41.541 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.42076246Z level=info msg="Migration successfully executed" id="Add unique index UQE_folder_org_id_parent_uid_title" duration=704.371µs 2024-09-14T03:57:41.541 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.421134512Z level=info msg="Executing migration" id="Remove index IDX_folder_parent_uid_org_id" 2024-09-14T03:57:41.541 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.421813928Z level=info msg="Migration successfully executed" id="Remove index IDX_folder_parent_uid_org_id" duration=679.102µs 2024-09-14T03:57:41.541 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.422127568Z level=info msg="Executing migration" id="create anon_device table" 2024-09-14T03:57:41.542 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.422698803Z level=info msg="Migration successfully executed" id="create anon_device table" duration=570.719µs 2024-09-14T03:57:41.542 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.423040317Z level=info msg="Executing migration" id="add unique index anon_device.device_id" 2024-09-14T03:57:41.542 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.423746053Z level=info msg="Migration successfully executed" id="add unique index anon_device.device_id" duration=703.53µs 2024-09-14T03:57:41.542 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.424074993Z level=info msg="Executing migration" id="add index anon_device.updated_at" 2024-09-14T03:57:41.542 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.424777642Z level=info msg="Migration successfully executed" id="add index anon_device.updated_at" duration=702.24µs 2024-09-14T03:57:41.542 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.425144623Z level=info msg="Executing migration" id="create signing_key table" 2024-09-14T03:57:41.542 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.425746974Z level=info msg="Migration successfully executed" id="create signing_key table" duration=601.844µs 2024-09-14T03:57:41.542 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.426133931Z level=info msg="Executing migration" id="add unique index signing_key.key_id" 2024-09-14T03:57:41.542 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.426910054Z level=info msg="Migration successfully executed" id="add unique index signing_key.key_id" duration=776.721µs 2024-09-14T03:57:41.542 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.427274533Z level=info msg="Executing migration" id="set legacy alert migration status in kvstore" 2024-09-14T03:57:41.542 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.427997381Z level=info msg="Migration successfully executed" id="set legacy alert migration status in kvstore" duration=722.194µs 2024-09-14T03:57:41.543 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.428317039Z level=info msg="Executing migration" id="migrate record of created folders during legacy migration to kvstore" 2024-09-14T03:57:41.543 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.428757947Z level=info msg="Migration successfully executed" id="migrate record of created folders during legacy migration to kvstore" duration=441.202µs 2024-09-14T03:57:41.543 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.429162566Z level=info msg="Executing migration" id="Add folder_uid for dashboard" 2024-09-14T03:57:41.543 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.43449398Z level=info msg="Migration successfully executed" id="Add folder_uid for dashboard" duration=5.329265ms 2024-09-14T03:57:41.543 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.434915995Z level=info msg="Executing migration" id="Populate dashboard folder_uid column" 2024-09-14T03:57:41.543 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.436978777Z level=info msg="Migration successfully executed" id="Populate dashboard folder_uid column" duration=2.0634ms 2024-09-14T03:57:41.543 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.437863582Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_folder_uid_title" 2024-09-14T03:57:41.543 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.439131558Z level=info msg="Migration successfully executed" id="Add unique index for dashboard_org_id_folder_uid_title" duration=1.27ms 2024-09-14T03:57:41.543 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.440559042Z level=info msg="Executing migration" id="Delete unique index for dashboard_org_id_folder_id_title" 2024-09-14T03:57:41.543 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.441656811Z level=info msg="Migration successfully executed" id="Delete unique index for dashboard_org_id_folder_id_title" duration=1.100885ms 2024-09-14T03:57:41.543 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.442174442Z level=info msg="Executing migration" id="Delete unique index for dashboard_org_id_folder_uid_title" 2024-09-14T03:57:41.543 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.443211211Z level=info msg="Migration successfully executed" id="Delete unique index for dashboard_org_id_folder_uid_title" duration=1.036502ms 2024-09-14T03:57:41.543 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.443625542Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_folder_uid_title_is_folder" 2024-09-14T03:57:41.544 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.444898117Z level=info msg="Migration successfully executed" id="Add unique index for dashboard_org_id_folder_uid_title_is_folder" duration=1.272266ms 2024-09-14T03:57:41.544 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.445272767Z level=info msg="Executing migration" id="Restore index for dashboard_org_id_folder_id_title" 2024-09-14T03:57:41.544 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.446094978Z level=info msg="Migration successfully executed" id="Restore index for dashboard_org_id_folder_id_title" duration=821.776µs 2024-09-14T03:57:41.544 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.44644934Z level=info msg="Executing migration" id="create sso_setting table" 2024-09-14T03:57:41.544 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.447234999Z level=info msg="Migration successfully executed" id="create sso_setting table" duration=785.256µs 2024-09-14T03:57:41.544 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.447611889Z level=info msg="Executing migration" id="copy kvstore migration status to each org" 2024-09-14T03:57:41.544 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.448281021Z level=info msg="Migration successfully executed" id="copy kvstore migration status to each org" duration=669.057µs 2024-09-14T03:57:41.544 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.448598446Z level=info msg="Executing migration" id="add back entry for orgid=0 migrated status" 2024-09-14T03:57:41.544 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.449021049Z level=info msg="Migration successfully executed" id="add back entry for orgid=0 migrated status" duration=422.571µs 2024-09-14T03:57:41.544 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.449409412Z level=info msg="Executing migration" id="alter kv_store.value to longtext" 2024-09-14T03:57:41.544 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.449454495Z level=info msg="Migration successfully executed" id="alter kv_store.value to longtext" duration=44.896µs 2024-09-14T03:57:41.544 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.449860185Z level=info msg="Executing migration" id="add notification_settings column to alert_rule table" 2024-09-14T03:57:41.545 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.455203066Z level=info msg="Migration successfully executed" id="add notification_settings column to alert_rule table" duration=5.342033ms 2024-09-14T03:57:41.545 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.455553784Z level=info msg="Executing migration" id="add notification_settings column to alert_rule_version table" 2024-09-14T03:57:41.545 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.460995711Z level=info msg="Migration successfully executed" id="add notification_settings column to alert_rule_version table" duration=5.439681ms 2024-09-14T03:57:41.545 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.461439874Z level=info msg="Executing migration" id="removing scope from alert.instances:read action migration" 2024-09-14T03:57:41.545 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.461718351Z level=info msg="Migration successfully executed" id="removing scope from alert.instances:read action migration" duration=277.276µs 2024-09-14T03:57:41.545 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=migrator t=2024-09-14T03:57:41.46209788Z level=info msg="migrations completed" performed=65 skipped=482 duration=150.655471ms 2024-09-14T03:57:41.545 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=sqlstore t=2024-09-14T03:57:41.462837439Z level=info msg="Created default organization" 2024-09-14T03:57:41.545 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=secrets t=2024-09-14T03:57:41.464912239Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-14T03:57:41.545 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=plugin.store t=2024-09-14T03:57:41.480003567Z level=info msg="Loading plugins..." 2024-09-14T03:57:41.545 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=local.finder t=2024-09-14T03:57:41.525261203Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled 2024-09-14T03:57:41.545 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=plugin.store t=2024-09-14T03:57:41.525282569Z level=info msg="Plugins loaded" count=55 duration=45.27881ms 2024-09-14T03:57:41.545 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=query_data t=2024-09-14T03:57:41.527334112Z level=info msg="Query Service initialization" 2024-09-14T03:57:41.546 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=live.push_http t=2024-09-14T03:57:41.530634086Z level=info msg="Live Push Gateway initialization" 2024-09-14T03:57:41.546 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=ngalert.migration t=2024-09-14T03:57:41.532252514Z level=info msg=Starting 2024-09-14T03:57:41.817 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=ngalert.state.manager t=2024-09-14T03:57:41.540413627Z level=info msg="Running in alternative execution of Error/NoData mode" 2024-09-14T03:57:41.817 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=infra.usagestats.collector t=2024-09-14T03:57:41.541867426Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-14T03:57:41.817 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=provisioning.datasources t=2024-09-14T03:57:41.544457751Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-09-14T03:57:41.817 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=provisioning.datasources t=2024-09-14T03:57:41.544724927Z level=info msg="inserting datasource from configuration" name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-14T03:57:41.817 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=provisioning.alerting t=2024-09-14T03:57:41.579213792Z level=info msg="starting to provision alerting" 2024-09-14T03:57:41.818 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=provisioning.alerting t=2024-09-14T03:57:41.579232322Z level=info msg="finished to provision alerting" 2024-09-14T03:57:41.818 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=grafanaStorageLogger t=2024-09-14T03:57:41.579373345Z level=info msg="Storage starting" 2024-09-14T03:57:41.818 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=ngalert.state.manager t=2024-09-14T03:57:41.579356608Z level=info msg="Warming state cache for startup" 2024-09-14T03:57:41.818 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=ngalert.multiorg.alertmanager t=2024-09-14T03:57:41.579668327Z level=info msg="Starting MultiOrg Alertmanager" 2024-09-14T03:57:41.818 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=ngalert.state.manager t=2024-09-14T03:57:41.579693923Z level=info msg="State cache has been initialized" states=0 duration=336.704µs 2024-09-14T03:57:41.818 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=ngalert.scheduler t=2024-09-14T03:57:41.579767195Z level=info msg="Starting scheduler" tickInterval=10s maxAttempts=1 2024-09-14T03:57:41.818 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=ticker t=2024-09-14T03:57:41.579839378Z level=info msg=starting first_tick=2024-09-14T03:57:50Z 2024-09-14T03:57:41.818 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=http.server t=2024-09-14T03:57:41.582652751Z level=info msg="HTTP Server TLS settings" MinTLSVersion=TLS1.2 configuredciphers=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA 2024-09-14T03:57:41.818 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=http.server t=2024-09-14T03:57:41.58320371Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-14T03:57:41.818 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=provisioning.dashboard t=2024-09-14T03:57:41.596038101Z level=info msg="starting to provision dashboards" 2024-09-14T03:57:41.818 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=sqlstore.transactions t=2024-09-14T03:57:41.611383091Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-14T03:57:41.818 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=plugins.update.checker t=2024-09-14T03:57:41.668713931Z level=info msg="Update check succeeded" duration=89.361886ms 2024-09-14T03:57:41.818 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=grafana.update.checker t=2024-09-14T03:57:41.671722512Z level=info msg="Update check succeeded" duration=92.364631ms 2024-09-14T03:57:41.819 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=sqlstore.transactions t=2024-09-14T03:57:41.689581152Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-14T03:57:41.819 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=provisioning.dashboard t=2024-09-14T03:57:41.815365585Z level=info msg="finished to provision dashboards" 2024-09-14T03:57:42.085 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:42 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=grafana-apiserver t=2024-09-14T03:57:42.035476385Z level=info msg="Adding GroupVersion playlist.grafana.app v0alpha1 to ResourceManager" 2024-09-14T03:57:42.085 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:57:42 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=grafana-apiserver t=2024-09-14T03:57:42.036074215Z level=info msg="Adding GroupVersion featuretoggle.grafana.app v0alpha1 to ResourceManager" 2024-09-14T03:57:42.328 INFO:tasks.rados.rados.0.smithi070.stdout:2754: done (1 left) 2024-09-14T03:57:42.328 INFO:tasks.rados.rados.0.smithi070.stdout:2755: done (0 left) 2024-09-14T03:57:42.329 INFO:tasks.rados.rados.0.smithi070.stdout:2756: rollback oid 43 current snap is 288 2024-09-14T03:57:42.329 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 43 to 215 2024-09-14T03:57:42.329 INFO:tasks.rados.rados.0.smithi070.stdout:2757: copy_from oid 32 from oid 46 current snap is 288 2024-09-14T03:57:42.329 INFO:tasks.rados.rados.0.smithi070.stdout:2758: copy_from oid 33 from oid 40 current snap is 288 2024-09-14T03:57:42.329 INFO:tasks.rados.rados.0.smithi070.stdout:2759: snap_remove snap 278 2024-09-14T03:57:42.344 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:42 smithi094 ceph-mon[86597]: osdmap e630: 8 total, 8 up, 8 in 2024-09-14T03:57:42.344 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:42 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:42.344 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:42 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:42.344 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:42 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:57:42.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:42 smithi070 ceph-mon[96704]: osdmap e630: 8 total, 8 up, 8 in 2024-09-14T03:57:42.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:42 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:42.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:42 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:42.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:42 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:57:42.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:42 smithi070 ceph-mon[98640]: osdmap e630: 8 total, 8 up, 8 in 2024-09-14T03:57:42.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:42 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:42.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:42 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:42.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:42 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:57:43.333 INFO:tasks.rados.rados.0.smithi070.stdout:2759: done (3 left) 2024-09-14T03:57:43.333 INFO:tasks.rados.rados.0.smithi070.stdout:2760: delete oid 29 current snap is 288 2024-09-14T03:57:43.333 INFO:tasks.rados.rados.0.smithi070.stdout:2756: finishing rollback tid 1 to smithi07085154-43 2024-09-14T03:57:43.333 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 2233 (ObjNum 33 snap 0 seq_num 2931205712) dirty dne 2024-09-14T03:57:43.333 INFO:tasks.rados.rados.0.smithi070.stdout:2758: finishing copy_from racing read to smithi07085154-33 2024-09-14T03:57:43.333 INFO:tasks.rados.rados.0.smithi070.stdout:2757: finishing copy_from racing read to smithi07085154-32 2024-09-14T03:57:43.334 INFO:tasks.rados.rados.0.smithi070.stdout:2757: finishing copy_from to smithi07085154-32 2024-09-14T03:57:43.334 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 3147 (ObjNum 576 snap 193 seq_num 576) dirty exists 2024-09-14T03:57:43.334 INFO:tasks.rados.rados.0.smithi070.stdout:2758: finishing copy_from to smithi07085154-33 2024-09-14T03:57:43.334 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 33 v 2085 (ObjNum 97 snap 0 seq_num 2930449744) dirty exists 2024-09-14T03:57:43.342 INFO:tasks.rados.rados.0.smithi070.stdout:2756: done (3 left) 2024-09-14T03:57:43.342 INFO:tasks.rados.rados.0.smithi070.stdout:2757: done (2 left) 2024-09-14T03:57:43.342 INFO:tasks.rados.rados.0.smithi070.stdout:2758: done (1 left) 2024-09-14T03:57:43.342 INFO:tasks.rados.rados.0.smithi070.stdout:2760: done (0 left) 2024-09-14T03:57:43.342 INFO:tasks.rados.rados.0.smithi070.stdout:2761: delete oid 21 current snap is 288 2024-09-14T03:57:43.348 INFO:tasks.rados.rados.0.smithi070.stdout:2761: done (0 left) 2024-09-14T03:57:43.348 INFO:tasks.rados.rados.0.smithi070.stdout:2762: copy_from oid 13 from oid 17 current snap is 288 2024-09-14T03:57:43.348 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 26 current snap is 288 2024-09-14T03:57:43.348 INFO:tasks.rados.rados.0.smithi070.stdout:2763: seq_num 854 ranges {2334720=712704,3047424=794624,3842048=450560,4292608=360448} 2024-09-14T03:57:43.365 INFO:tasks.rados.rados.0.smithi070.stdout:2763: writing smithi07085154-26 from 2334720 to 3047424 tid 1 2024-09-14T03:57:43.370 INFO:tasks.rados.rados.0.smithi070.stdout:2763: writing smithi07085154-26 from 3047424 to 3842048 tid 2 2024-09-14T03:57:43.372 INFO:tasks.rados.rados.0.smithi070.stdout:2763: writing smithi07085154-26 from 3842048 to 4292608 tid 3 2024-09-14T03:57:43.375 INFO:tasks.rados.rados.0.smithi070.stdout:2763: writing smithi07085154-26 from 4292608 to 4653056 tid 4 2024-09-14T03:57:43.375 INFO:tasks.rados.rados.0.smithi070.stdout:2764: snap_create 2024-09-14T03:57:43.375 INFO:tasks.rados.rados.0.smithi070.stdout:2762: finishing copy_from racing read to smithi07085154-13 2024-09-14T03:57:43.375 INFO:tasks.rados.rados.0.smithi070.stdout:2762: finishing copy_from to smithi07085154-13 2024-09-14T03:57:43.375 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 2224 (ObjNum 81 snap 0 seq_num 1953066355) dirty exists 2024-09-14T03:57:43.375 INFO:tasks.rados.rados.0.smithi070.stdout:2763: finishing write tid 1 to smithi07085154-26 2024-09-14T03:57:43.376 INFO:tasks.rados.rados.0.smithi070.stdout:2763: finishing write tid 2 to smithi07085154-26 2024-09-14T03:57:43.378 INFO:tasks.rados.rados.0.smithi070.stdout:2763: finishing write tid 3 to smithi07085154-26 2024-09-14T03:57:43.378 INFO:tasks.rados.rados.0.smithi070.stdout:2763: finishing write tid 4 to smithi07085154-26 2024-09-14T03:57:43.378 INFO:tasks.rados.rados.0.smithi070.stdout:2763: finishing write tid 5 to smithi07085154-26 2024-09-14T03:57:43.379 INFO:tasks.rados.rados.0.smithi070.stdout:2763: finishing write tid 6 to smithi07085154-26 2024-09-14T03:57:43.379 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 3202 (ObjNum 854 snap 288 seq_num 854) dirty exists 2024-09-14T03:57:43.380 INFO:tasks.rados.rados.0.smithi070.stdout:2763: left oid 26 (ObjNum 854 snap 288 seq_num 854) 2024-09-14T03:57:43.406 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:43 smithi070 ceph-mon[96704]: pgmap v542: 121 pgs: 121 active+clean; 1.1 GiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 4.7 MiB/s wr, 34 op/s 2024-09-14T03:57:43.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:43 smithi070 ceph-mon[96704]: osdmap e631: 8 total, 8 up, 8 in 2024-09-14T03:57:43.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:43 smithi070 ceph-mon[98640]: pgmap v542: 121 pgs: 121 active+clean; 1.1 GiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 4.7 MiB/s wr, 34 op/s 2024-09-14T03:57:43.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:43 smithi070 ceph-mon[98640]: osdmap e631: 8 total, 8 up, 8 in 2024-09-14T03:57:43.430 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:43 smithi094 ceph-mon[86597]: pgmap v542: 121 pgs: 121 active+clean; 1.1 GiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 4.7 MiB/s wr, 34 op/s 2024-09-14T03:57:43.430 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:43 smithi094 ceph-mon[86597]: osdmap e631: 8 total, 8 up, 8 in 2024-09-14T03:57:43.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:57:43 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:57:43] "GET /metrics HTTP/1.1" 200 36506 "" "Prometheus/2.51.0" 2024-09-14T03:57:44.088 INFO:tasks.rados.rados.0.smithi070.stdout:2762: done (2 left) 2024-09-14T03:57:44.088 INFO:tasks.rados.rados.0.smithi070.stdout:2763: done (1 left) 2024-09-14T03:57:44.088 INFO:tasks.rados.rados.0.smithi070.stdout:2764: done (0 left) 2024-09-14T03:57:44.088 INFO:tasks.rados.rados.0.smithi070.stdout:2765: copy_from oid 24 from oid 35 current snap is 289 2024-09-14T03:57:44.088 INFO:tasks.rados.rados.0.smithi070.stdout:2766: read oid 32 snap 276 2024-09-14T03:57:44.088 INFO:tasks.rados.rados.0.smithi070.stdout:2766: expect (ObjNum 718 snap 248 seq_num 718) 2024-09-14T03:57:44.088 INFO:tasks.rados.rados.0.smithi070.stdout:2767: read oid 38 snap -1 2024-09-14T03:57:44.088 INFO:tasks.rados.rados.0.smithi070.stdout:2767: expect deleted 2024-09-14T03:57:44.088 INFO:tasks.rados.rados.0.smithi070.stdout:2768: read oid 50 snap 286 2024-09-14T03:57:44.089 INFO:tasks.rados.rados.0.smithi070.stdout:2768: expect (ObjNum 799 snap 279 seq_num 799) 2024-09-14T03:57:44.089 INFO:tasks.rados.rados.0.smithi070.stdout:2769: copy_from oid 46 from oid 12 current snap is 289 2024-09-14T03:57:44.089 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 6 current snap is 289 2024-09-14T03:57:44.089 INFO:tasks.rados.rados.0.smithi070.stdout:2770: seq_num 855 ranges {11083776=729088,11812864=671744,12484608=663552,13148160=270336} 2024-09-14T03:57:44.153 INFO:tasks.rados.rados.0.smithi070.stdout:2770: writing smithi07085154-6 from 11083776 to 11812864 tid 1 2024-09-14T03:57:44.156 INFO:tasks.rados.rados.0.smithi070.stdout:2770: writing smithi07085154-6 from 11812864 to 12484608 tid 2 2024-09-14T03:57:44.160 INFO:tasks.rados.rados.0.smithi070.stdout:2770: writing smithi07085154-6 from 12484608 to 13148160 tid 3 2024-09-14T03:57:44.162 INFO:tasks.rados.rados.0.smithi070.stdout:2770: writing smithi07085154-6 from 13148160 to 13418496 tid 4 2024-09-14T03:57:44.162 INFO:tasks.rados.rados.0.smithi070.stdout:2771: delete oid 45 current snap is 289 2024-09-14T03:57:44.162 INFO:tasks.rados.rados.0.smithi070.stdout:2765: finishing copy_from racing read to smithi07085154-24 2024-09-14T03:57:44.162 INFO:tasks.rados.rados.0.smithi070.stdout:2765: finishing copy_from to smithi07085154-24 2024-09-14T03:57:44.162 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 1695 (ObjNum 65 snap 0 seq_num 1615881837) dirty exists 2024-09-14T03:57:44.162 INFO:tasks.rados.rados.0.smithi070.stdout:2768: expect (ObjNum 799 snap 279 seq_num 799) 2024-09-14T03:57:44.348 INFO:tasks.rados.rados.0.smithi070.stdout:2769: finishing copy_from racing read to smithi07085154-46 2024-09-14T03:57:44.348 INFO:tasks.rados.rados.0.smithi070.stdout:2769: finishing copy_from to smithi07085154-46 2024-09-14T03:57:44.348 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 46 v 2212 (ObjNum 800 snap 280 seq_num 800) dirty exists 2024-09-14T03:57:44.348 INFO:tasks.rados.rados.0.smithi070.stdout:2766: expect (ObjNum 718 snap 248 seq_num 718) 2024-09-14T03:57:44.670 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:44 smithi094 ceph-mon[86597]: osdmap e632: 8 total, 8 up, 8 in 2024-09-14T03:57:44.670 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:44 smithi094 ceph-mon[86597]: osdmap e633: 8 total, 8 up, 8 in 2024-09-14T03:57:44.670 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:44 smithi094 ceph-mon[86597]: pgmap v546: 121 pgs: 121 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 16 MiB/s rd, 4.2 MiB/s wr, 36 op/s 2024-09-14T03:57:44.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:44 smithi070 ceph-mon[96704]: osdmap e632: 8 total, 8 up, 8 in 2024-09-14T03:57:44.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:44 smithi070 ceph-mon[96704]: osdmap e633: 8 total, 8 up, 8 in 2024-09-14T03:57:44.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:44 smithi070 ceph-mon[96704]: pgmap v546: 121 pgs: 121 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 16 MiB/s rd, 4.2 MiB/s wr, 36 op/s 2024-09-14T03:57:44.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:44 smithi070 ceph-mon[98640]: osdmap e632: 8 total, 8 up, 8 in 2024-09-14T03:57:44.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:44 smithi070 ceph-mon[98640]: osdmap e633: 8 total, 8 up, 8 in 2024-09-14T03:57:44.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:44 smithi070 ceph-mon[98640]: pgmap v546: 121 pgs: 121 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 16 MiB/s rd, 4.2 MiB/s wr, 36 op/s 2024-09-14T03:57:45.267 INFO:tasks.rados.rados.0.smithi070.stdout:2770: finishing write tid 1 to smithi07085154-6 2024-09-14T03:57:45.268 INFO:tasks.rados.rados.0.smithi070.stdout:2770: finishing write tid 2 to smithi07085154-6 2024-09-14T03:57:45.268 INFO:tasks.rados.rados.0.smithi070.stdout:2770: finishing write tid 3 to smithi07085154-6 2024-09-14T03:57:45.268 INFO:tasks.rados.rados.0.smithi070.stdout:2770: finishing write tid 4 to smithi07085154-6 2024-09-14T03:57:45.268 INFO:tasks.rados.rados.0.smithi070.stdout:2770: finishing write tid 5 to smithi07085154-6 2024-09-14T03:57:45.268 INFO:tasks.rados.rados.0.smithi070.stdout:2770: finishing write tid 6 to smithi07085154-6 2024-09-14T03:57:45.268 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 2277 (ObjNum 855 snap 289 seq_num 855) dirty exists 2024-09-14T03:57:45.268 INFO:tasks.rados.rados.0.smithi070.stdout:2770: left oid 6 (ObjNum 855 snap 289 seq_num 855) 2024-09-14T03:57:45.268 INFO:tasks.rados.rados.0.smithi070.stdout:2765: done (6 left) 2024-09-14T03:57:45.268 INFO:tasks.rados.rados.0.smithi070.stdout:2766: done (5 left) 2024-09-14T03:57:45.268 INFO:tasks.rados.rados.0.smithi070.stdout:2767: done (4 left) 2024-09-14T03:57:45.268 INFO:tasks.rados.rados.0.smithi070.stdout:2768: done (3 left) 2024-09-14T03:57:45.268 INFO:tasks.rados.rados.0.smithi070.stdout:2769: done (2 left) 2024-09-14T03:57:45.269 INFO:tasks.rados.rados.0.smithi070.stdout:2770: done (1 left) 2024-09-14T03:57:45.269 INFO:tasks.rados.rados.0.smithi070.stdout:2771: done (0 left) 2024-09-14T03:57:45.269 INFO:tasks.rados.rados.0.smithi070.stdout:2772: copy_from oid 40 from oid 4 current snap is 289 2024-09-14T03:57:45.269 INFO:tasks.rados.rados.0.smithi070.stdout:2773: rollback oid 41 current snap is 289 2024-09-14T03:57:45.269 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 41 to 267 2024-09-14T03:57:45.269 INFO:tasks.rados.rados.0.smithi070.stdout:2774: snap_create 2024-09-14T03:57:45.270 INFO:tasks.rados.rados.0.smithi070.stdout:2773: finishing rollback tid 1 to smithi07085154-41 2024-09-14T03:57:45.270 INFO:tasks.rados.rados.0.smithi070.stdout:2773: finishing rollback tid 2 to smithi07085154-41 2024-09-14T03:57:45.270 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 2236 (ObjNum 724 snap 248 seq_num 724) dirty exists 2024-09-14T03:57:45.271 INFO:tasks.rados.rados.0.smithi070.stdout:2772: finishing copy_from racing read to smithi07085154-40 2024-09-14T03:57:45.271 INFO:tasks.rados.rados.0.smithi070.stdout:2772: finishing copy_from to smithi07085154-40 2024-09-14T03:57:45.271 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 3147 (ObjNum 576 snap 193 seq_num 576) dirty exists 2024-09-14T03:57:46.135 INFO:tasks.rados.rados.0.smithi070.stdout:2772: done (2 left) 2024-09-14T03:57:46.135 INFO:tasks.rados.rados.0.smithi070.stdout:2773: done (1 left) 2024-09-14T03:57:46.135 INFO:tasks.rados.rados.0.smithi070.stdout:2774: done (0 left) 2024-09-14T03:57:46.135 INFO:tasks.rados.rados.0.smithi070.stdout:2775: read oid 10 snap 215 2024-09-14T03:57:46.135 INFO:tasks.rados.rados.0.smithi070.stdout:2775: expect (ObjNum 508 snap 166 seq_num 508) 2024-09-14T03:57:46.135 INFO:tasks.rados.rados.0.smithi070.stdout:2776: snap_create 2024-09-14T03:57:46.154 INFO:tasks.rados.rados.0.smithi070.stdout:2775: expect (ObjNum 508 snap 166 seq_num 508) 2024-09-14T03:57:46.391 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:46 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:46.392 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:46 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:46.392 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:46 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:46.392 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:46 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:46.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:46 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:46.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:46 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:46.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:46 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:46.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:46 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:46.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:46 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:46.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:46 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:46.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:46 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:46.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:46 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:47.137 INFO:tasks.rados.rados.0.smithi070.stdout:2775: done (1 left) 2024-09-14T03:57:47.138 INFO:tasks.rados.rados.0.smithi070.stdout:2776: done (0 left) 2024-09-14T03:57:47.138 INFO:tasks.rados.rados.0.smithi070.stdout:2777: copy_from oid 43 from oid 41 current snap is 291 2024-09-14T03:57:47.138 INFO:tasks.rados.rados.0.smithi070.stdout:2778: delete oid 31 current snap is 291 2024-09-14T03:57:47.141 INFO:tasks.rados.rados.0.smithi070.stdout:2778: done (1 left) 2024-09-14T03:57:47.141 INFO:tasks.rados.rados.0.smithi070.stdout:2779: copy_from oid 10 from oid 31 current snap is 291 2024-09-14T03:57:47.141 INFO:tasks.rados.rados.0.smithi070.stdout:2780: read oid 4 snap -1 2024-09-14T03:57:47.141 INFO:tasks.rados.rados.0.smithi070.stdout:2780: expect (ObjNum 576 snap 193 seq_num 576) 2024-09-14T03:57:47.142 INFO:tasks.rados.rados.0.smithi070.stdout:2781: read oid 2 snap -1 2024-09-14T03:57:47.142 INFO:tasks.rados.rados.0.smithi070.stdout:2781: expect (ObjNum 847 snap 286 seq_num 847) 2024-09-14T03:57:47.142 INFO:tasks.rados.rados.0.smithi070.stdout:2782: read oid 7 snap -1 2024-09-14T03:57:47.142 INFO:tasks.rados.rados.0.smithi070.stdout:2782: expect deleted 2024-09-14T03:57:47.142 INFO:tasks.rados.rados.0.smithi070.stdout:2783: rollback oid 32 current snap is 291 2024-09-14T03:57:47.142 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 32 to 277 2024-09-14T03:57:47.142 INFO:tasks.rados.rados.0.smithi070.stdout:2784: delete oid 30 current snap is 291 2024-09-14T03:57:47.143 INFO:tasks.rados.rados.0.smithi070.stdout:2779: finishing copy_from to smithi07085154-10 2024-09-14T03:57:47.143 INFO:tasks.rados.rados.0.smithi070.stdout:2779: got expected ENOENT (src dne) 2024-09-14T03:57:47.146 INFO:tasks.rados.rados.0.smithi070.stdout:2779: finishing copy_from racing read to smithi07085154-10 2024-09-14T03:57:47.146 INFO:tasks.rados.rados.0.smithi070.stdout:2780: expect (ObjNum 576 snap 193 seq_num 576) 2024-09-14T03:57:47.149 INFO:tasks.rados.rados.0.smithi070.stdout:2779: done (6 left) 2024-09-14T03:57:47.149 INFO:tasks.rados.rados.0.smithi070.stdout:2780: done (5 left) 2024-09-14T03:57:47.150 INFO:tasks.rados.rados.0.smithi070.stdout:2782: done (4 left) 2024-09-14T03:57:47.150 INFO:tasks.rados.rados.0.smithi070.stdout:2784: done (3 left) 2024-09-14T03:57:47.150 INFO:tasks.rados.rados.0.smithi070.stdout:2785: setattr oid 44 current snap is 291 2024-09-14T03:57:47.150 INFO:tasks.rados.rados.0.smithi070.stdout:2783: finishing rollback tid 0 to smithi07085154-32 2024-09-14T03:57:47.151 INFO:tasks.rados.rados.0.smithi070.stdout:2783: finishing rollback tid 1 to smithi07085154-32 2024-09-14T03:57:47.151 INFO:tasks.rados.rados.0.smithi070.stdout:2783: finishing rollback tid 2 to smithi07085154-32 2024-09-14T03:57:47.151 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 3150 (ObjNum 718 snap 248 seq_num 718) dirty exists 2024-09-14T03:57:47.152 INFO:tasks.rados.rados.0.smithi070.stdout:2777: finishing copy_from racing read to smithi07085154-43 2024-09-14T03:57:47.152 INFO:tasks.rados.rados.0.smithi070.stdout:2777: finishing copy_from to smithi07085154-43 2024-09-14T03:57:47.152 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 2236 (ObjNum 724 snap 248 seq_num 724) dirty exists 2024-09-14T03:57:47.153 INFO:tasks.rados.rados.0.smithi070.stdout:2777: done (3 left) 2024-09-14T03:57:47.153 INFO:tasks.rados.rados.0.smithi070.stdout:2783: done (2 left) 2024-09-14T03:57:47.153 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 31 current snap is 291 2024-09-14T03:57:47.153 INFO:tasks.rados.rados.0.smithi070.stdout:2786: seq_num 859 ranges {0=794624,794624=614400,1409024=638976,2048000=352256} 2024-09-14T03:57:47.157 INFO:tasks.rados.rados.0.smithi070.stdout:2786: writing smithi07085154-31 from 0 to 794624 tid 1 2024-09-14T03:57:47.161 INFO:tasks.rados.rados.0.smithi070.stdout:2786: writing smithi07085154-31 from 794624 to 1409024 tid 2 2024-09-14T03:57:47.165 INFO:tasks.rados.rados.0.smithi070.stdout:2786: writing smithi07085154-31 from 1409024 to 2048000 tid 3 2024-09-14T03:57:47.168 INFO:tasks.rados.rados.0.smithi070.stdout:2786: writing smithi07085154-31 from 2048000 to 2400256 tid 4 2024-09-14T03:57:47.168 INFO:tasks.rados.rados.0.smithi070.stdout:2787: copy_from oid 32 from oid 47 current snap is 291 2024-09-14T03:57:47.168 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 3148 (ObjNum 843 snap 286 seq_num 843) dirty exists 2024-09-14T03:57:47.168 INFO:tasks.rados.rados.0.smithi070.stdout:2785: done (3 left) 2024-09-14T03:57:47.168 INFO:tasks.rados.rados.0.smithi070.stdout:2788: rollback oid 35 current snap is 291 2024-09-14T03:57:47.168 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 35 to 288 2024-09-14T03:57:47.168 INFO:tasks.rados.rados.0.smithi070.stdout:2786: finishing write tid 1 to smithi07085154-31 2024-09-14T03:57:47.168 INFO:tasks.rados.rados.0.smithi070.stdout:2786: finishing write tid 2 to smithi07085154-31 2024-09-14T03:57:47.169 INFO:tasks.rados.rados.0.smithi070.stdout:2789: setattr oid 30 current snap is 291 2024-09-14T03:57:47.170 INFO:tasks.rados.rados.0.smithi070.stdout:2790: snap_create 2024-09-14T03:57:47.170 INFO:tasks.rados.rados.0.smithi070.stdout:2781: expect (ObjNum 847 snap 286 seq_num 847) 2024-09-14T03:57:47.489 INFO:tasks.rados.rados.0.smithi070.stdout:2787: finishing copy_from to smithi07085154-32 2024-09-14T03:57:47.489 INFO:tasks.rados.rados.0.smithi070.stdout:2787: got expected ENOENT (src dne) 2024-09-14T03:57:47.489 INFO:tasks.rados.rados.0.smithi070.stdout:2787: finishing copy_from racing read to smithi07085154-32 2024-09-14T03:57:47.489 INFO:tasks.rados.rados.0.smithi070.stdout:2786: finishing write tid 3 to smithi07085154-31 2024-09-14T03:57:47.489 INFO:tasks.rados.rados.0.smithi070.stdout:2786: finishing write tid 4 to smithi07085154-31 2024-09-14T03:57:47.489 INFO:tasks.rados.rados.0.smithi070.stdout:2786: finishing write tid 5 to smithi07085154-31 2024-09-14T03:57:47.490 INFO:tasks.rados.rados.0.smithi070.stdout:2788: finishing rollback tid 0 to smithi07085154-35 2024-09-14T03:57:47.490 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 30 v 1735 (ObjNum 49 snap 0 seq_num 1953066355) dirty exists 2024-09-14T03:57:47.490 INFO:tasks.rados.rados.0.smithi070.stdout:2788: finishing rollback tid 1 to smithi07085154-35 2024-09-14T03:57:47.490 INFO:tasks.rados.rados.0.smithi070.stdout:2788: finishing rollback tid 2 to smithi07085154-35 2024-09-14T03:57:47.490 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 35 v 1698 (ObjNum 113 snap 0 seq_num 2931017952) dirty exists 2024-09-14T03:57:47.490 INFO:tasks.rados.rados.0.smithi070.stdout:2786: finishing write tid 6 to smithi07085154-31 2024-09-14T03:57:47.490 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 2283 (ObjNum 859 snap 291 seq_num 859) dirty exists 2024-09-14T03:57:47.490 INFO:tasks.rados.rados.0.smithi070.stdout:2786: left oid 31 (ObjNum 859 snap 291 seq_num 859) 2024-09-14T03:57:47.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:47 smithi094 ceph-mon[86597]: pgmap v547: 121 pgs: 121 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 14 MiB/s rd, 3.3 MiB/s wr, 30 op/s 2024-09-14T03:57:47.491 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:47 smithi094 ceph-mon[86597]: osdmap e634: 8 total, 8 up, 8 in 2024-09-14T03:57:47.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:47 smithi070 ceph-mon[96704]: pgmap v547: 121 pgs: 121 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 14 MiB/s rd, 3.3 MiB/s wr, 30 op/s 2024-09-14T03:57:47.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:47 smithi070 ceph-mon[96704]: osdmap e634: 8 total, 8 up, 8 in 2024-09-14T03:57:47.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:47 smithi070 ceph-mon[98640]: pgmap v547: 121 pgs: 121 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 14 MiB/s rd, 3.3 MiB/s wr, 30 op/s 2024-09-14T03:57:47.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:47 smithi070 ceph-mon[98640]: osdmap e634: 8 total, 8 up, 8 in 2024-09-14T03:57:48.148 INFO:tasks.rados.rados.0.smithi070.stdout:2781: done (5 left) 2024-09-14T03:57:48.148 INFO:tasks.rados.rados.0.smithi070.stdout:2786: done (4 left) 2024-09-14T03:57:48.148 INFO:tasks.rados.rados.0.smithi070.stdout:2787: done (3 left) 2024-09-14T03:57:48.148 INFO:tasks.rados.rados.0.smithi070.stdout:2788: done (2 left) 2024-09-14T03:57:48.148 INFO:tasks.rados.rados.0.smithi070.stdout:2789: done (1 left) 2024-09-14T03:57:48.148 INFO:tasks.rados.rados.0.smithi070.stdout:2790: done (0 left) 2024-09-14T03:57:48.148 INFO:tasks.rados.rados.0.smithi070.stdout:2791: read oid 18 snap -1 2024-09-14T03:57:48.148 INFO:tasks.rados.rados.0.smithi070.stdout:2791: expect (ObjNum 826 snap 283 seq_num 826) 2024-09-14T03:57:48.148 INFO:tasks.rados.rados.0.smithi070.stdout:2792: read oid 38 snap -1 2024-09-14T03:57:48.149 INFO:tasks.rados.rados.0.smithi070.stdout:2792: expect deleted 2024-09-14T03:57:48.149 INFO:tasks.rados.rados.0.smithi070.stdout:2793: read oid 10 snap -1 2024-09-14T03:57:48.149 INFO:tasks.rados.rados.0.smithi070.stdout:2793: expect (ObjNum 841 snap 286 seq_num 841) 2024-09-14T03:57:48.149 INFO:tasks.rados.rados.0.smithi070.stdout:2794: read oid 22 snap -1 2024-09-14T03:57:48.149 INFO:tasks.rados.rados.0.smithi070.stdout:2794: expect (ObjNum 576 snap 193 seq_num 576) 2024-09-14T03:57:48.149 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 42 current snap is 292 2024-09-14T03:57:48.149 INFO:tasks.rados.rados.0.smithi070.stdout:2795: seq_num 860 ranges {2555904=8192} 2024-09-14T03:57:48.162 INFO:tasks.rados.rados.0.smithi070.stdout:2795: writing smithi07085154-42 from 2555904 to 2564096 tid 1 2024-09-14T03:57:48.162 INFO:tasks.rados.rados.0.smithi070.stdout:2796: snap_remove snap 242 2024-09-14T03:57:48.511 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:48 smithi094 ceph-mon[86597]: osdmap e635: 8 total, 8 up, 8 in 2024-09-14T03:57:48.511 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:48 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:57:48.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:48 smithi070 ceph-mon[96704]: osdmap e635: 8 total, 8 up, 8 in 2024-09-14T03:57:48.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:48 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:57:48.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:48 smithi070 ceph-mon[98640]: osdmap e635: 8 total, 8 up, 8 in 2024-09-14T03:57:48.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:48 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:57:49.088 INFO:tasks.rados.rados.0.smithi070.stdout:2796: done (5 left) 2024-09-14T03:57:49.088 INFO:tasks.rados.rados.0.smithi070.stdout:2797: setattr oid 9 current snap is 292 2024-09-14T03:57:49.088 INFO:tasks.rados.rados.0.smithi070.stdout:2794: expect (ObjNum 576 snap 193 seq_num 576) 2024-09-14T03:57:49.093 INFO:tasks.rados.rados.0.smithi070.stdout:2791: expect (ObjNum 826 snap 283 seq_num 826) 2024-09-14T03:57:49.176 INFO:tasks.rados.rados.0.smithi070.stdout:2793: expect (ObjNum 841 snap 286 seq_num 841) 2024-09-14T03:57:49.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: pgmap v550: 121 pgs: 121 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 13 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-09-14T03:57:49.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: osdmap e636: 8 total, 8 up, 8 in 2024-09-14T03:57:49.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:49.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:49.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:57:49.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:57:49.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:49.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-14T03:57:49.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:57:49.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:57:49.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.529 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.529 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.529 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.529 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.529 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.529 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.529 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.529 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.529 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.529 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.529 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.530 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.530 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.530 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.530 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.530 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.530 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.530 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: Upgrade: Finalizing container_image settings 2024-09-14T03:57:49.530 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:49.530 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T03:57:49.530 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T03:57:49.530 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:57:49.530 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T03:57:49.531 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T03:57:49.531 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-14T03:57:49.531 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T03:57:49.531 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-14T03:57:49.531 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T03:57:49.531 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-14T03:57:49.531 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T03:57:49.531 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-14T03:57:49.531 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T03:57:49.531 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-14T03:57:49.531 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:57:49.531 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T03:57:49.532 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-14T03:57:49.532 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T03:57:49.532 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-14T03:57:49.532 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T03:57:49.532 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-14T03:57:49.532 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T03:57:49.532 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T03:57:49.532 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:57:49.532 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:57:49.532 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:57:49.532 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:57:49.532 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:57:49.533 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:57:49.533 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: Upgrade: Complete! 2024-09-14T03:57:49.533 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T03:57:49.533 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T03:57:49.533 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:57:49.533 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:57:49.533 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:57:49.534 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:49.534 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:49 smithi094 ceph-mon[86597]: osdmap e637: 8 total, 8 up, 8 in 2024-09-14T03:57:49.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: pgmap v550: 121 pgs: 121 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 13 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-09-14T03:57:49.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: osdmap e636: 8 total, 8 up, 8 in 2024-09-14T03:57:49.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:49.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:49.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:57:49.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:57:49.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:49.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-14T03:57:49.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:57:49.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:57:49.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: Upgrade: Finalizing container_image settings 2024-09-14T03:57:49.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:49.539 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: pgmap v550: 121 pgs: 121 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 13 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-09-14T03:57:49.539 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: osdmap e636: 8 total, 8 up, 8 in 2024-09-14T03:57:49.539 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:49.540 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:49.540 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:57:49.540 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:57:49.540 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:49.540 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-14T03:57:49.540 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:57:49.540 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:57:49.540 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.540 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.540 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.540 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.540 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.540 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.541 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.541 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.541 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.541 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.541 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.541 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.541 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.541 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.541 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.541 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.541 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.541 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:57:49.541 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: Upgrade: Finalizing container_image settings 2024-09-14T03:57:49.542 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:49.542 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T03:57:49.542 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T03:57:49.542 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:57:49.542 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T03:57:49.542 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T03:57:49.542 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-14T03:57:49.542 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T03:57:49.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-14T03:57:49.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T03:57:49.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-14T03:57:49.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T03:57:49.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-14T03:57:49.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T03:57:49.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-14T03:57:49.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:57:49.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T03:57:49.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-14T03:57:49.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T03:57:49.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-14T03:57:49.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T03:57:49.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-14T03:57:49.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T03:57:49.544 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T03:57:49.544 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:57:49.544 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:57:49.544 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:57:49.544 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:57:49.544 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:57:49.544 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:57:49.544 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: Upgrade: Complete! 2024-09-14T03:57:49.544 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T03:57:49.544 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T03:57:49.544 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:57:49.544 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:57:49.544 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:57:49.545 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:49.545 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[98640]: osdmap e637: 8 total, 8 up, 8 in 2024-09-14T03:57:49.545 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T03:57:49.545 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T03:57:49.545 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:57:49.545 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T03:57:49.545 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T03:57:49.545 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-14T03:57:49.546 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T03:57:49.546 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-14T03:57:49.546 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T03:57:49.546 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-14T03:57:49.546 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T03:57:49.546 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-14T03:57:49.546 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T03:57:49.546 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-14T03:57:49.546 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:57:49.546 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T03:57:49.546 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-14T03:57:49.546 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T03:57:49.546 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-14T03:57:49.547 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T03:57:49.547 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-14T03:57:49.547 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T03:57:49.547 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T03:57:49.547 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:57:49.547 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:57:49.547 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:57:49.547 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:57:49.547 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:57:49.547 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:57:49.547 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: Upgrade: Complete! 2024-09-14T03:57:49.547 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T03:57:49.547 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T03:57:49.548 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:57:49.548 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:57:49.548 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:57:49.548 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:49.548 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:49 smithi070 ceph-mon[96704]: osdmap e637: 8 total, 8 up, 8 in 2024-09-14T03:57:49.610 INFO:tasks.rados.rados.0.smithi070.stdout:2795: finishing write tid 1 to smithi07085154-42 2024-09-14T03:57:49.610 INFO:tasks.rados.rados.0.smithi070.stdout:2795: finishing write tid 2 to smithi07085154-42 2024-09-14T03:57:49.610 INFO:tasks.rados.rados.0.smithi070.stdout:2795: finishing write tid 3 to smithi07085154-42 2024-09-14T03:57:49.611 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 3207 (ObjNum 860 snap 292 seq_num 860) dirty exists 2024-09-14T03:57:49.611 INFO:tasks.rados.rados.0.smithi070.stdout:2795: left oid 42 (ObjNum 860 snap 292 seq_num 860) 2024-09-14T03:57:49.611 INFO:tasks.rados.rados.0.smithi070.stdout:2791: done (5 left) 2024-09-14T03:57:49.611 INFO:tasks.rados.rados.0.smithi070.stdout:2792: done (4 left) 2024-09-14T03:57:49.611 INFO:tasks.rados.rados.0.smithi070.stdout:2793: done (3 left) 2024-09-14T03:57:49.611 INFO:tasks.rados.rados.0.smithi070.stdout:2794: done (2 left) 2024-09-14T03:57:49.611 INFO:tasks.rados.rados.0.smithi070.stdout:2795: done (1 left) 2024-09-14T03:57:49.611 INFO:tasks.rados.rados.0.smithi070.stdout:2798: read oid 45 snap -1 2024-09-14T03:57:49.611 INFO:tasks.rados.rados.0.smithi070.stdout:2798: expect deleted 2024-09-14T03:57:49.612 INFO:tasks.rados.rados.0.smithi070.stdout:2799: rollback oid 2 current snap is 292 2024-09-14T03:57:49.612 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 2 to 282 2024-09-14T03:57:49.612 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 3209 (ObjNum 459 snap 155 seq_num 459) dirty exists 2024-09-14T03:57:49.612 INFO:tasks.rados.rados.0.smithi070.stdout:2797: done (2 left) 2024-09-14T03:57:49.612 INFO:tasks.rados.rados.0.smithi070.stdout:2800: delete oid 39 current snap is 292 2024-09-14T03:57:49.614 INFO:tasks.rados.rados.0.smithi070.stdout:2798: done (2 left) 2024-09-14T03:57:49.614 INFO:tasks.rados.rados.0.smithi070.stdout:2800: done (1 left) 2024-09-14T03:57:49.614 INFO:tasks.rados.rados.0.smithi070.stdout:2801: rollback oid 47 current snap is 292 2024-09-14T03:57:49.614 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 47 to 289 2024-09-14T03:57:49.615 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 33 current snap is 292 2024-09-14T03:57:49.615 INFO:tasks.rados.rados.0.smithi070.stdout:2802: seq_num 862 ranges {0=40960} 2024-09-14T03:57:49.615 INFO:tasks.rados.rados.0.smithi070.stdout:2802: writing smithi07085154-33 from 0 to 40960 tid 1 2024-09-14T03:57:49.615 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 4 current snap is 292 2024-09-14T03:57:49.615 INFO:tasks.rados.rados.0.smithi070.stdout:2803: seq_num 863 ranges {49152=57344} 2024-09-14T03:57:49.615 INFO:tasks.rados.rados.0.smithi070.stdout:2803: writing smithi07085154-4 from 49152 to 106496 tid 1 2024-09-14T03:57:49.615 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 41 current snap is 292 2024-09-14T03:57:49.615 INFO:tasks.rados.rados.0.smithi070.stdout:2804: seq_num 864 ranges {131072=73728} 2024-09-14T03:57:49.615 INFO:tasks.rados.rados.0.smithi070.stdout:2804: writing smithi07085154-41 from 131072 to 204800 tid 1 2024-09-14T03:57:49.615 INFO:tasks.rados.rados.0.smithi070.stdout:2805: snap_create 2024-09-14T03:57:49.615 INFO:tasks.rados.rados.0.smithi070.stdout:2801: finishing rollback tid 1 to smithi07085154-47 2024-09-14T03:57:49.615 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 47 v 1723 (ObjNum 81 snap 0 seq_num 1550674026) dirty dne 2024-09-14T03:57:49.616 INFO:tasks.rados.rados.0.smithi070.stdout:2799: finishing rollback tid 0 to smithi07085154-2 2024-09-14T03:57:49.616 INFO:tasks.rados.rados.0.smithi070.stdout:2799: finishing rollback tid 1 to smithi07085154-2 2024-09-14T03:57:49.616 INFO:tasks.rados.rados.0.smithi070.stdout:2799: finishing rollback tid 2 to smithi07085154-2 2024-09-14T03:57:49.617 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 2 v 3227 (ObjNum 759 snap 264 seq_num 759) dirty exists 2024-09-14T03:57:49.617 INFO:tasks.rados.rados.0.smithi070.stdout:2802: finishing write tid 1 to smithi07085154-33 2024-09-14T03:57:49.617 INFO:tasks.rados.rados.0.smithi070.stdout:2802: finishing write tid 2 to smithi07085154-33 2024-09-14T03:57:49.618 INFO:tasks.rados.rados.0.smithi070.stdout:2803: finishing write tid 1 to smithi07085154-4 2024-09-14T03:57:49.618 INFO:tasks.rados.rados.0.smithi070.stdout:2802: finishing write tid 3 to smithi07085154-33 2024-09-14T03:57:49.618 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 33 v 2088 (ObjNum 862 snap 292 seq_num 862) dirty exists 2024-09-14T03:57:49.618 INFO:tasks.rados.rados.0.smithi070.stdout:2802: left oid 33 (ObjNum 862 snap 292 seq_num 862) 2024-09-14T03:57:49.619 INFO:tasks.rados.rados.0.smithi070.stdout:2803: finishing write tid 2 to smithi07085154-4 2024-09-14T03:57:49.619 INFO:tasks.rados.rados.0.smithi070.stdout:2804: finishing write tid 1 to smithi07085154-41 2024-09-14T03:57:49.619 INFO:tasks.rados.rados.0.smithi070.stdout:2804: finishing write tid 2 to smithi07085154-41 2024-09-14T03:57:49.621 INFO:tasks.rados.rados.0.smithi070.stdout:2803: finishing write tid 3 to smithi07085154-4 2024-09-14T03:57:49.621 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 3149 (ObjNum 863 snap 292 seq_num 863) dirty exists 2024-09-14T03:57:49.621 INFO:tasks.rados.rados.0.smithi070.stdout:2803: left oid 4 (ObjNum 863 snap 292 seq_num 863) 2024-09-14T03:57:49.621 INFO:tasks.rados.rados.0.smithi070.stdout:2804: finishing write tid 3 to smithi07085154-41 2024-09-14T03:57:49.621 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 2238 (ObjNum 864 snap 292 seq_num 864) dirty exists 2024-09-14T03:57:49.621 INFO:tasks.rados.rados.0.smithi070.stdout:2804: left oid 41 (ObjNum 864 snap 292 seq_num 864) 2024-09-14T03:57:50.167 INFO:tasks.rados.rados.0.smithi070.stdout:2799: done (5 left) 2024-09-14T03:57:50.167 INFO:tasks.rados.rados.0.smithi070.stdout:2801: done (4 left) 2024-09-14T03:57:50.167 INFO:tasks.rados.rados.0.smithi070.stdout:2802: done (3 left) 2024-09-14T03:57:50.167 INFO:tasks.rados.rados.0.smithi070.stdout:2803: done (2 left) 2024-09-14T03:57:50.167 INFO:tasks.rados.rados.0.smithi070.stdout:2804: done (1 left) 2024-09-14T03:57:50.167 INFO:tasks.rados.rados.0.smithi070.stdout:2805: done (0 left) 2024-09-14T03:57:50.167 INFO:tasks.rados.rados.0.smithi070.stdout:2806: copy_from oid 48 from oid 43 current snap is 293 2024-09-14T03:57:50.167 INFO:tasks.rados.rados.0.smithi070.stdout:2807: setattr oid 7 current snap is 293 2024-09-14T03:57:50.168 INFO:tasks.rados.rados.0.smithi070.stdout:2808: read oid 36 snap 264 2024-09-14T03:57:50.169 INFO:tasks.rados.rados.0.smithi070.stdout:2808: expect deleted 2024-09-14T03:57:50.169 INFO:tasks.rados.rados.0.smithi070.stdout:2809: read oid 25 snap -1 2024-09-14T03:57:50.169 INFO:tasks.rados.rados.0.smithi070.stdout:2809: expect deleted 2024-09-14T03:57:50.169 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 44 current snap is 293 2024-09-14T03:57:50.169 INFO:tasks.rados.rados.0.smithi070.stdout:2810: seq_num 865 ranges {6094848=90112} 2024-09-14T03:57:50.201 INFO:tasks.rados.rados.0.smithi070.stdout:2810: writing smithi07085154-44 from 6094848 to 6184960 tid 1 2024-09-14T03:57:50.201 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 24 current snap is 293 2024-09-14T03:57:50.202 INFO:tasks.rados.rados.0.smithi070.stdout:2811: seq_num 866 ranges {0=106496} 2024-09-14T03:57:50.203 INFO:tasks.rados.rados.0.smithi070.stdout:2811: writing smithi07085154-24 from 0 to 106496 tid 1 2024-09-14T03:57:50.203 INFO:tasks.rados.rados.0.smithi070.stdout:2812: rmattr oid 37 current snap is 293 2024-09-14T03:57:50.203 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 3150 (ObjNum 65 snap 0 seq_num 2902963152) dirty exists 2024-09-14T03:57:50.203 INFO:tasks.rados.rados.0.smithi070.stdout:2806: finishing copy_from racing read to smithi07085154-48 2024-09-14T03:57:50.203 INFO:tasks.rados.rados.0.smithi070.stdout:2806: finishing copy_from to smithi07085154-48 2024-09-14T03:57:50.204 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 2236 (ObjNum 724 snap 248 seq_num 724) dirty exists 2024-09-14T03:57:50.204 INFO:tasks.rados.rados.0.smithi070.stdout:2806: done (6 left) 2024-09-14T03:57:50.204 INFO:tasks.rados.rados.0.smithi070.stdout:2807: done (5 left) 2024-09-14T03:57:50.204 INFO:tasks.rados.rados.0.smithi070.stdout:2808: done (4 left) 2024-09-14T03:57:50.204 INFO:tasks.rados.rados.0.smithi070.stdout:2809: done (3 left) 2024-09-14T03:57:50.204 INFO:tasks.rados.rados.0.smithi070.stdout:2812: done (2 left) 2024-09-14T03:57:50.204 INFO:tasks.rados.rados.0.smithi070.stdout:2813: rmattr oid 12 current snap is 293 2024-09-14T03:57:50.204 INFO:tasks.rados.rados.0.smithi070.stdout:2814: read oid 47 snap 226 2024-09-14T03:57:50.204 INFO:tasks.rados.rados.0.smithi070.stdout:2814: expect (ObjNum 592 snap 195 seq_num 592) 2024-09-14T03:57:50.204 INFO:tasks.rados.rados.0.smithi070.stdout:2815: snap_remove snap 201 2024-09-14T03:57:50.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:57:50 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:57:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:57:50.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:50 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:57:50.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:50 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:57:50.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:50 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:57:50.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:50 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:50.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:50 smithi094 ceph-mon[86597]: pgmap v553: 121 pgs: 121 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 16 MiB/s rd, 2.5 MiB/s wr, 19 op/s 2024-09-14T03:57:50.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:50 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:57:50.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:50 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:57:50.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:50 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:57:50.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:50 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:50.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:50 smithi070 ceph-mon[96704]: pgmap v553: 121 pgs: 121 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 16 MiB/s rd, 2.5 MiB/s wr, 19 op/s 2024-09-14T03:57:50.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:50 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:57:50.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:50 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:57:50.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:50 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:57:50.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:50 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:50.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:50 smithi070 ceph-mon[98640]: pgmap v553: 121 pgs: 121 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 16 MiB/s rd, 2.5 MiB/s wr, 19 op/s 2024-09-14T03:57:51.174 INFO:tasks.rados.rados.0.smithi070.stdout:2815: done (4 left) 2024-09-14T03:57:51.174 INFO:tasks.rados.rados.0.smithi070.stdout:2816: snap_create 2024-09-14T03:57:51.174 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 2237 (ObjNum 800 snap 280 seq_num 800) dirty exists 2024-09-14T03:57:51.174 INFO:tasks.rados.rados.0.smithi070.stdout:2810: finishing write tid 1 to smithi07085154-44 2024-09-14T03:57:51.174 INFO:tasks.rados.rados.0.smithi070.stdout:2811: finishing write tid 1 to smithi07085154-24 2024-09-14T03:57:51.174 INFO:tasks.rados.rados.0.smithi070.stdout:2811: finishing write tid 2 to smithi07085154-24 2024-09-14T03:57:51.175 INFO:tasks.rados.rados.0.smithi070.stdout:2810: finishing write tid 2 to smithi07085154-44 2024-09-14T03:57:51.175 INFO:tasks.rados.rados.0.smithi070.stdout:2811: finishing write tid 3 to smithi07085154-24 2024-09-14T03:57:51.175 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 2792 (ObjNum 866 snap 293 seq_num 866) dirty exists 2024-09-14T03:57:51.175 INFO:tasks.rados.rados.0.smithi070.stdout:2811: left oid 24 (ObjNum 866 snap 293 seq_num 866) 2024-09-14T03:57:51.175 INFO:tasks.rados.rados.0.smithi070.stdout:2810: finishing write tid 3 to smithi07085154-44 2024-09-14T03:57:51.175 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 3150 (ObjNum 865 snap 293 seq_num 865) dirty exists 2024-09-14T03:57:51.175 INFO:tasks.rados.rados.0.smithi070.stdout:2810: left oid 44 (ObjNum 865 snap 293 seq_num 865) 2024-09-14T03:57:51.175 INFO:tasks.rados.rados.0.smithi070.stdout:2814: expect (ObjNum 592 snap 195 seq_num 592) 2024-09-14T03:57:51.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:51 smithi094 ceph-mon[86597]: osdmap e638: 8 total, 8 up, 8 in 2024-09-14T03:57:51.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:51 smithi070 ceph-mon[96704]: osdmap e638: 8 total, 8 up, 8 in 2024-09-14T03:57:51.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:51 smithi070 ceph-mon[98640]: osdmap e638: 8 total, 8 up, 8 in 2024-09-14T03:57:52.177 INFO:tasks.rados.rados.0.smithi070.stdout:2810: done (4 left) 2024-09-14T03:57:52.178 INFO:tasks.rados.rados.0.smithi070.stdout:2811: done (3 left) 2024-09-14T03:57:52.178 INFO:tasks.rados.rados.0.smithi070.stdout:2813: done (2 left) 2024-09-14T03:57:52.178 INFO:tasks.rados.rados.0.smithi070.stdout:2814: done (1 left) 2024-09-14T03:57:52.178 INFO:tasks.rados.rados.0.smithi070.stdout:2816: done (0 left) 2024-09-14T03:57:52.178 INFO:tasks.rados.rados.0.smithi070.stdout:2817: delete oid 14 current snap is 294 2024-09-14T03:57:52.182 INFO:tasks.rados.rados.0.smithi070.stdout:2817: done (0 left) 2024-09-14T03:57:52.182 INFO:tasks.rados.rados.0.smithi070.stdout:2818: snap_create 2024-09-14T03:57:52.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:52 smithi094 ceph-mon[86597]: osdmap e639: 8 total, 8 up, 8 in 2024-09-14T03:57:52.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:52 smithi094 ceph-mon[86597]: pgmap v556: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 19 MiB/s rd, 1.2 MiB/s wr, 36 op/s 2024-09-14T03:57:52.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:52 smithi070 ceph-mon[96704]: osdmap e639: 8 total, 8 up, 8 in 2024-09-14T03:57:52.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:52 smithi070 ceph-mon[96704]: pgmap v556: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 19 MiB/s rd, 1.2 MiB/s wr, 36 op/s 2024-09-14T03:57:52.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:52 smithi070 ceph-mon[98640]: osdmap e639: 8 total, 8 up, 8 in 2024-09-14T03:57:52.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:52 smithi070 ceph-mon[98640]: pgmap v556: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 19 MiB/s rd, 1.2 MiB/s wr, 36 op/s 2024-09-14T03:57:53.180 INFO:tasks.rados.rados.0.smithi070.stdout:2818: done (0 left) 2024-09-14T03:57:53.180 INFO:tasks.rados.rados.0.smithi070.stdout:2819: copy_from oid 50 from oid 8 current snap is 295 2024-09-14T03:57:53.180 INFO:tasks.rados.rados.0.smithi070.stdout:2820: copy_from oid 29 from oid 43 current snap is 295 2024-09-14T03:57:53.181 INFO:tasks.rados.rados.0.smithi070.stdout:2821: read oid 40 snap -1 2024-09-14T03:57:53.181 INFO:tasks.rados.rados.0.smithi070.stdout:2821: expect (ObjNum 576 snap 193 seq_num 576) 2024-09-14T03:57:53.181 INFO:tasks.rados.rados.0.smithi070.stdout:2822: delete oid 10 current snap is 295 2024-09-14T03:57:53.183 INFO:tasks.rados.rados.0.smithi070.stdout:2821: expect (ObjNum 576 snap 193 seq_num 576) 2024-09-14T03:57:53.191 INFO:tasks.rados.rados.0.smithi070.stdout:2820: finishing copy_from racing read to smithi07085154-29 2024-09-14T03:57:53.191 INFO:tasks.rados.rados.0.smithi070.stdout:2819: finishing copy_from racing read to smithi07085154-50 2024-09-14T03:57:53.191 INFO:tasks.rados.rados.0.smithi070.stdout:2819: finishing copy_from to smithi07085154-50 2024-09-14T03:57:53.192 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 2234 (ObjNum 129 snap 0 seq_num 2933397616) dirty exists 2024-09-14T03:57:53.192 INFO:tasks.rados.rados.0.smithi070.stdout:2820: finishing copy_from to smithi07085154-29 2024-09-14T03:57:53.192 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 2236 (ObjNum 724 snap 248 seq_num 724) dirty exists 2024-09-14T03:57:53.192 INFO:tasks.rados.rados.0.smithi070.stdout:2819: done (3 left) 2024-09-14T03:57:53.192 INFO:tasks.rados.rados.0.smithi070.stdout:2820: done (2 left) 2024-09-14T03:57:53.192 INFO:tasks.rados.rados.0.smithi070.stdout:2821: done (1 left) 2024-09-14T03:57:53.193 INFO:tasks.rados.rados.0.smithi070.stdout:2822: done (0 left) 2024-09-14T03:57:53.193 INFO:tasks.rados.rados.0.smithi070.stdout:2823: snap_remove snap 226 2024-09-14T03:57:53.437 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:57:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:57:53] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.51.0" 2024-09-14T03:57:53.437 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:53 smithi070 ceph-mon[98640]: osdmap e640: 8 total, 8 up, 8 in 2024-09-14T03:57:53.438 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:53 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:53.438 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:53 smithi070 ceph-mon[96704]: osdmap e640: 8 total, 8 up, 8 in 2024-09-14T03:57:53.438 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:53 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:53.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:53 smithi094 ceph-mon[86597]: osdmap e640: 8 total, 8 up, 8 in 2024-09-14T03:57:53.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:53 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:57:54.090 INFO:tasks.rados.rados.0.smithi070.stdout:2823: done (0 left) 2024-09-14T03:57:54.090 INFO:tasks.rados.rados.0.smithi070.stdout:2824: copy_from oid 46 from oid 3 current snap is 295 2024-09-14T03:57:54.090 INFO:tasks.rados.rados.0.smithi070.stdout:2825: copy_from oid 50 from oid 40 current snap is 295 2024-09-14T03:57:54.090 INFO:tasks.rados.rados.0.smithi070.stdout:2826: rollback oid 47 current snap is 295 2024-09-14T03:57:54.090 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 47 to 268 2024-09-14T03:57:54.090 INFO:tasks.rados.rados.0.smithi070.stdout:2827: rmattr oid 36 current snap is 295 2024-09-14T03:57:54.090 INFO:tasks.rados.rados.0.smithi070.stdout:2828: snap_remove snap 215 2024-09-14T03:57:54.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:54 smithi094 ceph-mon[86597]: osdmap e641: 8 total, 8 up, 8 in 2024-09-14T03:57:54.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:54 smithi094 ceph-mon[86597]: osdmap e642: 8 total, 8 up, 8 in 2024-09-14T03:57:54.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:54 smithi094 ceph-mon[86597]: pgmap v560: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s rd, 97 KiB/s wr, 27 op/s 2024-09-14T03:57:54.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:54 smithi070 ceph-mon[98640]: osdmap e641: 8 total, 8 up, 8 in 2024-09-14T03:57:54.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:54 smithi070 ceph-mon[98640]: osdmap e642: 8 total, 8 up, 8 in 2024-09-14T03:57:54.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:54 smithi070 ceph-mon[98640]: pgmap v560: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s rd, 97 KiB/s wr, 27 op/s 2024-09-14T03:57:54.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:54 smithi070 ceph-mon[96704]: osdmap e641: 8 total, 8 up, 8 in 2024-09-14T03:57:54.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:54 smithi070 ceph-mon[96704]: osdmap e642: 8 total, 8 up, 8 in 2024-09-14T03:57:54.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:54 smithi070 ceph-mon[96704]: pgmap v560: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s rd, 97 KiB/s wr, 27 op/s 2024-09-14T03:57:55.094 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 36 v 3151 (ObjNum 784 snap 274 seq_num 784) dirty exists 2024-09-14T03:57:55.094 INFO:tasks.rados.rados.0.smithi070.stdout:2826: finishing rollback tid 1 to smithi07085154-47 2024-09-14T03:57:55.094 INFO:tasks.rados.rados.0.smithi070.stdout:2826: finishing rollback tid 2 to smithi07085154-47 2024-09-14T03:57:55.094 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 47 v 1737 (ObjNum 514 snap 171 seq_num 514) dirty exists 2024-09-14T03:57:55.094 INFO:tasks.rados.rados.0.smithi070.stdout:2825: finishing copy_from racing read to smithi07085154-50 2024-09-14T03:57:55.094 INFO:tasks.rados.rados.0.smithi070.stdout:2825: finishing copy_from to smithi07085154-50 2024-09-14T03:57:55.094 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 3147 (ObjNum 576 snap 193 seq_num 576) dirty exists 2024-09-14T03:57:55.094 INFO:tasks.rados.rados.0.smithi070.stdout:2824: finishing copy_from racing read to smithi07085154-46 2024-09-14T03:57:55.094 INFO:tasks.rados.rados.0.smithi070.stdout:2824: finishing copy_from to smithi07085154-46 2024-09-14T03:57:55.095 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 46 v 1728 (ObjNum 683 snap 230 seq_num 683) dirty exists 2024-09-14T03:57:55.095 INFO:tasks.rados.rados.0.smithi070.stdout:2824: done (4 left) 2024-09-14T03:57:55.095 INFO:tasks.rados.rados.0.smithi070.stdout:2825: done (3 left) 2024-09-14T03:57:55.095 INFO:tasks.rados.rados.0.smithi070.stdout:2826: done (2 left) 2024-09-14T03:57:55.095 INFO:tasks.rados.rados.0.smithi070.stdout:2827: done (1 left) 2024-09-14T03:57:55.095 INFO:tasks.rados.rados.0.smithi070.stdout:2828: done (0 left) 2024-09-14T03:57:55.095 INFO:tasks.rados.rados.0.smithi070.stdout:2829: read oid 10 snap -1 2024-09-14T03:57:55.095 INFO:tasks.rados.rados.0.smithi070.stdout:2829: expect deleted 2024-09-14T03:57:55.095 INFO:tasks.rados.rados.0.smithi070.stdout:2830: copy_from oid 38 from oid 48 current snap is 295 2024-09-14T03:57:55.095 INFO:tasks.rados.rados.0.smithi070.stdout:2831: copy_from oid 2 from oid 9 current snap is 295 2024-09-14T03:57:55.095 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 4 current snap is 295 2024-09-14T03:57:55.095 INFO:tasks.rados.rados.0.smithi070.stdout:2832: seq_num 869 ranges {106496=155648} 2024-09-14T03:57:55.096 INFO:tasks.rados.rados.0.smithi070.stdout:2832: writing smithi07085154-4 from 106496 to 262144 tid 1 2024-09-14T03:57:55.096 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 31 current snap is 295 2024-09-14T03:57:55.096 INFO:tasks.rados.rados.0.smithi070.stdout:2833: seq_num 870 ranges {2400256=172032} 2024-09-14T03:57:55.110 INFO:tasks.rados.rados.0.smithi070.stdout:2833: writing smithi07085154-31 from 2400256 to 2572288 tid 1 2024-09-14T03:57:55.111 INFO:tasks.rados.rados.0.smithi070.stdout:2834: snap_remove snap 261 2024-09-14T03:57:55.535 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:57:55 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2[113018]: 2024-09-14T03:57:55.112+0000 7f5c88c9d640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap 129 after trimming was started 2024-09-14T03:57:56.101 INFO:tasks.rados.rados.0.smithi070.stdout:2834: done (5 left) 2024-09-14T03:57:56.101 INFO:tasks.rados.rados.0.smithi070.stdout:2835: delete oid 20 current snap is 295 2024-09-14T03:57:56.101 INFO:tasks.rados.rados.0.smithi070.stdout:2832: finishing write tid 1 to smithi07085154-4 2024-09-14T03:57:56.101 INFO:tasks.rados.rados.0.smithi070.stdout:2830: finishing copy_from racing read to smithi07085154-38 2024-09-14T03:57:56.102 INFO:tasks.rados.rados.0.smithi070.stdout:2832: finishing write tid 2 to smithi07085154-4 2024-09-14T03:57:56.102 INFO:tasks.rados.rados.0.smithi070.stdout:2832: finishing write tid 3 to smithi07085154-4 2024-09-14T03:57:56.102 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 3152 (ObjNum 869 snap 295 seq_num 869) dirty exists 2024-09-14T03:57:56.102 INFO:tasks.rados.rados.0.smithi070.stdout:2832: left oid 4 (ObjNum 869 snap 295 seq_num 869) 2024-09-14T03:57:56.102 INFO:tasks.rados.rados.0.smithi070.stdout:2833: finishing write tid 1 to smithi07085154-31 2024-09-14T03:57:56.102 INFO:tasks.rados.rados.0.smithi070.stdout:2833: finishing write tid 2 to smithi07085154-31 2024-09-14T03:57:56.102 INFO:tasks.rados.rados.0.smithi070.stdout:2833: finishing write tid 3 to smithi07085154-31 2024-09-14T03:57:56.102 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 2285 (ObjNum 870 snap 295 seq_num 870) dirty exists 2024-09-14T03:57:56.102 INFO:tasks.rados.rados.0.smithi070.stdout:2833: left oid 31 (ObjNum 870 snap 295 seq_num 870) 2024-09-14T03:57:56.102 INFO:tasks.rados.rados.0.smithi070.stdout:2830: finishing copy_from to smithi07085154-38 2024-09-14T03:57:56.102 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 38 v 2236 (ObjNum 724 snap 248 seq_num 724) dirty exists 2024-09-14T03:57:56.102 INFO:tasks.rados.rados.0.smithi070.stdout:2831: finishing copy_from racing read to smithi07085154-2 2024-09-14T03:57:56.102 INFO:tasks.rados.rados.0.smithi070.stdout:2831: finishing copy_from to smithi07085154-2 2024-09-14T03:57:56.103 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 2 v 3209 (ObjNum 459 snap 155 seq_num 459) dirty exists 2024-09-14T03:57:56.119 INFO:tasks.rados.rados.0.smithi070.stdout:2829: done (5 left) 2024-09-14T03:57:56.119 INFO:tasks.rados.rados.0.smithi070.stdout:2830: done (4 left) 2024-09-14T03:57:56.119 INFO:tasks.rados.rados.0.smithi070.stdout:2831: done (3 left) 2024-09-14T03:57:56.119 INFO:tasks.rados.rados.0.smithi070.stdout:2832: done (2 left) 2024-09-14T03:57:56.119 INFO:tasks.rados.rados.0.smithi070.stdout:2833: done (1 left) 2024-09-14T03:57:56.120 INFO:tasks.rados.rados.0.smithi070.stdout:2835: done (0 left) 2024-09-14T03:57:56.120 INFO:tasks.rados.rados.0.smithi070.stdout:2836: rollback oid 31 current snap is 295 2024-09-14T03:57:56.120 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 31 to 174 2024-09-14T03:57:56.120 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 29 current snap is 295 2024-09-14T03:57:56.120 INFO:tasks.rados.rados.0.smithi070.stdout:2837: seq_num 872 ranges {131072=212992} 2024-09-14T03:57:56.121 INFO:tasks.rados.rados.0.smithi070.stdout:2837: writing smithi07085154-29 from 131072 to 344064 tid 1 2024-09-14T03:57:56.121 INFO:tasks.rados.rados.0.smithi070.stdout:2838: read oid 48 snap 271 2024-09-14T03:57:56.122 INFO:tasks.rados.rados.0.smithi070.stdout:2838: expect (ObjNum 757 snap 263 seq_num 757) 2024-09-14T03:57:56.122 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 12 current snap is 295 2024-09-14T03:57:56.122 INFO:tasks.rados.rados.0.smithi070.stdout:2839: seq_num 873 ranges {3227648=229376} 2024-09-14T03:57:56.142 INFO:tasks.rados.rados.0.smithi070.stdout:2839: writing smithi07085154-12 from 3227648 to 3457024 tid 1 2024-09-14T03:57:56.142 INFO:tasks.rados.rados.0.smithi070.stdout:2840: copy_from oid 24 from oid 39 current snap is 295 2024-09-14T03:57:56.142 INFO:tasks.rados.rados.0.smithi070.stdout:2836: finishing rollback tid 0 to smithi07085154-31 2024-09-14T03:57:56.142 INFO:tasks.rados.rados.0.smithi070.stdout:2836: finishing rollback tid 1 to smithi07085154-31 2024-09-14T03:57:56.142 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 2287 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-14T03:57:56.142 INFO:tasks.rados.rados.0.smithi070.stdout:2837: finishing write tid 1 to smithi07085154-29 2024-09-14T03:57:56.142 INFO:tasks.rados.rados.0.smithi070.stdout:2837: finishing write tid 2 to smithi07085154-29 2024-09-14T03:57:56.142 INFO:tasks.rados.rados.0.smithi070.stdout:2837: finishing write tid 3 to smithi07085154-29 2024-09-14T03:57:56.142 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 3153 (ObjNum 872 snap 295 seq_num 872) dirty exists 2024-09-14T03:57:56.143 INFO:tasks.rados.rados.0.smithi070.stdout:2837: left oid 29 (ObjNum 872 snap 295 seq_num 872) 2024-09-14T03:57:56.143 INFO:tasks.rados.rados.0.smithi070.stdout:2836: done (4 left) 2024-09-14T03:57:56.143 INFO:tasks.rados.rados.0.smithi070.stdout:2837: done (3 left) 2024-09-14T03:57:56.143 INFO:tasks.rados.rados.0.smithi070.stdout:2841: delete oid 34 current snap is 295 2024-09-14T03:57:56.143 INFO:tasks.rados.rados.0.smithi070.stdout:2840: finishing copy_from to smithi07085154-24 2024-09-14T03:57:56.143 INFO:tasks.rados.rados.0.smithi070.stdout:2840: got expected ENOENT (src dne) 2024-09-14T03:57:56.144 INFO:tasks.rados.rados.0.smithi070.stdout:2840: finishing copy_from racing read to smithi07085154-24 2024-09-14T03:57:56.144 INFO:tasks.rados.rados.0.smithi070.stdout:2838: expect (ObjNum 757 snap 263 seq_num 757) 2024-09-14T03:57:56.358 INFO:tasks.rados.rados.0.smithi070.stdout:2839: finishing write tid 1 to smithi07085154-12 2024-09-14T03:57:56.359 INFO:tasks.rados.rados.0.smithi070.stdout:2839: finishing write tid 2 to smithi07085154-12 2024-09-14T03:57:56.359 INFO:tasks.rados.rados.0.smithi070.stdout:2839: finishing write tid 3 to smithi07085154-12 2024-09-14T03:57:56.359 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 2239 (ObjNum 873 snap 295 seq_num 873) dirty exists 2024-09-14T03:57:56.359 INFO:tasks.rados.rados.0.smithi070.stdout:2839: left oid 12 (ObjNum 873 snap 295 seq_num 873) 2024-09-14T03:57:56.359 INFO:tasks.rados.rados.0.smithi070.stdout:2838: done (3 left) 2024-09-14T03:57:56.359 INFO:tasks.rados.rados.0.smithi070.stdout:2839: done (2 left) 2024-09-14T03:57:56.359 INFO:tasks.rados.rados.0.smithi070.stdout:2840: done (1 left) 2024-09-14T03:57:56.359 INFO:tasks.rados.rados.0.smithi070.stdout:2841: done (0 left) 2024-09-14T03:57:56.359 INFO:tasks.rados.rados.0.smithi070.stdout:2842: read oid 30 snap -1 2024-09-14T03:57:56.359 INFO:tasks.rados.rados.0.smithi070.stdout:2842: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:57:56.359 INFO:tasks.rados.rados.0.smithi070.stdout:2843: delete oid 38 current snap is 295 2024-09-14T03:57:56.360 INFO:tasks.rados.rados.0.smithi070.stdout:2842: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:57:56.361 INFO:tasks.rados.rados.0.smithi070.stdout:2842: done (1 left) 2024-09-14T03:57:56.362 INFO:tasks.rados.rados.0.smithi070.stdout:2843: done (0 left) 2024-09-14T03:57:56.362 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 30 current snap is 295 2024-09-14T03:57:56.362 INFO:tasks.rados.rados.0.smithi070.stdout:2844: seq_num 876 ranges {0=278528} 2024-09-14T03:57:56.363 INFO:tasks.rados.rados.0.smithi070.stdout:2844: writing smithi07085154-30 from 0 to 278528 tid 1 2024-09-14T03:57:56.363 INFO:tasks.rados.rados.0.smithi070.stdout:2845: rollback oid 6 current snap is 295 2024-09-14T03:57:56.363 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 6 to 281 2024-09-14T03:57:56.363 INFO:tasks.rados.rados.0.smithi070.stdout:2846: delete oid 46 current snap is 295 2024-09-14T03:57:56.366 INFO:tasks.rados.rados.0.smithi070.stdout:2846: done (2 left) 2024-09-14T03:57:56.367 INFO:tasks.rados.rados.0.smithi070.stdout:2847: read oid 13 snap -1 2024-09-14T03:57:56.367 INFO:tasks.rados.rados.0.smithi070.stdout:2847: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:57:56.367 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 9 current snap is 295 2024-09-14T03:57:56.367 INFO:tasks.rados.rados.0.smithi070.stdout:2848: seq_num 878 ranges {16171008=311296} 2024-09-14T03:57:56.454 INFO:tasks.rados.rados.0.smithi070.stdout:2848: writing smithi07085154-9 from 16171008 to 16482304 tid 1 2024-09-14T03:57:56.454 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 34 current snap is 295 2024-09-14T03:57:56.454 INFO:tasks.rados.rados.0.smithi070.stdout:2849: seq_num 879 ranges {0=327680} 2024-09-14T03:57:56.456 INFO:tasks.rados.rados.0.smithi070.stdout:2849: writing smithi07085154-34 from 0 to 327680 tid 1 2024-09-14T03:57:56.456 INFO:tasks.rados.rados.0.smithi070.stdout:2850: read oid 28 snap -1 2024-09-14T03:57:56.456 INFO:tasks.rados.rados.0.smithi070.stdout:2850: expect (ObjNum 639 snap 214 seq_num 639) 2024-09-14T03:57:56.456 INFO:tasks.rados.rados.0.smithi070.stdout:2851: read oid 45 snap -1 2024-09-14T03:57:56.456 INFO:tasks.rados.rados.0.smithi070.stdout:2851: expect deleted 2024-09-14T03:57:56.456 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 18 current snap is 295 2024-09-14T03:57:56.456 INFO:tasks.rados.rados.0.smithi070.stdout:2852: seq_num 880 ranges {2473984=344064} 2024-09-14T03:57:56.471 INFO:tasks.rados.rados.0.smithi070.stdout:2852: writing smithi07085154-18 from 2473984 to 2818048 tid 1 2024-09-14T03:57:56.472 INFO:tasks.rados.rados.0.smithi070.stdout:2853: delete oid 27 current snap is 295 2024-09-14T03:57:56.472 INFO:tasks.rados.rados.0.smithi070.stdout:2847: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:57:56.475 INFO:tasks.rados.rados.0.smithi070.stdout:2844: finishing write tid 1 to smithi07085154-30 2024-09-14T03:57:56.475 INFO:tasks.rados.rados.0.smithi070.stdout:2844: finishing write tid 2 to smithi07085154-30 2024-09-14T03:57:56.475 INFO:tasks.rados.rados.0.smithi070.stdout:2845: finishing rollback tid 0 to smithi07085154-6 2024-09-14T03:57:56.475 INFO:tasks.rados.rados.0.smithi070.stdout:2845: finishing rollback tid 1 to smithi07085154-6 2024-09-14T03:57:56.475 INFO:tasks.rados.rados.0.smithi070.stdout:2845: finishing rollback tid 2 to smithi07085154-6 2024-09-14T03:57:56.475 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 2290 (ObjNum 751 snap 260 seq_num 751) dirty exists 2024-09-14T03:57:56.475 INFO:tasks.rados.rados.0.smithi070.stdout:2844: finishing write tid 3 to smithi07085154-30 2024-09-14T03:57:56.475 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 30 v 2239 (ObjNum 876 snap 295 seq_num 876) dirty exists 2024-09-14T03:57:56.475 INFO:tasks.rados.rados.0.smithi070.stdout:2844: left oid 30 (ObjNum 876 snap 295 seq_num 876) 2024-09-14T03:57:56.475 INFO:tasks.rados.rados.0.smithi070.stdout:2844: done (8 left) 2024-09-14T03:57:56.475 INFO:tasks.rados.rados.0.smithi070.stdout:2845: done (7 left) 2024-09-14T03:57:56.476 INFO:tasks.rados.rados.0.smithi070.stdout:2847: done (6 left) 2024-09-14T03:57:56.476 INFO:tasks.rados.rados.0.smithi070.stdout:2853: done (5 left) 2024-09-14T03:57:56.476 INFO:tasks.rados.rados.0.smithi070.stdout:2854: setattr oid 12 current snap is 295 2024-09-14T03:57:56.476 INFO:tasks.rados.rados.0.smithi070.stdout:2848: finishing write tid 1 to smithi07085154-9 2024-09-14T03:57:56.476 INFO:tasks.rados.rados.0.smithi070.stdout:2849: finishing write tid 1 to smithi07085154-34 2024-09-14T03:57:56.476 INFO:tasks.rados.rados.0.smithi070.stdout:2849: finishing write tid 2 to smithi07085154-34 2024-09-14T03:57:56.476 INFO:tasks.rados.rados.0.smithi070.stdout:2850: expect (ObjNum 639 snap 214 seq_num 639) 2024-09-14T03:57:56.508 INFO:tasks.rados.rados.0.smithi070.stdout:2848: finishing write tid 2 to smithi07085154-9 2024-09-14T03:57:56.508 INFO:tasks.rados.rados.0.smithi070.stdout:2849: finishing write tid 3 to smithi07085154-34 2024-09-14T03:57:56.508 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 3155 (ObjNum 879 snap 295 seq_num 879) dirty exists 2024-09-14T03:57:56.508 INFO:tasks.rados.rados.0.smithi070.stdout:2849: left oid 34 (ObjNum 879 snap 295 seq_num 879) 2024-09-14T03:57:56.508 INFO:tasks.rados.rados.0.smithi070.stdout:2848: finishing write tid 3 to smithi07085154-9 2024-09-14T03:57:56.509 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 3291 (ObjNum 878 snap 295 seq_num 878) dirty exists 2024-09-14T03:57:56.509 INFO:tasks.rados.rados.0.smithi070.stdout:2848: left oid 9 (ObjNum 878 snap 295 seq_num 878) 2024-09-14T03:57:56.509 INFO:tasks.rados.rados.0.smithi070.stdout:2852: finishing write tid 1 to smithi07085154-18 2024-09-14T03:57:56.509 INFO:tasks.rados.rados.0.smithi070.stdout:2852: finishing write tid 2 to smithi07085154-18 2024-09-14T03:57:56.509 INFO:tasks.rados.rados.0.smithi070.stdout:2852: finishing write tid 3 to smithi07085154-18 2024-09-14T03:57:56.509 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 2261 (ObjNum 880 snap 295 seq_num 880) dirty exists 2024-09-14T03:57:56.509 INFO:tasks.rados.rados.0.smithi070.stdout:2852: left oid 18 (ObjNum 880 snap 295 seq_num 880) 2024-09-14T03:57:56.509 INFO:tasks.rados.rados.0.smithi070.stdout:2848: done (5 left) 2024-09-14T03:57:56.509 INFO:tasks.rados.rados.0.smithi070.stdout:2849: done (4 left) 2024-09-14T03:57:56.509 INFO:tasks.rados.rados.0.smithi070.stdout:2850: done (3 left) 2024-09-14T03:57:56.509 INFO:tasks.rados.rados.0.smithi070.stdout:2851: done (2 left) 2024-09-14T03:57:56.509 INFO:tasks.rados.rados.0.smithi070.stdout:2852: done (1 left) 2024-09-14T03:57:56.509 INFO:tasks.rados.rados.0.smithi070.stdout:2855: rollback oid 16 current snap is 295 2024-09-14T03:57:56.510 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 16 to 290 2024-09-14T03:57:56.510 INFO:tasks.rados.rados.0.smithi070.stdout:2856: copy_from oid 1 from oid 45 current snap is 295 2024-09-14T03:57:56.510 INFO:tasks.rados.rados.0.smithi070.stdout:2857: rollback oid 37 current snap is 295 2024-09-14T03:57:56.510 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 37 to 265 2024-09-14T03:57:56.510 INFO:tasks.rados.rados.0.smithi070.stdout:2858: read oid 3 snap -1 2024-09-14T03:57:56.510 INFO:tasks.rados.rados.0.smithi070.stdout:2858: expect (ObjNum 683 snap 230 seq_num 683) 2024-09-14T03:57:56.510 INFO:tasks.rados.rados.0.smithi070.stdout:2859: read oid 35 snap -1 2024-09-14T03:57:56.510 INFO:tasks.rados.rados.0.smithi070.stdout:2859: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:57:56.510 INFO:tasks.rados.rados.0.smithi070.stdout:2860: delete oid 23 current snap is 295 2024-09-14T03:57:56.511 INFO:tasks.rados.rados.0.smithi070.stdout:2859: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:57:56.511 INFO:tasks.rados.rados.0.smithi070.stdout:2856: finishing copy_from to smithi07085154-1 2024-09-14T03:57:56.511 INFO:tasks.rados.rados.0.smithi070.stdout:2856: got expected ENOENT (src dne) 2024-09-14T03:57:56.511 INFO:tasks.rados.rados.0.smithi070.stdout:2856: finishing copy_from racing read to smithi07085154-1 2024-09-14T03:57:56.512 INFO:tasks.rados.rados.0.smithi070.stdout:2857: finishing rollback tid 1 to smithi07085154-37 2024-09-14T03:57:56.512 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 37 v 2223 (ObjNum 129 snap 0 seq_num 2930011472) dirty dne 2024-09-14T03:57:56.512 INFO:tasks.rados.rados.0.smithi070.stdout:2856: done (6 left) 2024-09-14T03:57:56.512 INFO:tasks.rados.rados.0.smithi070.stdout:2857: done (5 left) 2024-09-14T03:57:56.512 INFO:tasks.rados.rados.0.smithi070.stdout:2859: done (4 left) 2024-09-14T03:57:56.512 INFO:tasks.rados.rados.0.smithi070.stdout:2860: done (3 left) 2024-09-14T03:57:56.513 INFO:tasks.rados.rados.0.smithi070.stdout:2861: copy_from oid 26 from oid 37 current snap is 295 2024-09-14T03:57:56.513 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 2240 (ObjNum 873 snap 295 seq_num 873) dirty exists 2024-09-14T03:57:56.513 INFO:tasks.rados.rados.0.smithi070.stdout:2854: done (3 left) 2024-09-14T03:57:56.513 INFO:tasks.rados.rados.0.smithi070.stdout:2862: snap_remove snap 285 2024-09-14T03:57:56.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:56 smithi094 ceph-mon[86597]: osdmap e643: 8 total, 8 up, 8 in 2024-09-14T03:57:56.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:56 smithi070 ceph-mon[96704]: osdmap e643: 8 total, 8 up, 8 in 2024-09-14T03:57:56.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:56 smithi070 ceph-mon[98640]: osdmap e643: 8 total, 8 up, 8 in 2024-09-14T03:57:57.105 INFO:tasks.rados.rados.0.smithi070.stdout:2862: done (3 left) 2024-09-14T03:57:57.105 INFO:tasks.rados.rados.0.smithi070.stdout:2863: rmattr oid 13 current snap is 295 2024-09-14T03:57:57.106 INFO:tasks.rados.rados.0.smithi070.stdout:2861: finishing copy_from to smithi07085154-26 2024-09-14T03:57:57.106 INFO:tasks.rados.rados.0.smithi070.stdout:2861: got expected ENOENT (src dne) 2024-09-14T03:57:57.106 INFO:tasks.rados.rados.0.smithi070.stdout:2861: finishing copy_from racing read to smithi07085154-26 2024-09-14T03:57:57.106 INFO:tasks.rados.rados.0.smithi070.stdout:2855: finishing rollback tid 0 to smithi07085154-16 2024-09-14T03:57:57.106 INFO:tasks.rados.rados.0.smithi070.stdout:2855: finishing rollback tid 1 to smithi07085154-16 2024-09-14T03:57:57.106 INFO:tasks.rados.rados.0.smithi070.stdout:2855: finishing rollback tid 2 to smithi07085154-16 2024-09-14T03:57:57.106 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 3297 (ObjNum 788 snap 274 seq_num 788) dirty exists 2024-09-14T03:57:57.106 INFO:tasks.rados.rados.0.smithi070.stdout:2858: expect (ObjNum 683 snap 230 seq_num 683) 2024-09-14T03:57:57.269 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 3151 (ObjNum 289 snap 0 seq_num 1) dirty exists 2024-09-14T03:57:57.269 INFO:tasks.rados.rados.0.smithi070.stdout:2855: done (3 left) 2024-09-14T03:57:57.269 INFO:tasks.rados.rados.0.smithi070.stdout:2858: done (2 left) 2024-09-14T03:57:57.269 INFO:tasks.rados.rados.0.smithi070.stdout:2861: done (1 left) 2024-09-14T03:57:57.269 INFO:tasks.rados.rados.0.smithi070.stdout:2863: done (0 left) 2024-09-14T03:57:57.269 INFO:tasks.rados.rados.0.smithi070.stdout:2864: setattr oid 17 current snap is 295 2024-09-14T03:57:57.270 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 47 current snap is 295 2024-09-14T03:57:57.270 INFO:tasks.rados.rados.0.smithi070.stdout:2865: seq_num 883 ranges {3612672=393216} 2024-09-14T03:57:57.292 INFO:tasks.rados.rados.0.smithi070.stdout:2865: writing smithi07085154-47 from 3612672 to 4005888 tid 1 2024-09-14T03:57:57.292 INFO:tasks.rados.rados.0.smithi070.stdout:2866: read oid 43 snap -1 2024-09-14T03:57:57.292 INFO:tasks.rados.rados.0.smithi070.stdout:2866: expect (ObjNum 724 snap 248 seq_num 724) 2024-09-14T03:57:57.292 INFO:tasks.rados.rados.0.smithi070.stdout:2867: snap_remove snap 281 2024-09-14T03:57:57.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:57 smithi094 ceph-mon[86597]: osdmap e644: 8 total, 8 up, 8 in 2024-09-14T03:57:57.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:57 smithi094 ceph-mon[86597]: pgmap v563: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 42 KiB/s rd, 65 KiB/s wr, 3 op/s 2024-09-14T03:57:57.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:57 smithi070 ceph-mon[98640]: osdmap e644: 8 total, 8 up, 8 in 2024-09-14T03:57:57.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:57 smithi070 ceph-mon[98640]: pgmap v563: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 42 KiB/s rd, 65 KiB/s wr, 3 op/s 2024-09-14T03:57:57.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:57 smithi070 ceph-mon[96704]: osdmap e644: 8 total, 8 up, 8 in 2024-09-14T03:57:57.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:57 smithi070 ceph-mon[96704]: pgmap v563: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 42 KiB/s rd, 65 KiB/s wr, 3 op/s 2024-09-14T03:57:58.115 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 2291 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-14T03:57:58.115 INFO:tasks.rados.rados.0.smithi070.stdout:2866: expect (ObjNum 724 snap 248 seq_num 724) 2024-09-14T03:57:58.118 INFO:tasks.rados.rados.0.smithi070.stdout:2865: finishing write tid 1 to smithi07085154-47 2024-09-14T03:57:58.118 INFO:tasks.rados.rados.0.smithi070.stdout:2865: finishing write tid 2 to smithi07085154-47 2024-09-14T03:57:58.118 INFO:tasks.rados.rados.0.smithi070.stdout:2865: finishing write tid 3 to smithi07085154-47 2024-09-14T03:57:58.118 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 47 v 2241 (ObjNum 883 snap 295 seq_num 883) dirty exists 2024-09-14T03:57:58.118 INFO:tasks.rados.rados.0.smithi070.stdout:2865: left oid 47 (ObjNum 883 snap 295 seq_num 883) 2024-09-14T03:57:58.119 INFO:tasks.rados.rados.0.smithi070.stdout:2864: done (3 left) 2024-09-14T03:57:58.119 INFO:tasks.rados.rados.0.smithi070.stdout:2865: done (2 left) 2024-09-14T03:57:58.119 INFO:tasks.rados.rados.0.smithi070.stdout:2866: done (1 left) 2024-09-14T03:57:58.119 INFO:tasks.rados.rados.0.smithi070.stdout:2867: done (0 left) 2024-09-14T03:57:58.119 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 50 current snap is 295 2024-09-14T03:57:58.119 INFO:tasks.rados.rados.0.smithi070.stdout:2868: seq_num 884 ranges {49152=409600} 2024-09-14T03:57:58.121 INFO:tasks.rados.rados.0.smithi070.stdout:2868: writing smithi07085154-50 from 49152 to 458752 tid 1 2024-09-14T03:57:58.122 INFO:tasks.rados.rados.0.smithi070.stdout:2869: snap_remove snap 294 2024-09-14T03:57:58.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:58 smithi094 ceph-mon[86597]: osdmap e645: 8 total, 8 up, 8 in 2024-09-14T03:57:58.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:58 smithi070 ceph-mon[96704]: osdmap e645: 8 total, 8 up, 8 in 2024-09-14T03:57:58.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:58 smithi070 ceph-mon[98640]: osdmap e645: 8 total, 8 up, 8 in 2024-09-14T03:57:59.093 INFO:tasks.rados.rados.0.smithi070.stdout:2869: done (1 left) 2024-09-14T03:57:59.093 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 18 current snap is 295 2024-09-14T03:57:59.093 INFO:tasks.rados.rados.0.smithi070.stdout:2870: seq_num 885 ranges {2818048=425984} 2024-09-14T03:57:59.112 INFO:tasks.rados.rados.0.smithi070.stdout:2870: writing smithi07085154-18 from 2818048 to 3244032 tid 1 2024-09-14T03:57:59.112 INFO:tasks.rados.rados.0.smithi070.stdout:2871: snap_create 2024-09-14T03:57:59.112 INFO:tasks.rados.rados.0.smithi070.stdout:2868: finishing write tid 1 to smithi07085154-50 2024-09-14T03:57:59.112 INFO:tasks.rados.rados.0.smithi070.stdout:2868: finishing write tid 2 to smithi07085154-50 2024-09-14T03:57:59.112 INFO:tasks.rados.rados.0.smithi070.stdout:2868: finishing write tid 3 to smithi07085154-50 2024-09-14T03:57:59.112 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 3157 (ObjNum 884 snap 295 seq_num 884) dirty exists 2024-09-14T03:57:59.112 INFO:tasks.rados.rados.0.smithi070.stdout:2868: left oid 50 (ObjNum 884 snap 295 seq_num 884) 2024-09-14T03:57:59.117 INFO:tasks.rados.rados.0.smithi070.stdout:2870: finishing write tid 1 to smithi07085154-18 2024-09-14T03:57:59.117 INFO:tasks.rados.rados.0.smithi070.stdout:2870: finishing write tid 2 to smithi07085154-18 2024-09-14T03:57:59.119 INFO:tasks.rados.rados.0.smithi070.stdout:2870: finishing write tid 3 to smithi07085154-18 2024-09-14T03:57:59.119 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 2274 (ObjNum 885 snap 295 seq_num 885) dirty exists 2024-09-14T03:57:59.119 INFO:tasks.rados.rados.0.smithi070.stdout:2870: left oid 18 (ObjNum 885 snap 295 seq_num 885) 2024-09-14T03:57:59.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:59 smithi094 ceph-mon[86597]: osdmap e646: 8 total, 8 up, 8 in 2024-09-14T03:57:59.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:59 smithi094 ceph-mon[86597]: pgmap v566: 121 pgs: 121 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 833 KiB/s rd, 5.1 MiB/s wr, 10 op/s 2024-09-14T03:57:59.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:57:59 smithi094 ceph-mon[86597]: osdmap e647: 8 total, 8 up, 8 in 2024-09-14T03:57:59.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:59 smithi070 ceph-mon[96704]: osdmap e646: 8 total, 8 up, 8 in 2024-09-14T03:57:59.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:59 smithi070 ceph-mon[96704]: pgmap v566: 121 pgs: 121 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 833 KiB/s rd, 5.1 MiB/s wr, 10 op/s 2024-09-14T03:57:59.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:57:59 smithi070 ceph-mon[96704]: osdmap e647: 8 total, 8 up, 8 in 2024-09-14T03:57:59.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:59 smithi070 ceph-mon[98640]: osdmap e646: 8 total, 8 up, 8 in 2024-09-14T03:57:59.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:59 smithi070 ceph-mon[98640]: pgmap v566: 121 pgs: 121 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 833 KiB/s rd, 5.1 MiB/s wr, 10 op/s 2024-09-14T03:57:59.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:57:59 smithi070 ceph-mon[98640]: osdmap e647: 8 total, 8 up, 8 in 2024-09-14T03:58:00.118 INFO:tasks.rados.rados.0.smithi070.stdout:2868: done (2 left) 2024-09-14T03:58:00.118 INFO:tasks.rados.rados.0.smithi070.stdout:2870: done (1 left) 2024-09-14T03:58:00.119 INFO:tasks.rados.rados.0.smithi070.stdout:2871: done (0 left) 2024-09-14T03:58:00.119 INFO:tasks.rados.rados.0.smithi070.stdout:2872: read oid 30 snap -1 2024-09-14T03:58:00.119 INFO:tasks.rados.rados.0.smithi070.stdout:2872: expect (ObjNum 876 snap 295 seq_num 876) 2024-09-14T03:58:00.119 INFO:tasks.rados.rados.0.smithi070.stdout:2873: snap_remove snap 288 2024-09-14T03:58:00.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:58:00 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:58:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:58:01.120 INFO:tasks.rados.rados.0.smithi070.stdout:2872: expect (ObjNum 876 snap 295 seq_num 876) 2024-09-14T03:58:01.137 INFO:tasks.rados.rados.0.smithi070.stdout:2872: done (1 left) 2024-09-14T03:58:01.137 INFO:tasks.rados.rados.0.smithi070.stdout:2873: done (0 left) 2024-09-14T03:58:01.137 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 11 current snap is 296 2024-09-14T03:58:01.137 INFO:tasks.rados.rados.0.smithi070.stdout:2874: seq_num 886 ranges {2310144=442368} 2024-09-14T03:58:01.153 INFO:tasks.rados.rados.0.smithi070.stdout:2874: writing smithi07085154-11 from 2310144 to 2752512 tid 1 2024-09-14T03:58:01.153 INFO:tasks.rados.rados.0.smithi070.stdout:2875: delete oid 14 current snap is 296 2024-09-14T03:58:01.155 INFO:tasks.rados.rados.0.smithi070.stdout:2875: done (1 left) 2024-09-14T03:58:01.155 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 32 current snap is 296 2024-09-14T03:58:01.155 INFO:tasks.rados.rados.0.smithi070.stdout:2876: seq_num 888 ranges {11255808=475136} 2024-09-14T03:58:01.217 INFO:tasks.rados.rados.0.smithi070.stdout:2876: writing smithi07085154-32 from 11255808 to 11730944 tid 1 2024-09-14T03:58:01.218 INFO:tasks.rados.rados.0.smithi070.stdout:2877: snap_remove snap 246 2024-09-14T03:58:01.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:01 smithi094 ceph-mon[86597]: pgmap v568: 121 pgs: 121 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.1 MiB/s rd, 5.3 MiB/s wr, 28 op/s 2024-09-14T03:58:01.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:01 smithi094 ceph-mon[86597]: osdmap e648: 8 total, 8 up, 8 in 2024-09-14T03:58:01.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:01 smithi070 ceph-mon[96704]: pgmap v568: 121 pgs: 121 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.1 MiB/s rd, 5.3 MiB/s wr, 28 op/s 2024-09-14T03:58:01.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:01 smithi070 ceph-mon[96704]: osdmap e648: 8 total, 8 up, 8 in 2024-09-14T03:58:01.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:01 smithi070 ceph-mon[98640]: pgmap v568: 121 pgs: 121 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.1 MiB/s rd, 5.3 MiB/s wr, 28 op/s 2024-09-14T03:58:01.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:01 smithi070 ceph-mon[98640]: osdmap e648: 8 total, 8 up, 8 in 2024-09-14T03:58:02.133 INFO:tasks.rados.rados.0.smithi070.stdout:2874: finishing write tid 1 to smithi07085154-11 2024-09-14T03:58:02.134 INFO:tasks.rados.rados.0.smithi070.stdout:2874: finishing write tid 2 to smithi07085154-11 2024-09-14T03:58:02.134 INFO:tasks.rados.rados.0.smithi070.stdout:2874: finishing write tid 3 to smithi07085154-11 2024-09-14T03:58:02.134 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 11 v 3153 (ObjNum 886 snap 296 seq_num 886) dirty exists 2024-09-14T03:58:02.134 INFO:tasks.rados.rados.0.smithi070.stdout:2874: left oid 11 (ObjNum 886 snap 296 seq_num 886) 2024-09-14T03:58:02.134 INFO:tasks.rados.rados.0.smithi070.stdout:2876: finishing write tid 1 to smithi07085154-32 2024-09-14T03:58:02.134 INFO:tasks.rados.rados.0.smithi070.stdout:2876: finishing write tid 2 to smithi07085154-32 2024-09-14T03:58:02.134 INFO:tasks.rados.rados.0.smithi070.stdout:2876: finishing write tid 3 to smithi07085154-32 2024-09-14T03:58:02.134 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 3155 (ObjNum 888 snap 296 seq_num 888) dirty exists 2024-09-14T03:58:02.134 INFO:tasks.rados.rados.0.smithi070.stdout:2876: left oid 32 (ObjNum 888 snap 296 seq_num 888) 2024-09-14T03:58:02.134 INFO:tasks.rados.rados.0.smithi070.stdout:2874: done (2 left) 2024-09-14T03:58:02.134 INFO:tasks.rados.rados.0.smithi070.stdout:2876: done (1 left) 2024-09-14T03:58:02.135 INFO:tasks.rados.rados.0.smithi070.stdout:2877: done (0 left) 2024-09-14T03:58:02.135 INFO:tasks.rados.rados.0.smithi070.stdout:2878: rmattr oid 46 current snap is 296 2024-09-14T03:58:02.135 INFO:tasks.rados.rados.0.smithi070.stdout:2878: done (0 left) 2024-09-14T03:58:02.135 INFO:tasks.rados.rados.0.smithi070.stdout:2879: delete oid 42 current snap is 296 2024-09-14T03:58:02.151 INFO:tasks.rados.rados.0.smithi070.stdout:2879: done (0 left) 2024-09-14T03:58:02.151 INFO:tasks.rados.rados.0.smithi070.stdout:2880: rollback oid 41 current snap is 296 2024-09-14T03:58:02.151 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 41 to 277 2024-09-14T03:58:02.152 INFO:tasks.rados.rados.0.smithi070.stdout:2881: read oid 26 snap -1 2024-09-14T03:58:02.152 INFO:tasks.rados.rados.0.smithi070.stdout:2881: expect (ObjNum 854 snap 288 seq_num 854) 2024-09-14T03:58:02.152 INFO:tasks.rados.rados.0.smithi070.stdout:2882: read oid 2 snap 284 2024-09-14T03:58:02.152 INFO:tasks.rados.rados.0.smithi070.stdout:2882: expect (ObjNum 830 snap 283 seq_num 830) 2024-09-14T03:58:02.152 INFO:tasks.rados.rados.0.smithi070.stdout:2883: rollback oid 40 current snap is 296 2024-09-14T03:58:02.152 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 40 to 292 2024-09-14T03:58:02.152 INFO:tasks.rados.rados.0.smithi070.stdout:2884: read oid 13 snap -1 2024-09-14T03:58:02.152 INFO:tasks.rados.rados.0.smithi070.stdout:2884: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:58:02.152 INFO:tasks.rados.rados.0.smithi070.stdout:2885: snap_create 2024-09-14T03:58:02.155 INFO:tasks.rados.rados.0.smithi070.stdout:2884: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:58:02.157 INFO:tasks.rados.rados.0.smithi070.stdout:2883: finishing rollback tid 0 to smithi07085154-40 2024-09-14T03:58:02.158 INFO:tasks.rados.rados.0.smithi070.stdout:2883: finishing rollback tid 1 to smithi07085154-40 2024-09-14T03:58:02.158 INFO:tasks.rados.rados.0.smithi070.stdout:2883: finishing rollback tid 2 to smithi07085154-40 2024-09-14T03:58:02.158 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 3154 (ObjNum 576 snap 193 seq_num 576) dirty exists 2024-09-14T03:58:02.160 INFO:tasks.rados.rados.0.smithi070.stdout:2880: finishing rollback tid 0 to smithi07085154-41 2024-09-14T03:58:02.161 INFO:tasks.rados.rados.0.smithi070.stdout:2880: finishing rollback tid 1 to smithi07085154-41 2024-09-14T03:58:02.161 INFO:tasks.rados.rados.0.smithi070.stdout:2880: finishing rollback tid 2 to smithi07085154-41 2024-09-14T03:58:02.161 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 2286 (ObjNum 724 snap 248 seq_num 724) dirty exists 2024-09-14T03:58:02.171 INFO:tasks.rados.rados.0.smithi070.stdout:2881: expect (ObjNum 854 snap 288 seq_num 854) 2024-09-14T03:58:02.329 INFO:tasks.rados.rados.0.smithi070.stdout:2882: expect (ObjNum 830 snap 283 seq_num 830) 2024-09-14T03:58:02.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:02 smithi094 ceph-mon[86597]: osdmap e649: 8 total, 8 up, 8 in 2024-09-14T03:58:02.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:02 smithi070 ceph-mon[98640]: osdmap e649: 8 total, 8 up, 8 in 2024-09-14T03:58:02.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:02 smithi070 ceph-mon[96704]: osdmap e649: 8 total, 8 up, 8 in 2024-09-14T03:58:03.133 INFO:tasks.rados.rados.0.smithi070.stdout:2880: done (5 left) 2024-09-14T03:58:03.133 INFO:tasks.rados.rados.0.smithi070.stdout:2881: done (4 left) 2024-09-14T03:58:03.133 INFO:tasks.rados.rados.0.smithi070.stdout:2882: done (3 left) 2024-09-14T03:58:03.133 INFO:tasks.rados.rados.0.smithi070.stdout:2883: done (2 left) 2024-09-14T03:58:03.133 INFO:tasks.rados.rados.0.smithi070.stdout:2884: done (1 left) 2024-09-14T03:58:03.133 INFO:tasks.rados.rados.0.smithi070.stdout:2885: done (0 left) 2024-09-14T03:58:03.133 INFO:tasks.rados.rados.0.smithi070.stdout:2886: read oid 39 snap -1 2024-09-14T03:58:03.133 INFO:tasks.rados.rados.0.smithi070.stdout:2886: expect deleted 2024-09-14T03:58:03.133 INFO:tasks.rados.rados.0.smithi070.stdout:2887: snap_create 2024-09-14T03:58:03.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:03 smithi070 ceph-mon[98640]: pgmap v571: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.5 MiB/s rd, 629 KiB/s wr, 29 op/s 2024-09-14T03:58:03.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:03 smithi070 ceph-mon[98640]: osdmap e650: 8 total, 8 up, 8 in 2024-09-14T03:58:03.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:03 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:58:03.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:03 smithi070 ceph-mon[96704]: pgmap v571: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.5 MiB/s rd, 629 KiB/s wr, 29 op/s 2024-09-14T03:58:03.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:03 smithi070 ceph-mon[96704]: osdmap e650: 8 total, 8 up, 8 in 2024-09-14T03:58:03.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:03 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:58:03.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:03 smithi094 ceph-mon[86597]: pgmap v571: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.5 MiB/s rd, 629 KiB/s wr, 29 op/s 2024-09-14T03:58:03.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:03 smithi094 ceph-mon[86597]: osdmap e650: 8 total, 8 up, 8 in 2024-09-14T03:58:03.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:03 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:58:03.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:58:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:58:03] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.51.0" 2024-09-14T03:58:04.092 INFO:tasks.rados.rados.0.smithi070.stdout:2886: done (1 left) 2024-09-14T03:58:04.093 INFO:tasks.rados.rados.0.smithi070.stdout:2887: done (0 left) 2024-09-14T03:58:04.093 INFO:tasks.rados.rados.0.smithi070.stdout:2888: read oid 39 snap -1 2024-09-14T03:58:04.093 INFO:tasks.rados.rados.0.smithi070.stdout:2888: expect deleted 2024-09-14T03:58:04.093 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 42 current snap is 298 2024-09-14T03:58:04.093 INFO:tasks.rados.rados.0.smithi070.stdout:2889: seq_num 890 ranges {0=507904} 2024-09-14T03:58:04.095 INFO:tasks.rados.rados.0.smithi070.stdout:2889: writing smithi07085154-42 from 0 to 507904 tid 1 2024-09-14T03:58:04.095 INFO:tasks.rados.rados.0.smithi070.stdout:2890: read oid 19 snap 277 2024-09-14T03:58:04.095 INFO:tasks.rados.rados.0.smithi070.stdout:2890: expect (ObjNum 747 snap 259 seq_num 747) 2024-09-14T03:58:04.095 INFO:tasks.rados.rados.0.smithi070.stdout:2891: setattr oid 26 current snap is 298 2024-09-14T03:58:04.096 INFO:tasks.rados.rados.0.smithi070.stdout:2888: done (3 left) 2024-09-14T03:58:04.096 INFO:tasks.rados.rados.0.smithi070.stdout:2892: copy_from oid 30 from oid 21 current snap is 298 2024-09-14T03:58:04.097 INFO:tasks.rados.rados.0.smithi070.stdout:2893: snap_remove snap 282 2024-09-14T03:58:04.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:04 smithi094 ceph-mon[86597]: osdmap e651: 8 total, 8 up, 8 in 2024-09-14T03:58:04.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:04 smithi094 ceph-mon[86597]: osdmap e652: 8 total, 8 up, 8 in 2024-09-14T03:58:04.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:04 smithi070 ceph-mon[98640]: osdmap e651: 8 total, 8 up, 8 in 2024-09-14T03:58:04.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:04 smithi070 ceph-mon[98640]: osdmap e652: 8 total, 8 up, 8 in 2024-09-14T03:58:04.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:04 smithi070 ceph-mon[96704]: osdmap e651: 8 total, 8 up, 8 in 2024-09-14T03:58:04.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:04 smithi070 ceph-mon[96704]: osdmap e652: 8 total, 8 up, 8 in 2024-09-14T03:58:05.098 INFO:tasks.rados.rados.0.smithi070.stdout:2893: done (4 left) 2024-09-14T03:58:05.098 INFO:tasks.rados.rados.0.smithi070.stdout:2894: rollback oid 48 current snap is 298 2024-09-14T03:58:05.098 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 48 to 276 2024-09-14T03:58:05.098 INFO:tasks.rados.rados.0.smithi070.stdout:2892: finishing copy_from to smithi07085154-30 2024-09-14T03:58:05.098 INFO:tasks.rados.rados.0.smithi070.stdout:2892: got expected ENOENT (src dne) 2024-09-14T03:58:05.098 INFO:tasks.rados.rados.0.smithi070.stdout:2892: finishing copy_from racing read to smithi07085154-30 2024-09-14T03:58:05.098 INFO:tasks.rados.rados.0.smithi070.stdout:2889: finishing write tid 1 to smithi07085154-42 2024-09-14T03:58:05.099 INFO:tasks.rados.rados.0.smithi070.stdout:2889: finishing write tid 2 to smithi07085154-42 2024-09-14T03:58:05.099 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 3375 (ObjNum 854 snap 288 seq_num 854) dirty exists 2024-09-14T03:58:05.099 INFO:tasks.rados.rados.0.smithi070.stdout:2889: finishing write tid 3 to smithi07085154-42 2024-09-14T03:58:05.099 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 3373 (ObjNum 890 snap 298 seq_num 890) dirty exists 2024-09-14T03:58:05.099 INFO:tasks.rados.rados.0.smithi070.stdout:2889: left oid 42 (ObjNum 890 snap 298 seq_num 890) 2024-09-14T03:58:05.099 INFO:tasks.rados.rados.0.smithi070.stdout:2890: expect (ObjNum 747 snap 259 seq_num 747) 2024-09-14T03:58:05.179 INFO:tasks.rados.rados.0.smithi070.stdout:2894: finishing rollback tid 0 to smithi07085154-48 2024-09-14T03:58:05.179 INFO:tasks.rados.rados.0.smithi070.stdout:2894: finishing rollback tid 1 to smithi07085154-48 2024-09-14T03:58:05.179 INFO:tasks.rados.rados.0.smithi070.stdout:2889: done (4 left) 2024-09-14T03:58:05.179 INFO:tasks.rados.rados.0.smithi070.stdout:2890: done (3 left) 2024-09-14T03:58:05.179 INFO:tasks.rados.rados.0.smithi070.stdout:2891: done (2 left) 2024-09-14T03:58:05.180 INFO:tasks.rados.rados.0.smithi070.stdout:2892: done (1 left) 2024-09-14T03:58:05.180 INFO:tasks.rados.rados.0.smithi070.stdout:2895: rollback oid 26 current snap is 298 2024-09-14T03:58:05.180 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 26 to 236 2024-09-14T03:58:05.180 INFO:tasks.rados.rados.0.smithi070.stdout:2894: finishing rollback tid 2 to smithi07085154-48 2024-09-14T03:58:05.180 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 2243 (ObjNum 757 snap 263 seq_num 757) dirty exists 2024-09-14T03:58:05.180 INFO:tasks.rados.rados.0.smithi070.stdout:2894: done (1 left) 2024-09-14T03:58:05.180 INFO:tasks.rados.rados.0.smithi070.stdout:2896: rollback oid 14 current snap is 298 2024-09-14T03:58:05.180 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 14 to 107 2024-09-14T03:58:05.180 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 2 current snap is 298 2024-09-14T03:58:05.180 INFO:tasks.rados.rados.0.smithi070.stdout:2897: seq_num 891 ranges {16171008=532480} 2024-09-14T03:58:05.267 INFO:tasks.rados.rados.0.smithi070.stdout:2897: writing smithi07085154-2 from 16171008 to 16703488 tid 1 2024-09-14T03:58:05.268 INFO:tasks.rados.rados.0.smithi070.stdout:2898: read oid 40 snap -1 2024-09-14T03:58:05.268 INFO:tasks.rados.rados.0.smithi070.stdout:2898: expect (ObjNum 576 snap 193 seq_num 576) 2024-09-14T03:58:05.268 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 46 current snap is 298 2024-09-14T03:58:05.268 INFO:tasks.rados.rados.0.smithi070.stdout:2896: finishing rollback tid 1 to smithi07085154-14 2024-09-14T03:58:05.268 INFO:tasks.rados.rados.0.smithi070.stdout:2896: finishing rollback tid 2 to smithi07085154-14 2024-09-14T03:58:05.268 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 2796 (ObjNum 313 snap 105 seq_num 313) dirty exists 2024-09-14T03:58:05.268 INFO:tasks.rados.rados.0.smithi070.stdout:2895: finishing rollback tid 0 to smithi07085154-26 2024-09-14T03:58:05.268 INFO:tasks.rados.rados.0.smithi070.stdout:2895: finishing rollback tid 1 to smithi07085154-26 2024-09-14T03:58:05.268 INFO:tasks.rados.rados.0.smithi070.stdout:2895: finishing rollback tid 2 to smithi07085154-26 2024-09-14T03:58:05.268 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 3394 (ObjNum 243 snap 69 seq_num 243) dirty exists 2024-09-14T03:58:05.269 INFO:tasks.rados.rados.0.smithi070.stdout:2899: seq_num 892 ranges {0=548864} 2024-09-14T03:58:05.270 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'ceph orch ps' 2024-09-14T03:58:05.271 INFO:tasks.rados.rados.0.smithi070.stdout:2899: writing smithi07085154-46 from 0 to 548864 tid 1 2024-09-14T03:58:05.271 INFO:tasks.rados.rados.0.smithi070.stdout:2895: done (4 left) 2024-09-14T03:58:05.271 INFO:tasks.rados.rados.0.smithi070.stdout:2896: done (3 left) 2024-09-14T03:58:05.271 INFO:tasks.rados.rados.0.smithi070.stdout:2900: read oid 20 snap -1 2024-09-14T03:58:05.272 INFO:tasks.rados.rados.0.smithi070.stdout:2900: expect deleted 2024-09-14T03:58:05.272 INFO:tasks.rados.rados.0.smithi070.stdout:2901: snap_create 2024-09-14T03:58:05.272 INFO:tasks.rados.rados.0.smithi070.stdout:2898: expect (ObjNum 576 snap 193 seq_num 576) 2024-09-14T03:58:05.276 INFO:tasks.rados.rados.0.smithi070.stdout:2897: finishing write tid 1 to smithi07085154-2 2024-09-14T03:58:05.276 INFO:tasks.rados.rados.0.smithi070.stdout:2897: finishing write tid 2 to smithi07085154-2 2024-09-14T03:58:05.276 INFO:tasks.rados.rados.0.smithi070.stdout:2899: finishing write tid 1 to smithi07085154-46 2024-09-14T03:58:05.277 INFO:tasks.rados.rados.0.smithi070.stdout:2899: finishing write tid 2 to smithi07085154-46 2024-09-14T03:58:05.279 INFO:tasks.rados.rados.0.smithi070.stdout:2897: finishing write tid 3 to smithi07085154-2 2024-09-14T03:58:05.279 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 2 v 3397 (ObjNum 891 snap 298 seq_num 891) dirty exists 2024-09-14T03:58:05.279 INFO:tasks.rados.rados.0.smithi070.stdout:2897: left oid 2 (ObjNum 891 snap 298 seq_num 891) 2024-09-14T03:58:05.279 INFO:tasks.rados.rados.0.smithi070.stdout:2899: finishing write tid 3 to smithi07085154-46 2024-09-14T03:58:05.279 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 46 v 3399 (ObjNum 892 snap 298 seq_num 892) dirty exists 2024-09-14T03:58:05.279 INFO:tasks.rados.rados.0.smithi070.stdout:2899: left oid 46 (ObjNum 892 snap 298 seq_num 892) 2024-09-14T03:58:05.521 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:58:05.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:05 smithi094 ceph-mon[86597]: pgmap v575: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.1 MiB/s rd, 344 KiB/s wr, 11 op/s 2024-09-14T03:58:05.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:05 smithi094 ceph-mon[86597]: osdmap e653: 8 total, 8 up, 8 in 2024-09-14T03:58:05.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:05 smithi070 ceph-mon[96704]: pgmap v575: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.1 MiB/s rd, 344 KiB/s wr, 11 op/s 2024-09-14T03:58:05.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:05 smithi070 ceph-mon[96704]: osdmap e653: 8 total, 8 up, 8 in 2024-09-14T03:58:05.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:05 smithi070 ceph-mon[98640]: pgmap v575: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.1 MiB/s rd, 344 KiB/s wr, 11 op/s 2024-09-14T03:58:05.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:05 smithi070 ceph-mon[98640]: osdmap e653: 8 total, 8 up, 8 in 2024-09-14T03:58:06.145 INFO:tasks.rados.rados.0.smithi070.stdout:2897: done (4 left) 2024-09-14T03:58:06.145 INFO:tasks.rados.rados.0.smithi070.stdout:2898: done (3 left) 2024-09-14T03:58:06.145 INFO:tasks.rados.rados.0.smithi070.stdout:2899: done (2 left) 2024-09-14T03:58:06.145 INFO:tasks.rados.rados.0.smithi070.stdout:2900: done (1 left) 2024-09-14T03:58:06.145 INFO:tasks.rados.rados.0.smithi070.stdout:2901: done (0 left) 2024-09-14T03:58:06.145 INFO:tasks.rados.rados.0.smithi070.stdout:2902: read oid 14 snap 107 2024-09-14T03:58:06.145 INFO:tasks.rados.rados.0.smithi070.stdout:2902: expect (ObjNum 313 snap 105 seq_num 313) 2024-09-14T03:58:06.145 INFO:tasks.rados.rados.0.smithi070.stdout:2903: rollback oid 10 current snap is 299 2024-09-14T03:58:06.145 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 10 to 290 2024-09-14T03:58:06.145 INFO:tasks.rados.rados.0.smithi070.stdout:2904: delete oid 6 current snap is 299 2024-09-14T03:58:06.153 INFO:tasks.rados.rados.0.smithi070.stdout:2904: done (2 left) 2024-09-14T03:58:06.153 INFO:tasks.rados.rados.0.smithi070.stdout:2905: snap_remove snap 260 2024-09-14T03:58:06.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:06 smithi070 ceph-mon[96704]: from='client.44486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:06.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:06 smithi070 ceph-mon[98640]: from='client.44486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:06.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:06 smithi094 ceph-mon[86597]: from='client.44486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:07.150 INFO:tasks.rados.rados.0.smithi070.stdout:2905: done (2 left) 2024-09-14T03:58:07.151 INFO:tasks.rados.rados.0.smithi070.stdout:2906: read oid 7 snap -1 2024-09-14T03:58:07.151 INFO:tasks.rados.rados.0.smithi070.stdout:2906: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:58:07.151 INFO:tasks.rados.rados.0.smithi070.stdout:2907: setattr oid 26 current snap is 299 2024-09-14T03:58:07.151 INFO:tasks.rados.rados.0.smithi070.stdout:2902: expect (ObjNum 313 snap 105 seq_num 313) 2024-09-14T03:58:07.173 INFO:tasks.rados.rados.0.smithi070.stdout:2902: done (3 left) 2024-09-14T03:58:07.173 INFO:tasks.rados.rados.0.smithi070.stdout:2908: delete oid 11 current snap is 299 2024-09-14T03:58:07.173 INFO:tasks.rados.rados.0.smithi070.stdout:2903: finishing rollback tid 1 to smithi07085154-10 2024-09-14T03:58:07.174 INFO:tasks.rados.rados.0.smithi070.stdout:2903: finishing rollback tid 2 to smithi07085154-10 2024-09-14T03:58:07.174 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 3401 (ObjNum 841 snap 286 seq_num 841) dirty exists 2024-09-14T03:58:07.174 INFO:tasks.rados.rados.0.smithi070.stdout:2906: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:58:07.175 INFO:tasks.rados.rados.0.smithi070.stdout:2903: done (3 left) 2024-09-14T03:58:07.175 INFO:tasks.rados.rados.0.smithi070.stdout:2906: done (2 left) 2024-09-14T03:58:07.175 INFO:tasks.rados.rados.0.smithi070.stdout:2908: done (1 left) 2024-09-14T03:58:07.176 INFO:tasks.rados.rados.0.smithi070.stdout:2909: copy_from oid 8 from oid 41 current snap is 299 2024-09-14T03:58:07.176 INFO:tasks.rados.rados.0.smithi070.stdout:2910: snap_remove snap 279 2024-09-14T03:58:07.282 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:07 smithi070 ceph-mon[98640]: pgmap v577: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-14T03:58:07.282 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:07 smithi070 ceph-mon[98640]: osdmap e654: 8 total, 8 up, 8 in 2024-09-14T03:58:07.283 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:07 smithi070 ceph-mon[96704]: pgmap v577: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-14T03:58:07.283 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:07 smithi070 ceph-mon[96704]: osdmap e654: 8 total, 8 up, 8 in 2024-09-14T03:58:07.456 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:58:07.456 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (8m) 67s ago 12m 21.2M - 0.27.0 11f11916f8cd 45e79e51074c 2024-09-14T03:58:07.456 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi094 *:3000 running (28s) 22s ago 12m 43.5M - 10.4.0 c8b91775d855 b7cb3ebd57f5 2024-09-14T03:58:07.456 INFO:teuthology.orchestra.run.smithi070.stdout:mds.a smithi070 running (71s) 67s ago 14m 15.5M - 19.3.0-4907-ga2782315 8801d2529e43 4fa138982c4f 2024-09-14T03:58:07.456 INFO:teuthology.orchestra.run.smithi070.stdout:mds.b smithi094 running (50s) 22s ago 14m 13.9M - 19.3.0-4907-ga2782315 8801d2529e43 9f7842c26184 2024-09-14T03:58:07.456 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi094 *:8443,9283,8765 running (6m) 22s ago 17m 456M - 19.3.0-4907-ga2782315 8801d2529e43 569d9c7e0f06 2024-09-14T03:58:07.456 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (7m) 67s ago 19m 529M - 19.3.0-4907-ga2782315 8801d2529e43 0ef4d8e0cdbe 2024-09-14T03:58:07.456 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (6m) 67s ago 19m 69.1M 2048M 19.3.0-4907-ga2782315 8801d2529e43 96542de31cac 2024-09-14T03:58:07.456 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi094 running (6m) 22s ago 18m 55.5M 2048M 19.3.0-4907-ga2782315 8801d2529e43 86827be98659 2024-09-14T03:58:07.457 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (6m) 67s ago 17m 55.7M 2048M 19.3.0-4907-ga2782315 8801d2529e43 5741e4fcf4b6 2024-09-14T03:58:07.457 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (7m) 67s ago 12m 11.2M - 1.7.0 72c9c2088986 e316760a30b3 2024-09-14T03:58:07.457 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi094 *:9100 running (7m) 22s ago 12m 11.4M - 1.7.0 72c9c2088986 767f8a093175 2024-09-14T03:58:07.457 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (5m) 67s ago 17m 315M 2480M 19.3.0-4907-ga2782315 8801d2529e43 2345ccac2085 2024-09-14T03:58:07.457 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (4m) 67s ago 16m 341M 2480M 19.3.0-4907-ga2782315 8801d2529e43 53868856512a 2024-09-14T03:58:07.457 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (4m) 67s ago 16m 219M 2480M 19.3.0-4907-ga2782315 8801d2529e43 39ef876d2fff 2024-09-14T03:58:07.457 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (3m) 67s ago 16m 149M 2480M 19.3.0-4907-ga2782315 8801d2529e43 8efc112edbe9 2024-09-14T03:58:07.457 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi094 running (3m) 22s ago 15m 274M 2480M 19.3.0-4907-ga2782315 8801d2529e43 09f583ed6c77 2024-09-14T03:58:07.457 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi094 running (2m) 22s ago 15m 260M 2480M 19.3.0-4907-ga2782315 8801d2529e43 162396f2e7b6 2024-09-14T03:58:07.457 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi094 running (2m) 22s ago 15m 143M 2480M 19.3.0-4907-ga2782315 8801d2529e43 cbdc02cdb8e4 2024-09-14T03:58:07.457 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi094 running (93s) 22s ago 14m 206M 2480M 19.3.0-4907-ga2782315 8801d2529e43 ea9896fb7dbc 2024-09-14T03:58:07.457 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi094 *:9095 running (7m) 22s ago 13m 54.0M - 2.51.0 1d3b7f56885b ce1b353828dc 2024-09-14T03:58:07.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:07 smithi094 ceph-mon[86597]: pgmap v577: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-14T03:58:07.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:07 smithi094 ceph-mon[86597]: osdmap e654: 8 total, 8 up, 8 in 2024-09-14T03:58:08.154 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:08 smithi070 ceph-mon[96704]: osdmap e655: 8 total, 8 up, 8 in 2024-09-14T03:58:08.154 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:08 smithi070 ceph-mon[96704]: from='client.34575 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:08.154 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:08 smithi070 ceph-mon[96704]: pgmap v580: 121 pgs: 121 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.5 MiB/s rd, 608 KiB/s wr, 14 op/s 2024-09-14T03:58:08.154 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:08 smithi070 ceph-mon[98640]: osdmap e655: 8 total, 8 up, 8 in 2024-09-14T03:58:08.155 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:08 smithi070 ceph-mon[98640]: from='client.34575 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:08.155 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:08 smithi070 ceph-mon[98640]: pgmap v580: 121 pgs: 121 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.5 MiB/s rd, 608 KiB/s wr, 14 op/s 2024-09-14T03:58:08.158 INFO:tasks.rados.rados.0.smithi070.stdout:2910: done (2 left) 2024-09-14T03:58:08.158 INFO:tasks.rados.rados.0.smithi070.stdout:2911: rmattr oid 40 current snap is 299 2024-09-14T03:58:08.158 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 17 current snap is 299 2024-09-14T03:58:08.158 INFO:tasks.rados.rados.0.smithi070.stdout:2912: seq_num 895 ranges {0=598016} 2024-09-14T03:58:08.160 INFO:tasks.rados.rados.0.smithi070.stdout:2912: writing smithi07085154-17 from 0 to 598016 tid 1 2024-09-14T03:58:08.160 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 27 current snap is 299 2024-09-14T03:58:08.160 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 3411 (ObjNum 243 snap 69 seq_num 243) dirty exists 2024-09-14T03:58:08.160 INFO:tasks.rados.rados.0.smithi070.stdout:2909: finishing copy_from racing read to smithi07085154-8 2024-09-14T03:58:08.160 INFO:tasks.rados.rados.0.smithi070.stdout:2909: finishing copy_from to smithi07085154-8 2024-09-14T03:58:08.160 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 8 v 2286 (ObjNum 724 snap 248 seq_num 724) dirty exists 2024-09-14T03:58:08.160 INFO:tasks.rados.rados.0.smithi070.stdout:2913: seq_num 896 ranges {0=614400} 2024-09-14T03:58:08.164 INFO:tasks.rados.rados.0.smithi070.stdout:2913: writing smithi07085154-27 from 0 to 614400 tid 1 2024-09-14T03:58:08.165 INFO:tasks.rados.rados.0.smithi070.stdout:2907: done (4 left) 2024-09-14T03:58:08.165 INFO:tasks.rados.rados.0.smithi070.stdout:2909: done (3 left) 2024-09-14T03:58:08.165 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 20 current snap is 299 2024-09-14T03:58:08.166 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 3155 (ObjNum 576 snap 193 seq_num 576) dirty exists 2024-09-14T03:58:08.166 INFO:tasks.rados.rados.0.smithi070.stdout:2914: seq_num 897 ranges {0=630784} 2024-09-14T03:58:08.171 INFO:tasks.rados.rados.0.smithi070.stdout:2914: writing smithi07085154-20 from 0 to 630784 tid 1 2024-09-14T03:58:08.172 INFO:tasks.rados.rados.0.smithi070.stdout:2911: done (3 left) 2024-09-14T03:58:08.172 INFO:tasks.rados.rados.0.smithi070.stdout:2915: copy_from oid 5 from oid 37 current snap is 299 2024-09-14T03:58:08.172 INFO:tasks.rados.rados.0.smithi070.stdout:2916: snap_create 2024-09-14T03:58:08.176 INFO:tasks.rados.rados.0.smithi070.stdout:2912: finishing write tid 1 to smithi07085154-17 2024-09-14T03:58:08.177 INFO:tasks.rados.rados.0.smithi070.stdout:2912: finishing write tid 2 to smithi07085154-17 2024-09-14T03:58:08.179 INFO:tasks.rados.rados.0.smithi070.stdout:2912: finishing write tid 3 to smithi07085154-17 2024-09-14T03:58:08.179 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 2294 (ObjNum 895 snap 299 seq_num 895) dirty exists 2024-09-14T03:58:08.179 INFO:tasks.rados.rados.0.smithi070.stdout:2912: left oid 17 (ObjNum 895 snap 299 seq_num 895) 2024-09-14T03:58:08.180 INFO:tasks.rados.rados.0.smithi070.stdout:2913: finishing write tid 1 to smithi07085154-27 2024-09-14T03:58:08.180 INFO:tasks.rados.rados.0.smithi070.stdout:2913: finishing write tid 2 to smithi07085154-27 2024-09-14T03:58:08.182 INFO:tasks.rados.rados.0.smithi070.stdout:2914: finishing write tid 1 to smithi07085154-20 2024-09-14T03:58:08.182 INFO:tasks.rados.rados.0.smithi070.stdout:2914: finishing write tid 2 to smithi07085154-20 2024-09-14T03:58:08.182 INFO:tasks.rados.rados.0.smithi070.stdout:2913: finishing write tid 3 to smithi07085154-27 2024-09-14T03:58:08.182 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 3421 (ObjNum 896 snap 299 seq_num 896) dirty exists 2024-09-14T03:58:08.182 INFO:tasks.rados.rados.0.smithi070.stdout:2913: left oid 27 (ObjNum 896 snap 299 seq_num 896) 2024-09-14T03:58:08.183 INFO:tasks.rados.rados.0.smithi070.stdout:2915: finishing copy_from to smithi07085154-5 2024-09-14T03:58:08.183 INFO:tasks.rados.rados.0.smithi070.stdout:2915: got expected ENOENT (src dne) 2024-09-14T03:58:08.183 INFO:tasks.rados.rados.0.smithi070.stdout:2915: finishing copy_from racing read to smithi07085154-5 2024-09-14T03:58:08.184 INFO:tasks.rados.rados.0.smithi070.stdout:2914: finishing write tid 3 to smithi07085154-20 2024-09-14T03:58:08.184 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 20 v 3155 (ObjNum 897 snap 299 seq_num 897) dirty exists 2024-09-14T03:58:08.184 INFO:tasks.rados.rados.0.smithi070.stdout:2914: left oid 20 (ObjNum 897 snap 299 seq_num 897) 2024-09-14T03:58:08.196 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'ceph versions' 2024-09-14T03:58:08.480 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:58:08.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:08 smithi094 ceph-mon[86597]: osdmap e655: 8 total, 8 up, 8 in 2024-09-14T03:58:08.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:08 smithi094 ceph-mon[86597]: from='client.34575 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:08.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:08 smithi094 ceph-mon[86597]: pgmap v580: 121 pgs: 121 active+clean; 1.1 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.5 MiB/s rd, 608 KiB/s wr, 14 op/s 2024-09-14T03:58:09.093 INFO:tasks.rados.rados.0.smithi070.stdout:2912: done (4 left) 2024-09-14T03:58:09.093 INFO:tasks.rados.rados.0.smithi070.stdout:2913: done (3 left) 2024-09-14T03:58:09.093 INFO:tasks.rados.rados.0.smithi070.stdout:2914: done (2 left) 2024-09-14T03:58:09.093 INFO:tasks.rados.rados.0.smithi070.stdout:2915: done (1 left) 2024-09-14T03:58:09.093 INFO:tasks.rados.rados.0.smithi070.stdout:2916: done (0 left) 2024-09-14T03:58:09.093 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 13 current snap is 300 2024-09-14T03:58:09.093 INFO:tasks.rados.rados.0.smithi070.stdout:2917: seq_num 898 ranges {0=647168} 2024-09-14T03:58:09.096 INFO:tasks.rados.rados.0.smithi070.stdout:2917: writing smithi07085154-13 from 0 to 647168 tid 1 2024-09-14T03:58:09.097 INFO:tasks.rados.rados.0.smithi070.stdout:2918: rollback oid 43 current snap is 300 2024-09-14T03:58:09.097 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 43 to 268 2024-09-14T03:58:09.097 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 30 current snap is 300 2024-09-14T03:58:09.097 INFO:tasks.rados.rados.0.smithi070.stdout:2919: seq_num 899 ranges {278528=663552} 2024-09-14T03:58:09.103 INFO:tasks.rados.rados.0.smithi070.stdout:2919: writing smithi07085154-30 from 278528 to 942080 tid 1 2024-09-14T03:58:09.103 INFO:tasks.rados.rados.0.smithi070.stdout:2920: snap_create 2024-09-14T03:58:09.103 INFO:tasks.rados.rados.0.smithi070.stdout:2917: finishing write tid 1 to smithi07085154-13 2024-09-14T03:58:09.103 INFO:tasks.rados.rados.0.smithi070.stdout:2917: finishing write tid 2 to smithi07085154-13 2024-09-14T03:58:09.104 INFO:tasks.rados.rados.0.smithi070.stdout:2917: finishing write tid 3 to smithi07085154-13 2024-09-14T03:58:09.104 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 3158 (ObjNum 898 snap 300 seq_num 898) dirty exists 2024-09-14T03:58:09.104 INFO:tasks.rados.rados.0.smithi070.stdout:2917: left oid 13 (ObjNum 898 snap 300 seq_num 898) 2024-09-14T03:58:09.105 INFO:tasks.rados.rados.0.smithi070.stdout:2918: finishing rollback tid 0 to smithi07085154-43 2024-09-14T03:58:09.105 INFO:tasks.rados.rados.0.smithi070.stdout:2918: finishing rollback tid 1 to smithi07085154-43 2024-09-14T03:58:09.105 INFO:tasks.rados.rados.0.smithi070.stdout:2918: finishing rollback tid 2 to smithi07085154-43 2024-09-14T03:58:09.105 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 2316 (ObjNum 681 snap 230 seq_num 681) dirty exists 2024-09-14T03:58:09.110 INFO:tasks.rados.rados.0.smithi070.stdout:2919: finishing write tid 1 to smithi07085154-30 2024-09-14T03:58:09.110 INFO:tasks.rados.rados.0.smithi070.stdout:2919: finishing write tid 2 to smithi07085154-30 2024-09-14T03:58:09.112 INFO:tasks.rados.rados.0.smithi070.stdout:2919: finishing write tid 3 to smithi07085154-30 2024-09-14T03:58:09.112 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 30 v 2243 (ObjNum 899 snap 300 seq_num 899) dirty exists 2024-09-14T03:58:09.112 INFO:tasks.rados.rados.0.smithi070.stdout:2919: left oid 30 (ObjNum 899 snap 300 seq_num 899) 2024-09-14T03:58:09.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:09 smithi070 ceph-mon[96704]: osdmap e656: 8 total, 8 up, 8 in 2024-09-14T03:58:09.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:09 smithi070 ceph-mon[96704]: osdmap e657: 8 total, 8 up, 8 in 2024-09-14T03:58:09.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:09 smithi070 ceph-mon[98640]: osdmap e656: 8 total, 8 up, 8 in 2024-09-14T03:58:09.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:09 smithi070 ceph-mon[98640]: osdmap e657: 8 total, 8 up, 8 in 2024-09-14T03:58:09.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:09 smithi094 ceph-mon[86597]: osdmap e656: 8 total, 8 up, 8 in 2024-09-14T03:58:09.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:09 smithi094 ceph-mon[86597]: osdmap e657: 8 total, 8 up, 8 in 2024-09-14T03:58:10.094 INFO:tasks.rados.rados.0.smithi070.stdout:2917: done (3 left) 2024-09-14T03:58:10.094 INFO:tasks.rados.rados.0.smithi070.stdout:2918: done (2 left) 2024-09-14T03:58:10.095 INFO:tasks.rados.rados.0.smithi070.stdout:2919: done (1 left) 2024-09-14T03:58:10.095 INFO:tasks.rados.rados.0.smithi070.stdout:2920: done (0 left) 2024-09-14T03:58:10.095 INFO:tasks.rados.rados.0.smithi070.stdout:2921: snap_remove snap 293 2024-09-14T03:58:10.437 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-09-14T03:58:10.437 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-09-14T03:58:10.437 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T03:58:10.437 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:58:10.437 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-09-14T03:58:10.437 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:58:10.437 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:58:10.437 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-09-14T03:58:10.437 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 8 2024-09-14T03:58:10.437 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:58:10.437 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": { 2024-09-14T03:58:10.437 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:58:10.438 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-09-14T03:58:10.438 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-09-14T03:58:10.438 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 15 2024-09-14T03:58:10.438 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-09-14T03:58:10.438 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-09-14T03:58:10.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:58:10 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:58:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:58:11.103 INFO:tasks.rados.rados.0.smithi070.stdout:2921: done (0 left) 2024-09-14T03:58:11.103 INFO:tasks.rados.rados.0.smithi070.stdout:2922: setattr oid 40 current snap is 301 2024-09-14T03:58:11.104 INFO:tasks.rados.rados.0.smithi070.stdout:2923: read oid 41 snap 297 2024-09-14T03:58:11.104 INFO:tasks.rados.rados.0.smithi070.stdout:2923: expect (ObjNum 724 snap 248 seq_num 724) 2024-09-14T03:58:11.104 INFO:tasks.rados.rados.0.smithi070.stdout:2924: snap_remove snap 174 2024-09-14T03:58:11.161 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-09-14T03:58:11.405 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:58:11.526 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:11 smithi070 ceph-mon[96704]: osdmap e658: 8 total, 8 up, 8 in 2024-09-14T03:58:11.526 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:11 smithi070 ceph-mon[96704]: pgmap v584: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.6 MiB/s rd, 570 KiB/s wr, 16 op/s 2024-09-14T03:58:11.526 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:11 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/417096513' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:58:11.526 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:11 smithi070 ceph-mon[98640]: osdmap e658: 8 total, 8 up, 8 in 2024-09-14T03:58:11.526 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:11 smithi070 ceph-mon[98640]: pgmap v584: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.6 MiB/s rd, 570 KiB/s wr, 16 op/s 2024-09-14T03:58:11.526 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:11 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/417096513' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:58:11.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:11 smithi094 ceph-mon[86597]: osdmap e658: 8 total, 8 up, 8 in 2024-09-14T03:58:11.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:11 smithi094 ceph-mon[86597]: pgmap v584: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.6 MiB/s rd, 570 KiB/s wr, 16 op/s 2024-09-14T03:58:11.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:11 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/417096513' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:58:12.110 INFO:tasks.rados.rados.0.smithi070.stdout:2924: done (2 left) 2024-09-14T03:58:12.110 INFO:tasks.rados.rados.0.smithi070.stdout:2925: copy_from oid 27 from oid 44 current snap is 301 2024-09-14T03:58:12.111 INFO:tasks.rados.rados.0.smithi070.stdout:2926: rmattr oid 4 current snap is 301 2024-09-14T03:58:12.111 INFO:tasks.rados.rados.0.smithi070.stdout:2927: rmattr oid 16 current snap is 301 2024-09-14T03:58:12.111 INFO:tasks.rados.rados.0.smithi070.stdout:2927: done (4 left) 2024-09-14T03:58:12.111 INFO:tasks.rados.rados.0.smithi070.stdout:2928: read oid 43 snap 254 2024-09-14T03:58:12.111 INFO:tasks.rados.rados.0.smithi070.stdout:2928: expect (ObjNum 730 snap 253 seq_num 730) 2024-09-14T03:58:12.111 INFO:tasks.rados.rados.0.smithi070.stdout:2929: copy_from oid 47 from oid 35 current snap is 301 2024-09-14T03:58:12.111 INFO:tasks.rados.rados.0.smithi070.stdout:2930: rollback oid 39 current snap is 301 2024-09-14T03:58:12.111 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 39 to 299 2024-09-14T03:58:12.111 INFO:tasks.rados.rados.0.smithi070.stdout:2931: read oid 34 snap -1 2024-09-14T03:58:12.111 INFO:tasks.rados.rados.0.smithi070.stdout:2931: expect (ObjNum 879 snap 295 seq_num 879) 2024-09-14T03:58:12.111 INFO:tasks.rados.rados.0.smithi070.stdout:2932: rollback oid 32 current snap is 301 2024-09-14T03:58:12.111 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 32 to 290 2024-09-14T03:58:12.111 INFO:tasks.rados.rados.0.smithi070.stdout:2933: copy_from oid 33 from oid 31 current snap is 301 2024-09-14T03:58:12.112 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 18 current snap is 301 2024-09-14T03:58:12.112 INFO:tasks.rados.rados.0.smithi070.stdout:2934: seq_num 900 ranges {3244032=679936} 2024-09-14T03:58:12.131 INFO:tasks.rados.rados.0.smithi070.stdout:2934: writing smithi07085154-18 from 3244032 to 3923968 tid 1 2024-09-14T03:58:12.132 INFO:tasks.rados.rados.0.smithi070.stdout:2935: snap_remove snap 276 2024-09-14T03:58:12.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:12 smithi094 ceph-mon[86597]: osdmap e659: 8 total, 8 up, 8 in 2024-09-14T03:58:12.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:12 smithi070 ceph-mon[96704]: osdmap e659: 8 total, 8 up, 8 in 2024-09-14T03:58:12.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:12 smithi070 ceph-mon[98640]: osdmap e659: 8 total, 8 up, 8 in 2024-09-14T03:58:13.026 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 03:58:12 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=infra.usagestats t=2024-09-14T03:58:12.58558135Z level=info msg="Usage stats are ready to report" 2024-09-14T03:58:13.110 INFO:tasks.rados.rados.0.smithi070.stdout:2935: done (11 left) 2024-09-14T03:58:13.111 INFO:tasks.rados.rados.0.smithi070.stdout:2936: rmattr oid 42 current snap is 301 2024-09-14T03:58:13.111 INFO:tasks.rados.rados.0.smithi070.stdout:2936: done (11 left) 2024-09-14T03:58:13.111 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 19 current snap is 301 2024-09-14T03:58:13.111 INFO:tasks.rados.rados.0.smithi070.stdout:2937: seq_num 901 ranges {9216000=696320} 2024-09-14T03:58:13.166 INFO:tasks.rados.rados.0.smithi070.stdout:2937: writing smithi07085154-19 from 9216000 to 9912320 tid 1 2024-09-14T03:58:13.166 INFO:tasks.rados.rados.0.smithi070.stdout:2938: read oid 11 snap -1 2024-09-14T03:58:13.166 INFO:tasks.rados.rados.0.smithi070.stdout:2938: expect deleted 2024-09-14T03:58:13.166 INFO:tasks.rados.rados.0.smithi070.stdout:2939: copy_from oid 50 from oid 28 current snap is 301 2024-09-14T03:58:13.166 INFO:tasks.rados.rados.0.smithi070.stdout:2923: expect (ObjNum 724 snap 248 seq_num 724) 2024-09-14T03:58:13.170 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 3156 (ObjNum 576 snap 193 seq_num 576) dirty exists 2024-09-14T03:58:13.170 INFO:tasks.rados.rados.0.smithi070.stdout:2930: finishing rollback tid 1 to smithi07085154-39 2024-09-14T03:58:13.171 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 39 v 2636 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-14T03:58:13.171 INFO:tasks.rados.rados.0.smithi070.stdout:2933: finishing copy_from to smithi07085154-33 2024-09-14T03:58:13.171 INFO:tasks.rados.rados.0.smithi070.stdout:2933: got expected ENOENT (src dne) 2024-09-14T03:58:13.171 INFO:tasks.rados.rados.0.smithi070.stdout:2933: finishing copy_from racing read to smithi07085154-33 2024-09-14T03:58:13.171 INFO:tasks.rados.rados.0.smithi070.stdout:2931: expect (ObjNum 879 snap 295 seq_num 879) 2024-09-14T03:58:13.180 INFO:tasks.rados.rados.0.smithi070.stdout:2929: finishing copy_from racing read to smithi07085154-47 2024-09-14T03:58:13.181 INFO:tasks.rados.rados.0.smithi070.stdout:2928: expect (ObjNum 730 snap 253 seq_num 730) 2024-09-14T03:58:13.267 INFO:tasks.rados.rados.0.smithi070.stdout:2929: finishing copy_from to smithi07085154-47 2024-09-14T03:58:13.267 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 47 v 1698 (ObjNum 65 snap 0 seq_num 2902963152) dirty exists 2024-09-14T03:58:13.267 INFO:tasks.rados.rados.0.smithi070.stdout:2932: finishing rollback tid 0 to smithi07085154-32 2024-09-14T03:58:13.267 INFO:tasks.rados.rados.0.smithi070.stdout:2922: done (13 left) 2024-09-14T03:58:13.267 INFO:tasks.rados.rados.0.smithi070.stdout:2923: done (12 left) 2024-09-14T03:58:13.267 INFO:tasks.rados.rados.0.smithi070.stdout:2928: done (11 left) 2024-09-14T03:58:13.267 INFO:tasks.rados.rados.0.smithi070.stdout:2929: done (10 left) 2024-09-14T03:58:13.267 INFO:tasks.rados.rados.0.smithi070.stdout:2930: done (9 left) 2024-09-14T03:58:13.268 INFO:tasks.rados.rados.0.smithi070.stdout:2931: done (8 left) 2024-09-14T03:58:13.268 INFO:tasks.rados.rados.0.smithi070.stdout:2933: done (7 left) 2024-09-14T03:58:13.268 INFO:tasks.rados.rados.0.smithi070.stdout:2940: copy_from oid 1 from oid 40 current snap is 301 2024-09-14T03:58:13.268 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 3156 (ObjNum 869 snap 295 seq_num 869) dirty exists 2024-09-14T03:58:13.268 INFO:tasks.rados.rados.0.smithi070.stdout:2932: finishing rollback tid 1 to smithi07085154-32 2024-09-14T03:58:13.268 INFO:tasks.rados.rados.0.smithi070.stdout:2932: finishing rollback tid 2 to smithi07085154-32 2024-09-14T03:58:13.268 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 3161 (ObjNum 576 snap 193 seq_num 576) dirty exists 2024-09-14T03:58:13.268 INFO:tasks.rados.rados.0.smithi070.stdout:2934: finishing write tid 1 to smithi07085154-18 2024-09-14T03:58:13.268 INFO:tasks.rados.rados.0.smithi070.stdout:2934: finishing write tid 2 to smithi07085154-18 2024-09-14T03:58:13.268 INFO:tasks.rados.rados.0.smithi070.stdout:2934: finishing write tid 3 to smithi07085154-18 2024-09-14T03:58:13.268 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 2335 (ObjNum 900 snap 301 seq_num 900) dirty exists 2024-09-14T03:58:13.268 INFO:tasks.rados.rados.0.smithi070.stdout:2934: left oid 18 (ObjNum 900 snap 301 seq_num 900) 2024-09-14T03:58:13.268 INFO:tasks.rados.rados.0.smithi070.stdout:2925: finishing copy_from racing read to smithi07085154-27 2024-09-14T03:58:13.269 INFO:tasks.rados.rados.0.smithi070.stdout:2925: finishing copy_from to smithi07085154-27 2024-09-14T03:58:13.269 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 3150 (ObjNum 865 snap 293 seq_num 865) dirty exists 2024-09-14T03:58:13.269 INFO:tasks.rados.rados.0.smithi070.stdout:2937: finishing write tid 1 to smithi07085154-19 2024-09-14T03:58:13.269 INFO:tasks.rados.rados.0.smithi070.stdout:2937: finishing write tid 2 to smithi07085154-19 2024-09-14T03:58:13.269 INFO:tasks.rados.rados.0.smithi070.stdout:2937: finishing write tid 3 to smithi07085154-19 2024-09-14T03:58:13.269 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 19 v 3159 (ObjNum 901 snap 301 seq_num 901) dirty exists 2024-09-14T03:58:13.269 INFO:tasks.rados.rados.0.smithi070.stdout:2937: left oid 19 (ObjNum 901 snap 301 seq_num 901) 2024-09-14T03:58:13.269 INFO:tasks.rados.rados.0.smithi070.stdout:2939: finishing copy_from racing read to smithi07085154-50 2024-09-14T03:58:13.269 INFO:tasks.rados.rados.0.smithi070.stdout:2939: finishing copy_from to smithi07085154-50 2024-09-14T03:58:13.269 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 2066 (ObjNum 639 snap 214 seq_num 639) dirty exists 2024-09-14T03:58:13.269 INFO:tasks.rados.rados.0.smithi070.stdout:2925: done (7 left) 2024-09-14T03:58:13.269 INFO:tasks.rados.rados.0.smithi070.stdout:2926: done (6 left) 2024-09-14T03:58:13.269 INFO:tasks.rados.rados.0.smithi070.stdout:2932: done (5 left) 2024-09-14T03:58:13.269 INFO:tasks.rados.rados.0.smithi070.stdout:2934: done (4 left) 2024-09-14T03:58:13.270 INFO:tasks.rados.rados.0.smithi070.stdout:2937: done (3 left) 2024-09-14T03:58:13.270 INFO:tasks.rados.rados.0.smithi070.stdout:2938: done (2 left) 2024-09-14T03:58:13.270 INFO:tasks.rados.rados.0.smithi070.stdout:2939: done (1 left) 2024-09-14T03:58:13.270 INFO:tasks.rados.rados.0.smithi070.stdout:2941: snap_remove snap 283 2024-09-14T03:58:13.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:13 smithi070 ceph-mon[96704]: osdmap e660: 8 total, 8 up, 8 in 2024-09-14T03:58:13.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:13 smithi070 ceph-mon[96704]: pgmap v587: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 396 KiB/s rd, 809 KiB/s wr, 15 op/s 2024-09-14T03:58:13.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:13 smithi070 ceph-mon[98640]: osdmap e660: 8 total, 8 up, 8 in 2024-09-14T03:58:13.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:13 smithi070 ceph-mon[98640]: pgmap v587: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 396 KiB/s rd, 809 KiB/s wr, 15 op/s 2024-09-14T03:58:13.417 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-09-14T03:58:13.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:13 smithi094 ceph-mon[86597]: osdmap e660: 8 total, 8 up, 8 in 2024-09-14T03:58:13.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:13 smithi094 ceph-mon[86597]: pgmap v587: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 396 KiB/s rd, 809 KiB/s wr, 15 op/s 2024-09-14T03:58:13.767 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:58:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:58:13] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.51.0" 2024-09-14T03:58:14.049 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6380d42-724a-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-09-14T03:58:14.094 INFO:tasks.rados.rados.0.smithi070.stdout:2941: done (1 left) 2024-09-14T03:58:14.094 INFO:tasks.rados.rados.0.smithi070.stdout:2942: rmattr oid 4 current snap is 301 2024-09-14T03:58:14.094 INFO:tasks.rados.rados.0.smithi070.stdout:2940: finishing copy_from racing read to smithi07085154-1 2024-09-14T03:58:14.095 INFO:tasks.rados.rados.0.smithi070.stdout:2940: finishing copy_from to smithi07085154-1 2024-09-14T03:58:14.095 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 1 v 3156 (ObjNum 576 snap 193 seq_num 576) dirty exists 2024-09-14T03:58:14.095 INFO:tasks.rados.rados.0.smithi070.stdout:2940: done (1 left) 2024-09-14T03:58:14.095 INFO:tasks.rados.rados.0.smithi070.stdout:2943: snap_remove snap 295 2024-09-14T03:58:14.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:14 smithi070 ceph-mon[98640]: osdmap e661: 8 total, 8 up, 8 in 2024-09-14T03:58:14.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:14 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/2633640420' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:58:14.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:14 smithi070 ceph-mon[98640]: osdmap e662: 8 total, 8 up, 8 in 2024-09-14T03:58:14.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:14 smithi070 ceph-mon[96704]: osdmap e661: 8 total, 8 up, 8 in 2024-09-14T03:58:14.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:14 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/2633640420' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:58:14.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:14 smithi070 ceph-mon[96704]: osdmap e662: 8 total, 8 up, 8 in 2024-09-14T03:58:14.312 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:58:14.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:14 smithi094 ceph-mon[86597]: osdmap e661: 8 total, 8 up, 8 in 2024-09-14T03:58:14.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:14 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/2633640420' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:58:14.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:14 smithi094 ceph-mon[86597]: osdmap e662: 8 total, 8 up, 8 in 2024-09-14T03:58:15.096 INFO:tasks.rados.rados.0.smithi070.stdout:2943: done (1 left) 2024-09-14T03:58:15.096 INFO:tasks.rados.rados.0.smithi070.stdout:2944: rollback oid 44 current snap is 301 2024-09-14T03:58:15.096 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 44 to 234 2024-09-14T03:58:15.096 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 3157 (ObjNum 869 snap 295 seq_num 869) dirty exists 2024-09-14T03:58:15.096 INFO:tasks.rados.rados.0.smithi070.stdout:2942: done (1 left) 2024-09-14T03:58:15.096 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 4 current snap is 301 2024-09-14T03:58:15.097 INFO:tasks.rados.rados.0.smithi070.stdout:2945: seq_num 902 ranges {262144=712704} 2024-09-14T03:58:15.102 INFO:tasks.rados.rados.0.smithi070.stdout:2945: writing smithi07085154-4 from 262144 to 974848 tid 1 2024-09-14T03:58:15.102 INFO:tasks.rados.rados.0.smithi070.stdout:2946: read oid 19 snap -1 2024-09-14T03:58:15.102 INFO:tasks.rados.rados.0.smithi070.stdout:2946: expect (ObjNum 901 snap 301 seq_num 901) 2024-09-14T03:58:15.103 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 21 current snap is 301 2024-09-14T03:58:15.103 INFO:tasks.rados.rados.0.smithi070.stdout:2947: seq_num 903 ranges {0=729088} 2024-09-14T03:58:15.108 INFO:tasks.rados.rados.0.smithi070.stdout:2947: writing smithi07085154-21 from 0 to 729088 tid 1 2024-09-14T03:58:15.108 INFO:tasks.rados.rados.0.smithi070.stdout:2948: read oid 15 snap -1 2024-09-14T03:58:15.108 INFO:tasks.rados.rados.0.smithi070.stdout:2948: expect (ObjNum 805 snap 281 seq_num 805) 2024-09-14T03:58:15.108 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 33 current snap is 301 2024-09-14T03:58:15.108 INFO:tasks.rados.rados.0.smithi070.stdout:2949: seq_num 904 ranges {40960=745472} 2024-09-14T03:58:15.113 INFO:tasks.rados.rados.0.smithi070.stdout:2949: writing smithi07085154-33 from 40960 to 786432 tid 1 2024-09-14T03:58:15.113 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 36 current snap is 301 2024-09-14T03:58:15.113 INFO:tasks.rados.rados.0.smithi070.stdout:2944: finishing rollback tid 0 to smithi07085154-44 2024-09-14T03:58:15.113 INFO:tasks.rados.rados.0.smithi070.stdout:2950: seq_num 905 ranges {1138688=761856} 2024-09-14T03:58:15.125 INFO:tasks.rados.rados.0.smithi070.stdout:2950: writing smithi07085154-36 from 1138688 to 1900544 tid 1 2024-09-14T03:58:15.125 INFO:tasks.rados.rados.0.smithi070.stdout:2951: snap_create 2024-09-14T03:58:15.125 INFO:tasks.rados.rados.0.smithi070.stdout:2947: finishing write tid 1 to smithi07085154-21 2024-09-14T03:58:15.125 INFO:tasks.rados.rados.0.smithi070.stdout:2945: finishing write tid 1 to smithi07085154-4 2024-09-14T03:58:15.125 INFO:tasks.rados.rados.0.smithi070.stdout:2947: finishing write tid 2 to smithi07085154-21 2024-09-14T03:58:15.126 INFO:tasks.rados.rados.0.smithi070.stdout:2945: finishing write tid 2 to smithi07085154-4 2024-09-14T03:58:15.126 INFO:tasks.rados.rados.0.smithi070.stdout:2947: finishing write tid 3 to smithi07085154-21 2024-09-14T03:58:15.126 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 1717 (ObjNum 903 snap 301 seq_num 903) dirty exists 2024-09-14T03:58:15.126 INFO:tasks.rados.rados.0.smithi070.stdout:2947: left oid 21 (ObjNum 903 snap 301 seq_num 903) 2024-09-14T03:58:15.126 INFO:tasks.rados.rados.0.smithi070.stdout:2944: finishing rollback tid 1 to smithi07085154-44 2024-09-14T03:58:15.126 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 3158 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-14T03:58:15.126 INFO:tasks.rados.rados.0.smithi070.stdout:2945: finishing write tid 3 to smithi07085154-4 2024-09-14T03:58:15.126 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 3159 (ObjNum 902 snap 301 seq_num 902) dirty exists 2024-09-14T03:58:15.126 INFO:tasks.rados.rados.0.smithi070.stdout:2945: left oid 4 (ObjNum 902 snap 301 seq_num 902) 2024-09-14T03:58:15.127 INFO:tasks.rados.rados.0.smithi070.stdout:2948: expect (ObjNum 805 snap 281 seq_num 805) 2024-09-14T03:58:15.176 INFO:tasks.rados.rados.0.smithi070.stdout:2949: finishing write tid 1 to smithi07085154-33 2024-09-14T03:58:15.176 INFO:tasks.rados.rados.0.smithi070.stdout:2949: finishing write tid 2 to smithi07085154-33 2024-09-14T03:58:15.176 INFO:tasks.rados.rados.0.smithi070.stdout:2949: finishing write tid 3 to smithi07085154-33 2024-09-14T03:58:15.176 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 33 v 2090 (ObjNum 904 snap 301 seq_num 904) dirty exists 2024-09-14T03:58:15.176 INFO:tasks.rados.rados.0.smithi070.stdout:2949: left oid 33 (ObjNum 904 snap 301 seq_num 904) 2024-09-14T03:58:15.176 INFO:tasks.rados.rados.0.smithi070.stdout:2950: finishing write tid 1 to smithi07085154-36 2024-09-14T03:58:15.176 INFO:tasks.rados.rados.0.smithi070.stdout:2950: finishing write tid 2 to smithi07085154-36 2024-09-14T03:58:15.176 INFO:tasks.rados.rados.0.smithi070.stdout:2950: finishing write tid 3 to smithi07085154-36 2024-09-14T03:58:15.176 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 36 v 3160 (ObjNum 905 snap 301 seq_num 905) dirty exists 2024-09-14T03:58:15.177 INFO:tasks.rados.rados.0.smithi070.stdout:2950: left oid 36 (ObjNum 905 snap 301 seq_num 905) 2024-09-14T03:58:15.177 INFO:tasks.rados.rados.0.smithi070.stdout:2946: expect (ObjNum 901 snap 301 seq_num 901) 2024-09-14T03:58:15.456 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:58:15 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2[113018]: 2024-09-14T03:58:15.116+0000 7f5c8cca5640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap 179 after trimming was started 2024-09-14T03:58:15.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:15 smithi070 ceph-mon[98640]: pgmap v590: 121 pgs: 121 active+clean; 1.1 GiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 351 KiB/s rd, 624 KiB/s wr, 15 op/s 2024-09-14T03:58:15.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:15 smithi070 ceph-mon[98640]: osdmap e663: 8 total, 8 up, 8 in 2024-09-14T03:58:15.457 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:15 smithi070 ceph-mon[96704]: pgmap v590: 121 pgs: 121 active+clean; 1.1 GiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 351 KiB/s rd, 624 KiB/s wr, 15 op/s 2024-09-14T03:58:15.457 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:15 smithi070 ceph-mon[96704]: osdmap e663: 8 total, 8 up, 8 in 2024-09-14T03:58:15.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:15 smithi094 ceph-mon[86597]: pgmap v590: 121 pgs: 121 active+clean; 1.1 GiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 351 KiB/s rd, 624 KiB/s wr, 15 op/s 2024-09-14T03:58:15.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:15 smithi094 ceph-mon[86597]: osdmap e663: 8 total, 8 up, 8 in 2024-09-14T03:58:16.127 INFO:tasks.rados.rados.0.smithi070.stdout:2944: done (7 left) 2024-09-14T03:58:16.128 INFO:tasks.rados.rados.0.smithi070.stdout:2945: done (6 left) 2024-09-14T03:58:16.128 INFO:tasks.rados.rados.0.smithi070.stdout:2946: done (5 left) 2024-09-14T03:58:16.128 INFO:tasks.rados.rados.0.smithi070.stdout:2947: done (4 left) 2024-09-14T03:58:16.128 INFO:tasks.rados.rados.0.smithi070.stdout:2948: done (3 left) 2024-09-14T03:58:16.128 INFO:tasks.rados.rados.0.smithi070.stdout:2949: done (2 left) 2024-09-14T03:58:16.128 INFO:tasks.rados.rados.0.smithi070.stdout:2950: done (1 left) 2024-09-14T03:58:16.128 INFO:tasks.rados.rados.0.smithi070.stdout:2951: done (0 left) 2024-09-14T03:58:16.128 INFO:tasks.rados.rados.0.smithi070.stdout:2952: read oid 28 snap -1 2024-09-14T03:58:16.128 INFO:tasks.rados.rados.0.smithi070.stdout:2952: expect (ObjNum 639 snap 214 seq_num 639) 2024-09-14T03:58:16.128 INFO:tasks.rados.rados.0.smithi070.stdout:2953: read oid 20 snap -1 2024-09-14T03:58:16.128 INFO:tasks.rados.rados.0.smithi070.stdout:2953: expect (ObjNum 897 snap 299 seq_num 897) 2024-09-14T03:58:16.128 INFO:tasks.rados.rados.0.smithi070.stdout:2954: snap_remove snap 265 2024-09-14T03:58:16.312 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)" 2024-09-14T03:58:16.956 INFO:teuthology.task.sequential:In sequential, running task print... 2024-09-14T03:58:16.957 INFO:teuthology.task.print:**** done end upgrade, wait... 2024-09-14T03:58:16.957 DEBUG:teuthology.parallel:result is None 2024-09-14T03:58:17.131 INFO:tasks.rados.rados.0.smithi070.stdout:2954: done (2 left) 2024-09-14T03:58:17.131 INFO:tasks.rados.rados.0.smithi070.stdout:2955: snap_create 2024-09-14T03:58:17.131 INFO:tasks.rados.rados.0.smithi070.stdout:2952: expect (ObjNum 639 snap 214 seq_num 639) 2024-09-14T03:58:17.177 INFO:tasks.rados.rados.0.smithi070.stdout:2953: expect (ObjNum 897 snap 299 seq_num 897) 2024-09-14T03:58:17.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:17 smithi070 ceph-mon[96704]: pgmap v592: 121 pgs: 121 active+clean; 1.0 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 1.6 MiB/s rd, 165 KiB/s wr, 7 op/s 2024-09-14T03:58:17.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:17 smithi070 ceph-mon[96704]: osdmap e664: 8 total, 8 up, 8 in 2024-09-14T03:58:17.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:17 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/1353502837' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:58:17.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:17 smithi070 ceph-mon[98640]: pgmap v592: 121 pgs: 121 active+clean; 1.0 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 1.6 MiB/s rd, 165 KiB/s wr, 7 op/s 2024-09-14T03:58:17.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:17 smithi070 ceph-mon[98640]: osdmap e664: 8 total, 8 up, 8 in 2024-09-14T03:58:17.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:17 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/1353502837' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:58:17.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:17 smithi094 ceph-mon[86597]: pgmap v592: 121 pgs: 121 active+clean; 1.0 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 1.6 MiB/s rd, 165 KiB/s wr, 7 op/s 2024-09-14T03:58:17.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:17 smithi094 ceph-mon[86597]: osdmap e664: 8 total, 8 up, 8 in 2024-09-14T03:58:17.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:17 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/1353502837' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:58:18.131 INFO:tasks.rados.rados.0.smithi070.stdout:2952: done (2 left) 2024-09-14T03:58:18.131 INFO:tasks.rados.rados.0.smithi070.stdout:2953: done (1 left) 2024-09-14T03:58:18.132 INFO:tasks.rados.rados.0.smithi070.stdout:2955: done (0 left) 2024-09-14T03:58:18.132 INFO:tasks.rados.rados.0.smithi070.stdout:2956: read oid 39 snap -1 2024-09-14T03:58:18.132 INFO:tasks.rados.rados.0.smithi070.stdout:2956: expect deleted 2024-09-14T03:58:18.132 INFO:tasks.rados.rados.0.smithi070.stdout:2957: delete oid 30 current snap is 303 2024-09-14T03:58:18.138 INFO:tasks.rados.rados.0.smithi070.stdout:2956: done (1 left) 2024-09-14T03:58:18.138 INFO:tasks.rados.rados.0.smithi070.stdout:2957: done (0 left) 2024-09-14T03:58:18.138 INFO:tasks.rados.rados.0.smithi070.stdout:2958: setattr oid 41 current snap is 303 2024-09-14T03:58:18.141 INFO:tasks.rados.rados.0.smithi070.stdout:2959: setattr oid 7 current snap is 303 2024-09-14T03:58:18.145 INFO:tasks.rados.rados.0.smithi070.stdout:2960: read oid 4 snap 301 2024-09-14T03:58:18.145 INFO:tasks.rados.rados.0.smithi070.stdout:2960: expect (ObjNum 902 snap 301 seq_num 902) 2024-09-14T03:58:18.145 INFO:tasks.rados.rados.0.smithi070.stdout:2961: snap_create 2024-09-14T03:58:18.145 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 2369 (ObjNum 724 snap 248 seq_num 724) dirty exists 2024-09-14T03:58:18.151 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 3160 (ObjNum 161 snap 0 seq_num 0) dirty exists 2024-09-14T03:58:18.155 INFO:tasks.rados.rados.0.smithi070.stdout:2960: expect (ObjNum 902 snap 301 seq_num 902) 2024-09-14T03:58:18.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:18 smithi094 ceph-mon[86597]: osdmap e665: 8 total, 8 up, 8 in 2024-09-14T03:58:18.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:18 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:58:18.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:18 smithi094 ceph-mon[86597]: osdmap e666: 8 total, 8 up, 8 in 2024-09-14T03:58:18.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:18 smithi070 ceph-mon[98640]: osdmap e665: 8 total, 8 up, 8 in 2024-09-14T03:58:18.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:18 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:58:18.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:18 smithi070 ceph-mon[98640]: osdmap e666: 8 total, 8 up, 8 in 2024-09-14T03:58:18.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:18 smithi070 ceph-mon[96704]: osdmap e665: 8 total, 8 up, 8 in 2024-09-14T03:58:18.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:18 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:58:18.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:18 smithi070 ceph-mon[96704]: osdmap e666: 8 total, 8 up, 8 in 2024-09-14T03:58:19.098 INFO:tasks.rados.rados.0.smithi070.stdout:2958: done (3 left) 2024-09-14T03:58:19.098 INFO:tasks.rados.rados.0.smithi070.stdout:2959: done (2 left) 2024-09-14T03:58:19.098 INFO:tasks.rados.rados.0.smithi070.stdout:2960: done (1 left) 2024-09-14T03:58:19.098 INFO:tasks.rados.rados.0.smithi070.stdout:2961: done (0 left) 2024-09-14T03:58:19.098 INFO:tasks.rados.rados.0.smithi070.stdout:2962: rmattr oid 7 current snap is 304 2024-09-14T03:58:19.098 INFO:tasks.rados.rados.0.smithi070.stdout:2963: copy_from oid 47 from oid 38 current snap is 304 2024-09-14T03:58:19.098 INFO:tasks.rados.rados.0.smithi070.stdout:2964: read oid 6 snap -1 2024-09-14T03:58:19.098 INFO:tasks.rados.rados.0.smithi070.stdout:2964: expect deleted 2024-09-14T03:58:19.098 INFO:tasks.rados.rados.0.smithi070.stdout:2965: copy_from oid 25 from oid 4 current snap is 304 2024-09-14T03:58:19.098 INFO:tasks.rados.rados.0.smithi070.stdout:2966: copy_from oid 29 from oid 41 current snap is 304 2024-09-14T03:58:19.098 INFO:tasks.rados.rados.0.smithi070.stdout:2967: snap_remove snap 277 2024-09-14T03:58:19.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:19 smithi094 ceph-mon[86597]: pgmap v595: 121 pgs: 121 active+clean; 998 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 2.4 MiB/s wr, 12 op/s 2024-09-14T03:58:19.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:19 smithi094 ceph-mon[86597]: osdmap e667: 8 total, 8 up, 8 in 2024-09-14T03:58:19.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:19 smithi070 ceph-mon[96704]: pgmap v595: 121 pgs: 121 active+clean; 998 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 2.4 MiB/s wr, 12 op/s 2024-09-14T03:58:19.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:19 smithi070 ceph-mon[96704]: osdmap e667: 8 total, 8 up, 8 in 2024-09-14T03:58:19.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:19 smithi070 ceph-mon[98640]: pgmap v595: 121 pgs: 121 active+clean; 998 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 2.4 MiB/s wr, 12 op/s 2024-09-14T03:58:19.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:19 smithi070 ceph-mon[98640]: osdmap e667: 8 total, 8 up, 8 in 2024-09-14T03:58:20.105 INFO:tasks.rados.rados.0.smithi070.stdout:2967: done (5 left) 2024-09-14T03:58:20.105 INFO:tasks.rados.rados.0.smithi070.stdout:2968: read oid 9 snap -1 2024-09-14T03:58:20.105 INFO:tasks.rados.rados.0.smithi070.stdout:2968: expect (ObjNum 878 snap 295 seq_num 878) 2024-09-14T03:58:20.105 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 26 current snap is 304 2024-09-14T03:58:20.105 INFO:tasks.rados.rados.0.smithi070.stdout:2969: seq_num 907 ranges {1679360=794624} 2024-09-14T03:58:20.115 INFO:tasks.rados.rados.0.smithi070.stdout:2969: writing smithi07085154-26 from 1679360 to 2473984 tid 1 2024-09-14T03:58:20.115 INFO:tasks.rados.rados.0.smithi070.stdout:2970: read oid 14 snap -1 2024-09-14T03:58:20.115 INFO:tasks.rados.rados.0.smithi070.stdout:2970: expect (ObjNum 313 snap 105 seq_num 313) 2024-09-14T03:58:20.116 INFO:tasks.rados.rados.0.smithi070.stdout:2971: snap_remove snap 297 2024-09-14T03:58:20.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:58:20 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:58:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:58:21.104 INFO:tasks.rados.rados.0.smithi070.stdout:2971: done (8 left) 2024-09-14T03:58:21.104 INFO:tasks.rados.rados.0.smithi070.stdout:2972: read oid 2 snap -1 2024-09-14T03:58:21.104 INFO:tasks.rados.rados.0.smithi070.stdout:2972: expect (ObjNum 891 snap 298 seq_num 891) 2024-09-14T03:58:21.104 INFO:tasks.rados.rados.0.smithi070.stdout:2973: rollback oid 46 current snap is 304 2024-09-14T03:58:21.104 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 46 to 267 2024-09-14T03:58:21.104 INFO:tasks.rados.rados.0.smithi070.stdout:2963: finishing copy_from to smithi07085154-47 2024-09-14T03:58:21.104 INFO:tasks.rados.rados.0.smithi070.stdout:2963: got expected ENOENT (src dne) 2024-09-14T03:58:21.104 INFO:tasks.rados.rados.0.smithi070.stdout:2963: finishing copy_from racing read to smithi07085154-47 2024-09-14T03:58:21.104 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 3161 (ObjNum 33 snap 0 seq_num 2930699408) dirty exists 2024-09-14T03:58:21.105 INFO:tasks.rados.rados.0.smithi070.stdout:2965: finishing copy_from racing read to smithi07085154-25 2024-09-14T03:58:21.105 INFO:tasks.rados.rados.0.smithi070.stdout:2966: finishing copy_from racing read to smithi07085154-29 2024-09-14T03:58:21.105 INFO:tasks.rados.rados.0.smithi070.stdout:2966: finishing copy_from to smithi07085154-29 2024-09-14T03:58:21.105 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 2369 (ObjNum 724 snap 248 seq_num 724) dirty exists 2024-09-14T03:58:21.105 INFO:tasks.rados.rados.0.smithi070.stdout:2965: finishing copy_from to smithi07085154-25 2024-09-14T03:58:21.105 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 3159 (ObjNum 902 snap 301 seq_num 902) dirty exists 2024-09-14T03:58:21.105 INFO:tasks.rados.rados.0.smithi070.stdout:2970: expect (ObjNum 313 snap 105 seq_num 313) 2024-09-14T03:58:21.124 INFO:tasks.rados.rados.0.smithi070.stdout:2969: finishing write tid 1 to smithi07085154-26 2024-09-14T03:58:21.125 INFO:tasks.rados.rados.0.smithi070.stdout:2969: finishing write tid 2 to smithi07085154-26 2024-09-14T03:58:21.125 INFO:tasks.rados.rados.0.smithi070.stdout:2968: expect (ObjNum 878 snap 295 seq_num 878) 2024-09-14T03:58:21.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:21 smithi094 ceph-mon[86597]: osdmap e668: 8 total, 8 up, 8 in 2024-09-14T03:58:21.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:21 smithi094 ceph-mon[86597]: pgmap v599: 121 pgs: 121 active+clean; 999 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.7 MiB/s rd, 2.6 MiB/s wr, 18 op/s 2024-09-14T03:58:21.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:21 smithi070 ceph-mon[98640]: osdmap e668: 8 total, 8 up, 8 in 2024-09-14T03:58:21.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:21 smithi070 ceph-mon[98640]: pgmap v599: 121 pgs: 121 active+clean; 999 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.7 MiB/s rd, 2.6 MiB/s wr, 18 op/s 2024-09-14T03:58:21.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:21 smithi070 ceph-mon[96704]: osdmap e668: 8 total, 8 up, 8 in 2024-09-14T03:58:21.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:21 smithi070 ceph-mon[96704]: pgmap v599: 121 pgs: 121 active+clean; 999 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.7 MiB/s rd, 2.6 MiB/s wr, 18 op/s 2024-09-14T03:58:22.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:22 smithi094 ceph-mon[86597]: osdmap e669: 8 total, 8 up, 8 in 2024-09-14T03:58:22.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:22 smithi070 ceph-mon[96704]: osdmap e669: 8 total, 8 up, 8 in 2024-09-14T03:58:22.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:22 smithi070 ceph-mon[98640]: osdmap e669: 8 total, 8 up, 8 in 2024-09-14T03:58:23.041 INFO:tasks.rados.rados.0.smithi070.stdout:2969: finishing write tid 3 to smithi07085154-26 2024-09-14T03:58:23.042 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 3522 (ObjNum 907 snap 304 seq_num 907) dirty exists 2024-09-14T03:58:23.042 INFO:tasks.rados.rados.0.smithi070.stdout:2969: left oid 26 (ObjNum 907 snap 304 seq_num 907) 2024-09-14T03:58:23.042 INFO:tasks.rados.rados.0.smithi070.stdout:2962: done (9 left) 2024-09-14T03:58:23.042 INFO:tasks.rados.rados.0.smithi070.stdout:2963: done (8 left) 2024-09-14T03:58:23.042 INFO:tasks.rados.rados.0.smithi070.stdout:2964: done (7 left) 2024-09-14T03:58:23.042 INFO:tasks.rados.rados.0.smithi070.stdout:2965: done (6 left) 2024-09-14T03:58:23.042 INFO:tasks.rados.rados.0.smithi070.stdout:2966: done (5 left) 2024-09-14T03:58:23.042 INFO:tasks.rados.rados.0.smithi070.stdout:2968: done (4 left) 2024-09-14T03:58:23.042 INFO:tasks.rados.rados.0.smithi070.stdout:2969: done (3 left) 2024-09-14T03:58:23.042 INFO:tasks.rados.rados.0.smithi070.stdout:2970: done (2 left) 2024-09-14T03:58:23.042 INFO:tasks.rados.rados.0.smithi070.stdout:2974: rmattr oid 38 current snap is 304 2024-09-14T03:58:23.043 INFO:tasks.rados.rados.0.smithi070.stdout:2974: done (2 left) 2024-09-14T03:58:23.043 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 48 current snap is 304 2024-09-14T03:58:23.043 INFO:tasks.rados.rados.0.smithi070.stdout:2973: finishing rollback tid 0 to smithi07085154-46 2024-09-14T03:58:23.043 INFO:tasks.rados.rados.0.smithi070.stdout:2975: seq_num 908 ranges {4538368=409600,4947968=401408} 2024-09-14T03:58:23.067 INFO:tasks.rados.rados.0.smithi070.stdout:2975: writing smithi07085154-48 from 4538368 to 4947968 tid 1 2024-09-14T03:58:23.069 INFO:tasks.rados.rados.0.smithi070.stdout:2975: writing smithi07085154-48 from 4947968 to 5349376 tid 2 2024-09-14T03:58:23.069 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 35 current snap is 304 2024-09-14T03:58:23.069 INFO:tasks.rados.rados.0.smithi070.stdout:2973: finishing rollback tid 1 to smithi07085154-46 2024-09-14T03:58:23.070 INFO:tasks.rados.rados.0.smithi070.stdout:2973: finishing rollback tid 2 to smithi07085154-46 2024-09-14T03:58:23.070 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 46 v 3542 (ObjNum 576 snap 193 seq_num 576) dirty exists 2024-09-14T03:58:23.070 INFO:tasks.rados.rados.0.smithi070.stdout:2972: expect (ObjNum 891 snap 298 seq_num 891) 2024-09-14T03:58:23.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:23 smithi070 ceph-mon[96704]: pgmap v601: 121 pgs: 121 active+clean; 992 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 26 MiB/s rd, 606 KiB/s wr, 22 op/s 2024-09-14T03:58:23.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:23 smithi070 ceph-mon[98640]: pgmap v601: 121 pgs: 121 active+clean; 992 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 26 MiB/s rd, 606 KiB/s wr, 22 op/s 2024-09-14T03:58:23.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:23 smithi094 ceph-mon[86597]: pgmap v601: 121 pgs: 121 active+clean; 992 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 26 MiB/s rd, 606 KiB/s wr, 22 op/s 2024-09-14T03:58:23.670 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:58:23 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:58:23] "GET /metrics HTTP/1.1" 200 36505 "" "Prometheus/2.51.0" 2024-09-14T03:58:25.023 INFO:tasks.rados.rados.0.smithi070.stdout:2975: finishing write tid 1 to smithi07085154-48 2024-09-14T03:58:25.023 INFO:tasks.rados.rados.0.smithi070.stdout:2975: finishing write tid 2 to smithi07085154-48 2024-09-14T03:58:25.023 INFO:tasks.rados.rados.0.smithi070.stdout:2975: finishing write tid 3 to smithi07085154-48 2024-09-14T03:58:25.023 INFO:tasks.rados.rados.0.smithi070.stdout:2975: finishing write tid 4 to smithi07085154-48 2024-09-14T03:58:25.023 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 2246 (ObjNum 908 snap 304 seq_num 908) dirty exists 2024-09-14T03:58:25.023 INFO:tasks.rados.rados.0.smithi070.stdout:2975: left oid 48 (ObjNum 908 snap 304 seq_num 908) 2024-09-14T03:58:25.023 INFO:tasks.rados.rados.0.smithi070.stdout:2976: seq_num 909 ranges {0=425984,425984=401408} 2024-09-14T03:58:25.025 INFO:tasks.rados.rados.0.smithi070.stdout:2976: writing smithi07085154-35 from 0 to 425984 tid 1 2024-09-14T03:58:25.028 INFO:tasks.rados.rados.0.smithi070.stdout:2976: writing smithi07085154-35 from 425984 to 827392 tid 2 2024-09-14T03:58:25.028 INFO:tasks.rados.rados.0.smithi070.stdout:2972: done (3 left) 2024-09-14T03:58:25.028 INFO:tasks.rados.rados.0.smithi070.stdout:2973: done (2 left) 2024-09-14T03:58:25.028 INFO:tasks.rados.rados.0.smithi070.stdout:2975: done (1 left) 2024-09-14T03:58:25.028 INFO:tasks.rados.rados.0.smithi070.stdout:2977: copy_from oid 38 from oid 31 current snap is 304 2024-09-14T03:58:25.028 INFO:tasks.rados.rados.0.smithi070.stdout:2978: read oid 33 snap 299 2024-09-14T03:58:25.028 INFO:tasks.rados.rados.0.smithi070.stdout:2978: expect (ObjNum 862 snap 292 seq_num 862) 2024-09-14T03:58:25.028 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 17 current snap is 304 2024-09-14T03:58:25.028 INFO:tasks.rados.rados.0.smithi070.stdout:2979: seq_num 910 ranges {598016=442368,1040384=401408} 2024-09-14T03:58:25.033 INFO:tasks.rados.rados.0.smithi070.stdout:2979: writing smithi07085154-17 from 598016 to 1040384 tid 1 2024-09-14T03:58:25.036 INFO:tasks.rados.rados.0.smithi070.stdout:2979: writing smithi07085154-17 from 1040384 to 1441792 tid 2 2024-09-14T03:58:25.036 INFO:tasks.rados.rados.0.smithi070.stdout:2980: snap_remove snap 286 2024-09-14T03:58:25.105 INFO:tasks.rados.rados.0.smithi070.stdout:2976: finishing write tid 1 to smithi07085154-35 2024-09-14T03:58:25.105 INFO:tasks.rados.rados.0.smithi070.stdout:2977: finishing copy_from to smithi07085154-38 2024-09-14T03:58:25.105 INFO:tasks.rados.rados.0.smithi070.stdout:2977: got expected ENOENT (src dne) 2024-09-14T03:58:25.105 INFO:tasks.rados.rados.0.smithi070.stdout:2977: finishing copy_from racing read to smithi07085154-38 2024-09-14T03:58:25.105 INFO:tasks.rados.rados.0.smithi070.stdout:2978: expect (ObjNum 862 snap 292 seq_num 862) 2024-09-14T03:58:25.111 INFO:tasks.rados.rados.0.smithi070.stdout:2976: finishing write tid 2 to smithi07085154-35 2024-09-14T03:58:25.111 INFO:tasks.rados.rados.0.smithi070.stdout:2976: finishing write tid 3 to smithi07085154-35 2024-09-14T03:58:25.111 INFO:tasks.rados.rados.0.smithi070.stdout:2976: finishing write tid 4 to smithi07085154-35 2024-09-14T03:58:25.111 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 35 v 1701 (ObjNum 909 snap 304 seq_num 909) dirty exists 2024-09-14T03:58:25.111 INFO:tasks.rados.rados.0.smithi070.stdout:2976: left oid 35 (ObjNum 909 snap 304 seq_num 909) 2024-09-14T03:58:25.111 INFO:tasks.rados.rados.0.smithi070.stdout:2979: finishing write tid 1 to smithi07085154-17 2024-09-14T03:58:25.111 INFO:tasks.rados.rados.0.smithi070.stdout:2979: finishing write tid 2 to smithi07085154-17 2024-09-14T03:58:25.111 INFO:tasks.rados.rados.0.smithi070.stdout:2979: finishing write tid 3 to smithi07085154-17 2024-09-14T03:58:25.111 INFO:tasks.rados.rados.0.smithi070.stdout:2979: finishing write tid 4 to smithi07085154-17 2024-09-14T03:58:25.111 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 3162 (ObjNum 910 snap 304 seq_num 910) dirty exists 2024-09-14T03:58:25.112 INFO:tasks.rados.rados.0.smithi070.stdout:2979: left oid 17 (ObjNum 910 snap 304 seq_num 910) 2024-09-14T03:58:25.112 INFO:tasks.rados.rados.0.smithi070.stdout:2976: done (4 left) 2024-09-14T03:58:25.112 INFO:tasks.rados.rados.0.smithi070.stdout:2977: done (3 left) 2024-09-14T03:58:25.112 INFO:tasks.rados.rados.0.smithi070.stdout:2978: done (2 left) 2024-09-14T03:58:25.112 INFO:tasks.rados.rados.0.smithi070.stdout:2979: done (1 left) 2024-09-14T03:58:25.113 INFO:tasks.rados.rados.0.smithi070.stdout:2980: done (0 left) 2024-09-14T03:58:25.113 INFO:tasks.rados.rados.0.smithi070.stdout:2981: snap_create 2024-09-14T03:58:25.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:25 smithi094 ceph-mon[86597]: osdmap e670: 8 total, 8 up, 8 in 2024-09-14T03:58:25.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:25 smithi094 ceph-mon[86597]: pgmap v603: 121 pgs: 121 active+clean; 994 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 21 MiB/s rd, 672 KiB/s wr, 17 op/s 2024-09-14T03:58:25.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:25 smithi070 ceph-mon[98640]: osdmap e670: 8 total, 8 up, 8 in 2024-09-14T03:58:25.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:25 smithi070 ceph-mon[98640]: pgmap v603: 121 pgs: 121 active+clean; 994 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 21 MiB/s rd, 672 KiB/s wr, 17 op/s 2024-09-14T03:58:25.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:25 smithi070 ceph-mon[96704]: osdmap e670: 8 total, 8 up, 8 in 2024-09-14T03:58:25.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:25 smithi070 ceph-mon[96704]: pgmap v603: 121 pgs: 121 active+clean; 994 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 21 MiB/s rd, 672 KiB/s wr, 17 op/s 2024-09-14T03:58:26.108 INFO:tasks.rados.rados.0.smithi070.stdout:2981: done (0 left) 2024-09-14T03:58:26.108 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 34 current snap is 305 2024-09-14T03:58:26.108 INFO:tasks.rados.rados.0.smithi070.stdout:2982: seq_num 911 ranges {327680=466944,794624=401408} 2024-09-14T03:58:26.112 INFO:tasks.rados.rados.0.smithi070.stdout:2982: writing smithi07085154-34 from 327680 to 794624 tid 1 2024-09-14T03:58:26.114 INFO:tasks.rados.rados.0.smithi070.stdout:2982: writing smithi07085154-34 from 794624 to 1196032 tid 2 2024-09-14T03:58:26.114 INFO:tasks.rados.rados.0.smithi070.stdout:2983: setattr oid 41 current snap is 305 2024-09-14T03:58:26.116 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 40 current snap is 305 2024-09-14T03:58:26.117 INFO:tasks.rados.rados.0.smithi070.stdout:2984: seq_num 912 ranges {49152=483328,532480=401408} 2024-09-14T03:58:26.120 INFO:tasks.rados.rados.0.smithi070.stdout:2984: writing smithi07085154-40 from 49152 to 532480 tid 1 2024-09-14T03:58:26.123 INFO:tasks.rados.rados.0.smithi070.stdout:2984: writing smithi07085154-40 from 532480 to 933888 tid 2 2024-09-14T03:58:26.124 INFO:tasks.rados.rados.0.smithi070.stdout:2985: read oid 47 snap 245 2024-09-14T03:58:26.124 INFO:tasks.rados.rados.0.smithi070.stdout:2985: expect (ObjNum 460 snap 155 seq_num 460) 2024-09-14T03:58:26.124 INFO:tasks.rados.rados.0.smithi070.stdout:2986: snap_create 2024-09-14T03:58:26.124 INFO:tasks.rados.rados.0.smithi070.stdout:2982: finishing write tid 1 to smithi07085154-34 2024-09-14T03:58:26.124 INFO:tasks.rados.rados.0.smithi070.stdout:2982: finishing write tid 2 to smithi07085154-34 2024-09-14T03:58:26.124 INFO:tasks.rados.rados.0.smithi070.stdout:2982: finishing write tid 3 to smithi07085154-34 2024-09-14T03:58:26.124 INFO:tasks.rados.rados.0.smithi070.stdout:2982: finishing write tid 4 to smithi07085154-34 2024-09-14T03:58:26.124 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 3162 (ObjNum 911 snap 305 seq_num 911) dirty exists 2024-09-14T03:58:26.124 INFO:tasks.rados.rados.0.smithi070.stdout:2982: left oid 34 (ObjNum 911 snap 305 seq_num 911) 2024-09-14T03:58:26.125 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 2393 (ObjNum 724 snap 248 seq_num 724) dirty exists 2024-09-14T03:58:26.128 INFO:tasks.rados.rados.0.smithi070.stdout:2984: finishing write tid 1 to smithi07085154-40 2024-09-14T03:58:26.128 INFO:tasks.rados.rados.0.smithi070.stdout:2984: finishing write tid 2 to smithi07085154-40 2024-09-14T03:58:26.128 INFO:tasks.rados.rados.0.smithi070.stdout:2984: finishing write tid 3 to smithi07085154-40 2024-09-14T03:58:26.131 INFO:tasks.rados.rados.0.smithi070.stdout:2984: finishing write tid 4 to smithi07085154-40 2024-09-14T03:58:26.131 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 3163 (ObjNum 912 snap 305 seq_num 912) dirty exists 2024-09-14T03:58:26.131 INFO:tasks.rados.rados.0.smithi070.stdout:2984: left oid 40 (ObjNum 912 snap 305 seq_num 912) 2024-09-14T03:58:26.155 INFO:tasks.rados.rados.0.smithi070.stdout:2985: expect (ObjNum 460 snap 155 seq_num 460) 2024-09-14T03:58:26.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:26 smithi094 ceph-mon[86597]: osdmap e671: 8 total, 8 up, 8 in 2024-09-14T03:58:26.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:26 smithi070 ceph-mon[96704]: osdmap e671: 8 total, 8 up, 8 in 2024-09-14T03:58:26.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:26 smithi070 ceph-mon[98640]: osdmap e671: 8 total, 8 up, 8 in 2024-09-14T03:58:27.112 INFO:tasks.rados.rados.0.smithi070.stdout:2982: done (4 left) 2024-09-14T03:58:27.113 INFO:tasks.rados.rados.0.smithi070.stdout:2983: done (3 left) 2024-09-14T03:58:27.113 INFO:tasks.rados.rados.0.smithi070.stdout:2984: done (2 left) 2024-09-14T03:58:27.113 INFO:tasks.rados.rados.0.smithi070.stdout:2985: done (1 left) 2024-09-14T03:58:27.113 INFO:tasks.rados.rados.0.smithi070.stdout:2986: done (0 left) 2024-09-14T03:58:27.113 INFO:tasks.rados.rados.0.smithi070.stdout:2987: setattr oid 18 current snap is 306 2024-09-14T03:58:27.114 INFO:tasks.rados.rados.0.smithi070.stdout:2988: copy_from oid 28 from oid 41 current snap is 306 2024-09-14T03:58:27.114 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 3 current snap is 306 2024-09-14T03:58:27.114 INFO:tasks.rados.rados.0.smithi070.stdout:2989: seq_num 913 ranges {4022272=499712,4521984=401408} 2024-09-14T03:58:27.138 INFO:tasks.rados.rados.0.smithi070.stdout:2989: writing smithi07085154-3 from 4022272 to 4521984 tid 1 2024-09-14T03:58:27.140 INFO:tasks.rados.rados.0.smithi070.stdout:2989: writing smithi07085154-3 from 4521984 to 4923392 tid 2 2024-09-14T03:58:27.141 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 5 current snap is 306 2024-09-14T03:58:27.141 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 2395 (ObjNum 900 snap 301 seq_num 900) dirty exists 2024-09-14T03:58:27.141 INFO:tasks.rados.rados.0.smithi070.stdout:2988: finishing copy_from racing read to smithi07085154-28 2024-09-14T03:58:27.141 INFO:tasks.rados.rados.0.smithi070.stdout:2988: finishing copy_from to smithi07085154-28 2024-09-14T03:58:27.141 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 28 v 2393 (ObjNum 724 snap 248 seq_num 724) dirty exists 2024-09-14T03:58:27.141 INFO:tasks.rados.rados.0.smithi070.stdout:2990: seq_num 914 ranges {0=516096,516096=401408} 2024-09-14T03:58:27.143 INFO:tasks.rados.rados.0.smithi070.stdout:2990: writing smithi07085154-5 from 0 to 516096 tid 1 2024-09-14T03:58:27.146 INFO:tasks.rados.rados.0.smithi070.stdout:2990: writing smithi07085154-5 from 516096 to 917504 tid 2 2024-09-14T03:58:27.146 INFO:tasks.rados.rados.0.smithi070.stdout:2987: done (3 left) 2024-09-14T03:58:27.146 INFO:tasks.rados.rados.0.smithi070.stdout:2988: done (2 left) 2024-09-14T03:58:27.146 INFO:tasks.rados.rados.0.smithi070.stdout:2991: snap_remove snap 234 2024-09-14T03:58:27.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:27 smithi094 ceph-mon[86597]: osdmap e672: 8 total, 8 up, 8 in 2024-09-14T03:58:27.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:27 smithi094 ceph-mon[86597]: pgmap v606: 121 pgs: 121 active+clean; 994 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 528 KiB/s wr, 12 op/s 2024-09-14T03:58:27.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:27 smithi070 ceph-mon[98640]: osdmap e672: 8 total, 8 up, 8 in 2024-09-14T03:58:27.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:27 smithi070 ceph-mon[98640]: pgmap v606: 121 pgs: 121 active+clean; 994 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 528 KiB/s wr, 12 op/s 2024-09-14T03:58:27.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:27 smithi070 ceph-mon[96704]: osdmap e672: 8 total, 8 up, 8 in 2024-09-14T03:58:27.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:27 smithi070 ceph-mon[96704]: pgmap v606: 121 pgs: 121 active+clean; 994 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 528 KiB/s wr, 12 op/s 2024-09-14T03:58:28.118 INFO:tasks.rados.rados.0.smithi070.stdout:2991: done (2 left) 2024-09-14T03:58:28.118 INFO:tasks.rados.rados.0.smithi070.stdout:2992: rollback oid 28 current snap is 306 2024-09-14T03:58:28.118 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 28 to 289 2024-09-14T03:58:28.119 INFO:tasks.rados.rados.0.smithi070.stdout:2990: finishing write tid 1 to smithi07085154-5 2024-09-14T03:58:28.119 INFO:tasks.rados.rados.0.smithi070.stdout:2989: finishing write tid 1 to smithi07085154-3 2024-09-14T03:58:28.119 INFO:tasks.rados.rados.0.smithi070.stdout:2989: finishing write tid 2 to smithi07085154-3 2024-09-14T03:58:28.119 INFO:tasks.rados.rados.0.smithi070.stdout:2989: finishing write tid 3 to smithi07085154-3 2024-09-14T03:58:28.119 INFO:tasks.rados.rados.0.smithi070.stdout:2990: finishing write tid 2 to smithi07085154-5 2024-09-14T03:58:28.119 INFO:tasks.rados.rados.0.smithi070.stdout:2990: finishing write tid 3 to smithi07085154-5 2024-09-14T03:58:28.119 INFO:tasks.rados.rados.0.smithi070.stdout:2989: finishing write tid 4 to smithi07085154-3 2024-09-14T03:58:28.119 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 3 v 2247 (ObjNum 913 snap 306 seq_num 913) dirty exists 2024-09-14T03:58:28.119 INFO:tasks.rados.rados.0.smithi070.stdout:2989: left oid 3 (ObjNum 913 snap 306 seq_num 913) 2024-09-14T03:58:28.119 INFO:tasks.rados.rados.0.smithi070.stdout:2990: finishing write tid 4 to smithi07085154-5 2024-09-14T03:58:28.119 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 578 (ObjNum 914 snap 306 seq_num 914) dirty exists 2024-09-14T03:58:28.119 INFO:tasks.rados.rados.0.smithi070.stdout:2990: left oid 5 (ObjNum 914 snap 306 seq_num 914) 2024-09-14T03:58:28.119 INFO:tasks.rados.rados.0.smithi070.stdout:2989: done (2 left) 2024-09-14T03:58:28.119 INFO:tasks.rados.rados.0.smithi070.stdout:2990: done (1 left) 2024-09-14T03:58:28.120 INFO:tasks.rados.rados.0.smithi070.stdout:2993: setattr oid 16 current snap is 306 2024-09-14T03:58:28.120 INFO:tasks.rados.rados.0.smithi070.stdout:2994: copy_from oid 45 from oid 35 current snap is 306 2024-09-14T03:58:28.120 INFO:tasks.rados.rados.0.smithi070.stdout:2995: read oid 15 snap -1 2024-09-14T03:58:28.121 INFO:tasks.rados.rados.0.smithi070.stdout:2995: expect (ObjNum 805 snap 281 seq_num 805) 2024-09-14T03:58:28.121 INFO:tasks.rados.rados.0.smithi070.stdout:2996: rollback oid 18 current snap is 306 2024-09-14T03:58:28.121 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 18 to 267 2024-09-14T03:58:28.121 INFO:tasks.rados.rados.0.smithi070.stdout:2997: snap_remove snap 264 2024-09-14T03:58:28.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:28 smithi094 ceph-mon[86597]: osdmap e673: 8 total, 8 up, 8 in 2024-09-14T03:58:28.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:28 smithi070 ceph-mon[98640]: osdmap e673: 8 total, 8 up, 8 in 2024-09-14T03:58:28.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:28 smithi070 ceph-mon[96704]: osdmap e673: 8 total, 8 up, 8 in 2024-09-14T03:58:29.103 INFO:tasks.rados.rados.0.smithi070.stdout:2997: done (5 left) 2024-09-14T03:58:29.103 INFO:tasks.rados.rados.0.smithi070.stdout:2998: rollback oid 7 current snap is 306 2024-09-14T03:58:29.103 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 7 to 299 2024-09-14T03:58:29.103 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 3564 (ObjNum 788 snap 274 seq_num 788) dirty exists 2024-09-14T03:58:29.103 INFO:tasks.rados.rados.0.smithi070.stdout:2996: finishing rollback tid 0 to smithi07085154-18 2024-09-14T03:58:29.103 INFO:tasks.rados.rados.0.smithi070.stdout:2992: finishing rollback tid 0 to smithi07085154-28 2024-09-14T03:58:29.103 INFO:tasks.rados.rados.0.smithi070.stdout:2992: finishing rollback tid 1 to smithi07085154-28 2024-09-14T03:58:29.103 INFO:tasks.rados.rados.0.smithi070.stdout:2995: expect (ObjNum 805 snap 281 seq_num 805) 2024-09-14T03:58:29.156 INFO:tasks.rados.rados.0.smithi070.stdout:2992: finishing rollback tid 2 to smithi07085154-28 2024-09-14T03:58:29.156 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 28 v 3164 (ObjNum 639 snap 214 seq_num 639) dirty exists 2024-09-14T03:58:29.156 INFO:tasks.rados.rados.0.smithi070.stdout:2996: finishing rollback tid 1 to smithi07085154-18 2024-09-14T03:58:29.156 INFO:tasks.rados.rados.0.smithi070.stdout:2992: done (5 left) 2024-09-14T03:58:29.156 INFO:tasks.rados.rados.0.smithi070.stdout:2993: done (4 left) 2024-09-14T03:58:29.156 INFO:tasks.rados.rados.0.smithi070.stdout:2995: done (3 left) 2024-09-14T03:58:29.156 INFO:tasks.rados.rados.0.smithi070.stdout:2999: rmattr oid 21 current snap is 306 2024-09-14T03:58:29.156 INFO:tasks.rados.rados.0.smithi070.stdout:2999: done (3 left) 2024-09-14T03:58:29.156 INFO:tasks.rados.rados.0.smithi070.stdout:3000: snap_remove snap 287 2024-09-14T03:58:29.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:29 smithi094 ceph-mon[86597]: osdmap e674: 8 total, 8 up, 8 in 2024-09-14T03:58:29.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:29 smithi094 ceph-mon[86597]: pgmap v609: 121 pgs: 121 active+clean; 996 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.0 MiB/s rd, 414 KiB/s wr, 2 op/s 2024-09-14T03:58:29.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:29 smithi094 ceph-mon[86597]: osdmap e675: 8 total, 8 up, 8 in 2024-09-14T03:58:29.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:29 smithi070 ceph-mon[96704]: osdmap e674: 8 total, 8 up, 8 in 2024-09-14T03:58:29.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:29 smithi070 ceph-mon[96704]: pgmap v609: 121 pgs: 121 active+clean; 996 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.0 MiB/s rd, 414 KiB/s wr, 2 op/s 2024-09-14T03:58:29.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:29 smithi070 ceph-mon[96704]: osdmap e675: 8 total, 8 up, 8 in 2024-09-14T03:58:29.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:29 smithi070 ceph-mon[98640]: osdmap e674: 8 total, 8 up, 8 in 2024-09-14T03:58:29.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:29 smithi070 ceph-mon[98640]: pgmap v609: 121 pgs: 121 active+clean; 996 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.0 MiB/s rd, 414 KiB/s wr, 2 op/s 2024-09-14T03:58:29.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:29 smithi070 ceph-mon[98640]: osdmap e675: 8 total, 8 up, 8 in 2024-09-14T03:58:30.144 INFO:tasks.rados.rados.0.smithi070.stdout:3000: done (3 left) 2024-09-14T03:58:30.144 INFO:tasks.rados.rados.0.smithi070.stdout:3001: snap_remove snap 245 2024-09-14T03:58:30.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:58:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:58:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:58:31.145 INFO:tasks.rados.rados.0.smithi070.stdout:3001: done (3 left) 2024-09-14T03:58:31.145 INFO:tasks.rados.rados.0.smithi070.stdout:3002: read oid 13 snap 284 2024-09-14T03:58:31.145 INFO:tasks.rados.rados.0.smithi070.stdout:3002: expect (ObjNum 459 snap 155 seq_num 459) 2024-09-14T03:58:31.145 INFO:tasks.rados.rados.0.smithi070.stdout:3003: rollback oid 22 current snap is 306 2024-09-14T03:58:31.145 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 22 to 303 2024-09-14T03:58:31.145 INFO:tasks.rados.rados.0.smithi070.stdout:2996: finishing rollback tid 2 to smithi07085154-18 2024-09-14T03:58:31.145 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 2406 (ObjNum 754 snap 260 seq_num 754) dirty exists 2024-09-14T03:58:31.145 INFO:tasks.rados.rados.0.smithi070.stdout:2994: finishing copy_from racing read to smithi07085154-45 2024-09-14T03:58:31.145 INFO:tasks.rados.rados.0.smithi070.stdout:2994: finishing copy_from to smithi07085154-45 2024-09-14T03:58:31.146 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 45 v 1701 (ObjNum 909 snap 304 seq_num 909) dirty exists 2024-09-14T03:58:31.146 INFO:tasks.rados.rados.0.smithi070.stdout:2998: finishing rollback tid 0 to smithi07085154-7 2024-09-14T03:58:31.146 INFO:tasks.rados.rados.0.smithi070.stdout:2998: finishing rollback tid 1 to smithi07085154-7 2024-09-14T03:58:31.146 INFO:tasks.rados.rados.0.smithi070.stdout:2998: finishing rollback tid 2 to smithi07085154-7 2024-09-14T03:58:31.146 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 3167 (ObjNum 289 snap 0 seq_num 1) dirty exists 2024-09-14T03:58:31.146 INFO:tasks.rados.rados.0.smithi070.stdout:2994: done (4 left) 2024-09-14T03:58:31.146 INFO:tasks.rados.rados.0.smithi070.stdout:2996: done (3 left) 2024-09-14T03:58:31.146 INFO:tasks.rados.rados.0.smithi070.stdout:2998: done (2 left) 2024-09-14T03:58:31.147 INFO:tasks.rados.rados.0.smithi070.stdout:3004: snap_create 2024-09-14T03:58:31.157 INFO:tasks.rados.rados.0.smithi070.stdout:3003: finishing rollback tid 0 to smithi07085154-22 2024-09-14T03:58:31.158 INFO:tasks.rados.rados.0.smithi070.stdout:3003: finishing rollback tid 1 to smithi07085154-22 2024-09-14T03:58:31.161 INFO:tasks.rados.rados.0.smithi070.stdout:3003: finishing rollback tid 2 to smithi07085154-22 2024-09-14T03:58:31.161 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 3165 (ObjNum 576 snap 193 seq_num 576) dirty exists 2024-09-14T03:58:31.219 INFO:tasks.rados.rados.0.smithi070.stdout:3002: expect (ObjNum 459 snap 155 seq_num 459) 2024-09-14T03:58:31.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:31 smithi094 ceph-mon[86597]: pgmap v611: 121 pgs: 121 active+clean; 996 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.1 MiB/s rd, 1.0 MiB/s wr, 17 op/s 2024-09-14T03:58:31.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:31 smithi094 ceph-mon[86597]: osdmap e676: 8 total, 8 up, 8 in 2024-09-14T03:58:31.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:31 smithi070 ceph-mon[96704]: pgmap v611: 121 pgs: 121 active+clean; 996 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.1 MiB/s rd, 1.0 MiB/s wr, 17 op/s 2024-09-14T03:58:31.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:31 smithi070 ceph-mon[96704]: osdmap e676: 8 total, 8 up, 8 in 2024-09-14T03:58:31.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:31 smithi070 ceph-mon[98640]: pgmap v611: 121 pgs: 121 active+clean; 996 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.1 MiB/s rd, 1.0 MiB/s wr, 17 op/s 2024-09-14T03:58:31.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:31 smithi070 ceph-mon[98640]: osdmap e676: 8 total, 8 up, 8 in 2024-09-14T03:58:32.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:32 smithi094 ceph-mon[86597]: osdmap e677: 8 total, 8 up, 8 in 2024-09-14T03:58:32.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:32 smithi070 ceph-mon[98640]: osdmap e677: 8 total, 8 up, 8 in 2024-09-14T03:58:32.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:32 smithi070 ceph-mon[96704]: osdmap e677: 8 total, 8 up, 8 in 2024-09-14T03:58:32.866 INFO:tasks.rados.rados.0.smithi070.stdout:3002: done (2 left) 2024-09-14T03:58:32.866 INFO:tasks.rados.rados.0.smithi070.stdout:3003: done (1 left) 2024-09-14T03:58:32.866 INFO:tasks.rados.rados.0.smithi070.stdout:3004: done (0 left) 2024-09-14T03:58:32.866 INFO:tasks.rados.rados.0.smithi070.stdout:3005: read oid 13 snap -1 2024-09-14T03:58:32.866 INFO:tasks.rados.rados.0.smithi070.stdout:3005: expect (ObjNum 898 snap 300 seq_num 898) 2024-09-14T03:58:32.866 INFO:tasks.rados.rados.0.smithi070.stdout:3006: read oid 42 snap -1 2024-09-14T03:58:32.866 INFO:tasks.rados.rados.0.smithi070.stdout:3006: expect (ObjNum 890 snap 298 seq_num 890) 2024-09-14T03:58:32.866 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 33 current snap is 307 2024-09-14T03:58:32.866 INFO:tasks.rados.rados.0.smithi070.stdout:3007: seq_num 915 ranges {786432=532480,1318912=401408} 2024-09-14T03:58:32.873 INFO:tasks.rados.rados.0.smithi070.stdout:3007: writing smithi07085154-33 from 786432 to 1318912 tid 1 2024-09-14T03:58:32.875 INFO:tasks.rados.rados.0.smithi070.stdout:3007: writing smithi07085154-33 from 1318912 to 1720320 tid 2 2024-09-14T03:58:32.875 INFO:tasks.rados.rados.0.smithi070.stdout:3008: rollback oid 27 current snap is 307 2024-09-14T03:58:32.875 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 27 to 304 2024-09-14T03:58:32.876 INFO:tasks.rados.rados.0.smithi070.stdout:3006: expect (ObjNum 890 snap 298 seq_num 890) 2024-09-14T03:58:32.890 INFO:tasks.rados.rados.0.smithi070.stdout:3005: expect (ObjNum 898 snap 300 seq_num 898) 2024-09-14T03:58:32.907 INFO:tasks.rados.rados.0.smithi070.stdout:3007: finishing write tid 1 to smithi07085154-33 2024-09-14T03:58:32.908 INFO:tasks.rados.rados.0.smithi070.stdout:3007: finishing write tid 2 to smithi07085154-33 2024-09-14T03:58:32.908 INFO:tasks.rados.rados.0.smithi070.stdout:3005: done (3 left) 2024-09-14T03:58:32.908 INFO:tasks.rados.rados.0.smithi070.stdout:3006: done (2 left) 2024-09-14T03:58:32.908 INFO:tasks.rados.rados.0.smithi070.stdout:3009: snap_create 2024-09-14T03:58:32.908 INFO:tasks.rados.rados.0.smithi070.stdout:3007: finishing write tid 3 to smithi07085154-33 2024-09-14T03:58:32.908 INFO:tasks.rados.rados.0.smithi070.stdout:3007: finishing write tid 4 to smithi07085154-33 2024-09-14T03:58:32.908 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 33 v 2093 (ObjNum 915 snap 307 seq_num 915) dirty exists 2024-09-14T03:58:32.908 INFO:tasks.rados.rados.0.smithi070.stdout:3007: left oid 33 (ObjNum 915 snap 307 seq_num 915) 2024-09-14T03:58:32.908 INFO:tasks.rados.rados.0.smithi070.stdout:3008: finishing rollback tid 0 to smithi07085154-27 2024-09-14T03:58:32.908 INFO:tasks.rados.rados.0.smithi070.stdout:3008: finishing rollback tid 1 to smithi07085154-27 2024-09-14T03:58:32.908 INFO:tasks.rados.rados.0.smithi070.stdout:3008: finishing rollback tid 2 to smithi07085154-27 2024-09-14T03:58:32.909 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 3626 (ObjNum 865 snap 293 seq_num 865) dirty exists 2024-09-14T03:58:33.157 INFO:tasks.rados.rados.0.smithi070.stdout:3007: done (2 left) 2024-09-14T03:58:33.157 INFO:tasks.rados.rados.0.smithi070.stdout:3008: done (1 left) 2024-09-14T03:58:33.157 INFO:tasks.rados.rados.0.smithi070.stdout:3009: done (0 left) 2024-09-14T03:58:33.157 INFO:tasks.rados.rados.0.smithi070.stdout:3010: snap_remove snap 271 2024-09-14T03:58:33.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:33 smithi070 ceph-mon[98640]: pgmap v614: 121 pgs: 1 active+clean+snaptrim_wait, 120 active+clean; 989 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.1 MiB/s wr, 23 op/s 2024-09-14T03:58:33.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:33 smithi070 ceph-mon[98640]: osdmap e678: 8 total, 8 up, 8 in 2024-09-14T03:58:33.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:33 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:58:33.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:33 smithi070 ceph-mon[96704]: pgmap v614: 121 pgs: 1 active+clean+snaptrim_wait, 120 active+clean; 989 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.1 MiB/s wr, 23 op/s 2024-09-14T03:58:33.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:33 smithi070 ceph-mon[96704]: osdmap e678: 8 total, 8 up, 8 in 2024-09-14T03:58:33.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:33 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:58:33.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:33 smithi094 ceph-mon[86597]: pgmap v614: 121 pgs: 1 active+clean+snaptrim_wait, 120 active+clean; 989 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.1 MiB/s wr, 23 op/s 2024-09-14T03:58:33.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:33 smithi094 ceph-mon[86597]: osdmap e678: 8 total, 8 up, 8 in 2024-09-14T03:58:33.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:33 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:58:33.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:58:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:58:33] "GET /metrics HTTP/1.1" 200 36500 "" "Prometheus/2.51.0" 2024-09-14T03:58:34.101 INFO:tasks.rados.rados.0.smithi070.stdout:3010: done (0 left) 2024-09-14T03:58:34.101 INFO:tasks.rados.rados.0.smithi070.stdout:3011: delete oid 16 current snap is 308 2024-09-14T03:58:34.123 INFO:tasks.rados.rados.0.smithi070.stdout:3011: done (0 left) 2024-09-14T03:58:34.123 INFO:tasks.rados.rados.0.smithi070.stdout:3012: read oid 10 snap -1 2024-09-14T03:58:34.123 INFO:tasks.rados.rados.0.smithi070.stdout:3012: expect (ObjNum 841 snap 286 seq_num 841) 2024-09-14T03:58:34.123 INFO:tasks.rados.rados.0.smithi070.stdout:3013: rmattr oid 43 current snap is 308 2024-09-14T03:58:34.123 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 12 current snap is 308 2024-09-14T03:58:34.123 INFO:tasks.rados.rados.0.smithi070.stdout:3014: seq_num 917 ranges {3457024=565248,4022272=401408} 2024-09-14T03:58:34.146 INFO:tasks.rados.rados.0.smithi070.stdout:3014: writing smithi07085154-12 from 3457024 to 4022272 tid 1 2024-09-14T03:58:34.148 INFO:tasks.rados.rados.0.smithi070.stdout:3014: writing smithi07085154-12 from 4022272 to 4423680 tid 2 2024-09-14T03:58:34.149 INFO:tasks.rados.rados.0.smithi070.stdout:3015: rmattr oid 38 current snap is 308 2024-09-14T03:58:34.149 INFO:tasks.rados.rados.0.smithi070.stdout:3015: done (3 left) 2024-09-14T03:58:34.149 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 26 current snap is 308 2024-09-14T03:58:34.149 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 2438 (ObjNum 681 snap 230 seq_num 681) dirty exists 2024-09-14T03:58:34.149 INFO:tasks.rados.rados.0.smithi070.stdout:3016: seq_num 918 ranges {2473984=581632,3055616=401408} 2024-09-14T03:58:34.166 INFO:tasks.rados.rados.0.smithi070.stdout:3016: writing smithi07085154-26 from 2473984 to 3055616 tid 1 2024-09-14T03:58:34.169 INFO:tasks.rados.rados.0.smithi070.stdout:3016: writing smithi07085154-26 from 3055616 to 3457024 tid 2 2024-09-14T03:58:34.169 INFO:tasks.rados.rados.0.smithi070.stdout:3013: done (3 left) 2024-09-14T03:58:34.169 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 9 current snap is 308 2024-09-14T03:58:34.169 INFO:tasks.rados.rados.0.smithi070.stdout:3014: finishing write tid 1 to smithi07085154-12 2024-09-14T03:58:34.169 INFO:tasks.rados.rados.0.smithi070.stdout:3014: finishing write tid 2 to smithi07085154-12 2024-09-14T03:58:34.169 INFO:tasks.rados.rados.0.smithi070.stdout:3014: finishing write tid 3 to smithi07085154-12 2024-09-14T03:58:34.169 INFO:tasks.rados.rados.0.smithi070.stdout:3017: seq_num 919 ranges {16482304=598016,17080320=401408} 2024-09-14T03:58:34.260 INFO:tasks.rados.rados.0.smithi070.stdout:3017: writing smithi07085154-9 from 16482304 to 17080320 tid 1 2024-09-14T03:58:34.262 INFO:tasks.rados.rados.0.smithi070.stdout:3017: writing smithi07085154-9 from 17080320 to 17481728 tid 2 2024-09-14T03:58:34.262 INFO:tasks.rados.rados.0.smithi070.stdout:3018: delete oid 46 current snap is 308 2024-09-14T03:58:34.262 INFO:tasks.rados.rados.0.smithi070.stdout:3014: finishing write tid 4 to smithi07085154-12 2024-09-14T03:58:34.262 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 2249 (ObjNum 917 snap 308 seq_num 917) dirty exists 2024-09-14T03:58:34.262 INFO:tasks.rados.rados.0.smithi070.stdout:3014: left oid 12 (ObjNum 917 snap 308 seq_num 917) 2024-09-14T03:58:34.262 INFO:tasks.rados.rados.0.smithi070.stdout:3012: expect (ObjNum 841 snap 286 seq_num 841) 2024-09-14T03:58:34.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:34 smithi094 ceph-mon[86597]: osdmap e679: 8 total, 8 up, 8 in 2024-09-14T03:58:34.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:34 smithi094 ceph-mon[86597]: osdmap e680: 8 total, 8 up, 8 in 2024-09-14T03:58:34.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:34 smithi094 ceph-mon[86597]: pgmap v618: 121 pgs: 1 active+clean+snaptrim_wait, 120 active+clean; 989 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 514 B/s rd, 478 KiB/s wr, 9 op/s 2024-09-14T03:58:34.535 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 03:58:34 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2[113018]: 2024-09-14T03:58:34.179+0000 7f5c8a4a0640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap 161 after trimming was started 2024-09-14T03:58:34.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:34 smithi070 ceph-mon[96704]: osdmap e679: 8 total, 8 up, 8 in 2024-09-14T03:58:34.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:34 smithi070 ceph-mon[96704]: osdmap e680: 8 total, 8 up, 8 in 2024-09-14T03:58:34.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:34 smithi070 ceph-mon[96704]: pgmap v618: 121 pgs: 1 active+clean+snaptrim_wait, 120 active+clean; 989 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 514 B/s rd, 478 KiB/s wr, 9 op/s 2024-09-14T03:58:34.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:34 smithi070 ceph-mon[98640]: osdmap e679: 8 total, 8 up, 8 in 2024-09-14T03:58:34.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:34 smithi070 ceph-mon[98640]: osdmap e680: 8 total, 8 up, 8 in 2024-09-14T03:58:34.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:34 smithi070 ceph-mon[98640]: pgmap v618: 121 pgs: 1 active+clean+snaptrim_wait, 120 active+clean; 989 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 514 B/s rd, 478 KiB/s wr, 9 op/s 2024-09-14T03:58:34.695 INFO:tasks.rados.rados.0.smithi070.stdout:3016: finishing write tid 1 to smithi07085154-26 2024-09-14T03:58:34.695 INFO:tasks.rados.rados.0.smithi070.stdout:3016: finishing write tid 2 to smithi07085154-26 2024-09-14T03:58:34.695 INFO:tasks.rados.rados.0.smithi070.stdout:3016: finishing write tid 3 to smithi07085154-26 2024-09-14T03:58:34.695 INFO:tasks.rados.rados.0.smithi070.stdout:3016: finishing write tid 4 to smithi07085154-26 2024-09-14T03:58:34.695 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 3644 (ObjNum 918 snap 308 seq_num 918) dirty exists 2024-09-14T03:58:34.695 INFO:tasks.rados.rados.0.smithi070.stdout:3016: left oid 26 (ObjNum 918 snap 308 seq_num 918) 2024-09-14T03:58:34.696 INFO:tasks.rados.rados.0.smithi070.stdout:3017: finishing write tid 1 to smithi07085154-9 2024-09-14T03:58:34.696 INFO:tasks.rados.rados.0.smithi070.stdout:3017: finishing write tid 2 to smithi07085154-9 2024-09-14T03:58:34.696 INFO:tasks.rados.rados.0.smithi070.stdout:3017: finishing write tid 3 to smithi07085154-9 2024-09-14T03:58:34.696 INFO:tasks.rados.rados.0.smithi070.stdout:3017: finishing write tid 4 to smithi07085154-9 2024-09-14T03:58:34.696 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 3650 (ObjNum 919 snap 308 seq_num 919) dirty exists 2024-09-14T03:58:34.696 INFO:tasks.rados.rados.0.smithi070.stdout:3017: left oid 9 (ObjNum 919 snap 308 seq_num 919) 2024-09-14T03:58:34.696 INFO:tasks.rados.rados.0.smithi070.stdout:3012: done (4 left) 2024-09-14T03:58:34.696 INFO:tasks.rados.rados.0.smithi070.stdout:3014: done (3 left) 2024-09-14T03:58:34.696 INFO:tasks.rados.rados.0.smithi070.stdout:3016: done (2 left) 2024-09-14T03:58:34.696 INFO:tasks.rados.rados.0.smithi070.stdout:3017: done (1 left) 2024-09-14T03:58:34.696 INFO:tasks.rados.rados.0.smithi070.stdout:3018: done (0 left) 2024-09-14T03:58:34.696 INFO:tasks.rados.rados.0.smithi070.stdout:3019: read oid 4 snap -1 2024-09-14T03:58:34.696 INFO:tasks.rados.rados.0.smithi070.stdout:3019: expect (ObjNum 902 snap 301 seq_num 902) 2024-09-14T03:58:34.697 INFO:tasks.rados.rados.0.smithi070.stdout:3020: read oid 40 snap -1 2024-09-14T03:58:34.697 INFO:tasks.rados.rados.0.smithi070.stdout:3020: expect (ObjNum 912 snap 305 seq_num 912) 2024-09-14T03:58:34.697 INFO:tasks.rados.rados.0.smithi070.stdout:3021: rmattr oid 13 current snap is 308 2024-09-14T03:58:34.697 INFO:tasks.rados.rados.0.smithi070.stdout:3022: read oid 38 snap -1 2024-09-14T03:58:34.697 INFO:tasks.rados.rados.0.smithi070.stdout:3022: expect deleted 2024-09-14T03:58:34.697 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 45 current snap is 308 2024-09-14T03:58:34.697 INFO:tasks.rados.rados.0.smithi070.stdout:3023: seq_num 921 ranges {827392=630784,1458176=401408} 2024-09-14T03:58:34.703 INFO:tasks.rados.rados.0.smithi070.stdout:3023: writing smithi07085154-45 from 827392 to 1458176 tid 1 2024-09-14T03:58:34.705 INFO:tasks.rados.rados.0.smithi070.stdout:3023: writing smithi07085154-45 from 1458176 to 1859584 tid 2 2024-09-14T03:58:34.705 INFO:tasks.rados.rados.0.smithi070.stdout:3024: rollback oid 11 current snap is 308 2024-09-14T03:58:34.705 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 11 to 292 2024-09-14T03:58:34.705 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 3162 (ObjNum 898 snap 300 seq_num 898) dirty exists 2024-09-14T03:58:34.705 INFO:tasks.rados.rados.0.smithi070.stdout:3019: expect (ObjNum 902 snap 301 seq_num 902) 2024-09-14T03:58:34.739 INFO:tasks.rados.rados.0.smithi070.stdout:3020: expect (ObjNum 912 snap 305 seq_num 912) 2024-09-14T03:58:34.767 INFO:tasks.rados.rados.0.smithi070.stdout:3023: finishing write tid 1 to smithi07085154-45 2024-09-14T03:58:34.767 INFO:tasks.rados.rados.0.smithi070.stdout:3024: finishing rollback tid 1 to smithi07085154-11 2024-09-14T03:58:34.767 INFO:tasks.rados.rados.0.smithi070.stdout:3023: finishing write tid 2 to smithi07085154-45 2024-09-14T03:58:34.767 INFO:tasks.rados.rados.0.smithi070.stdout:3023: finishing write tid 3 to smithi07085154-45 2024-09-14T03:58:34.768 INFO:tasks.rados.rados.0.smithi070.stdout:3024: finishing rollback tid 2 to smithi07085154-11 2024-09-14T03:58:34.768 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 11 v 3164 (ObjNum 293 snap 91 seq_num 293) dirty exists 2024-09-14T03:58:34.768 INFO:tasks.rados.rados.0.smithi070.stdout:3023: finishing write tid 4 to smithi07085154-45 2024-09-14T03:58:34.768 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 45 v 2096 (ObjNum 921 snap 308 seq_num 921) dirty exists 2024-09-14T03:58:34.768 INFO:tasks.rados.rados.0.smithi070.stdout:3023: left oid 45 (ObjNum 921 snap 308 seq_num 921) 2024-09-14T03:58:34.768 INFO:tasks.rados.rados.0.smithi070.stdout:3019: done (5 left) 2024-09-14T03:58:34.768 INFO:tasks.rados.rados.0.smithi070.stdout:3020: done (4 left) 2024-09-14T03:58:34.768 INFO:tasks.rados.rados.0.smithi070.stdout:3021: done (3 left) 2024-09-14T03:58:34.768 INFO:tasks.rados.rados.0.smithi070.stdout:3022: done (2 left) 2024-09-14T03:58:34.768 INFO:tasks.rados.rados.0.smithi070.stdout:3023: done (1 left) 2024-09-14T03:58:34.768 INFO:tasks.rados.rados.0.smithi070.stdout:3024: done (0 left) 2024-09-14T03:58:34.768 INFO:tasks.rados.rados.0.smithi070.stdout:3025: read oid 27 snap -1 2024-09-14T03:58:34.768 INFO:tasks.rados.rados.0.smithi070.stdout:3025: expect (ObjNum 865 snap 293 seq_num 865) 2024-09-14T03:58:34.768 INFO:tasks.rados.rados.0.smithi070.stdout:3026: read oid 9 snap -1 2024-09-14T03:58:34.768 INFO:tasks.rados.rados.0.smithi070.stdout:3026: expect (ObjNum 919 snap 308 seq_num 919) 2024-09-14T03:58:34.769 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 43 current snap is 308 2024-09-14T03:58:34.769 INFO:tasks.rados.rados.0.smithi070.stdout:3027: seq_num 922 ranges {1818624=647168,2465792=401408} 2024-09-14T03:58:34.781 INFO:tasks.rados.rados.0.smithi070.stdout:3027: writing smithi07085154-43 from 1818624 to 2465792 tid 1 2024-09-14T03:58:34.783 INFO:tasks.rados.rados.0.smithi070.stdout:3027: writing smithi07085154-43 from 2465792 to 2867200 tid 2 2024-09-14T03:58:34.783 INFO:tasks.rados.rados.0.smithi070.stdout:3028: snap_create 2024-09-14T03:58:34.789 INFO:tasks.rados.rados.0.smithi070.stdout:3027: finishing write tid 1 to smithi07085154-43 2024-09-14T03:58:34.789 INFO:tasks.rados.rados.0.smithi070.stdout:3027: finishing write tid 2 to smithi07085154-43 2024-09-14T03:58:34.789 INFO:tasks.rados.rados.0.smithi070.stdout:3027: finishing write tid 3 to smithi07085154-43 2024-09-14T03:58:34.791 INFO:tasks.rados.rados.0.smithi070.stdout:3027: finishing write tid 4 to smithi07085154-43 2024-09-14T03:58:34.791 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 2441 (ObjNum 922 snap 308 seq_num 922) dirty exists 2024-09-14T03:58:34.791 INFO:tasks.rados.rados.0.smithi070.stdout:3027: left oid 43 (ObjNum 922 snap 308 seq_num 922) 2024-09-14T03:58:34.804 INFO:tasks.rados.rados.0.smithi070.stdout:3025: expect (ObjNum 865 snap 293 seq_num 865) 2024-09-14T03:58:35.196 INFO:tasks.rados.rados.0.smithi070.stdout:3026: expect (ObjNum 919 snap 308 seq_num 919) 2024-09-14T03:58:36.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:36 smithi094 ceph-mon[86597]: osdmap e681: 8 total, 8 up, 8 in 2024-09-14T03:58:36.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:36 smithi094 ceph-mon[86597]: pgmap v620: 121 pgs: 1 active+clean+snaptrim_wait, 120 active+clean; 989 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 484 KiB/s wr, 2 op/s 2024-09-14T03:58:36.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:36 smithi070 ceph-mon[98640]: osdmap e681: 8 total, 8 up, 8 in 2024-09-14T03:58:36.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:36 smithi070 ceph-mon[98640]: pgmap v620: 121 pgs: 1 active+clean+snaptrim_wait, 120 active+clean; 989 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 484 KiB/s wr, 2 op/s 2024-09-14T03:58:36.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:36 smithi070 ceph-mon[96704]: osdmap e681: 8 total, 8 up, 8 in 2024-09-14T03:58:36.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:36 smithi070 ceph-mon[96704]: pgmap v620: 121 pgs: 1 active+clean+snaptrim_wait, 120 active+clean; 989 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 484 KiB/s wr, 2 op/s 2024-09-14T03:58:37.383 INFO:tasks.rados.rados.0.smithi070.stdout:3025: done (3 left) 2024-09-14T03:58:37.383 INFO:tasks.rados.rados.0.smithi070.stdout:3026: done (2 left) 2024-09-14T03:58:37.384 INFO:tasks.rados.rados.0.smithi070.stdout:3027: done (1 left) 2024-09-14T03:58:37.384 INFO:tasks.rados.rados.0.smithi070.stdout:3028: done (0 left) 2024-09-14T03:58:37.384 INFO:tasks.rados.rados.0.smithi070.stdout:3029: copy_from oid 24 from oid 15 current snap is 309 2024-09-14T03:58:37.384 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 8 current snap is 309 2024-09-14T03:58:37.384 INFO:tasks.rados.rados.0.smithi070.stdout:3030: seq_num 923 ranges {131072=663552,794624=401408} 2024-09-14T03:58:37.387 INFO:tasks.rados.rados.0.smithi070.stdout:3030: writing smithi07085154-8 from 131072 to 794624 tid 1 2024-09-14T03:58:37.390 INFO:tasks.rados.rados.0.smithi070.stdout:3030: writing smithi07085154-8 from 794624 to 1196032 tid 2 2024-09-14T03:58:37.391 INFO:tasks.rados.rados.0.smithi070.stdout:3031: delete oid 50 current snap is 309 2024-09-14T03:58:37.393 INFO:tasks.rados.rados.0.smithi070.stdout:3029: finishing copy_from racing read to smithi07085154-24 2024-09-14T03:58:37.393 INFO:tasks.rados.rados.0.smithi070.stdout:3030: finishing write tid 1 to smithi07085154-8 2024-09-14T03:58:37.396 INFO:tasks.rados.rados.0.smithi070.stdout:3031: done (2 left) 2024-09-14T03:58:37.396 INFO:tasks.rados.rados.0.smithi070.stdout:3032: read oid 4 snap -1 2024-09-14T03:58:37.396 INFO:tasks.rados.rados.0.smithi070.stdout:3032: expect (ObjNum 902 snap 301 seq_num 902) 2024-09-14T03:58:37.396 INFO:tasks.rados.rados.0.smithi070.stdout:3033: rollback oid 17 current snap is 309 2024-09-14T03:58:37.396 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 17 to 232 2024-09-14T03:58:37.396 INFO:tasks.rados.rados.0.smithi070.stdout:3029: finishing copy_from to smithi07085154-24 2024-09-14T03:58:37.397 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 2227 (ObjNum 805 snap 281 seq_num 805) dirty exists 2024-09-14T03:58:37.397 INFO:tasks.rados.rados.0.smithi070.stdout:3029: done (3 left) 2024-09-14T03:58:37.397 INFO:tasks.rados.rados.0.smithi070.stdout:3034: snap_create 2024-09-14T03:58:37.397 INFO:tasks.rados.rados.0.smithi070.stdout:3030: finishing write tid 2 to smithi07085154-8 2024-09-14T03:58:37.397 INFO:tasks.rados.rados.0.smithi070.stdout:3030: finishing write tid 3 to smithi07085154-8 2024-09-14T03:58:37.397 INFO:tasks.rados.rados.0.smithi070.stdout:3030: finishing write tid 4 to smithi07085154-8 2024-09-14T03:58:37.397 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 8 v 2445 (ObjNum 923 snap 309 seq_num 923) dirty exists 2024-09-14T03:58:37.397 INFO:tasks.rados.rados.0.smithi070.stdout:3030: left oid 8 (ObjNum 923 snap 309 seq_num 923) 2024-09-14T03:58:37.398 INFO:tasks.rados.rados.0.smithi070.stdout:3032: expect (ObjNum 902 snap 301 seq_num 902) 2024-09-14T03:58:37.429 INFO:tasks.rados.rados.0.smithi070.stdout:3033: finishing rollback tid 0 to smithi07085154-17 2024-09-14T03:58:37.429 INFO:tasks.rados.rados.0.smithi070.stdout:3033: finishing rollback tid 1 to smithi07085154-17 2024-09-14T03:58:37.429 INFO:tasks.rados.rados.0.smithi070.stdout:3033: finishing rollback tid 2 to smithi07085154-17 2024-09-14T03:58:37.429 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 3165 (ObjNum 672 snap 228 seq_num 672) dirty exists 2024-09-14T03:58:37.449 INFO:tasks.rados.rados.0.smithi070.stdout:3030: done (3 left) 2024-09-14T03:58:37.450 INFO:tasks.rados.rados.0.smithi070.stdout:3032: done (2 left) 2024-09-14T03:58:37.450 INFO:tasks.rados.rados.0.smithi070.stdout:3033: done (1 left) 2024-09-14T03:58:37.450 INFO:tasks.rados.rados.0.smithi070.stdout:3034: done (0 left) 2024-09-14T03:58:37.450 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 50 current snap is 310 2024-09-14T03:58:37.450 INFO:tasks.rados.rados.0.smithi070.stdout:3035: seq_num 925 ranges {0=696320,696320=401408} 2024-09-14T03:58:37.453 INFO:tasks.rados.rados.0.smithi070.stdout:3035: writing smithi07085154-50 from 0 to 696320 tid 1 2024-09-14T03:58:37.455 INFO:tasks.rados.rados.0.smithi070.stdout:3035: writing smithi07085154-50 from 696320 to 1097728 tid 2 2024-09-14T03:58:37.455 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 32 current snap is 310 2024-09-14T03:58:37.455 INFO:tasks.rados.rados.0.smithi070.stdout:3036: seq_num 926 ranges {49152=712704,761856=401408} 2024-09-14T03:58:37.460 INFO:tasks.rados.rados.0.smithi070.stdout:3036: writing smithi07085154-32 from 49152 to 761856 tid 1 2024-09-14T03:58:37.463 INFO:tasks.rados.rados.0.smithi070.stdout:3036: writing smithi07085154-32 from 761856 to 1163264 tid 2 2024-09-14T03:58:37.463 INFO:tasks.rados.rados.0.smithi070.stdout:3037: copy_from oid 14 from oid 7 current snap is 310 2024-09-14T03:58:37.463 INFO:tasks.rados.rados.0.smithi070.stdout:3035: finishing write tid 1 to smithi07085154-50 2024-09-14T03:58:37.463 INFO:tasks.rados.rados.0.smithi070.stdout:3035: finishing write tid 2 to smithi07085154-50 2024-09-14T03:58:37.463 INFO:tasks.rados.rados.0.smithi070.stdout:3035: finishing write tid 3 to smithi07085154-50 2024-09-14T03:58:37.463 INFO:tasks.rados.rados.0.smithi070.stdout:3038: copy_from oid 34 from oid 37 current snap is 310 2024-09-14T03:58:37.463 INFO:tasks.rados.rados.0.smithi070.stdout:3039: copy_from oid 20 from oid 19 current snap is 310 2024-09-14T03:58:37.463 INFO:tasks.rados.rados.0.smithi070.stdout:3040: snap_remove snap 303 2024-09-14T03:58:38.459 INFO:tasks.rados.rados.0.smithi070.stdout:3040: done (5 left) 2024-09-14T03:58:38.460 INFO:tasks.rados.rados.0.smithi070.stdout:3041: rollback oid 25 current snap is 310 2024-09-14T03:58:38.460 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 25 to 304 2024-09-14T03:58:38.460 INFO:tasks.rados.rados.0.smithi070.stdout:3035: finishing write tid 4 to smithi07085154-50 2024-09-14T03:58:38.460 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 3169 (ObjNum 925 snap 310 seq_num 925) dirty exists 2024-09-14T03:58:38.460 INFO:tasks.rados.rados.0.smithi070.stdout:3035: left oid 50 (ObjNum 925 snap 310 seq_num 925) 2024-09-14T03:58:38.460 INFO:tasks.rados.rados.0.smithi070.stdout:3038: finishing copy_from to smithi07085154-34 2024-09-14T03:58:38.460 INFO:tasks.rados.rados.0.smithi070.stdout:3038: got expected ENOENT (src dne) 2024-09-14T03:58:38.460 INFO:tasks.rados.rados.0.smithi070.stdout:3038: finishing copy_from racing read to smithi07085154-34 2024-09-14T03:58:38.460 INFO:tasks.rados.rados.0.smithi070.stdout:3036: finishing write tid 1 to smithi07085154-32 2024-09-14T03:58:38.460 INFO:tasks.rados.rados.0.smithi070.stdout:3037: finishing copy_from racing read to smithi07085154-14 2024-09-14T03:58:38.460 INFO:tasks.rados.rados.0.smithi070.stdout:3037: finishing copy_from to smithi07085154-14 2024-09-14T03:58:38.461 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 3167 (ObjNum 97 snap 0 seq_num 2931329344) dirty exists 2024-09-14T03:58:38.461 INFO:tasks.rados.rados.0.smithi070.stdout:3036: finishing write tid 2 to smithi07085154-32 2024-09-14T03:58:38.461 INFO:tasks.rados.rados.0.smithi070.stdout:3036: finishing write tid 3 to smithi07085154-32 2024-09-14T03:58:38.461 INFO:tasks.rados.rados.0.smithi070.stdout:3036: finishing write tid 4 to smithi07085154-32 2024-09-14T03:58:38.461 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 3167 (ObjNum 926 snap 310 seq_num 926) dirty exists 2024-09-14T03:58:38.461 INFO:tasks.rados.rados.0.smithi070.stdout:3036: left oid 32 (ObjNum 926 snap 310 seq_num 926) 2024-09-14T03:58:38.461 INFO:tasks.rados.rados.0.smithi070.stdout:3039: finishing copy_from racing read to smithi07085154-20 2024-09-14T03:58:38.461 INFO:tasks.rados.rados.0.smithi070.stdout:3039: finishing copy_from to smithi07085154-20 2024-09-14T03:58:38.461 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 20 v 3159 (ObjNum 901 snap 301 seq_num 901) dirty exists 2024-09-14T03:58:38.461 INFO:tasks.rados.rados.0.smithi070.stdout:3035: done (5 left) 2024-09-14T03:58:38.461 INFO:tasks.rados.rados.0.smithi070.stdout:3036: done (4 left) 2024-09-14T03:58:38.461 INFO:tasks.rados.rados.0.smithi070.stdout:3037: done (3 left) 2024-09-14T03:58:38.461 INFO:tasks.rados.rados.0.smithi070.stdout:3038: done (2 left) 2024-09-14T03:58:38.462 INFO:tasks.rados.rados.0.smithi070.stdout:3039: done (1 left) 2024-09-14T03:58:38.462 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 21 current snap is 310 2024-09-14T03:58:38.462 INFO:tasks.rados.rados.0.smithi070.stdout:3042: seq_num 927 ranges {729088=729088,1458176=401408} 2024-09-14T03:58:38.467 INFO:tasks.rados.rados.0.smithi070.stdout:3042: writing smithi07085154-21 from 729088 to 1458176 tid 1 2024-09-14T03:58:38.471 INFO:tasks.rados.rados.0.smithi070.stdout:3042: writing smithi07085154-21 from 1458176 to 1859584 tid 2 2024-09-14T03:58:38.471 INFO:tasks.rados.rados.0.smithi070.stdout:3043: read oid 44 snap -1 2024-09-14T03:58:38.471 INFO:tasks.rados.rados.0.smithi070.stdout:3043: expect deleted 2024-09-14T03:58:38.471 INFO:tasks.rados.rados.0.smithi070.stdout:3044: delete oid 38 current snap is 310 2024-09-14T03:58:38.471 INFO:tasks.rados.rados.0.smithi070.stdout:3041: finishing rollback tid 0 to smithi07085154-25 2024-09-14T03:58:38.471 INFO:tasks.rados.rados.0.smithi070.stdout:3041: finishing rollback tid 1 to smithi07085154-25 2024-09-14T03:58:38.473 INFO:tasks.rados.rados.0.smithi070.stdout:3041: finishing rollback tid 2 to smithi07085154-25 2024-09-14T03:58:38.473 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 3168 (ObjNum 902 snap 301 seq_num 902) dirty exists 2024-09-14T03:58:38.477 INFO:tasks.rados.rados.0.smithi070.stdout:3042: finishing write tid 1 to smithi07085154-21 2024-09-14T03:58:38.478 INFO:tasks.rados.rados.0.smithi070.stdout:3042: finishing write tid 2 to smithi07085154-21 2024-09-14T03:58:38.478 INFO:tasks.rados.rados.0.smithi070.stdout:3042: finishing write tid 3 to smithi07085154-21 2024-09-14T03:58:38.479 INFO:tasks.rados.rados.0.smithi070.stdout:3041: done (3 left) 2024-09-14T03:58:38.479 INFO:tasks.rados.rados.0.smithi070.stdout:3043: done (2 left) 2024-09-14T03:58:38.479 INFO:tasks.rados.rados.0.smithi070.stdout:3044: done (1 left) 2024-09-14T03:58:38.479 INFO:tasks.rados.rados.0.smithi070.stdout:3045: rollback oid 1 current snap is 310 2024-09-14T03:58:38.480 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 1 to 284 2024-09-14T03:58:38.480 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 34 current snap is 310 2024-09-14T03:58:38.480 INFO:tasks.rados.rados.0.smithi070.stdout:3046: seq_num 929 ranges {1196032=761856,1957888=401408} 2024-09-14T03:58:38.490 INFO:tasks.rados.rados.0.smithi070.stdout:3046: writing smithi07085154-34 from 1196032 to 1957888 tid 1 2024-09-14T03:58:38.493 INFO:tasks.rados.rados.0.smithi070.stdout:3046: writing smithi07085154-34 from 1957888 to 2359296 tid 2 2024-09-14T03:58:38.493 INFO:tasks.rados.rados.0.smithi070.stdout:3047: snap_create 2024-09-14T03:58:38.493 INFO:tasks.rados.rados.0.smithi070.stdout:3042: finishing write tid 4 to smithi07085154-21 2024-09-14T03:58:38.493 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 1720 (ObjNum 927 snap 310 seq_num 927) dirty exists 2024-09-14T03:58:38.493 INFO:tasks.rados.rados.0.smithi070.stdout:3042: left oid 21 (ObjNum 927 snap 310 seq_num 927) 2024-09-14T03:58:38.493 INFO:tasks.rados.rados.0.smithi070.stdout:3045: finishing rollback tid 0 to smithi07085154-1 2024-09-14T03:58:38.494 INFO:tasks.rados.rados.0.smithi070.stdout:3045: finishing rollback tid 1 to smithi07085154-1 2024-09-14T03:58:38.494 INFO:tasks.rados.rados.0.smithi070.stdout:3045: finishing rollback tid 2 to smithi07085154-1 2024-09-14T03:58:38.494 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 1 v 3166 (ObjNum 712 snap 243 seq_num 712) dirty exists 2024-09-14T03:58:38.497 INFO:tasks.rados.rados.0.smithi070.stdout:3046: finishing write tid 1 to smithi07085154-34 2024-09-14T03:58:38.497 INFO:tasks.rados.rados.0.smithi070.stdout:3046: finishing write tid 2 to smithi07085154-34 2024-09-14T03:58:38.497 INFO:tasks.rados.rados.0.smithi070.stdout:3046: finishing write tid 3 to smithi07085154-34 2024-09-14T03:58:38.498 INFO:tasks.rados.rados.0.smithi070.stdout:3046: finishing write tid 4 to smithi07085154-34 2024-09-14T03:58:38.498 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 3172 (ObjNum 929 snap 310 seq_num 929) dirty exists 2024-09-14T03:58:38.498 INFO:tasks.rados.rados.0.smithi070.stdout:3046: left oid 34 (ObjNum 929 snap 310 seq_num 929) 2024-09-14T03:58:38.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:38 smithi094 ceph-mon[86597]: osdmap e682: 8 total, 8 up, 8 in 2024-09-14T03:58:38.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:38 smithi094 ceph-mon[86597]: pgmap v622: 121 pgs: 121 active+clean; 993 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 29 MiB/s rd, 780 KiB/s wr, 11 op/s 2024-09-14T03:58:38.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:38 smithi070 ceph-mon[96704]: osdmap e682: 8 total, 8 up, 8 in 2024-09-14T03:58:38.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:38 smithi070 ceph-mon[96704]: pgmap v622: 121 pgs: 121 active+clean; 993 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 29 MiB/s rd, 780 KiB/s wr, 11 op/s 2024-09-14T03:58:38.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:38 smithi070 ceph-mon[98640]: osdmap e682: 8 total, 8 up, 8 in 2024-09-14T03:58:38.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:38 smithi070 ceph-mon[98640]: pgmap v622: 121 pgs: 121 active+clean; 993 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 29 MiB/s rd, 780 KiB/s wr, 11 op/s 2024-09-14T03:58:39.104 INFO:tasks.rados.rados.0.smithi070.stdout:3042: done (3 left) 2024-09-14T03:58:39.104 INFO:tasks.rados.rados.0.smithi070.stdout:3045: done (2 left) 2024-09-14T03:58:39.104 INFO:tasks.rados.rados.0.smithi070.stdout:3046: done (1 left) 2024-09-14T03:58:39.104 INFO:tasks.rados.rados.0.smithi070.stdout:3047: done (0 left) 2024-09-14T03:58:39.104 INFO:tasks.rados.rados.0.smithi070.stdout:3048: rmattr oid 32 current snap is 311 2024-09-14T03:58:39.104 INFO:tasks.rados.rados.0.smithi070.stdout:3049: rollback oid 17 current snap is 311 2024-09-14T03:58:39.104 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 17 to 290 2024-09-14T03:58:39.104 INFO:tasks.rados.rados.0.smithi070.stdout:3050: copy_from oid 9 from oid 8 current snap is 311 2024-09-14T03:58:39.104 INFO:tasks.rados.rados.0.smithi070.stdout:3051: read oid 30 snap 298 2024-09-14T03:58:39.105 INFO:tasks.rados.rados.0.smithi070.stdout:3051: expect (ObjNum 876 snap 295 seq_num 876) 2024-09-14T03:58:39.105 INFO:tasks.rados.rados.0.smithi070.stdout:3052: snap_create 2024-09-14T03:58:39.110 INFO:tasks.rados.rados.0.smithi070.stdout:3049: finishing rollback tid 0 to smithi07085154-17 2024-09-14T03:58:39.111 INFO:tasks.rados.rados.0.smithi070.stdout:3051: expect (ObjNum 876 snap 295 seq_num 876) 2024-09-14T03:58:39.123 INFO:tasks.rados.rados.0.smithi070.stdout:3049: finishing rollback tid 1 to smithi07085154-17 2024-09-14T03:58:39.123 INFO:tasks.rados.rados.0.smithi070.stdout:3049: finishing rollback tid 2 to smithi07085154-17 2024-09-14T03:58:39.123 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 3171 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-14T03:58:39.123 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 3168 (ObjNum 926 snap 310 seq_num 926) dirty exists 2024-09-14T03:58:39.123 INFO:tasks.rados.rados.0.smithi070.stdout:3050: finishing copy_from racing read to smithi07085154-9 2024-09-14T03:58:39.123 INFO:tasks.rados.rados.0.smithi070.stdout:3050: finishing copy_from to smithi07085154-9 2024-09-14T03:58:39.123 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 2445 (ObjNum 923 snap 309 seq_num 923) dirty exists 2024-09-14T03:58:39.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:39 smithi094 ceph-mon[86597]: osdmap e683: 8 total, 8 up, 8 in 2024-09-14T03:58:39.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:39 smithi094 ceph-mon[86597]: osdmap e684: 8 total, 8 up, 8 in 2024-09-14T03:58:39.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:39 smithi070 ceph-mon[98640]: osdmap e683: 8 total, 8 up, 8 in 2024-09-14T03:58:39.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:39 smithi070 ceph-mon[98640]: osdmap e684: 8 total, 8 up, 8 in 2024-09-14T03:58:39.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:39 smithi070 ceph-mon[96704]: osdmap e683: 8 total, 8 up, 8 in 2024-09-14T03:58:39.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:39 smithi070 ceph-mon[96704]: osdmap e684: 8 total, 8 up, 8 in 2024-09-14T03:58:40.105 INFO:tasks.rados.rados.0.smithi070.stdout:3048: done (4 left) 2024-09-14T03:58:40.106 INFO:tasks.rados.rados.0.smithi070.stdout:3049: done (3 left) 2024-09-14T03:58:40.106 INFO:tasks.rados.rados.0.smithi070.stdout:3050: done (2 left) 2024-09-14T03:58:40.106 INFO:tasks.rados.rados.0.smithi070.stdout:3051: done (1 left) 2024-09-14T03:58:40.106 INFO:tasks.rados.rados.0.smithi070.stdout:3052: done (0 left) 2024-09-14T03:58:40.106 INFO:tasks.rados.rados.0.smithi070.stdout:3053: snap_remove snap 268 2024-09-14T03:58:40.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:58:40 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:58:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:58:41.112 INFO:tasks.rados.rados.0.smithi070.stdout:3053: done (0 left) 2024-09-14T03:58:41.112 INFO:tasks.rados.rados.0.smithi070.stdout:3054: copy_from oid 20 from oid 32 current snap is 312 2024-09-14T03:58:41.112 INFO:tasks.rados.rados.0.smithi070.stdout:3055: read oid 42 snap -1 2024-09-14T03:58:41.112 INFO:tasks.rados.rados.0.smithi070.stdout:3055: expect (ObjNum 890 snap 298 seq_num 890) 2024-09-14T03:58:41.112 INFO:tasks.rados.rados.0.smithi070.stdout:3056: read oid 19 snap -1 2024-09-14T03:58:41.112 INFO:tasks.rados.rados.0.smithi070.stdout:3056: expect (ObjNum 901 snap 301 seq_num 901) 2024-09-14T03:58:41.113 INFO:tasks.rados.rados.0.smithi070.stdout:3057: setattr oid 13 current snap is 312 2024-09-14T03:58:41.115 INFO:tasks.rados.rados.0.smithi070.stdout:3058: read oid 14 snap -1 2024-09-14T03:58:41.115 INFO:tasks.rados.rados.0.smithi070.stdout:3058: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:58:41.115 INFO:tasks.rados.rados.0.smithi070.stdout:3059: read oid 27 snap -1 2024-09-14T03:58:41.115 INFO:tasks.rados.rados.0.smithi070.stdout:3059: expect (ObjNum 865 snap 293 seq_num 865) 2024-09-14T03:58:41.115 INFO:tasks.rados.rados.0.smithi070.stdout:3060: snap_create 2024-09-14T03:58:41.120 INFO:tasks.rados.rados.0.smithi070.stdout:3058: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:58:41.129 INFO:tasks.rados.rados.0.smithi070.stdout:3055: expect (ObjNum 890 snap 298 seq_num 890) 2024-09-14T03:58:41.150 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 3169 (ObjNum 898 snap 300 seq_num 898) dirty exists 2024-09-14T03:58:41.150 INFO:tasks.rados.rados.0.smithi070.stdout:3054: finishing copy_from racing read to smithi07085154-20 2024-09-14T03:58:41.153 INFO:tasks.rados.rados.0.smithi070.stdout:3054: finishing copy_from to smithi07085154-20 2024-09-14T03:58:41.153 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 20 v 3168 (ObjNum 926 snap 310 seq_num 926) dirty exists 2024-09-14T03:58:41.158 INFO:tasks.rados.rados.0.smithi070.stdout:3059: expect (ObjNum 865 snap 293 seq_num 865) 2024-09-14T03:58:41.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:41 smithi094 ceph-mon[86597]: osdmap e685: 8 total, 8 up, 8 in 2024-09-14T03:58:41.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:41 smithi094 ceph-mon[86597]: pgmap v626: 121 pgs: 121 active+clean; 1000 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 37 MiB/s rd, 1.2 MiB/s wr, 15 op/s 2024-09-14T03:58:41.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:41 smithi070 ceph-mon[98640]: osdmap e685: 8 total, 8 up, 8 in 2024-09-14T03:58:41.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:41 smithi070 ceph-mon[98640]: pgmap v626: 121 pgs: 121 active+clean; 1000 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 37 MiB/s rd, 1.2 MiB/s wr, 15 op/s 2024-09-14T03:58:41.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:41 smithi070 ceph-mon[96704]: osdmap e685: 8 total, 8 up, 8 in 2024-09-14T03:58:41.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:41 smithi070 ceph-mon[96704]: pgmap v626: 121 pgs: 121 active+clean; 1000 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 37 MiB/s rd, 1.2 MiB/s wr, 15 op/s 2024-09-14T03:58:41.545 INFO:tasks.rados.rados.0.smithi070.stdout:3056: expect (ObjNum 901 snap 301 seq_num 901) 2024-09-14T03:58:42.347 INFO:tasks.rados.rados.0.smithi070.stdout:3054: done (6 left) 2024-09-14T03:58:42.347 INFO:tasks.rados.rados.0.smithi070.stdout:3055: done (5 left) 2024-09-14T03:58:42.347 INFO:tasks.rados.rados.0.smithi070.stdout:3056: done (4 left) 2024-09-14T03:58:42.347 INFO:tasks.rados.rados.0.smithi070.stdout:3057: done (3 left) 2024-09-14T03:58:42.347 INFO:tasks.rados.rados.0.smithi070.stdout:3058: done (2 left) 2024-09-14T03:58:42.348 INFO:tasks.rados.rados.0.smithi070.stdout:3059: done (1 left) 2024-09-14T03:58:42.348 INFO:tasks.rados.rados.0.smithi070.stdout:3060: done (0 left) 2024-09-14T03:58:42.348 INFO:tasks.rados.rados.0.smithi070.stdout:3061: copy_from oid 49 from oid 13 current snap is 313 2024-09-14T03:58:42.348 INFO:tasks.rados.rados.0.smithi070.stdout:3062: copy_from oid 46 from oid 17 current snap is 313 2024-09-14T03:58:42.348 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 25 current snap is 313 2024-09-14T03:58:42.348 INFO:tasks.rados.rados.0.smithi070.stdout:3063: seq_num 930 ranges {974848=786432,1761280=401408} 2024-09-14T03:58:42.357 INFO:tasks.rados.rados.0.smithi070.stdout:3063: writing smithi07085154-25 from 974848 to 1761280 tid 1 2024-09-14T03:58:42.359 INFO:tasks.rados.rados.0.smithi070.stdout:3063: writing smithi07085154-25 from 1761280 to 2162688 tid 2 2024-09-14T03:58:42.359 INFO:tasks.rados.rados.0.smithi070.stdout:3064: read oid 11 snap -1 2024-09-14T03:58:42.360 INFO:tasks.rados.rados.0.smithi070.stdout:3064: expect (ObjNum 293 snap 91 seq_num 293) 2024-09-14T03:58:42.360 INFO:tasks.rados.rados.0.smithi070.stdout:3065: setattr oid 40 current snap is 313 2024-09-14T03:58:42.360 INFO:tasks.rados.rados.0.smithi070.stdout:3062: finishing copy_from racing read to smithi07085154-46 2024-09-14T03:58:42.360 INFO:tasks.rados.rados.0.smithi070.stdout:3062: finishing copy_from to smithi07085154-46 2024-09-14T03:58:42.360 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 46 v 3171 (ObjNum 65 snap 0 seq_num 1347376734) dirty exists 2024-09-14T03:58:42.360 INFO:tasks.rados.rados.0.smithi070.stdout:3061: finishing copy_from racing read to smithi07085154-49 2024-09-14T03:58:42.360 INFO:tasks.rados.rados.0.smithi070.stdout:3061: finishing copy_from to smithi07085154-49 2024-09-14T03:58:42.360 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 49 v 3169 (ObjNum 898 snap 300 seq_num 898) dirty exists 2024-09-14T03:58:42.361 INFO:tasks.rados.rados.0.smithi070.stdout:3061: done (4 left) 2024-09-14T03:58:42.362 INFO:tasks.rados.rados.0.smithi070.stdout:3062: done (3 left) 2024-09-14T03:58:42.362 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 33 current snap is 313 2024-09-14T03:58:42.362 INFO:tasks.rados.rados.0.smithi070.stdout:3066: seq_num 931 ranges {1720320=802816,2523136=401408} 2024-09-14T03:58:42.375 INFO:tasks.rados.rados.0.smithi070.stdout:3066: writing smithi07085154-33 from 1720320 to 2523136 tid 1 2024-09-14T03:58:42.378 INFO:tasks.rados.rados.0.smithi070.stdout:3066: writing smithi07085154-33 from 2523136 to 2924544 tid 2 2024-09-14T03:58:42.378 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 35 current snap is 313 2024-09-14T03:58:42.378 INFO:tasks.rados.rados.0.smithi070.stdout:3063: finishing write tid 1 to smithi07085154-25 2024-09-14T03:58:42.378 INFO:tasks.rados.rados.0.smithi070.stdout:3063: finishing write tid 2 to smithi07085154-25 2024-09-14T03:58:42.378 INFO:tasks.rados.rados.0.smithi070.stdout:3067: seq_num 932 ranges {827392=417792,1245184=663552,1908736=139264} 2024-09-14T03:58:42.384 INFO:tasks.rados.rados.0.smithi070.stdout:3067: writing smithi07085154-35 from 827392 to 1245184 tid 1 2024-09-14T03:58:42.388 INFO:tasks.rados.rados.0.smithi070.stdout:3067: writing smithi07085154-35 from 1245184 to 1908736 tid 2 2024-09-14T03:58:42.389 INFO:tasks.rados.rados.0.smithi070.stdout:3067: writing smithi07085154-35 from 1908736 to 2048000 tid 3 2024-09-14T03:58:42.389 INFO:tasks.rados.rados.0.smithi070.stdout:3068: snap_remove snap 305 2024-09-14T03:58:42.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:42 smithi094 ceph-mon[86597]: osdmap e686: 8 total, 8 up, 8 in 2024-09-14T03:58:42.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:42 smithi070 ceph-mon[96704]: osdmap e686: 8 total, 8 up, 8 in 2024-09-14T03:58:42.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:42 smithi070 ceph-mon[98640]: osdmap e686: 8 total, 8 up, 8 in 2024-09-14T03:58:43.121 INFO:tasks.rados.rados.0.smithi070.stdout:3068: done (5 left) 2024-09-14T03:58:43.121 INFO:tasks.rados.rados.0.smithi070.stdout:3069: read oid 13 snap -1 2024-09-14T03:58:43.121 INFO:tasks.rados.rados.0.smithi070.stdout:3069: expect (ObjNum 898 snap 300 seq_num 898) 2024-09-14T03:58:43.122 INFO:tasks.rados.rados.0.smithi070.stdout:3070: rmattr oid 6 current snap is 313 2024-09-14T03:58:43.122 INFO:tasks.rados.rados.0.smithi070.stdout:3070: done (6 left) 2024-09-14T03:58:43.122 INFO:tasks.rados.rados.0.smithi070.stdout:3071: snap_remove snap 298 2024-09-14T03:58:43.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:43 smithi070 ceph-mon[96704]: osdmap e687: 8 total, 8 up, 8 in 2024-09-14T03:58:43.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:43 smithi070 ceph-mon[96704]: pgmap v629: 121 pgs: 121 active+clean; 1016 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 14 MiB/s rd, 5.3 MiB/s wr, 22 op/s 2024-09-14T03:58:43.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:43 smithi070 ceph-mon[98640]: osdmap e687: 8 total, 8 up, 8 in 2024-09-14T03:58:43.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:43 smithi070 ceph-mon[98640]: pgmap v629: 121 pgs: 121 active+clean; 1016 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 14 MiB/s rd, 5.3 MiB/s wr, 22 op/s 2024-09-14T03:58:43.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:43 smithi094 ceph-mon[86597]: osdmap e687: 8 total, 8 up, 8 in 2024-09-14T03:58:43.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:43 smithi094 ceph-mon[86597]: pgmap v629: 121 pgs: 121 active+clean; 1016 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 14 MiB/s rd, 5.3 MiB/s wr, 22 op/s 2024-09-14T03:58:43.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:58:43 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:58:43] "GET /metrics HTTP/1.1" 200 36500 "" "Prometheus/2.51.0" 2024-09-14T03:58:44.106 INFO:tasks.rados.rados.0.smithi070.stdout:3071: done (6 left) 2024-09-14T03:58:44.106 INFO:tasks.rados.rados.0.smithi070.stdout:3072: copy_from oid 15 from oid 34 current snap is 313 2024-09-14T03:58:44.106 INFO:tasks.rados.rados.0.smithi070.stdout:3073: snap_create 2024-09-14T03:58:44.106 INFO:tasks.rados.rados.0.smithi070.stdout:3063: finishing write tid 3 to smithi07085154-25 2024-09-14T03:58:44.106 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 3167 (ObjNum 912 snap 305 seq_num 912) dirty exists 2024-09-14T03:58:44.106 INFO:tasks.rados.rados.0.smithi070.stdout:3063: finishing write tid 4 to smithi07085154-25 2024-09-14T03:58:44.106 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 3174 (ObjNum 930 snap 313 seq_num 930) dirty exists 2024-09-14T03:58:44.106 INFO:tasks.rados.rados.0.smithi070.stdout:3063: left oid 25 (ObjNum 930 snap 313 seq_num 930) 2024-09-14T03:58:44.106 INFO:tasks.rados.rados.0.smithi070.stdout:3064: expect (ObjNum 293 snap 91 seq_num 293) 2024-09-14T03:58:44.230 INFO:tasks.rados.rados.0.smithi070.stdout:3066: finishing write tid 1 to smithi07085154-33 2024-09-14T03:58:44.230 INFO:tasks.rados.rados.0.smithi070.stdout:3066: finishing write tid 2 to smithi07085154-33 2024-09-14T03:58:44.230 INFO:tasks.rados.rados.0.smithi070.stdout:3066: finishing write tid 3 to smithi07085154-33 2024-09-14T03:58:44.230 INFO:tasks.rados.rados.0.smithi070.stdout:3066: finishing write tid 4 to smithi07085154-33 2024-09-14T03:58:44.231 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 33 v 3172 (ObjNum 931 snap 313 seq_num 931) dirty exists 2024-09-14T03:58:44.231 INFO:tasks.rados.rados.0.smithi070.stdout:3066: left oid 33 (ObjNum 931 snap 313 seq_num 931) 2024-09-14T03:58:44.231 INFO:tasks.rados.rados.0.smithi070.stdout:3067: finishing write tid 1 to smithi07085154-35 2024-09-14T03:58:44.231 INFO:tasks.rados.rados.0.smithi070.stdout:3067: finishing write tid 2 to smithi07085154-35 2024-09-14T03:58:44.231 INFO:tasks.rados.rados.0.smithi070.stdout:3067: finishing write tid 3 to smithi07085154-35 2024-09-14T03:58:44.231 INFO:tasks.rados.rados.0.smithi070.stdout:3067: finishing write tid 4 to smithi07085154-35 2024-09-14T03:58:44.232 INFO:tasks.rados.rados.0.smithi070.stdout:3067: finishing write tid 5 to smithi07085154-35 2024-09-14T03:58:44.232 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 35 v 1705 (ObjNum 932 snap 313 seq_num 932) dirty exists 2024-09-14T03:58:44.232 INFO:tasks.rados.rados.0.smithi070.stdout:3067: left oid 35 (ObjNum 932 snap 313 seq_num 932) 2024-09-14T03:58:44.232 INFO:tasks.rados.rados.0.smithi070.stdout:3069: expect (ObjNum 898 snap 300 seq_num 898) 2024-09-14T03:58:44.248 INFO:tasks.rados.rados.0.smithi070.stdout:3072: finishing copy_from racing read to smithi07085154-15 2024-09-14T03:58:44.249 INFO:tasks.rados.rados.0.smithi070.stdout:3072: finishing copy_from to smithi07085154-15 2024-09-14T03:58:44.249 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 15 v 3172 (ObjNum 929 snap 310 seq_num 929) dirty exists 2024-09-14T03:58:44.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:44 smithi094 ceph-mon[86597]: osdmap e688: 8 total, 8 up, 8 in 2024-09-14T03:58:44.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:44 smithi094 ceph-mon[86597]: osdmap e689: 8 total, 8 up, 8 in 2024-09-14T03:58:44.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:44 smithi070 ceph-mon[96704]: osdmap e688: 8 total, 8 up, 8 in 2024-09-14T03:58:44.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:44 smithi070 ceph-mon[96704]: osdmap e689: 8 total, 8 up, 8 in 2024-09-14T03:58:44.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:44 smithi070 ceph-mon[98640]: osdmap e688: 8 total, 8 up, 8 in 2024-09-14T03:58:44.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:44 smithi070 ceph-mon[98640]: osdmap e689: 8 total, 8 up, 8 in 2024-09-14T03:58:45.106 INFO:tasks.rados.rados.0.smithi070.stdout:3063: done (7 left) 2024-09-14T03:58:45.106 INFO:tasks.rados.rados.0.smithi070.stdout:3064: done (6 left) 2024-09-14T03:58:45.107 INFO:tasks.rados.rados.0.smithi070.stdout:3065: done (5 left) 2024-09-14T03:58:45.107 INFO:tasks.rados.rados.0.smithi070.stdout:3066: done (4 left) 2024-09-14T03:58:45.107 INFO:tasks.rados.rados.0.smithi070.stdout:3067: done (3 left) 2024-09-14T03:58:45.107 INFO:tasks.rados.rados.0.smithi070.stdout:3069: done (2 left) 2024-09-14T03:58:45.107 INFO:tasks.rados.rados.0.smithi070.stdout:3072: done (1 left) 2024-09-14T03:58:45.107 INFO:tasks.rados.rados.0.smithi070.stdout:3073: done (0 left) 2024-09-14T03:58:45.107 INFO:tasks.rados.rados.0.smithi070.stdout:3074: snap_create 2024-09-14T03:58:45.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:45 smithi094 ceph-mon[86597]: pgmap v632: 121 pgs: 121 active+clean; 1019 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 13 MiB/s rd, 4.6 MiB/s wr, 26 op/s 2024-09-14T03:58:45.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:45 smithi094 ceph-mon[86597]: osdmap e690: 8 total, 8 up, 8 in 2024-09-14T03:58:45.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:45 smithi070 ceph-mon[96704]: pgmap v632: 121 pgs: 121 active+clean; 1019 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 13 MiB/s rd, 4.6 MiB/s wr, 26 op/s 2024-09-14T03:58:45.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:45 smithi070 ceph-mon[96704]: osdmap e690: 8 total, 8 up, 8 in 2024-09-14T03:58:45.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:45 smithi070 ceph-mon[98640]: pgmap v632: 121 pgs: 121 active+clean; 1019 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 13 MiB/s rd, 4.6 MiB/s wr, 26 op/s 2024-09-14T03:58:45.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:45 smithi070 ceph-mon[98640]: osdmap e690: 8 total, 8 up, 8 in 2024-09-14T03:58:46.108 INFO:tasks.rados.rados.0.smithi070.stdout:3074: done (0 left) 2024-09-14T03:58:46.108 INFO:tasks.rados.rados.0.smithi070.stdout:3075: snap_create 2024-09-14T03:58:47.115 INFO:tasks.rados.rados.0.smithi070.stdout:3075: done (0 left) 2024-09-14T03:58:47.115 INFO:tasks.rados.rados.0.smithi070.stdout:3076: rmattr oid 31 current snap is 316 2024-09-14T03:58:47.115 INFO:tasks.rados.rados.0.smithi070.stdout:3076: done (0 left) 2024-09-14T03:58:47.115 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 24 current snap is 316 2024-09-14T03:58:47.115 INFO:tasks.rados.rados.0.smithi070.stdout:3077: seq_num 933 ranges {1490944=434176,1925120=548864,2473984=253952} 2024-09-14T03:58:47.125 INFO:tasks.rados.rados.0.smithi070.stdout:3077: writing smithi07085154-24 from 1490944 to 1925120 tid 1 2024-09-14T03:58:47.128 INFO:tasks.rados.rados.0.smithi070.stdout:3077: writing smithi07085154-24 from 1925120 to 2473984 tid 2 2024-09-14T03:58:47.130 INFO:tasks.rados.rados.0.smithi070.stdout:3077: writing smithi07085154-24 from 2473984 to 2727936 tid 3 2024-09-14T03:58:47.130 INFO:tasks.rados.rados.0.smithi070.stdout:3078: copy_from oid 13 from oid 50 current snap is 316 2024-09-14T03:58:47.130 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 6 current snap is 316 2024-09-14T03:58:47.130 INFO:tasks.rados.rados.0.smithi070.stdout:3079: seq_num 934 ranges {0=450560,450560=434176,884736=368640} 2024-09-14T03:58:47.133 INFO:tasks.rados.rados.0.smithi070.stdout:3079: writing smithi07085154-6 from 0 to 450560 tid 1 2024-09-14T03:58:47.135 INFO:tasks.rados.rados.0.smithi070.stdout:3079: writing smithi07085154-6 from 450560 to 884736 tid 2 2024-09-14T03:58:47.138 INFO:tasks.rados.rados.0.smithi070.stdout:3079: writing smithi07085154-6 from 884736 to 1253376 tid 3 2024-09-14T03:58:47.138 INFO:tasks.rados.rados.0.smithi070.stdout:3080: copy_from oid 2 from oid 10 current snap is 316 2024-09-14T03:58:47.139 INFO:tasks.rados.rados.0.smithi070.stdout:3077: finishing write tid 1 to smithi07085154-24 2024-09-14T03:58:47.139 INFO:tasks.rados.rados.0.smithi070.stdout:3077: finishing write tid 2 to smithi07085154-24 2024-09-14T03:58:47.139 INFO:tasks.rados.rados.0.smithi070.stdout:3077: finishing write tid 3 to smithi07085154-24 2024-09-14T03:58:47.139 INFO:tasks.rados.rados.0.smithi070.stdout:3077: finishing write tid 4 to smithi07085154-24 2024-09-14T03:58:47.139 INFO:tasks.rados.rados.0.smithi070.stdout:3081: read oid 18 snap -1 2024-09-14T03:58:47.139 INFO:tasks.rados.rados.0.smithi070.stdout:3081: expect (ObjNum 754 snap 260 seq_num 754) 2024-09-14T03:58:47.139 INFO:tasks.rados.rados.0.smithi070.stdout:3082: read oid 41 snap -1 2024-09-14T03:58:47.140 INFO:tasks.rados.rados.0.smithi070.stdout:3082: expect (ObjNum 724 snap 248 seq_num 724) 2024-09-14T03:58:47.140 INFO:tasks.rados.rados.0.smithi070.stdout:3083: snap_remove snap 302 2024-09-14T03:58:47.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:47 smithi094 ceph-mon[86597]: osdmap e691: 8 total, 8 up, 8 in 2024-09-14T03:58:47.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:47 smithi094 ceph-mon[86597]: pgmap v635: 121 pgs: 121 active+clean; 1023 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 1.3 MiB/s wr, 9 op/s 2024-09-14T03:58:47.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:47 smithi070 ceph-mon[96704]: osdmap e691: 8 total, 8 up, 8 in 2024-09-14T03:58:47.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:47 smithi070 ceph-mon[96704]: pgmap v635: 121 pgs: 121 active+clean; 1023 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 1.3 MiB/s wr, 9 op/s 2024-09-14T03:58:47.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:47 smithi070 ceph-mon[98640]: osdmap e691: 8 total, 8 up, 8 in 2024-09-14T03:58:47.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:47 smithi070 ceph-mon[98640]: pgmap v635: 121 pgs: 121 active+clean; 1023 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 1.3 MiB/s wr, 9 op/s 2024-09-14T03:58:48.122 INFO:tasks.rados.rados.0.smithi070.stdout:3077: finishing write tid 5 to smithi07085154-24 2024-09-14T03:58:48.122 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 3171 (ObjNum 933 snap 316 seq_num 933) dirty exists 2024-09-14T03:58:48.122 INFO:tasks.rados.rados.0.smithi070.stdout:3077: left oid 24 (ObjNum 933 snap 316 seq_num 933) 2024-09-14T03:58:48.122 INFO:tasks.rados.rados.0.smithi070.stdout:3078: finishing copy_from racing read to smithi07085154-13 2024-09-14T03:58:48.122 INFO:tasks.rados.rados.0.smithi070.stdout:3078: finishing copy_from to smithi07085154-13 2024-09-14T03:58:48.122 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 3169 (ObjNum 925 snap 310 seq_num 925) dirty exists 2024-09-14T03:58:48.122 INFO:tasks.rados.rados.0.smithi070.stdout:3079: finishing write tid 1 to smithi07085154-6 2024-09-14T03:58:48.123 INFO:tasks.rados.rados.0.smithi070.stdout:3079: finishing write tid 2 to smithi07085154-6 2024-09-14T03:58:48.123 INFO:tasks.rados.rados.0.smithi070.stdout:3079: finishing write tid 3 to smithi07085154-6 2024-09-14T03:58:48.123 INFO:tasks.rados.rados.0.smithi070.stdout:3079: finishing write tid 4 to smithi07085154-6 2024-09-14T03:58:48.123 INFO:tasks.rados.rados.0.smithi070.stdout:3079: finishing write tid 5 to smithi07085154-6 2024-09-14T03:58:48.123 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 3178 (ObjNum 934 snap 316 seq_num 934) dirty exists 2024-09-14T03:58:48.123 INFO:tasks.rados.rados.0.smithi070.stdout:3079: left oid 6 (ObjNum 934 snap 316 seq_num 934) 2024-09-14T03:58:48.123 INFO:tasks.rados.rados.0.smithi070.stdout:3081: expect (ObjNum 754 snap 260 seq_num 754) 2024-09-14T03:58:48.149 INFO:tasks.rados.rados.0.smithi070.stdout:3082: expect (ObjNum 724 snap 248 seq_num 724) 2024-09-14T03:58:48.162 INFO:tasks.rados.rados.0.smithi070.stdout:3080: finishing copy_from racing read to smithi07085154-2 2024-09-14T03:58:48.162 INFO:tasks.rados.rados.0.smithi070.stdout:3080: finishing copy_from to smithi07085154-2 2024-09-14T03:58:48.163 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 2 v 3401 (ObjNum 841 snap 286 seq_num 841) dirty exists 2024-09-14T03:58:48.163 INFO:tasks.rados.rados.0.smithi070.stdout:3077: done (6 left) 2024-09-14T03:58:48.163 INFO:tasks.rados.rados.0.smithi070.stdout:3078: done (5 left) 2024-09-14T03:58:48.163 INFO:tasks.rados.rados.0.smithi070.stdout:3079: done (4 left) 2024-09-14T03:58:48.163 INFO:tasks.rados.rados.0.smithi070.stdout:3080: done (3 left) 2024-09-14T03:58:48.163 INFO:tasks.rados.rados.0.smithi070.stdout:3081: done (2 left) 2024-09-14T03:58:48.163 INFO:tasks.rados.rados.0.smithi070.stdout:3082: done (1 left) 2024-09-14T03:58:48.163 INFO:tasks.rados.rados.0.smithi070.stdout:3083: done (0 left) 2024-09-14T03:58:48.163 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 37 current snap is 316 2024-09-14T03:58:48.163 INFO:tasks.rados.rados.0.smithi070.stdout:3084: seq_num 935 ranges {0=466944,466944=720896,1187840=81920} 2024-09-14T03:58:48.165 INFO:tasks.rados.rados.0.smithi070.stdout:3084: writing smithi07085154-37 from 0 to 466944 tid 1 2024-09-14T03:58:48.169 INFO:tasks.rados.rados.0.smithi070.stdout:3084: writing smithi07085154-37 from 466944 to 1187840 tid 2 2024-09-14T03:58:48.170 INFO:tasks.rados.rados.0.smithi070.stdout:3084: writing smithi07085154-37 from 1187840 to 1269760 tid 3 2024-09-14T03:58:48.170 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 26 current snap is 316 2024-09-14T03:58:48.170 INFO:tasks.rados.rados.0.smithi070.stdout:3084: finishing write tid 1 to smithi07085154-37 2024-09-14T03:58:48.170 INFO:tasks.rados.rados.0.smithi070.stdout:3085: seq_num 936 ranges {3457024=483328,3940352=655360,4595712=147456} 2024-09-14T03:58:48.192 INFO:tasks.rados.rados.0.smithi070.stdout:3085: writing smithi07085154-26 from 3457024 to 3940352 tid 1 2024-09-14T03:58:48.195 INFO:tasks.rados.rados.0.smithi070.stdout:3085: writing smithi07085154-26 from 3940352 to 4595712 tid 2 2024-09-14T03:58:48.197 INFO:tasks.rados.rados.0.smithi070.stdout:3085: writing smithi07085154-26 from 4595712 to 4743168 tid 3 2024-09-14T03:58:48.197 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 7 current snap is 316 2024-09-14T03:58:48.197 INFO:tasks.rados.rados.0.smithi070.stdout:3086: seq_num 937 ranges {0=499712,499712=532480,1032192=270336} 2024-09-14T03:58:48.200 INFO:tasks.rados.rados.0.smithi070.stdout:3086: writing smithi07085154-7 from 0 to 499712 tid 1 2024-09-14T03:58:48.203 INFO:tasks.rados.rados.0.smithi070.stdout:3086: writing smithi07085154-7 from 499712 to 1032192 tid 2 2024-09-14T03:58:48.206 INFO:tasks.rados.rados.0.smithi070.stdout:3086: writing smithi07085154-7 from 1032192 to 1302528 tid 3 2024-09-14T03:58:48.206 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 34 current snap is 316 2024-09-14T03:58:48.206 INFO:tasks.rados.rados.0.smithi070.stdout:3084: finishing write tid 2 to smithi07085154-37 2024-09-14T03:58:48.206 INFO:tasks.rados.rados.0.smithi070.stdout:3084: finishing write tid 3 to smithi07085154-37 2024-09-14T03:58:48.206 INFO:tasks.rados.rados.0.smithi070.stdout:3084: finishing write tid 4 to smithi07085154-37 2024-09-14T03:58:48.206 INFO:tasks.rados.rados.0.smithi070.stdout:3084: finishing write tid 5 to smithi07085154-37 2024-09-14T03:58:48.206 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 37 v 3176 (ObjNum 935 snap 316 seq_num 935) dirty exists 2024-09-14T03:58:48.206 INFO:tasks.rados.rados.0.smithi070.stdout:3084: left oid 37 (ObjNum 935 snap 316 seq_num 935) 2024-09-14T03:58:48.206 INFO:tasks.rados.rados.0.smithi070.stdout:3085: finishing write tid 1 to smithi07085154-26 2024-09-14T03:58:48.206 INFO:tasks.rados.rados.0.smithi070.stdout:3087: seq_num 938 ranges {2359296=516096,2875392=417792,3293184=385024} 2024-09-14T03:58:48.221 INFO:tasks.rados.rados.0.smithi070.stdout:3087: writing smithi07085154-34 from 2359296 to 2875392 tid 1 2024-09-14T03:58:48.223 INFO:tasks.rados.rados.0.smithi070.stdout:3087: writing smithi07085154-34 from 2875392 to 3293184 tid 2 2024-09-14T03:58:48.226 INFO:tasks.rados.rados.0.smithi070.stdout:3087: writing smithi07085154-34 from 3293184 to 3678208 tid 3 2024-09-14T03:58:48.226 INFO:tasks.rados.rados.0.smithi070.stdout:3084: done (3 left) 2024-09-14T03:58:48.226 INFO:tasks.rados.rados.0.smithi070.stdout:3088: read oid 28 snap -1 2024-09-14T03:58:48.226 INFO:tasks.rados.rados.0.smithi070.stdout:3088: expect (ObjNum 639 snap 214 seq_num 639) 2024-09-14T03:58:48.226 INFO:tasks.rados.rados.0.smithi070.stdout:3089: snap_remove snap 300 2024-09-14T03:58:48.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:48 smithi094 ceph-mon[86597]: osdmap e692: 8 total, 8 up, 8 in 2024-09-14T03:58:48.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:48 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:58:48.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:48 smithi070 ceph-mon[96704]: osdmap e692: 8 total, 8 up, 8 in 2024-09-14T03:58:48.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:48 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:58:48.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:48 smithi070 ceph-mon[98640]: osdmap e692: 8 total, 8 up, 8 in 2024-09-14T03:58:48.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:48 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:58:49.108 INFO:tasks.rados.rados.0.smithi070.stdout:3089: done (4 left) 2024-09-14T03:58:49.109 INFO:tasks.rados.rados.0.smithi070.stdout:3090: rollback oid 20 current snap is 316 2024-09-14T03:58:49.109 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 20 to 273 2024-09-14T03:58:49.109 INFO:tasks.rados.rados.0.smithi070.stdout:3085: finishing write tid 2 to smithi07085154-26 2024-09-14T03:58:49.109 INFO:tasks.rados.rados.0.smithi070.stdout:3085: finishing write tid 3 to smithi07085154-26 2024-09-14T03:58:49.109 INFO:tasks.rados.rados.0.smithi070.stdout:3085: finishing write tid 4 to smithi07085154-26 2024-09-14T03:58:49.109 INFO:tasks.rados.rados.0.smithi070.stdout:3086: finishing write tid 1 to smithi07085154-7 2024-09-14T03:58:49.109 INFO:tasks.rados.rados.0.smithi070.stdout:3085: finishing write tid 5 to smithi07085154-26 2024-09-14T03:58:49.109 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 3716 (ObjNum 936 snap 316 seq_num 936) dirty exists 2024-09-14T03:58:49.109 INFO:tasks.rados.rados.0.smithi070.stdout:3085: left oid 26 (ObjNum 936 snap 316 seq_num 936) 2024-09-14T03:58:49.110 INFO:tasks.rados.rados.0.smithi070.stdout:3086: finishing write tid 2 to smithi07085154-7 2024-09-14T03:58:49.110 INFO:tasks.rados.rados.0.smithi070.stdout:3086: finishing write tid 3 to smithi07085154-7 2024-09-14T03:58:49.110 INFO:tasks.rados.rados.0.smithi070.stdout:3086: finishing write tid 4 to smithi07085154-7 2024-09-14T03:58:49.110 INFO:tasks.rados.rados.0.smithi070.stdout:3086: finishing write tid 5 to smithi07085154-7 2024-09-14T03:58:49.110 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 3172 (ObjNum 937 snap 316 seq_num 937) dirty exists 2024-09-14T03:58:49.110 INFO:tasks.rados.rados.0.smithi070.stdout:3086: left oid 7 (ObjNum 937 snap 316 seq_num 937) 2024-09-14T03:58:49.110 INFO:tasks.rados.rados.0.smithi070.stdout:3087: finishing write tid 1 to smithi07085154-34 2024-09-14T03:58:49.110 INFO:tasks.rados.rados.0.smithi070.stdout:3087: finishing write tid 2 to smithi07085154-34 2024-09-14T03:58:49.110 INFO:tasks.rados.rados.0.smithi070.stdout:3088: expect (ObjNum 639 snap 214 seq_num 639) 2024-09-14T03:58:49.155 INFO:tasks.rados.rados.0.smithi070.stdout:3087: finishing write tid 3 to smithi07085154-34 2024-09-14T03:58:49.155 INFO:tasks.rados.rados.0.smithi070.stdout:3087: finishing write tid 4 to smithi07085154-34 2024-09-14T03:58:49.155 INFO:tasks.rados.rados.0.smithi070.stdout:3087: finishing write tid 5 to smithi07085154-34 2024-09-14T03:58:49.155 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 3176 (ObjNum 938 snap 316 seq_num 938) dirty exists 2024-09-14T03:58:49.155 INFO:tasks.rados.rados.0.smithi070.stdout:3087: left oid 34 (ObjNum 938 snap 316 seq_num 938) 2024-09-14T03:58:49.156 INFO:tasks.rados.rados.0.smithi070.stdout:3085: done (4 left) 2024-09-14T03:58:49.156 INFO:tasks.rados.rados.0.smithi070.stdout:3086: done (3 left) 2024-09-14T03:58:49.156 INFO:tasks.rados.rados.0.smithi070.stdout:3087: done (2 left) 2024-09-14T03:58:49.156 INFO:tasks.rados.rados.0.smithi070.stdout:3088: done (1 left) 2024-09-14T03:58:49.156 INFO:tasks.rados.rados.0.smithi070.stdout:3091: copy_from oid 33 from oid 15 current snap is 316 2024-09-14T03:58:49.156 INFO:tasks.rados.rados.0.smithi070.stdout:3090: finishing rollback tid 0 to smithi07085154-20 2024-09-14T03:58:49.156 INFO:tasks.rados.rados.0.smithi070.stdout:3090: finishing rollback tid 1 to smithi07085154-20 2024-09-14T03:58:49.156 INFO:tasks.rados.rados.0.smithi070.stdout:3090: finishing rollback tid 2 to smithi07085154-20 2024-09-14T03:58:49.156 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 20 v 3175 (ObjNum 524 snap 174 seq_num 524) dirty exists 2024-09-14T03:58:49.156 INFO:tasks.rados.rados.0.smithi070.stdout:3090: done (1 left) 2024-09-14T03:58:49.156 INFO:tasks.rados.rados.0.smithi070.stdout:3092: snap_remove snap 308 2024-09-14T03:58:49.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:49 smithi094 ceph-mon[86597]: osdmap e693: 8 total, 8 up, 8 in 2024-09-14T03:58:49.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:49 smithi094 ceph-mon[86597]: pgmap v638: 121 pgs: 121 active+clean; 1.0 GiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.3 MiB/s wr, 5 op/s 2024-09-14T03:58:49.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:49 smithi094 ceph-mon[86597]: osdmap e694: 8 total, 8 up, 8 in 2024-09-14T03:58:49.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:49 smithi070 ceph-mon[98640]: osdmap e693: 8 total, 8 up, 8 in 2024-09-14T03:58:49.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:49 smithi070 ceph-mon[98640]: pgmap v638: 121 pgs: 121 active+clean; 1.0 GiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.3 MiB/s wr, 5 op/s 2024-09-14T03:58:49.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:49 smithi070 ceph-mon[98640]: osdmap e694: 8 total, 8 up, 8 in 2024-09-14T03:58:49.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:49 smithi070 ceph-mon[96704]: osdmap e693: 8 total, 8 up, 8 in 2024-09-14T03:58:49.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:49 smithi070 ceph-mon[96704]: pgmap v638: 121 pgs: 121 active+clean; 1.0 GiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.3 MiB/s wr, 5 op/s 2024-09-14T03:58:49.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:49 smithi070 ceph-mon[96704]: osdmap e694: 8 total, 8 up, 8 in 2024-09-14T03:58:50.136 INFO:tasks.rados.rados.0.smithi070.stdout:3092: done (1 left) 2024-09-14T03:58:50.137 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 14 current snap is 316 2024-09-14T03:58:50.137 INFO:tasks.rados.rados.0.smithi070.stdout:3093: seq_num 939 ranges {0=532480,532480=704512,1236992=98304} 2024-09-14T03:58:50.139 INFO:tasks.rados.rados.0.smithi070.stdout:3093: writing smithi07085154-14 from 0 to 532480 tid 1 2024-09-14T03:58:50.144 INFO:tasks.rados.rados.0.smithi070.stdout:3093: writing smithi07085154-14 from 532480 to 1236992 tid 2 2024-09-14T03:58:50.145 INFO:tasks.rados.rados.0.smithi070.stdout:3093: writing smithi07085154-14 from 1236992 to 1335296 tid 3 2024-09-14T03:58:50.145 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 50 current snap is 316 2024-09-14T03:58:50.145 INFO:tasks.rados.rados.0.smithi070.stdout:3091: finishing copy_from racing read to smithi07085154-33 2024-09-14T03:58:50.145 INFO:tasks.rados.rados.0.smithi070.stdout:3094: seq_num 940 ranges {1097728=548864,1646592=589824,2236416=212992} 2024-09-14T03:58:50.154 INFO:tasks.rados.rados.0.smithi070.stdout:3094: writing smithi07085154-50 from 1097728 to 1646592 tid 1 2024-09-14T03:58:50.157 INFO:tasks.rados.rados.0.smithi070.stdout:3094: writing smithi07085154-50 from 1646592 to 2236416 tid 2 2024-09-14T03:58:50.159 INFO:tasks.rados.rados.0.smithi070.stdout:3094: writing smithi07085154-50 from 2236416 to 2449408 tid 3 2024-09-14T03:58:50.159 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 41 current snap is 316 2024-09-14T03:58:50.159 INFO:tasks.rados.rados.0.smithi070.stdout:3091: finishing copy_from to smithi07085154-33 2024-09-14T03:58:50.159 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 33 v 3172 (ObjNum 929 snap 310 seq_num 929) dirty exists 2024-09-14T03:58:50.159 INFO:tasks.rados.rados.0.smithi070.stdout:3093: finishing write tid 1 to smithi07085154-14 2024-09-14T03:58:50.159 INFO:tasks.rados.rados.0.smithi070.stdout:3093: finishing write tid 2 to smithi07085154-14 2024-09-14T03:58:50.160 INFO:tasks.rados.rados.0.smithi070.stdout:3095: seq_num 941 ranges {131072=565248,696320=475136,1171456=327680} 2024-09-14T03:58:50.162 INFO:tasks.rados.rados.0.smithi070.stdout:3095: writing smithi07085154-41 from 131072 to 696320 tid 1 2024-09-14T03:58:50.166 INFO:tasks.rados.rados.0.smithi070.stdout:3095: writing smithi07085154-41 from 696320 to 1171456 tid 2 2024-09-14T03:58:50.168 INFO:tasks.rados.rados.0.smithi070.stdout:3095: writing smithi07085154-41 from 1171456 to 1499136 tid 3 2024-09-14T03:58:50.168 INFO:tasks.rados.rados.0.smithi070.stdout:3091: done (3 left) 2024-09-14T03:58:50.168 INFO:tasks.rados.rados.0.smithi070.stdout:3096: delete oid 35 current snap is 316 2024-09-14T03:58:50.169 INFO:tasks.rados.rados.0.smithi070.stdout:3093: finishing write tid 3 to smithi07085154-14 2024-09-14T03:58:50.169 INFO:tasks.rados.rados.0.smithi070.stdout:3093: finishing write tid 4 to smithi07085154-14 2024-09-14T03:58:50.169 INFO:tasks.rados.rados.0.smithi070.stdout:3093: finishing write tid 5 to smithi07085154-14 2024-09-14T03:58:50.169 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 3175 (ObjNum 939 snap 316 seq_num 939) dirty exists 2024-09-14T03:58:50.169 INFO:tasks.rados.rados.0.smithi070.stdout:3093: left oid 14 (ObjNum 939 snap 316 seq_num 939) 2024-09-14T03:58:50.169 INFO:tasks.rados.rados.0.smithi070.stdout:3094: finishing write tid 1 to smithi07085154-50 2024-09-14T03:58:50.169 INFO:tasks.rados.rados.0.smithi070.stdout:3094: finishing write tid 2 to smithi07085154-50 2024-09-14T03:58:50.169 INFO:tasks.rados.rados.0.smithi070.stdout:3094: finishing write tid 3 to smithi07085154-50 2024-09-14T03:58:50.169 INFO:tasks.rados.rados.0.smithi070.stdout:3094: finishing write tid 4 to smithi07085154-50 2024-09-14T03:58:50.169 INFO:tasks.rados.rados.0.smithi070.stdout:3094: finishing write tid 5 to smithi07085154-50 2024-09-14T03:58:50.169 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 3180 (ObjNum 940 snap 316 seq_num 940) dirty exists 2024-09-14T03:58:50.169 INFO:tasks.rados.rados.0.smithi070.stdout:3094: left oid 50 (ObjNum 940 snap 316 seq_num 940) 2024-09-14T03:58:50.172 INFO:tasks.rados.rados.0.smithi070.stdout:3095: finishing write tid 1 to smithi07085154-41 2024-09-14T03:58:50.173 INFO:tasks.rados.rados.0.smithi070.stdout:3095: finishing write tid 2 to smithi07085154-41 2024-09-14T03:58:50.175 INFO:tasks.rados.rados.0.smithi070.stdout:3095: finishing write tid 3 to smithi07085154-41 2024-09-14T03:58:50.175 INFO:tasks.rados.rados.0.smithi070.stdout:3095: finishing write tid 4 to smithi07085154-41 2024-09-14T03:58:50.176 INFO:tasks.rados.rados.0.smithi070.stdout:3093: done (3 left) 2024-09-14T03:58:50.176 INFO:tasks.rados.rados.0.smithi070.stdout:3094: done (2 left) 2024-09-14T03:58:50.176 INFO:tasks.rados.rados.0.smithi070.stdout:3096: done (1 left) 2024-09-14T03:58:50.176 INFO:tasks.rados.rados.0.smithi070.stdout:3097: snap_create 2024-09-14T03:58:50.177 INFO:tasks.rados.rados.0.smithi070.stdout:3095: finishing write tid 5 to smithi07085154-41 2024-09-14T03:58:50.177 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 3180 (ObjNum 941 snap 316 seq_num 941) dirty exists 2024-09-14T03:58:50.178 INFO:tasks.rados.rados.0.smithi070.stdout:3095: left oid 41 (ObjNum 941 snap 316 seq_num 941) 2024-09-14T03:58:50.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:58:50 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:58:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:58:50.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:50 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:58:50.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:50 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:58:50.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:50 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:58:50.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:50 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:58:50.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:50 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:58:50.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:50 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:58:50.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:50 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:58:50.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:50 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:58:50.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:50 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:58:50.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:50 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:58:50.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:50 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:58:50.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:50 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:58:51.141 INFO:tasks.rados.rados.0.smithi070.stdout:3095: done (1 left) 2024-09-14T03:58:51.141 INFO:tasks.rados.rados.0.smithi070.stdout:3097: done (0 left) 2024-09-14T03:58:51.142 INFO:tasks.rados.rados.0.smithi070.stdout:3098: read oid 21 snap 292 2024-09-14T03:58:51.142 INFO:tasks.rados.rados.0.smithi070.stdout:3098: expect deleted 2024-09-14T03:58:51.142 INFO:tasks.rados.rados.0.smithi070.stdout:3099: delete oid 3 current snap is 317 2024-09-14T03:58:51.147 INFO:tasks.rados.rados.0.smithi070.stdout:3098: done (1 left) 2024-09-14T03:58:51.147 INFO:tasks.rados.rados.0.smithi070.stdout:3099: done (0 left) 2024-09-14T03:58:51.147 INFO:tasks.rados.rados.0.smithi070.stdout:3100: read oid 23 snap 267 2024-09-14T03:58:51.147 INFO:tasks.rados.rados.0.smithi070.stdout:3100: expect deleted 2024-09-14T03:58:51.147 INFO:tasks.rados.rados.0.smithi070.stdout:3101: setattr oid 29 current snap is 317 2024-09-14T03:58:51.148 INFO:tasks.rados.rados.0.smithi070.stdout:3102: copy_from oid 21 from oid 16 current snap is 317 2024-09-14T03:58:51.149 INFO:tasks.rados.rados.0.smithi070.stdout:3103: delete oid 4 current snap is 317 2024-09-14T03:58:51.149 INFO:tasks.rados.rados.0.smithi070.stdout:3102: finishing copy_from to smithi07085154-21 2024-09-14T03:58:51.149 INFO:tasks.rados.rados.0.smithi070.stdout:3102: got expected ENOENT (src dne) 2024-09-14T03:58:51.150 INFO:tasks.rados.rados.0.smithi070.stdout:3102: finishing copy_from racing read to smithi07085154-21 2024-09-14T03:58:51.152 INFO:tasks.rados.rados.0.smithi070.stdout:3100: done (3 left) 2024-09-14T03:58:51.152 INFO:tasks.rados.rados.0.smithi070.stdout:3102: done (2 left) 2024-09-14T03:58:51.152 INFO:tasks.rados.rados.0.smithi070.stdout:3103: done (1 left) 2024-09-14T03:58:51.152 INFO:tasks.rados.rados.0.smithi070.stdout:3104: snap_create 2024-09-14T03:58:51.154 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 3181 (ObjNum 724 snap 248 seq_num 724) dirty exists 2024-09-14T03:58:51.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:51 smithi094 ceph-mon[86597]: pgmap v640: 121 pgs: 121 active+clean; 1.0 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.7 MiB/s rd, 910 KiB/s wr, 8 op/s 2024-09-14T03:58:51.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:51 smithi094 ceph-mon[86597]: osdmap e695: 8 total, 8 up, 8 in 2024-09-14T03:58:51.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:51 smithi070 ceph-mon[96704]: pgmap v640: 121 pgs: 121 active+clean; 1.0 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.7 MiB/s rd, 910 KiB/s wr, 8 op/s 2024-09-14T03:58:51.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:51 smithi070 ceph-mon[96704]: osdmap e695: 8 total, 8 up, 8 in 2024-09-14T03:58:51.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:51 smithi070 ceph-mon[98640]: pgmap v640: 121 pgs: 121 active+clean; 1.0 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.7 MiB/s rd, 910 KiB/s wr, 8 op/s 2024-09-14T03:58:51.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:51 smithi070 ceph-mon[98640]: osdmap e695: 8 total, 8 up, 8 in 2024-09-14T03:58:52.146 INFO:tasks.rados.rados.0.smithi070.stdout:3101: done (1 left) 2024-09-14T03:58:52.147 INFO:tasks.rados.rados.0.smithi070.stdout:3104: done (0 left) 2024-09-14T03:58:52.147 INFO:tasks.rados.rados.0.smithi070.stdout:3105: copy_from oid 14 from oid 19 current snap is 318 2024-09-14T03:58:52.147 INFO:tasks.rados.rados.0.smithi070.stdout:3106: rmattr oid 49 current snap is 318 2024-09-14T03:58:52.147 INFO:tasks.rados.rados.0.smithi070.stdout:3107: delete oid 31 current snap is 318 2024-09-14T03:58:52.150 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 49 v 3173 (ObjNum 898 snap 300 seq_num 898) dirty exists 2024-09-14T03:58:52.151 INFO:tasks.rados.rados.0.smithi070.stdout:3106: done (2 left) 2024-09-14T03:58:52.151 INFO:tasks.rados.rados.0.smithi070.stdout:3107: done (1 left) 2024-09-14T03:58:52.151 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 39 current snap is 318 2024-09-14T03:58:52.151 INFO:tasks.rados.rados.0.smithi070.stdout:3108: seq_num 946 ranges {0=647168,647168=745472,1392640=57344} 2024-09-14T03:58:52.154 INFO:tasks.rados.rados.0.smithi070.stdout:3108: writing smithi07085154-39 from 0 to 647168 tid 1 2024-09-14T03:58:52.158 INFO:tasks.rados.rados.0.smithi070.stdout:3108: writing smithi07085154-39 from 647168 to 1392640 tid 2 2024-09-14T03:58:52.159 INFO:tasks.rados.rados.0.smithi070.stdout:3108: writing smithi07085154-39 from 1392640 to 1449984 tid 3 2024-09-14T03:58:52.159 INFO:tasks.rados.rados.0.smithi070.stdout:3109: delete oid 5 current snap is 318 2024-09-14T03:58:52.159 INFO:tasks.rados.rados.0.smithi070.stdout:3108: finishing write tid 1 to smithi07085154-39 2024-09-14T03:58:52.163 INFO:tasks.rados.rados.0.smithi070.stdout:3109: done (2 left) 2024-09-14T03:58:52.163 INFO:tasks.rados.rados.0.smithi070.stdout:3110: rmattr oid 7 current snap is 318 2024-09-14T03:58:52.164 INFO:tasks.rados.rados.0.smithi070.stdout:3111: read oid 35 snap 292 2024-09-14T03:58:52.164 INFO:tasks.rados.rados.0.smithi070.stdout:3111: expect (ObjNum -1365232688 snap 21849 seq_num 2930188880) 2024-09-14T03:58:52.164 INFO:tasks.rados.rados.0.smithi070.stdout:3112: delete oid 33 current snap is 318 2024-09-14T03:58:52.164 INFO:tasks.rados.rados.0.smithi070.stdout:3108: finishing write tid 2 to smithi07085154-39 2024-09-14T03:58:52.164 INFO:tasks.rados.rados.0.smithi070.stdout:3108: finishing write tid 3 to smithi07085154-39 2024-09-14T03:58:52.164 INFO:tasks.rados.rados.0.smithi070.stdout:3108: finishing write tid 4 to smithi07085154-39 2024-09-14T03:58:52.164 INFO:tasks.rados.rados.0.smithi070.stdout:3111: expect (ObjNum -1365232688 snap 21849 seq_num 2930188880) 2024-09-14T03:58:52.165 INFO:tasks.rados.rados.0.smithi070.stdout:3108: finishing write tid 5 to smithi07085154-39 2024-09-14T03:58:52.165 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 39 v 2640 (ObjNum 946 snap 318 seq_num 946) dirty exists 2024-09-14T03:58:52.165 INFO:tasks.rados.rados.0.smithi070.stdout:3108: left oid 39 (ObjNum 946 snap 318 seq_num 946) 2024-09-14T03:58:52.166 INFO:tasks.rados.rados.0.smithi070.stdout:3108: done (4 left) 2024-09-14T03:58:52.166 INFO:tasks.rados.rados.0.smithi070.stdout:3111: done (3 left) 2024-09-14T03:58:52.166 INFO:tasks.rados.rados.0.smithi070.stdout:3112: done (2 left) 2024-09-14T03:58:52.166 INFO:tasks.rados.rados.0.smithi070.stdout:3113: copy_from oid 42 from oid 18 current snap is 318 2024-09-14T03:58:52.167 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 9 current snap is 318 2024-09-14T03:58:52.167 INFO:tasks.rados.rados.0.smithi070.stdout:3114: seq_num 949 ranges {1196032=704512,1900544=794624,2695168=8192} 2024-09-14T03:58:52.177 INFO:tasks.rados.rados.0.smithi070.stdout:3114: writing smithi07085154-9 from 1196032 to 1900544 tid 1 2024-09-14T03:58:52.182 INFO:tasks.rados.rados.0.smithi070.stdout:3114: writing smithi07085154-9 from 1900544 to 2695168 tid 2 2024-09-14T03:58:52.182 INFO:tasks.rados.rados.0.smithi070.stdout:3114: writing smithi07085154-9 from 2695168 to 2703360 tid 3 2024-09-14T03:58:52.182 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 24 current snap is 318 2024-09-14T03:58:52.182 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 3177 (ObjNum 937 snap 316 seq_num 937) dirty exists 2024-09-14T03:58:52.182 INFO:tasks.rados.rados.0.smithi070.stdout:3115: seq_num 950 ranges {2727936=720896,3448832=679936,4128768=122880} 2024-09-14T03:58:52.200 INFO:tasks.rados.rados.0.smithi070.stdout:3115: writing smithi07085154-24 from 2727936 to 3448832 tid 1 2024-09-14T03:58:52.204 INFO:tasks.rados.rados.0.smithi070.stdout:3115: writing smithi07085154-24 from 3448832 to 4128768 tid 2 2024-09-14T03:58:52.206 INFO:tasks.rados.rados.0.smithi070.stdout:3115: writing smithi07085154-24 from 4128768 to 4251648 tid 3 2024-09-14T03:58:52.206 INFO:tasks.rados.rados.0.smithi070.stdout:3110: done (4 left) 2024-09-14T03:58:52.206 INFO:tasks.rados.rados.0.smithi070.stdout:3116: rollback oid 27 current snap is 318 2024-09-14T03:58:52.206 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 27 to 284 2024-09-14T03:58:52.206 INFO:tasks.rados.rados.0.smithi070.stdout:3113: finishing copy_from racing read to smithi07085154-42 2024-09-14T03:58:52.206 INFO:tasks.rados.rados.0.smithi070.stdout:3113: finishing copy_from to smithi07085154-42 2024-09-14T03:58:52.206 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 2406 (ObjNum 754 snap 260 seq_num 754) dirty exists 2024-09-14T03:58:52.206 INFO:tasks.rados.rados.0.smithi070.stdout:3114: finishing write tid 1 to smithi07085154-9 2024-09-14T03:58:52.206 INFO:tasks.rados.rados.0.smithi070.stdout:3114: finishing write tid 2 to smithi07085154-9 2024-09-14T03:58:52.206 INFO:tasks.rados.rados.0.smithi070.stdout:3114: finishing write tid 3 to smithi07085154-9 2024-09-14T03:58:52.206 INFO:tasks.rados.rados.0.smithi070.stdout:3114: finishing write tid 4 to smithi07085154-9 2024-09-14T03:58:52.207 INFO:tasks.rados.rados.0.smithi070.stdout:3114: finishing write tid 5 to smithi07085154-9 2024-09-14T03:58:52.207 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 3741 (ObjNum 949 snap 318 seq_num 949) dirty exists 2024-09-14T03:58:52.207 INFO:tasks.rados.rados.0.smithi070.stdout:3114: left oid 9 (ObjNum 949 snap 318 seq_num 949) 2024-09-14T03:58:52.207 INFO:tasks.rados.rados.0.smithi070.stdout:3105: finishing copy_from racing read to smithi07085154-14 2024-09-14T03:58:52.207 INFO:tasks.rados.rados.0.smithi070.stdout:3105: finishing copy_from to smithi07085154-14 2024-09-14T03:58:52.207 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 3159 (ObjNum 901 snap 301 seq_num 901) dirty exists 2024-09-14T03:58:52.207 INFO:tasks.rados.rados.0.smithi070.stdout:3115: finishing write tid 1 to smithi07085154-24 2024-09-14T03:58:52.207 INFO:tasks.rados.rados.0.smithi070.stdout:3105: done (4 left) 2024-09-14T03:58:52.207 INFO:tasks.rados.rados.0.smithi070.stdout:3113: done (3 left) 2024-09-14T03:58:52.207 INFO:tasks.rados.rados.0.smithi070.stdout:3114: done (2 left) 2024-09-14T03:58:52.207 INFO:tasks.rados.rados.0.smithi070.stdout:3117: rmattr oid 48 current snap is 318 2024-09-14T03:58:52.207 INFO:tasks.rados.rados.0.smithi070.stdout:3117: done (2 left) 2024-09-14T03:58:52.207 INFO:tasks.rados.rados.0.smithi070.stdout:3118: setattr oid 32 current snap is 318 2024-09-14T03:58:52.208 INFO:tasks.rados.rados.0.smithi070.stdout:3119: snap_create 2024-09-14T03:58:52.209 INFO:tasks.rados.rados.0.smithi070.stdout:3115: finishing write tid 2 to smithi07085154-24 2024-09-14T03:58:52.209 INFO:tasks.rados.rados.0.smithi070.stdout:3115: finishing write tid 3 to smithi07085154-24 2024-09-14T03:58:52.209 INFO:tasks.rados.rados.0.smithi070.stdout:3115: finishing write tid 4 to smithi07085154-24 2024-09-14T03:58:52.209 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 3170 (ObjNum 926 snap 310 seq_num 926) dirty exists 2024-09-14T03:58:52.210 INFO:tasks.rados.rados.0.smithi070.stdout:3115: finishing write tid 5 to smithi07085154-24 2024-09-14T03:58:52.211 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 3179 (ObjNum 950 snap 318 seq_num 950) dirty exists 2024-09-14T03:58:52.211 INFO:tasks.rados.rados.0.smithi070.stdout:3115: left oid 24 (ObjNum 950 snap 318 seq_num 950) 2024-09-14T03:58:52.212 INFO:tasks.rados.rados.0.smithi070.stdout:3116: finishing rollback tid 0 to smithi07085154-27 2024-09-14T03:58:52.212 INFO:tasks.rados.rados.0.smithi070.stdout:3116: finishing rollback tid 1 to smithi07085154-27 2024-09-14T03:58:52.212 INFO:tasks.rados.rados.0.smithi070.stdout:3116: finishing rollback tid 2 to smithi07085154-27 2024-09-14T03:58:52.212 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 3745 (ObjNum 829 snap 283 seq_num 829) dirty exists 2024-09-14T03:58:52.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:52 smithi094 ceph-mon[86597]: osdmap e696: 8 total, 8 up, 8 in 2024-09-14T03:58:52.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:52 smithi070 ceph-mon[98640]: osdmap e696: 8 total, 8 up, 8 in 2024-09-14T03:58:52.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:52 smithi070 ceph-mon[96704]: osdmap e696: 8 total, 8 up, 8 in 2024-09-14T03:58:53.148 INFO:tasks.rados.rados.0.smithi070.stdout:3115: done (3 left) 2024-09-14T03:58:53.148 INFO:tasks.rados.rados.0.smithi070.stdout:3116: done (2 left) 2024-09-14T03:58:53.148 INFO:tasks.rados.rados.0.smithi070.stdout:3118: done (1 left) 2024-09-14T03:58:53.148 INFO:tasks.rados.rados.0.smithi070.stdout:3119: done (0 left) 2024-09-14T03:58:53.148 INFO:tasks.rados.rados.0.smithi070.stdout:3120: delete oid 42 current snap is 319 2024-09-14T03:58:53.157 INFO:tasks.rados.rados.0.smithi070.stdout:3120: done (0 left) 2024-09-14T03:58:53.157 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 26 current snap is 319 2024-09-14T03:58:53.158 INFO:tasks.rados.rados.0.smithi070.stdout:3121: seq_num 952 ranges {4743168=753664,5496832=499712,5996544=303104} 2024-09-14T03:58:53.186 INFO:tasks.rados.rados.0.smithi070.stdout:3121: writing smithi07085154-26 from 4743168 to 5496832 tid 1 2024-09-14T03:58:53.189 INFO:tasks.rados.rados.0.smithi070.stdout:3121: writing smithi07085154-26 from 5496832 to 5996544 tid 2 2024-09-14T03:58:53.191 INFO:tasks.rados.rados.0.smithi070.stdout:3121: writing smithi07085154-26 from 5996544 to 6299648 tid 3 2024-09-14T03:58:53.192 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 42 current snap is 319 2024-09-14T03:58:53.192 INFO:tasks.rados.rados.0.smithi070.stdout:3122: seq_num 953 ranges {0=770048,770048=786432,1556480=16384} 2024-09-14T03:58:53.195 INFO:tasks.rados.rados.0.smithi070.stdout:3122: writing smithi07085154-42 from 0 to 770048 tid 1 2024-09-14T03:58:53.200 INFO:tasks.rados.rados.0.smithi070.stdout:3122: writing smithi07085154-42 from 770048 to 1556480 tid 2 2024-09-14T03:58:53.200 INFO:tasks.rados.rados.0.smithi070.stdout:3122: writing smithi07085154-42 from 1556480 to 1572864 tid 3 2024-09-14T03:58:53.201 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 22 current snap is 319 2024-09-14T03:58:53.201 INFO:tasks.rados.rados.0.smithi070.stdout:3121: finishing write tid 1 to smithi07085154-26 2024-09-14T03:58:53.201 INFO:tasks.rados.rados.0.smithi070.stdout:3121: finishing write tid 2 to smithi07085154-26 2024-09-14T03:58:53.201 INFO:tasks.rados.rados.0.smithi070.stdout:3121: finishing write tid 3 to smithi07085154-26 2024-09-14T03:58:53.201 INFO:tasks.rados.rados.0.smithi070.stdout:3121: finishing write tid 4 to smithi07085154-26 2024-09-14T03:58:53.202 INFO:tasks.rados.rados.0.smithi070.stdout:3123: seq_num 954 ranges {49152=786432,835584=671744,1507328=131072} 2024-09-14T03:58:53.204 INFO:tasks.rados.rados.0.smithi070.stdout:3123: writing smithi07085154-22 from 49152 to 835584 tid 1 2024-09-14T03:58:53.208 INFO:tasks.rados.rados.0.smithi070.stdout:3123: writing smithi07085154-22 from 835584 to 1507328 tid 2 2024-09-14T03:58:53.210 INFO:tasks.rados.rados.0.smithi070.stdout:3123: writing smithi07085154-22 from 1507328 to 1638400 tid 3 2024-09-14T03:58:53.210 INFO:tasks.rados.rados.0.smithi070.stdout:3124: delete oid 44 current snap is 319 2024-09-14T03:58:53.210 INFO:tasks.rados.rados.0.smithi070.stdout:3121: finishing write tid 5 to smithi07085154-26 2024-09-14T03:58:53.210 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 3752 (ObjNum 952 snap 319 seq_num 952) dirty exists 2024-09-14T03:58:53.210 INFO:tasks.rados.rados.0.smithi070.stdout:3121: left oid 26 (ObjNum 952 snap 319 seq_num 952) 2024-09-14T03:58:53.210 INFO:tasks.rados.rados.0.smithi070.stdout:3122: finishing write tid 1 to smithi07085154-42 2024-09-14T03:58:53.210 INFO:tasks.rados.rados.0.smithi070.stdout:3122: finishing write tid 2 to smithi07085154-42 2024-09-14T03:58:53.210 INFO:tasks.rados.rados.0.smithi070.stdout:3122: finishing write tid 3 to smithi07085154-42 2024-09-14T03:58:53.211 INFO:tasks.rados.rados.0.smithi070.stdout:3122: finishing write tid 4 to smithi07085154-42 2024-09-14T03:58:53.211 INFO:tasks.rados.rados.0.smithi070.stdout:3122: finishing write tid 5 to smithi07085154-42 2024-09-14T03:58:53.211 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 3756 (ObjNum 953 snap 319 seq_num 953) dirty exists 2024-09-14T03:58:53.211 INFO:tasks.rados.rados.0.smithi070.stdout:3122: left oid 42 (ObjNum 953 snap 319 seq_num 953) 2024-09-14T03:58:53.212 INFO:tasks.rados.rados.0.smithi070.stdout:3123: finishing write tid 1 to smithi07085154-22 2024-09-14T03:58:53.215 INFO:tasks.rados.rados.0.smithi070.stdout:3123: finishing write tid 2 to smithi07085154-22 2024-09-14T03:58:53.215 INFO:tasks.rados.rados.0.smithi070.stdout:3123: finishing write tid 3 to smithi07085154-22 2024-09-14T03:58:53.215 INFO:tasks.rados.rados.0.smithi070.stdout:3123: finishing write tid 4 to smithi07085154-22 2024-09-14T03:58:53.217 INFO:tasks.rados.rados.0.smithi070.stdout:3121: done (3 left) 2024-09-14T03:58:53.217 INFO:tasks.rados.rados.0.smithi070.stdout:3122: done (2 left) 2024-09-14T03:58:53.217 INFO:tasks.rados.rados.0.smithi070.stdout:3124: done (1 left) 2024-09-14T03:58:53.217 INFO:tasks.rados.rados.0.smithi070.stdout:3125: read oid 18 snap -1 2024-09-14T03:58:53.217 INFO:tasks.rados.rados.0.smithi070.stdout:3125: expect (ObjNum 754 snap 260 seq_num 754) 2024-09-14T03:58:53.217 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 46 current snap is 319 2024-09-14T03:58:53.217 INFO:tasks.rados.rados.0.smithi070.stdout:3126: seq_num 956 ranges {0=417792,417792=434176,851968=770048} 2024-09-14T03:58:53.219 INFO:tasks.rados.rados.0.smithi070.stdout:3126: writing smithi07085154-46 from 0 to 417792 tid 1 2024-09-14T03:58:53.221 INFO:tasks.rados.rados.0.smithi070.stdout:3126: writing smithi07085154-46 from 417792 to 851968 tid 2 2024-09-14T03:58:53.226 INFO:tasks.rados.rados.0.smithi070.stdout:3126: writing smithi07085154-46 from 851968 to 1622016 tid 3 2024-09-14T03:58:53.227 INFO:tasks.rados.rados.0.smithi070.stdout:3127: read oid 29 snap -1 2024-09-14T03:58:53.227 INFO:tasks.rados.rados.0.smithi070.stdout:3127: expect (ObjNum 724 snap 248 seq_num 724) 2024-09-14T03:58:53.227 INFO:tasks.rados.rados.0.smithi070.stdout:3128: setattr oid 30 current snap is 319 2024-09-14T03:58:53.227 INFO:tasks.rados.rados.0.smithi070.stdout:3123: finishing write tid 5 to smithi07085154-22 2024-09-14T03:58:53.227 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 3185 (ObjNum 954 snap 319 seq_num 954) dirty exists 2024-09-14T03:58:53.227 INFO:tasks.rados.rados.0.smithi070.stdout:3123: left oid 22 (ObjNum 954 snap 319 seq_num 954) 2024-09-14T03:58:53.227 INFO:tasks.rados.rados.0.smithi070.stdout:3125: expect (ObjNum 754 snap 260 seq_num 754) 2024-09-14T03:58:53.243 INFO:tasks.rados.rados.0.smithi070.stdout:3126: finishing write tid 1 to smithi07085154-46 2024-09-14T03:58:53.243 INFO:tasks.rados.rados.0.smithi070.stdout:3126: finishing write tid 2 to smithi07085154-46 2024-09-14T03:58:53.243 INFO:tasks.rados.rados.0.smithi070.stdout:3127: expect (ObjNum 724 snap 248 seq_num 724) 2024-09-14T03:58:53.254 INFO:tasks.rados.rados.0.smithi070.stdout:3126: finishing write tid 3 to smithi07085154-46 2024-09-14T03:58:53.254 INFO:tasks.rados.rados.0.smithi070.stdout:3126: finishing write tid 4 to smithi07085154-46 2024-09-14T03:58:53.254 INFO:tasks.rados.rados.0.smithi070.stdout:3126: finishing write tid 5 to smithi07085154-46 2024-09-14T03:58:53.254 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 46 v 3761 (ObjNum 956 snap 319 seq_num 956) dirty exists 2024-09-14T03:58:53.254 INFO:tasks.rados.rados.0.smithi070.stdout:3126: left oid 46 (ObjNum 956 snap 319 seq_num 956) 2024-09-14T03:58:53.254 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 30 v 2249 (ObjNum 177 snap 0 seq_num 4194278784) dirty exists 2024-09-14T03:58:53.254 INFO:tasks.rados.rados.0.smithi070.stdout:3123: done (4 left) 2024-09-14T03:58:53.254 INFO:tasks.rados.rados.0.smithi070.stdout:3125: done (3 left) 2024-09-14T03:58:53.254 INFO:tasks.rados.rados.0.smithi070.stdout:3126: done (2 left) 2024-09-14T03:58:53.255 INFO:tasks.rados.rados.0.smithi070.stdout:3127: done (1 left) 2024-09-14T03:58:53.255 INFO:tasks.rados.rados.0.smithi070.stdout:3128: done (0 left) 2024-09-14T03:58:53.255 INFO:tasks.rados.rados.0.smithi070.stdout:3129: read oid 25 snap -1 2024-09-14T03:58:53.255 INFO:tasks.rados.rados.0.smithi070.stdout:3129: expect (ObjNum 930 snap 313 seq_num 930) 2024-09-14T03:58:53.255 INFO:tasks.rados.rados.0.smithi070.stdout:3130: rmattr oid 6 current snap is 319 2024-09-14T03:58:53.255 INFO:tasks.rados.rados.0.smithi070.stdout:3130: done (1 left) 2024-09-14T03:58:53.255 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 48 current snap is 319 2024-09-14T03:58:53.255 INFO:tasks.rados.rados.0.smithi070.stdout:3131: seq_num 957 ranges {5349376=434176,5783552=720896,6504448=483328} 2024-09-14T03:58:53.284 INFO:tasks.rados.rados.0.smithi070.stdout:3131: writing smithi07085154-48 from 5349376 to 5783552 tid 1 2024-09-14T03:58:53.288 INFO:tasks.rados.rados.0.smithi070.stdout:3131: writing smithi07085154-48 from 5783552 to 6504448 tid 2 2024-09-14T03:58:53.291 INFO:tasks.rados.rados.0.smithi070.stdout:3131: writing smithi07085154-48 from 6504448 to 6987776 tid 3 2024-09-14T03:58:53.291 INFO:tasks.rados.rados.0.smithi070.stdout:3132: rollback oid 45 current snap is 319 2024-09-14T03:58:53.291 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 45 to 310 2024-09-14T03:58:53.291 INFO:tasks.rados.rados.0.smithi070.stdout:3129: expect (ObjNum 930 snap 313 seq_num 930) 2024-09-14T03:58:53.370 INFO:tasks.rados.rados.0.smithi070.stdout:3131: finishing write tid 1 to smithi07085154-48 2024-09-14T03:58:53.370 INFO:tasks.rados.rados.0.smithi070.stdout:3129: done (2 left) 2024-09-14T03:58:53.370 INFO:tasks.rados.rados.0.smithi070.stdout:3133: read oid 19 snap -1 2024-09-14T03:58:53.370 INFO:tasks.rados.rados.0.smithi070.stdout:3133: expect (ObjNum 901 snap 301 seq_num 901) 2024-09-14T03:58:53.370 INFO:tasks.rados.rados.0.smithi070.stdout:3134: delete oid 43 current snap is 319 2024-09-14T03:58:53.370 INFO:tasks.rados.rados.0.smithi070.stdout:3131: finishing write tid 2 to smithi07085154-48 2024-09-14T03:58:53.370 INFO:tasks.rados.rados.0.smithi070.stdout:3131: finishing write tid 3 to smithi07085154-48 2024-09-14T03:58:53.370 INFO:tasks.rados.rados.0.smithi070.stdout:3131: finishing write tid 4 to smithi07085154-48 2024-09-14T03:58:53.370 INFO:tasks.rados.rados.0.smithi070.stdout:3132: finishing rollback tid 0 to smithi07085154-45 2024-09-14T03:58:53.370 INFO:tasks.rados.rados.0.smithi070.stdout:3131: finishing write tid 5 to smithi07085154-48 2024-09-14T03:58:53.370 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 2253 (ObjNum 957 snap 319 seq_num 957) dirty exists 2024-09-14T03:58:53.371 INFO:tasks.rados.rados.0.smithi070.stdout:3131: left oid 48 (ObjNum 957 snap 319 seq_num 957) 2024-09-14T03:58:53.371 INFO:tasks.rados.rados.0.smithi070.stdout:3132: finishing rollback tid 1 to smithi07085154-45 2024-09-14T03:58:53.371 INFO:tasks.rados.rados.0.smithi070.stdout:3132: finishing rollback tid 2 to smithi07085154-45 2024-09-14T03:58:53.371 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 45 v 3177 (ObjNum 921 snap 308 seq_num 921) dirty exists 2024-09-14T03:58:53.373 INFO:tasks.rados.rados.0.smithi070.stdout:3131: done (3 left) 2024-09-14T03:58:53.373 INFO:tasks.rados.rados.0.smithi070.stdout:3132: done (2 left) 2024-09-14T03:58:53.374 INFO:tasks.rados.rados.0.smithi070.stdout:3134: done (1 left) 2024-09-14T03:58:53.374 INFO:tasks.rados.rados.0.smithi070.stdout:3135: copy_from oid 47 from oid 37 current snap is 319 2024-09-14T03:58:53.374 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 1 current snap is 319 2024-09-14T03:58:53.374 INFO:tasks.rados.rados.0.smithi070.stdout:3136: seq_num 959 ranges {2334720=466944,2801664=540672,3342336=557056,3899392=106496} 2024-09-14T03:58:53.388 INFO:tasks.rados.rados.0.smithi070.stdout:3136: writing smithi07085154-1 from 2334720 to 2801664 tid 1 2024-09-14T03:58:53.391 INFO:tasks.rados.rados.0.smithi070.stdout:3136: writing smithi07085154-1 from 2801664 to 3342336 tid 2 2024-09-14T03:58:53.394 INFO:tasks.rados.rados.0.smithi070.stdout:3136: writing smithi07085154-1 from 3342336 to 3899392 tid 3 2024-09-14T03:58:53.395 INFO:tasks.rados.rados.0.smithi070.stdout:3136: writing smithi07085154-1 from 3899392 to 4005888 tid 4 2024-09-14T03:58:53.395 INFO:tasks.rados.rados.0.smithi070.stdout:3137: read oid 50 snap -1 2024-09-14T03:58:53.395 INFO:tasks.rados.rados.0.smithi070.stdout:3137: expect (ObjNum 940 snap 316 seq_num 940) 2024-09-14T03:58:53.395 INFO:tasks.rados.rados.0.smithi070.stdout:3138: delete oid 18 current snap is 319 2024-09-14T03:58:53.396 INFO:tasks.rados.rados.0.smithi070.stdout:3135: finishing copy_from racing read to smithi07085154-47 2024-09-14T03:58:53.396 INFO:tasks.rados.rados.0.smithi070.stdout:3135: finishing copy_from to smithi07085154-47 2024-09-14T03:58:53.396 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 47 v 3176 (ObjNum 935 snap 316 seq_num 935) dirty exists 2024-09-14T03:58:53.399 INFO:tasks.rados.rados.0.smithi070.stdout:3135: done (4 left) 2024-09-14T03:58:53.399 INFO:tasks.rados.rados.0.smithi070.stdout:3138: done (3 left) 2024-09-14T03:58:53.399 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 31 current snap is 319 2024-09-14T03:58:53.399 INFO:tasks.rados.rados.0.smithi070.stdout:3139: seq_num 961 ranges {0=499712,499712=712704,1212416=491520} 2024-09-14T03:58:53.402 INFO:tasks.rados.rados.0.smithi070.stdout:3139: writing smithi07085154-31 from 0 to 499712 tid 1 2024-09-14T03:58:53.406 INFO:tasks.rados.rados.0.smithi070.stdout:3139: writing smithi07085154-31 from 499712 to 1212416 tid 2 2024-09-14T03:58:53.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:53 smithi070 ceph-mon[96704]: pgmap v643: 121 pgs: 121 active+clean; 1.0 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 1.4 MiB/s rd, 4.7 MiB/s wr, 20 op/s 2024-09-14T03:58:53.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:53 smithi070 ceph-mon[96704]: osdmap e697: 8 total, 8 up, 8 in 2024-09-14T03:58:53.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:53 smithi070 ceph-mon[98640]: pgmap v643: 121 pgs: 121 active+clean; 1.0 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 1.4 MiB/s rd, 4.7 MiB/s wr, 20 op/s 2024-09-14T03:58:53.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:53 smithi070 ceph-mon[98640]: osdmap e697: 8 total, 8 up, 8 in 2024-09-14T03:58:53.410 INFO:tasks.rados.rados.0.smithi070.stdout:3139: writing smithi07085154-31 from 1212416 to 1703936 tid 3 2024-09-14T03:58:53.411 INFO:tasks.rados.rados.0.smithi070.stdout:3140: rollback oid 15 current snap is 319 2024-09-14T03:58:53.411 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 15 to 312 2024-09-14T03:58:53.411 INFO:tasks.rados.rados.0.smithi070.stdout:3136: finishing write tid 1 to smithi07085154-1 2024-09-14T03:58:53.411 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 42 current snap is 319 2024-09-14T03:58:53.411 INFO:tasks.rados.rados.0.smithi070.stdout:3136: finishing write tid 2 to smithi07085154-1 2024-09-14T03:58:53.411 INFO:tasks.rados.rados.0.smithi070.stdout:3136: finishing write tid 3 to smithi07085154-1 2024-09-14T03:58:53.411 INFO:tasks.rados.rados.0.smithi070.stdout:3136: finishing write tid 4 to smithi07085154-1 2024-09-14T03:58:53.411 INFO:tasks.rados.rados.0.smithi070.stdout:3136: finishing write tid 5 to smithi07085154-1 2024-09-14T03:58:53.411 INFO:tasks.rados.rados.0.smithi070.stdout:3136: finishing write tid 6 to smithi07085154-1 2024-09-14T03:58:53.411 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 1 v 3172 (ObjNum 959 snap 319 seq_num 959) dirty exists 2024-09-14T03:58:53.411 INFO:tasks.rados.rados.0.smithi070.stdout:3136: left oid 1 (ObjNum 959 snap 319 seq_num 959) 2024-09-14T03:58:53.411 INFO:tasks.rados.rados.0.smithi070.stdout:3139: finishing write tid 1 to smithi07085154-31 2024-09-14T03:58:53.412 INFO:tasks.rados.rados.0.smithi070.stdout:3141: seq_num 962 ranges {1572864=516096,2088960=589824,2678784=614400} 2024-09-14T03:58:53.422 INFO:tasks.rados.rados.0.smithi070.stdout:3141: writing smithi07085154-42 from 1572864 to 2088960 tid 1 2024-09-14T03:58:53.425 INFO:tasks.rados.rados.0.smithi070.stdout:3141: writing smithi07085154-42 from 2088960 to 2678784 tid 2 2024-09-14T03:58:53.429 INFO:tasks.rados.rados.0.smithi070.stdout:3141: writing smithi07085154-42 from 2678784 to 3293184 tid 3 2024-09-14T03:58:53.429 INFO:tasks.rados.rados.0.smithi070.stdout:3136: done (5 left) 2024-09-14T03:58:53.429 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 7 current snap is 319 2024-09-14T03:58:53.429 INFO:tasks.rados.rados.0.smithi070.stdout:3139: finishing write tid 2 to smithi07085154-31 2024-09-14T03:58:53.429 INFO:tasks.rados.rados.0.smithi070.stdout:3133: expect (ObjNum 901 snap 301 seq_num 901) 2024-09-14T03:58:53.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:53 smithi094 ceph-mon[86597]: pgmap v643: 121 pgs: 121 active+clean; 1.0 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 1.4 MiB/s rd, 4.7 MiB/s wr, 20 op/s 2024-09-14T03:58:53.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:53 smithi094 ceph-mon[86597]: osdmap e697: 8 total, 8 up, 8 in 2024-09-14T03:58:53.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:58:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:58:53] "GET /metrics HTTP/1.1" 200 36506 "" "Prometheus/2.51.0" 2024-09-14T03:58:54.231 INFO:tasks.rados.rados.0.smithi070.stdout:3139: finishing write tid 3 to smithi07085154-31 2024-09-14T03:58:54.231 INFO:tasks.rados.rados.0.smithi070.stdout:3139: finishing write tid 4 to smithi07085154-31 2024-09-14T03:58:54.232 INFO:tasks.rados.rados.0.smithi070.stdout:3140: finishing rollback tid 0 to smithi07085154-15 2024-09-14T03:58:54.232 INFO:tasks.rados.rados.0.smithi070.stdout:3140: finishing rollback tid 1 to smithi07085154-15 2024-09-14T03:58:54.232 INFO:tasks.rados.rados.0.smithi070.stdout:3140: finishing rollback tid 2 to smithi07085154-15 2024-09-14T03:58:54.232 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 15 v 3185 (ObjNum 805 snap 281 seq_num 805) dirty exists 2024-09-14T03:58:54.232 INFO:tasks.rados.rados.0.smithi070.stdout:3139: finishing write tid 5 to smithi07085154-31 2024-09-14T03:58:54.232 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 3182 (ObjNum 961 snap 319 seq_num 961) dirty exists 2024-09-14T03:58:54.232 INFO:tasks.rados.rados.0.smithi070.stdout:3139: left oid 31 (ObjNum 961 snap 319 seq_num 961) 2024-09-14T03:58:54.232 INFO:tasks.rados.rados.0.smithi070.stdout:3137: expect (ObjNum 940 snap 316 seq_num 940) 2024-09-14T03:58:54.309 INFO:tasks.rados.rados.0.smithi070.stdout:3141: finishing write tid 1 to smithi07085154-42 2024-09-14T03:58:54.309 INFO:tasks.rados.rados.0.smithi070.stdout:3141: finishing write tid 2 to smithi07085154-42 2024-09-14T03:58:54.309 INFO:tasks.rados.rados.0.smithi070.stdout:3141: finishing write tid 3 to smithi07085154-42 2024-09-14T03:58:54.309 INFO:tasks.rados.rados.0.smithi070.stdout:3141: finishing write tid 4 to smithi07085154-42 2024-09-14T03:58:54.309 INFO:tasks.rados.rados.0.smithi070.stdout:3141: finishing write tid 5 to smithi07085154-42 2024-09-14T03:58:54.309 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 3765 (ObjNum 962 snap 319 seq_num 962) dirty exists 2024-09-14T03:58:54.310 INFO:tasks.rados.rados.0.smithi070.stdout:3141: left oid 42 (ObjNum 962 snap 319 seq_num 962) 2024-09-14T03:58:54.310 INFO:tasks.rados.rados.0.smithi070.stdout:3142: seq_num 963 ranges {1302528=532480,1835008=475136,2310144=532480,2842624=196608} 2024-09-14T03:58:54.319 INFO:tasks.rados.rados.0.smithi070.stdout:3142: writing smithi07085154-7 from 1302528 to 1835008 tid 1 2024-09-14T03:58:54.322 INFO:tasks.rados.rados.0.smithi070.stdout:3142: writing smithi07085154-7 from 1835008 to 2310144 tid 2 2024-09-14T03:58:54.325 INFO:tasks.rados.rados.0.smithi070.stdout:3142: writing smithi07085154-7 from 2310144 to 2842624 tid 3 2024-09-14T03:58:54.328 INFO:tasks.rados.rados.0.smithi070.stdout:3142: writing smithi07085154-7 from 2842624 to 3039232 tid 4 2024-09-14T03:58:54.328 INFO:tasks.rados.rados.0.smithi070.stdout:3133: done (5 left) 2024-09-14T03:58:54.328 INFO:tasks.rados.rados.0.smithi070.stdout:3137: done (4 left) 2024-09-14T03:58:54.328 INFO:tasks.rados.rados.0.smithi070.stdout:3139: done (3 left) 2024-09-14T03:58:54.328 INFO:tasks.rados.rados.0.smithi070.stdout:3140: done (2 left) 2024-09-14T03:58:54.328 INFO:tasks.rados.rados.0.smithi070.stdout:3141: done (1 left) 2024-09-14T03:58:54.328 INFO:tasks.rados.rados.0.smithi070.stdout:3143: snap_create 2024-09-14T03:58:54.328 INFO:tasks.rados.rados.0.smithi070.stdout:3142: finishing write tid 1 to smithi07085154-7 2024-09-14T03:58:54.328 INFO:tasks.rados.rados.0.smithi070.stdout:3142: finishing write tid 2 to smithi07085154-7 2024-09-14T03:58:54.330 INFO:tasks.rados.rados.0.smithi070.stdout:3142: finishing write tid 3 to smithi07085154-7 2024-09-14T03:58:54.330 INFO:tasks.rados.rados.0.smithi070.stdout:3142: finishing write tid 4 to smithi07085154-7 2024-09-14T03:58:54.331 INFO:tasks.rados.rados.0.smithi070.stdout:3142: finishing write tid 5 to smithi07085154-7 2024-09-14T03:58:54.332 INFO:tasks.rados.rados.0.smithi070.stdout:3142: finishing write tid 6 to smithi07085154-7 2024-09-14T03:58:54.332 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 3182 (ObjNum 963 snap 319 seq_num 963) dirty exists 2024-09-14T03:58:54.332 INFO:tasks.rados.rados.0.smithi070.stdout:3142: left oid 7 (ObjNum 963 snap 319 seq_num 963) 2024-09-14T03:58:54.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:54 smithi094 ceph-mon[86597]: osdmap e698: 8 total, 8 up, 8 in 2024-09-14T03:58:54.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:54 smithi094 ceph-mon[86597]: osdmap e699: 8 total, 8 up, 8 in 2024-09-14T03:58:54.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:54 smithi070 ceph-mon[96704]: osdmap e698: 8 total, 8 up, 8 in 2024-09-14T03:58:54.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:54 smithi070 ceph-mon[96704]: osdmap e699: 8 total, 8 up, 8 in 2024-09-14T03:58:54.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:54 smithi070 ceph-mon[98640]: osdmap e698: 8 total, 8 up, 8 in 2024-09-14T03:58:54.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:54 smithi070 ceph-mon[98640]: osdmap e699: 8 total, 8 up, 8 in 2024-09-14T03:58:55.159 INFO:tasks.rados.rados.0.smithi070.stdout:3142: done (1 left) 2024-09-14T03:58:55.159 INFO:tasks.rados.rados.0.smithi070.stdout:3143: done (0 left) 2024-09-14T03:58:55.159 INFO:tasks.rados.rados.0.smithi070.stdout:3144: delete oid 47 current snap is 320 2024-09-14T03:58:55.164 INFO:tasks.rados.rados.0.smithi070.stdout:3144: done (0 left) 2024-09-14T03:58:55.165 INFO:tasks.rados.rados.0.smithi070.stdout:3145: snap_create 2024-09-14T03:58:55.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:55 smithi094 ceph-mon[86597]: pgmap v647: 121 pgs: 121 active+clean; 1.0 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 829 KiB/s rd, 4.5 MiB/s wr, 17 op/s 2024-09-14T03:58:55.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:55 smithi070 ceph-mon[96704]: pgmap v647: 121 pgs: 121 active+clean; 1.0 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 829 KiB/s rd, 4.5 MiB/s wr, 17 op/s 2024-09-14T03:58:55.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:55 smithi070 ceph-mon[98640]: pgmap v647: 121 pgs: 121 active+clean; 1.0 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 829 KiB/s rd, 4.5 MiB/s wr, 17 op/s 2024-09-14T03:58:56.165 INFO:tasks.rados.rados.0.smithi070.stdout:3145: done (0 left) 2024-09-14T03:58:56.165 INFO:tasks.rados.rados.0.smithi070.stdout:3146: read oid 39 snap -1 2024-09-14T03:58:56.165 INFO:tasks.rados.rados.0.smithi070.stdout:3146: expect (ObjNum 946 snap 318 seq_num 946) 2024-09-14T03:58:56.165 INFO:tasks.rados.rados.0.smithi070.stdout:3147: snap_create 2024-09-14T03:58:56.174 INFO:tasks.rados.rados.0.smithi070.stdout:3146: expect (ObjNum 946 snap 318 seq_num 946) 2024-09-14T03:58:56.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:56 smithi094 ceph-mon[86597]: osdmap e700: 8 total, 8 up, 8 in 2024-09-14T03:58:56.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:56 smithi094 ceph-mon[86597]: pgmap v649: 121 pgs: 121 active+clean; 1.0 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 463 KiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-14T03:58:56.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:56 smithi070 ceph-mon[96704]: osdmap e700: 8 total, 8 up, 8 in 2024-09-14T03:58:56.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:56 smithi070 ceph-mon[96704]: pgmap v649: 121 pgs: 121 active+clean; 1.0 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 463 KiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-14T03:58:56.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:56 smithi070 ceph-mon[98640]: osdmap e700: 8 total, 8 up, 8 in 2024-09-14T03:58:56.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:56 smithi070 ceph-mon[98640]: pgmap v649: 121 pgs: 121 active+clean; 1.0 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 463 KiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-14T03:58:57.166 INFO:tasks.rados.rados.0.smithi070.stdout:3146: done (1 left) 2024-09-14T03:58:57.166 INFO:tasks.rados.rados.0.smithi070.stdout:3147: done (0 left) 2024-09-14T03:58:57.166 INFO:tasks.rados.rados.0.smithi070.stdout:3148: delete oid 11 current snap is 322 2024-09-14T03:58:57.170 INFO:tasks.rados.rados.0.smithi070.stdout:3148: done (0 left) 2024-09-14T03:58:57.170 INFO:tasks.rados.rados.0.smithi070.stdout:3149: delete oid 18 current snap is 322 2024-09-14T03:58:57.175 INFO:tasks.rados.rados.0.smithi070.stdout:3149: done (0 left) 2024-09-14T03:58:57.175 INFO:tasks.rados.rados.0.smithi070.stdout:3150: read oid 38 snap -1 2024-09-14T03:58:57.175 INFO:tasks.rados.rados.0.smithi070.stdout:3150: expect deleted 2024-09-14T03:58:57.176 INFO:tasks.rados.rados.0.smithi070.stdout:3151: read oid 31 snap -1 2024-09-14T03:58:57.176 INFO:tasks.rados.rados.0.smithi070.stdout:3151: expect (ObjNum 961 snap 319 seq_num 961) 2024-09-14T03:58:57.176 INFO:tasks.rados.rados.0.smithi070.stdout:3152: snap_remove snap 319 2024-09-14T03:58:57.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:57 smithi094 ceph-mon[86597]: osdmap e701: 8 total, 8 up, 8 in 2024-09-14T03:58:57.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:57 smithi070 ceph-mon[96704]: osdmap e701: 8 total, 8 up, 8 in 2024-09-14T03:58:57.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:57 smithi070 ceph-mon[98640]: osdmap e701: 8 total, 8 up, 8 in 2024-09-14T03:58:58.177 INFO:tasks.rados.rados.0.smithi070.stdout:3152: done (2 left) 2024-09-14T03:58:58.177 INFO:tasks.rados.rados.0.smithi070.stdout:3153: setattr oid 12 current snap is 322 2024-09-14T03:58:58.177 INFO:tasks.rados.rados.0.smithi070.stdout:3151: expect (ObjNum 961 snap 319 seq_num 961) 2024-09-14T03:58:58.220 INFO:tasks.rados.rados.0.smithi070.stdout:3150: done (2 left) 2024-09-14T03:58:58.221 INFO:tasks.rados.rados.0.smithi070.stdout:3151: done (1 left) 2024-09-14T03:58:58.221 INFO:tasks.rados.rados.0.smithi070.stdout:3154: rollback oid 38 current snap is 322 2024-09-14T03:58:58.221 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 38 to 312 2024-09-14T03:58:58.221 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 29 current snap is 322 2024-09-14T03:58:58.221 INFO:tasks.rados.rados.0.smithi070.stdout:3155: seq_num 967 ranges {131072=598016,729088=466944,1196032=516096,1712128=221184} 2024-09-14T03:58:58.225 INFO:tasks.rados.rados.0.smithi070.stdout:3155: writing smithi07085154-29 from 131072 to 729088 tid 1 2024-09-14T03:58:58.228 INFO:tasks.rados.rados.0.smithi070.stdout:3155: writing smithi07085154-29 from 729088 to 1196032 tid 2 2024-09-14T03:58:58.231 INFO:tasks.rados.rados.0.smithi070.stdout:3155: writing smithi07085154-29 from 1196032 to 1712128 tid 3 2024-09-14T03:58:58.233 INFO:tasks.rados.rados.0.smithi070.stdout:3155: writing smithi07085154-29 from 1712128 to 1933312 tid 4 2024-09-14T03:58:58.233 INFO:tasks.rados.rados.0.smithi070.stdout:3156: copy_from oid 40 from oid 47 current snap is 322 2024-09-14T03:58:58.233 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 2254 (ObjNum 917 snap 308 seq_num 917) dirty exists 2024-09-14T03:58:58.233 INFO:tasks.rados.rados.0.smithi070.stdout:3154: finishing rollback tid 1 to smithi07085154-38 2024-09-14T03:58:58.233 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 38 v 2237 (ObjNum 257 snap 0 seq_num 403) dirty dne 2024-09-14T03:58:58.233 INFO:tasks.rados.rados.0.smithi070.stdout:3153: done (3 left) 2024-09-14T03:58:58.234 INFO:tasks.rados.rados.0.smithi070.stdout:3154: done (2 left) 2024-09-14T03:58:58.234 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 22 current snap is 322 2024-09-14T03:58:58.234 INFO:tasks.rados.rados.0.smithi070.stdout:3157: seq_num 968 ranges {1638400=614400,2252800=745472,2998272=458752} 2024-09-14T03:58:58.248 INFO:tasks.rados.rados.0.smithi070.stdout:3157: writing smithi07085154-22 from 1638400 to 2252800 tid 1 2024-09-14T03:58:58.250 INFO:tasks.rados.rados.0.smithi070.stdout:3157: writing smithi07085154-22 from 2252800 to 2998272 tid 2 2024-09-14T03:58:58.253 INFO:tasks.rados.rados.0.smithi070.stdout:3157: writing smithi07085154-22 from 2998272 to 3457024 tid 3 2024-09-14T03:58:58.253 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 43 current snap is 322 2024-09-14T03:58:58.253 INFO:tasks.rados.rados.0.smithi070.stdout:3156: finishing copy_from to smithi07085154-40 2024-09-14T03:58:58.253 INFO:tasks.rados.rados.0.smithi070.stdout:3156: got expected ENOENT (src dne) 2024-09-14T03:58:58.253 INFO:tasks.rados.rados.0.smithi070.stdout:3156: finishing copy_from racing read to smithi07085154-40 2024-09-14T03:58:58.253 INFO:tasks.rados.rados.0.smithi070.stdout:3155: finishing write tid 1 to smithi07085154-29 2024-09-14T03:58:58.253 INFO:tasks.rados.rados.0.smithi070.stdout:3155: finishing write tid 2 to smithi07085154-29 2024-09-14T03:58:58.253 INFO:tasks.rados.rados.0.smithi070.stdout:3155: finishing write tid 3 to smithi07085154-29 2024-09-14T03:58:58.253 INFO:tasks.rados.rados.0.smithi070.stdout:3155: finishing write tid 4 to smithi07085154-29 2024-09-14T03:58:58.253 INFO:tasks.rados.rados.0.smithi070.stdout:3158: seq_num 969 ranges {0=638976,638976=630784,1269760=532480,1802240=40960} 2024-09-14T03:58:58.256 INFO:tasks.rados.rados.0.smithi070.stdout:3158: writing smithi07085154-43 from 0 to 638976 tid 1 2024-09-14T03:58:58.260 INFO:tasks.rados.rados.0.smithi070.stdout:3158: writing smithi07085154-43 from 638976 to 1269760 tid 2 2024-09-14T03:58:58.265 INFO:tasks.rados.rados.0.smithi070.stdout:3158: writing smithi07085154-43 from 1269760 to 1802240 tid 3 2024-09-14T03:58:58.265 INFO:tasks.rados.rados.0.smithi070.stdout:3158: writing smithi07085154-43 from 1802240 to 1843200 tid 4 2024-09-14T03:58:58.265 INFO:tasks.rados.rados.0.smithi070.stdout:3156: done (3 left) 2024-09-14T03:58:58.265 INFO:tasks.rados.rados.0.smithi070.stdout:3159: read oid 8 snap -1 2024-09-14T03:58:58.266 INFO:tasks.rados.rados.0.smithi070.stdout:3159: expect (ObjNum 923 snap 309 seq_num 923) 2024-09-14T03:58:58.266 INFO:tasks.rados.rados.0.smithi070.stdout:3160: read oid 18 snap 284 2024-09-14T03:58:58.266 INFO:tasks.rados.rados.0.smithi070.stdout:3160: expect (ObjNum 826 snap 283 seq_num 826) 2024-09-14T03:58:58.266 INFO:tasks.rados.rados.0.smithi070.stdout:3161: rmattr oid 11 current snap is 322 2024-09-14T03:58:58.266 INFO:tasks.rados.rados.0.smithi070.stdout:3161: done (5 left) 2024-09-14T03:58:58.266 INFO:tasks.rados.rados.0.smithi070.stdout:3162: rollback oid 26 current snap is 322 2024-09-14T03:58:58.266 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 26 to 310 2024-09-14T03:58:58.266 INFO:tasks.rados.rados.0.smithi070.stdout:3155: finishing write tid 5 to smithi07085154-29 2024-09-14T03:58:58.266 INFO:tasks.rados.rados.0.smithi070.stdout:3155: finishing write tid 6 to smithi07085154-29 2024-09-14T03:58:58.266 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 3190 (ObjNum 967 snap 322 seq_num 967) dirty exists 2024-09-14T03:58:58.267 INFO:tasks.rados.rados.0.smithi070.stdout:3155: left oid 29 (ObjNum 967 snap 322 seq_num 967) 2024-09-14T03:58:58.267 INFO:tasks.rados.rados.0.smithi070.stdout:3157: finishing write tid 1 to smithi07085154-22 2024-09-14T03:58:58.267 INFO:tasks.rados.rados.0.smithi070.stdout:3155: done (5 left) 2024-09-14T03:58:58.267 INFO:tasks.rados.rados.0.smithi070.stdout:3163: snap_create 2024-09-14T03:58:58.267 INFO:tasks.rados.rados.0.smithi070.stdout:3157: finishing write tid 2 to smithi07085154-22 2024-09-14T03:58:58.267 INFO:tasks.rados.rados.0.smithi070.stdout:3157: finishing write tid 3 to smithi07085154-22 2024-09-14T03:58:58.267 INFO:tasks.rados.rados.0.smithi070.stdout:3157: finishing write tid 4 to smithi07085154-22 2024-09-14T03:58:58.267 INFO:tasks.rados.rados.0.smithi070.stdout:3157: finishing write tid 5 to smithi07085154-22 2024-09-14T03:58:58.267 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 3194 (ObjNum 968 snap 322 seq_num 968) dirty exists 2024-09-14T03:58:58.267 INFO:tasks.rados.rados.0.smithi070.stdout:3157: left oid 22 (ObjNum 968 snap 322 seq_num 968) 2024-09-14T03:58:58.267 INFO:tasks.rados.rados.0.smithi070.stdout:3158: finishing write tid 1 to smithi07085154-43 2024-09-14T03:58:58.267 INFO:tasks.rados.rados.0.smithi070.stdout:3158: finishing write tid 2 to smithi07085154-43 2024-09-14T03:58:58.268 INFO:tasks.rados.rados.0.smithi070.stdout:3158: finishing write tid 3 to smithi07085154-43 2024-09-14T03:58:58.268 INFO:tasks.rados.rados.0.smithi070.stdout:3158: finishing write tid 4 to smithi07085154-43 2024-09-14T03:58:58.270 INFO:tasks.rados.rados.0.smithi070.stdout:3158: finishing write tid 5 to smithi07085154-43 2024-09-14T03:58:58.271 INFO:tasks.rados.rados.0.smithi070.stdout:3162: finishing rollback tid 0 to smithi07085154-26 2024-09-14T03:58:58.272 INFO:tasks.rados.rados.0.smithi070.stdout:3162: finishing rollback tid 1 to smithi07085154-26 2024-09-14T03:58:58.272 INFO:tasks.rados.rados.0.smithi070.stdout:3162: finishing rollback tid 2 to smithi07085154-26 2024-09-14T03:58:58.273 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 3769 (ObjNum 918 snap 308 seq_num 918) dirty exists 2024-09-14T03:58:58.274 INFO:tasks.rados.rados.0.smithi070.stdout:3158: finishing write tid 6 to smithi07085154-43 2024-09-14T03:58:58.274 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 3190 (ObjNum 969 snap 322 seq_num 969) dirty exists 2024-09-14T03:58:58.274 INFO:tasks.rados.rados.0.smithi070.stdout:3158: left oid 43 (ObjNum 969 snap 322 seq_num 969) 2024-09-14T03:58:58.278 INFO:tasks.rados.rados.0.smithi070.stdout:3159: expect (ObjNum 923 snap 309 seq_num 923) 2024-09-14T03:58:58.312 INFO:tasks.rados.rados.0.smithi070.stdout:3160: expect (ObjNum 826 snap 283 seq_num 826) 2024-09-14T03:58:58.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:58 smithi094 ceph-mon[86597]: osdmap e702: 8 total, 8 up, 8 in 2024-09-14T03:58:58.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:58 smithi094 ceph-mon[86597]: pgmap v652: 121 pgs: 121 active+clean; 1.1 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.4 MiB/s rd, 6.7 MiB/s wr, 36 op/s 2024-09-14T03:58:58.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:58 smithi070 ceph-mon[96704]: osdmap e702: 8 total, 8 up, 8 in 2024-09-14T03:58:58.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:58 smithi070 ceph-mon[96704]: pgmap v652: 121 pgs: 121 active+clean; 1.1 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.4 MiB/s rd, 6.7 MiB/s wr, 36 op/s 2024-09-14T03:58:58.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:58 smithi070 ceph-mon[98640]: osdmap e702: 8 total, 8 up, 8 in 2024-09-14T03:58:58.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:58 smithi070 ceph-mon[98640]: pgmap v652: 121 pgs: 121 active+clean; 1.1 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.4 MiB/s rd, 6.7 MiB/s wr, 36 op/s 2024-09-14T03:58:59.108 INFO:tasks.rados.rados.0.smithi070.stdout:3157: done (5 left) 2024-09-14T03:58:59.109 INFO:tasks.rados.rados.0.smithi070.stdout:3158: done (4 left) 2024-09-14T03:58:59.109 INFO:tasks.rados.rados.0.smithi070.stdout:3159: done (3 left) 2024-09-14T03:58:59.109 INFO:tasks.rados.rados.0.smithi070.stdout:3160: done (2 left) 2024-09-14T03:58:59.109 INFO:tasks.rados.rados.0.smithi070.stdout:3162: done (1 left) 2024-09-14T03:58:59.109 INFO:tasks.rados.rados.0.smithi070.stdout:3163: done (0 left) 2024-09-14T03:58:59.109 INFO:tasks.rados.rados.0.smithi070.stdout:3164: delete oid 12 current snap is 323 2024-09-14T03:58:59.116 INFO:tasks.rados.rados.0.smithi070.stdout:3164: done (0 left) 2024-09-14T03:58:59.116 INFO:tasks.rados.rados.0.smithi070.stdout:3165: setattr oid 33 current snap is 323 2024-09-14T03:58:59.117 INFO:tasks.rados.rados.0.smithi070.stdout:3166: delete oid 45 current snap is 323 2024-09-14T03:58:59.121 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 33 v 3178 (ObjNum 33 snap 0 seq_num 2929780960) dirty exists 2024-09-14T03:58:59.121 INFO:tasks.rados.rados.0.smithi070.stdout:3165: done (1 left) 2024-09-14T03:58:59.121 INFO:tasks.rados.rados.0.smithi070.stdout:3166: done (0 left) 2024-09-14T03:58:59.121 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 14 current snap is 323 2024-09-14T03:58:59.121 INFO:tasks.rados.rados.0.smithi070.stdout:3167: seq_num 972 ranges {9912320=688128,10600448=688128,11288576=516096} 2024-09-14T03:58:59.177 INFO:tasks.rados.rados.0.smithi070.stdout:3167: writing smithi07085154-14 from 9912320 to 10600448 tid 1 2024-09-14T03:58:59.181 INFO:tasks.rados.rados.0.smithi070.stdout:3167: writing smithi07085154-14 from 10600448 to 11288576 tid 2 2024-09-14T03:58:59.184 INFO:tasks.rados.rados.0.smithi070.stdout:3167: writing smithi07085154-14 from 11288576 to 11804672 tid 3 2024-09-14T03:58:59.184 INFO:tasks.rados.rados.0.smithi070.stdout:3168: delete oid 48 current snap is 323 2024-09-14T03:58:59.187 INFO:tasks.rados.rados.0.smithi070.stdout:3168: done (1 left) 2024-09-14T03:58:59.187 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 5 current snap is 323 2024-09-14T03:58:59.187 INFO:tasks.rados.rados.0.smithi070.stdout:3169: seq_num 974 ranges {0=720896,720896=507904,1228800=671744,1900544=24576} 2024-09-14T03:58:59.191 INFO:tasks.rados.rados.0.smithi070.stdout:3169: writing smithi07085154-5 from 0 to 720896 tid 1 2024-09-14T03:58:59.194 INFO:tasks.rados.rados.0.smithi070.stdout:3169: writing smithi07085154-5 from 720896 to 1228800 tid 2 2024-09-14T03:58:59.198 INFO:tasks.rados.rados.0.smithi070.stdout:3169: writing smithi07085154-5 from 1228800 to 1900544 tid 3 2024-09-14T03:58:59.198 INFO:tasks.rados.rados.0.smithi070.stdout:3169: writing smithi07085154-5 from 1900544 to 1925120 tid 4 2024-09-14T03:58:59.198 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 48 current snap is 323 2024-09-14T03:58:59.198 INFO:tasks.rados.rados.0.smithi070.stdout:3167: finishing write tid 1 to smithi07085154-14 2024-09-14T03:58:59.199 INFO:tasks.rados.rados.0.smithi070.stdout:3167: finishing write tid 2 to smithi07085154-14 2024-09-14T03:58:59.199 INFO:tasks.rados.rados.0.smithi070.stdout:3167: finishing write tid 3 to smithi07085154-14 2024-09-14T03:58:59.199 INFO:tasks.rados.rados.0.smithi070.stdout:3167: finishing write tid 4 to smithi07085154-14 2024-09-14T03:58:59.199 INFO:tasks.rados.rados.0.smithi070.stdout:3167: finishing write tid 5 to smithi07085154-14 2024-09-14T03:58:59.199 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 3183 (ObjNum 972 snap 323 seq_num 972) dirty exists 2024-09-14T03:58:59.199 INFO:tasks.rados.rados.0.smithi070.stdout:3167: left oid 14 (ObjNum 972 snap 323 seq_num 972) 2024-09-14T03:58:59.199 INFO:tasks.rados.rados.0.smithi070.stdout:3169: finishing write tid 1 to smithi07085154-5 2024-09-14T03:58:59.199 INFO:tasks.rados.rados.0.smithi070.stdout:3169: finishing write tid 2 to smithi07085154-5 2024-09-14T03:58:59.199 INFO:tasks.rados.rados.0.smithi070.stdout:3170: seq_num 975 ranges {0=737280,737280=786432,1523712=417792} 2024-09-14T03:58:59.202 INFO:tasks.rados.rados.0.smithi070.stdout:3170: writing smithi07085154-48 from 0 to 737280 tid 1 2024-09-14T03:58:59.207 INFO:tasks.rados.rados.0.smithi070.stdout:3170: writing smithi07085154-48 from 737280 to 1523712 tid 2 2024-09-14T03:58:59.210 INFO:tasks.rados.rados.0.smithi070.stdout:3170: writing smithi07085154-48 from 1523712 to 1941504 tid 3 2024-09-14T03:58:59.210 INFO:tasks.rados.rados.0.smithi070.stdout:3167: done (2 left) 2024-09-14T03:58:59.210 INFO:tasks.rados.rados.0.smithi070.stdout:3171: rollback oid 25 current snap is 323 2024-09-14T03:58:59.211 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 25 to 316 2024-09-14T03:58:59.211 INFO:tasks.rados.rados.0.smithi070.stdout:3169: finishing write tid 3 to smithi07085154-5 2024-09-14T03:58:59.211 INFO:tasks.rados.rados.0.smithi070.stdout:3169: finishing write tid 4 to smithi07085154-5 2024-09-14T03:58:59.211 INFO:tasks.rados.rados.0.smithi070.stdout:3169: finishing write tid 5 to smithi07085154-5 2024-09-14T03:58:59.211 INFO:tasks.rados.rados.0.smithi070.stdout:3169: finishing write tid 6 to smithi07085154-5 2024-09-14T03:58:59.211 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 584 (ObjNum 974 snap 323 seq_num 974) dirty exists 2024-09-14T03:58:59.211 INFO:tasks.rados.rados.0.smithi070.stdout:3169: left oid 5 (ObjNum 974 snap 323 seq_num 974) 2024-09-14T03:58:59.211 INFO:tasks.rados.rados.0.smithi070.stdout:3170: finishing write tid 1 to smithi07085154-48 2024-09-14T03:58:59.211 INFO:tasks.rados.rados.0.smithi070.stdout:3169: done (2 left) 2024-09-14T03:58:59.211 INFO:tasks.rados.rados.0.smithi070.stdout:3172: delete oid 32 current snap is 323 2024-09-14T03:58:59.212 INFO:tasks.rados.rados.0.smithi070.stdout:3172: done (2 left) 2024-09-14T03:58:59.212 INFO:tasks.rados.rados.0.smithi070.stdout:3173: delete oid 33 current snap is 323 2024-09-14T03:58:59.213 INFO:tasks.rados.rados.0.smithi070.stdout:3170: finishing write tid 2 to smithi07085154-48 2024-09-14T03:58:59.213 INFO:tasks.rados.rados.0.smithi070.stdout:3170: finishing write tid 3 to smithi07085154-48 2024-09-14T03:58:59.214 INFO:tasks.rados.rados.0.smithi070.stdout:3170: finishing write tid 4 to smithi07085154-48 2024-09-14T03:58:59.215 INFO:tasks.rados.rados.0.smithi070.stdout:3170: finishing write tid 5 to smithi07085154-48 2024-09-14T03:58:59.215 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 2260 (ObjNum 975 snap 323 seq_num 975) dirty exists 2024-09-14T03:58:59.215 INFO:tasks.rados.rados.0.smithi070.stdout:3170: left oid 48 (ObjNum 975 snap 323 seq_num 975) 2024-09-14T03:58:59.215 INFO:tasks.rados.rados.0.smithi070.stdout:3171: finishing rollback tid 0 to smithi07085154-25 2024-09-14T03:58:59.215 INFO:tasks.rados.rados.0.smithi070.stdout:3170: done (2 left) 2024-09-14T03:58:59.215 INFO:tasks.rados.rados.0.smithi070.stdout:3173: done (1 left) 2024-09-14T03:58:59.215 INFO:tasks.rados.rados.0.smithi070.stdout:3174: read oid 23 snap 301 2024-09-14T03:58:59.215 INFO:tasks.rados.rados.0.smithi070.stdout:3174: expect deleted 2024-09-14T03:58:59.215 INFO:tasks.rados.rados.0.smithi070.stdout:3175: snap_remove snap 292 2024-09-14T03:58:59.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:59 smithi094 ceph-mon[86597]: osdmap e703: 8 total, 8 up, 8 in 2024-09-14T03:58:59.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:58:59 smithi094 ceph-mon[86597]: osdmap e704: 8 total, 8 up, 8 in 2024-09-14T03:58:59.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:59 smithi070 ceph-mon[96704]: osdmap e703: 8 total, 8 up, 8 in 2024-09-14T03:58:59.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:58:59 smithi070 ceph-mon[96704]: osdmap e704: 8 total, 8 up, 8 in 2024-09-14T03:58:59.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:59 smithi070 ceph-mon[98640]: osdmap e703: 8 total, 8 up, 8 in 2024-09-14T03:58:59.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:58:59 smithi070 ceph-mon[98640]: osdmap e704: 8 total, 8 up, 8 in 2024-09-14T03:59:00.186 INFO:tasks.rados.rados.0.smithi070.stdout:3171: finishing rollback tid 1 to smithi07085154-25 2024-09-14T03:59:00.186 INFO:tasks.rados.rados.0.smithi070.stdout:3171: finishing rollback tid 2 to smithi07085154-25 2024-09-14T03:59:00.186 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 3185 (ObjNum 930 snap 313 seq_num 930) dirty exists 2024-09-14T03:59:00.186 INFO:tasks.rados.rados.0.smithi070.stdout:3171: done (2 left) 2024-09-14T03:59:00.186 INFO:tasks.rados.rados.0.smithi070.stdout:3174: done (1 left) 2024-09-14T03:59:00.186 INFO:tasks.rados.rados.0.smithi070.stdout:3175: done (0 left) 2024-09-14T03:59:00.186 INFO:tasks.rados.rados.0.smithi070.stdout:3176: delete oid 43 current snap is 323 2024-09-14T03:59:00.198 INFO:tasks.rados.rados.0.smithi070.stdout:3176: done (0 left) 2024-09-14T03:59:00.198 INFO:tasks.rados.rados.0.smithi070.stdout:3177: delete oid 48 current snap is 323 2024-09-14T03:59:00.204 INFO:tasks.rados.rados.0.smithi070.stdout:3177: done (0 left) 2024-09-14T03:59:00.204 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 7 current snap is 323 2024-09-14T03:59:00.204 INFO:tasks.rados.rados.0.smithi070.stdout:3178: seq_num 980 ranges {3039232=417792,3457024=614400,4071424=663552,4734976=327680} 2024-09-14T03:59:00.224 INFO:tasks.rados.rados.0.smithi070.stdout:3178: writing smithi07085154-7 from 3039232 to 3457024 tid 1 2024-09-14T03:59:00.227 INFO:tasks.rados.rados.0.smithi070.stdout:3178: writing smithi07085154-7 from 3457024 to 4071424 tid 2 2024-09-14T03:59:00.232 INFO:tasks.rados.rados.0.smithi070.stdout:3178: writing smithi07085154-7 from 4071424 to 4734976 tid 3 2024-09-14T03:59:00.234 INFO:tasks.rados.rados.0.smithi070.stdout:3178: writing smithi07085154-7 from 4734976 to 5062656 tid 4 2024-09-14T03:59:00.234 INFO:tasks.rados.rados.0.smithi070.stdout:3179: rollback oid 19 current snap is 323 2024-09-14T03:59:00.234 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 19 to 322 2024-09-14T03:59:00.234 INFO:tasks.rados.rados.0.smithi070.stdout:3178: finishing write tid 1 to smithi07085154-7 2024-09-14T03:59:00.234 INFO:tasks.rados.rados.0.smithi070.stdout:3178: finishing write tid 2 to smithi07085154-7 2024-09-14T03:59:00.235 INFO:tasks.rados.rados.0.smithi070.stdout:3180: rollback oid 24 current snap is 323 2024-09-14T03:59:00.235 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 24 to 232 2024-09-14T03:59:00.235 INFO:tasks.rados.rados.0.smithi070.stdout:3181: snap_create 2024-09-14T03:59:00.237 INFO:tasks.rados.rados.0.smithi070.stdout:3180: finishing rollback tid 0 to smithi07085154-24 2024-09-14T03:59:00.238 INFO:tasks.rados.rados.0.smithi070.stdout:3180: finishing rollback tid 1 to smithi07085154-24 2024-09-14T03:59:00.238 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 3185 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-14T03:59:00.238 INFO:tasks.rados.rados.0.smithi070.stdout:3178: finishing write tid 3 to smithi07085154-7 2024-09-14T03:59:00.238 INFO:tasks.rados.rados.0.smithi070.stdout:3178: finishing write tid 4 to smithi07085154-7 2024-09-14T03:59:00.238 INFO:tasks.rados.rados.0.smithi070.stdout:3178: finishing write tid 5 to smithi07085154-7 2024-09-14T03:59:00.240 INFO:tasks.rados.rados.0.smithi070.stdout:3179: finishing rollback tid 0 to smithi07085154-19 2024-09-14T03:59:00.240 INFO:tasks.rados.rados.0.smithi070.stdout:3178: finishing write tid 6 to smithi07085154-7 2024-09-14T03:59:00.240 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 3187 (ObjNum 980 snap 323 seq_num 980) dirty exists 2024-09-14T03:59:00.240 INFO:tasks.rados.rados.0.smithi070.stdout:3178: left oid 7 (ObjNum 980 snap 323 seq_num 980) 2024-09-14T03:59:00.240 INFO:tasks.rados.rados.0.smithi070.stdout:3179: finishing rollback tid 1 to smithi07085154-19 2024-09-14T03:59:00.240 INFO:tasks.rados.rados.0.smithi070.stdout:3179: finishing rollback tid 2 to smithi07085154-19 2024-09-14T03:59:00.240 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 19 v 3197 (ObjNum 901 snap 301 seq_num 901) dirty exists 2024-09-14T03:59:00.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:00 smithi094 ceph-mon[86597]: pgmap v655: 121 pgs: 121 active+clean; 1.1 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 7.5 MiB/s wr, 45 op/s 2024-09-14T03:59:00.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:59:00 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:59:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:59:00.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:00 smithi070 ceph-mon[96704]: pgmap v655: 121 pgs: 121 active+clean; 1.1 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 7.5 MiB/s wr, 45 op/s 2024-09-14T03:59:00.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:00 smithi070 ceph-mon[98640]: pgmap v655: 121 pgs: 121 active+clean; 1.1 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 7.5 MiB/s wr, 45 op/s 2024-09-14T03:59:01.188 INFO:tasks.rados.rados.0.smithi070.stdout:3178: done (3 left) 2024-09-14T03:59:01.188 INFO:tasks.rados.rados.0.smithi070.stdout:3179: done (2 left) 2024-09-14T03:59:01.188 INFO:tasks.rados.rados.0.smithi070.stdout:3180: done (1 left) 2024-09-14T03:59:01.189 INFO:tasks.rados.rados.0.smithi070.stdout:3181: done (0 left) 2024-09-14T03:59:01.189 INFO:tasks.rados.rados.0.smithi070.stdout:3182: read oid 49 snap -1 2024-09-14T03:59:01.189 INFO:tasks.rados.rados.0.smithi070.stdout:3182: expect (ObjNum 898 snap 300 seq_num 898) 2024-09-14T03:59:01.189 INFO:tasks.rados.rados.0.smithi070.stdout:3183: setattr oid 36 current snap is 324 2024-09-14T03:59:01.191 INFO:tasks.rados.rados.0.smithi070.stdout:3184: delete oid 7 current snap is 324 2024-09-14T03:59:01.195 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 36 v 3173 (ObjNum 905 snap 301 seq_num 905) dirty exists 2024-09-14T03:59:01.197 INFO:tasks.rados.rados.0.smithi070.stdout:3182: expect (ObjNum 898 snap 300 seq_num 898) 2024-09-14T03:59:01.216 INFO:tasks.rados.rados.0.smithi070.stdout:3182: done (2 left) 2024-09-14T03:59:01.217 INFO:tasks.rados.rados.0.smithi070.stdout:3183: done (1 left) 2024-09-14T03:59:01.217 INFO:tasks.rados.rados.0.smithi070.stdout:3184: done (0 left) 2024-09-14T03:59:01.217 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 30 current snap is 324 2024-09-14T03:59:01.217 INFO:tasks.rados.rados.0.smithi070.stdout:3185: seq_num 982 ranges {0=450560,450560=425984,876544=679936,1556480=466944,2023424=32768} 2024-09-14T03:59:01.219 INFO:tasks.rados.rados.0.smithi070.stdout:3185: writing smithi07085154-30 from 0 to 450560 tid 1 2024-09-14T03:59:01.221 INFO:tasks.rados.rados.0.smithi070.stdout:3185: writing smithi07085154-30 from 450560 to 876544 tid 2 2024-09-14T03:59:01.224 INFO:tasks.rados.rados.0.smithi070.stdout:3185: writing smithi07085154-30 from 876544 to 1556480 tid 3 2024-09-14T03:59:01.227 INFO:tasks.rados.rados.0.smithi070.stdout:3185: writing smithi07085154-30 from 1556480 to 2023424 tid 4 2024-09-14T03:59:01.228 INFO:tasks.rados.rados.0.smithi070.stdout:3185: writing smithi07085154-30 from 2023424 to 2056192 tid 5 2024-09-14T03:59:01.228 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 7 current snap is 324 2024-09-14T03:59:01.228 INFO:tasks.rados.rados.0.smithi070.stdout:3185: finishing write tid 1 to smithi07085154-30 2024-09-14T03:59:01.228 INFO:tasks.rados.rados.0.smithi070.stdout:3185: finishing write tid 2 to smithi07085154-30 2024-09-14T03:59:01.228 INFO:tasks.rados.rados.0.smithi070.stdout:3186: seq_num 983 ranges {0=466944,466944=712704,1179648=434176,1613824=458752} 2024-09-14T03:59:01.230 INFO:tasks.rados.rados.0.smithi070.stdout:3186: writing smithi07085154-7 from 0 to 466944 tid 1 2024-09-14T03:59:01.234 INFO:tasks.rados.rados.0.smithi070.stdout:3186: writing smithi07085154-7 from 466944 to 1179648 tid 2 2024-09-14T03:59:01.237 INFO:tasks.rados.rados.0.smithi070.stdout:3186: writing smithi07085154-7 from 1179648 to 1613824 tid 3 2024-09-14T03:59:01.240 INFO:tasks.rados.rados.0.smithi070.stdout:3186: writing smithi07085154-7 from 1613824 to 2072576 tid 4 2024-09-14T03:59:01.241 INFO:tasks.rados.rados.0.smithi070.stdout:3187: setattr oid 36 current snap is 324 2024-09-14T03:59:01.241 INFO:tasks.rados.rados.0.smithi070.stdout:3185: finishing write tid 3 to smithi07085154-30 2024-09-14T03:59:01.241 INFO:tasks.rados.rados.0.smithi070.stdout:3185: finishing write tid 4 to smithi07085154-30 2024-09-14T03:59:01.241 INFO:tasks.rados.rados.0.smithi070.stdout:3185: finishing write tid 5 to smithi07085154-30 2024-09-14T03:59:01.241 INFO:tasks.rados.rados.0.smithi070.stdout:3185: finishing write tid 6 to smithi07085154-30 2024-09-14T03:59:01.241 INFO:tasks.rados.rados.0.smithi070.stdout:3185: finishing write tid 7 to smithi07085154-30 2024-09-14T03:59:01.241 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 30 v 3183 (ObjNum 982 snap 324 seq_num 982) dirty exists 2024-09-14T03:59:01.241 INFO:tasks.rados.rados.0.smithi070.stdout:3185: left oid 30 (ObjNum 982 snap 324 seq_num 982) 2024-09-14T03:59:01.241 INFO:tasks.rados.rados.0.smithi070.stdout:3186: finishing write tid 1 to smithi07085154-7 2024-09-14T03:59:01.241 INFO:tasks.rados.rados.0.smithi070.stdout:3186: finishing write tid 2 to smithi07085154-7 2024-09-14T03:59:01.241 INFO:tasks.rados.rados.0.smithi070.stdout:3186: finishing write tid 3 to smithi07085154-7 2024-09-14T03:59:01.243 INFO:tasks.rados.rados.0.smithi070.stdout:3185: done (2 left) 2024-09-14T03:59:01.243 INFO:tasks.rados.rados.0.smithi070.stdout:3188: rollback oid 14 current snap is 324 2024-09-14T03:59:01.243 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 14 to 307 2024-09-14T03:59:01.243 INFO:tasks.rados.rados.0.smithi070.stdout:3189: setattr oid 22 current snap is 324 2024-09-14T03:59:01.243 INFO:tasks.rados.rados.0.smithi070.stdout:3186: finishing write tid 4 to smithi07085154-7 2024-09-14T03:59:01.244 INFO:tasks.rados.rados.0.smithi070.stdout:3186: finishing write tid 5 to smithi07085154-7 2024-09-14T03:59:01.245 INFO:tasks.rados.rados.0.smithi070.stdout:3186: finishing write tid 6 to smithi07085154-7 2024-09-14T03:59:01.245 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 3193 (ObjNum 983 snap 324 seq_num 983) dirty exists 2024-09-14T03:59:01.245 INFO:tasks.rados.rados.0.smithi070.stdout:3186: left oid 7 (ObjNum 983 snap 324 seq_num 983) 2024-09-14T03:59:01.246 INFO:tasks.rados.rados.0.smithi070.stdout:3186: done (3 left) 2024-09-14T03:59:01.246 INFO:tasks.rados.rados.0.smithi070.stdout:3190: snap_create 2024-09-14T03:59:01.249 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 36 v 3174 (ObjNum 905 snap 301 seq_num 905) dirty exists 2024-09-14T03:59:01.250 INFO:tasks.rados.rados.0.smithi070.stdout:3188: finishing rollback tid 0 to smithi07085154-14 2024-09-14T03:59:01.250 INFO:tasks.rados.rados.0.smithi070.stdout:3188: finishing rollback tid 1 to smithi07085154-14 2024-09-14T03:59:01.250 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 3198 (ObjNum 968 snap 322 seq_num 968) dirty exists 2024-09-14T03:59:01.250 INFO:tasks.rados.rados.0.smithi070.stdout:3188: finishing rollback tid 2 to smithi07085154-14 2024-09-14T03:59:01.250 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 3188 (ObjNum 313 snap 105 seq_num 313) dirty exists 2024-09-14T03:59:01.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:01 smithi094 ceph-mon[86597]: osdmap e705: 8 total, 8 up, 8 in 2024-09-14T03:59:01.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:01 smithi070 ceph-mon[96704]: osdmap e705: 8 total, 8 up, 8 in 2024-09-14T03:59:01.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:01 smithi070 ceph-mon[98640]: osdmap e705: 8 total, 8 up, 8 in 2024-09-14T03:59:02.195 INFO:tasks.rados.rados.0.smithi070.stdout:3187: done (3 left) 2024-09-14T03:59:02.195 INFO:tasks.rados.rados.0.smithi070.stdout:3188: done (2 left) 2024-09-14T03:59:02.195 INFO:tasks.rados.rados.0.smithi070.stdout:3189: done (1 left) 2024-09-14T03:59:02.195 INFO:tasks.rados.rados.0.smithi070.stdout:3190: done (0 left) 2024-09-14T03:59:02.196 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 45 current snap is 325 2024-09-14T03:59:02.196 INFO:tasks.rados.rados.0.smithi070.stdout:3191: seq_num 984 ranges {0=483328,483328=598016,1081344=647168,1728512=360448} 2024-09-14T03:59:02.197 INFO:tasks.rados.rados.0.smithi070.stdout:3191: writing smithi07085154-45 from 0 to 483328 tid 1 2024-09-14T03:59:02.201 INFO:tasks.rados.rados.0.smithi070.stdout:3191: writing smithi07085154-45 from 483328 to 1081344 tid 2 2024-09-14T03:59:02.206 INFO:tasks.rados.rados.0.smithi070.stdout:3191: writing smithi07085154-45 from 1081344 to 1728512 tid 3 2024-09-14T03:59:02.208 INFO:tasks.rados.rados.0.smithi070.stdout:3191: writing smithi07085154-45 from 1728512 to 2088960 tid 4 2024-09-14T03:59:02.208 INFO:tasks.rados.rados.0.smithi070.stdout:3192: read oid 41 snap 284 2024-09-14T03:59:02.208 INFO:tasks.rados.rados.0.smithi070.stdout:3192: expect deleted 2024-09-14T03:59:02.208 INFO:tasks.rados.rados.0.smithi070.stdout:3193: delete oid 6 current snap is 325 2024-09-14T03:59:02.209 INFO:tasks.rados.rados.0.smithi070.stdout:3191: finishing write tid 1 to smithi07085154-45 2024-09-14T03:59:02.209 INFO:tasks.rados.rados.0.smithi070.stdout:3191: finishing write tid 2 to smithi07085154-45 2024-09-14T03:59:02.211 INFO:tasks.rados.rados.0.smithi070.stdout:3191: finishing write tid 3 to smithi07085154-45 2024-09-14T03:59:02.211 INFO:tasks.rados.rados.0.smithi070.stdout:3193: done (2 left) 2024-09-14T03:59:02.211 INFO:tasks.rados.rados.0.smithi070.stdout:3194: copy_from oid 5 from oid 32 current snap is 325 2024-09-14T03:59:02.211 INFO:tasks.rados.rados.0.smithi070.stdout:3195: setattr oid 24 current snap is 325 2024-09-14T03:59:02.212 INFO:tasks.rados.rados.0.smithi070.stdout:3191: finishing write tid 4 to smithi07085154-45 2024-09-14T03:59:02.212 INFO:tasks.rados.rados.0.smithi070.stdout:3191: finishing write tid 5 to smithi07085154-45 2024-09-14T03:59:02.213 INFO:tasks.rados.rados.0.smithi070.stdout:3192: done (3 left) 2024-09-14T03:59:02.213 INFO:tasks.rados.rados.0.smithi070.stdout:3196: read oid 38 snap 273 2024-09-14T03:59:02.213 INFO:tasks.rados.rados.0.smithi070.stdout:3196: expect (ObjNum 540 snap 176 seq_num 540) 2024-09-14T03:59:02.213 INFO:tasks.rados.rados.0.smithi070.stdout:3197: copy_from oid 9 from oid 34 current snap is 325 2024-09-14T03:59:02.213 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 29 current snap is 325 2024-09-14T03:59:02.213 INFO:tasks.rados.rados.0.smithi070.stdout:3198: seq_num 986 ranges {1933312=516096,2449408=770048,3219456=655360,3874816=180224} 2024-09-14T03:59:02.226 INFO:tasks.rados.rados.0.smithi070.stdout:3198: writing smithi07085154-29 from 1933312 to 2449408 tid 1 2024-09-14T03:59:02.231 INFO:tasks.rados.rados.0.smithi070.stdout:3198: writing smithi07085154-29 from 2449408 to 3219456 tid 2 2024-09-14T03:59:02.235 INFO:tasks.rados.rados.0.smithi070.stdout:3198: writing smithi07085154-29 from 3219456 to 3874816 tid 3 2024-09-14T03:59:02.236 INFO:tasks.rados.rados.0.smithi070.stdout:3198: writing smithi07085154-29 from 3874816 to 4055040 tid 4 2024-09-14T03:59:02.237 INFO:tasks.rados.rados.0.smithi070.stdout:3199: rollback oid 18 current snap is 325 2024-09-14T03:59:02.237 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 18 to 320 2024-09-14T03:59:02.237 INFO:tasks.rados.rados.0.smithi070.stdout:3200: snap_remove snap 317 2024-09-14T03:59:02.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:02 smithi094 ceph-mon[86597]: osdmap e706: 8 total, 8 up, 8 in 2024-09-14T03:59:02.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:02 smithi094 ceph-mon[86597]: pgmap v658: 121 pgs: 121 active+clean; 1.1 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.4 MiB/s rd, 4.4 MiB/s wr, 27 op/s 2024-09-14T03:59:02.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:02 smithi070 ceph-mon[96704]: osdmap e706: 8 total, 8 up, 8 in 2024-09-14T03:59:02.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:02 smithi070 ceph-mon[96704]: pgmap v658: 121 pgs: 121 active+clean; 1.1 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.4 MiB/s rd, 4.4 MiB/s wr, 27 op/s 2024-09-14T03:59:02.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:02 smithi070 ceph-mon[98640]: osdmap e706: 8 total, 8 up, 8 in 2024-09-14T03:59:02.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:02 smithi070 ceph-mon[98640]: pgmap v658: 121 pgs: 121 active+clean; 1.1 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.4 MiB/s rd, 4.4 MiB/s wr, 27 op/s 2024-09-14T03:59:03.202 INFO:tasks.rados.rados.0.smithi070.stdout:3200: done (7 left) 2024-09-14T03:59:03.202 INFO:tasks.rados.rados.0.smithi070.stdout:3201: rollback oid 50 current snap is 325 2024-09-14T03:59:03.202 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 50 to 311 2024-09-14T03:59:03.202 INFO:tasks.rados.rados.0.smithi070.stdout:3191: finishing write tid 6 to smithi07085154-45 2024-09-14T03:59:03.202 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 45 v 3185 (ObjNum 984 snap 325 seq_num 984) dirty exists 2024-09-14T03:59:03.202 INFO:tasks.rados.rados.0.smithi070.stdout:3191: left oid 45 (ObjNum 984 snap 325 seq_num 984) 2024-09-14T03:59:03.202 INFO:tasks.rados.rados.0.smithi070.stdout:3194: finishing copy_from to smithi07085154-5 2024-09-14T03:59:03.202 INFO:tasks.rados.rados.0.smithi070.stdout:3194: got expected ENOENT (src dne) 2024-09-14T03:59:03.203 INFO:tasks.rados.rados.0.smithi070.stdout:3194: finishing copy_from racing read to smithi07085154-5 2024-09-14T03:59:03.203 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 3189 (ObjNum 65 snap 0 seq_num 1464755809) dirty exists 2024-09-14T03:59:03.203 INFO:tasks.rados.rados.0.smithi070.stdout:3196: expect (ObjNum 540 snap 176 seq_num 540) 2024-09-14T03:59:03.264 INFO:tasks.rados.rados.0.smithi070.stdout:3198: finishing write tid 1 to smithi07085154-29 2024-09-14T03:59:03.264 INFO:tasks.rados.rados.0.smithi070.stdout:3199: finishing rollback tid 1 to smithi07085154-18 2024-09-14T03:59:03.264 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 3182 (ObjNum 33 snap 0 seq_num 2902963576) dirty dne 2024-09-14T03:59:03.264 INFO:tasks.rados.rados.0.smithi070.stdout:3197: finishing copy_from racing read to smithi07085154-9 2024-09-14T03:59:03.264 INFO:tasks.rados.rados.0.smithi070.stdout:3198: finishing write tid 2 to smithi07085154-29 2024-09-14T03:59:03.265 INFO:tasks.rados.rados.0.smithi070.stdout:3198: finishing write tid 3 to smithi07085154-29 2024-09-14T03:59:03.265 INFO:tasks.rados.rados.0.smithi070.stdout:3198: finishing write tid 4 to smithi07085154-29 2024-09-14T03:59:03.265 INFO:tasks.rados.rados.0.smithi070.stdout:3198: finishing write tid 5 to smithi07085154-29 2024-09-14T03:59:03.265 INFO:tasks.rados.rados.0.smithi070.stdout:3197: finishing copy_from to smithi07085154-9 2024-09-14T03:59:03.265 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 3176 (ObjNum 938 snap 316 seq_num 938) dirty exists 2024-09-14T03:59:03.265 INFO:tasks.rados.rados.0.smithi070.stdout:3198: finishing write tid 6 to smithi07085154-29 2024-09-14T03:59:03.265 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 3203 (ObjNum 986 snap 325 seq_num 986) dirty exists 2024-09-14T03:59:03.265 INFO:tasks.rados.rados.0.smithi070.stdout:3198: left oid 29 (ObjNum 986 snap 325 seq_num 986) 2024-09-14T03:59:03.265 INFO:tasks.rados.rados.0.smithi070.stdout:3201: finishing rollback tid 0 to smithi07085154-50 2024-09-14T03:59:03.265 INFO:tasks.rados.rados.0.smithi070.stdout:3201: finishing rollback tid 1 to smithi07085154-50 2024-09-14T03:59:03.265 INFO:tasks.rados.rados.0.smithi070.stdout:3201: finishing rollback tid 2 to smithi07085154-50 2024-09-14T03:59:03.265 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 3206 (ObjNum 925 snap 310 seq_num 925) dirty exists 2024-09-14T03:59:03.265 INFO:tasks.rados.rados.0.smithi070.stdout:3191: done (7 left) 2024-09-14T03:59:03.266 INFO:tasks.rados.rados.0.smithi070.stdout:3194: done (6 left) 2024-09-14T03:59:03.266 INFO:tasks.rados.rados.0.smithi070.stdout:3195: done (5 left) 2024-09-14T03:59:03.266 INFO:tasks.rados.rados.0.smithi070.stdout:3196: done (4 left) 2024-09-14T03:59:03.266 INFO:tasks.rados.rados.0.smithi070.stdout:3197: done (3 left) 2024-09-14T03:59:03.266 INFO:tasks.rados.rados.0.smithi070.stdout:3198: done (2 left) 2024-09-14T03:59:03.266 INFO:tasks.rados.rados.0.smithi070.stdout:3199: done (1 left) 2024-09-14T03:59:03.266 INFO:tasks.rados.rados.0.smithi070.stdout:3201: done (0 left) 2024-09-14T03:59:03.266 INFO:tasks.rados.rados.0.smithi070.stdout:3202: read oid 16 snap -1 2024-09-14T03:59:03.266 INFO:tasks.rados.rados.0.smithi070.stdout:3202: expect deleted 2024-09-14T03:59:03.266 INFO:tasks.rados.rados.0.smithi070.stdout:3203: delete oid 12 current snap is 325 2024-09-14T03:59:03.266 INFO:tasks.rados.rados.0.smithi070.stdout:3202: done (1 left) 2024-09-14T03:59:03.267 INFO:tasks.rados.rados.0.smithi070.stdout:3203: done (0 left) 2024-09-14T03:59:03.267 INFO:tasks.rados.rados.0.smithi070.stdout:3204: delete oid 12 current snap is 325 2024-09-14T03:59:03.267 INFO:tasks.rados.rados.0.smithi070.stdout:3204: done (0 left) 2024-09-14T03:59:03.268 INFO:tasks.rados.rados.0.smithi070.stdout:3205: read oid 45 snap -1 2024-09-14T03:59:03.268 INFO:tasks.rados.rados.0.smithi070.stdout:3205: expect (ObjNum 984 snap 325 seq_num 984) 2024-09-14T03:59:03.268 INFO:tasks.rados.rados.0.smithi070.stdout:3206: copy_from oid 44 from oid 31 current snap is 325 2024-09-14T03:59:03.268 INFO:tasks.rados.rados.0.smithi070.stdout:3207: rollback oid 4 current snap is 325 2024-09-14T03:59:03.268 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 4 to 284 2024-09-14T03:59:03.268 INFO:tasks.rados.rados.0.smithi070.stdout:3208: snap_create 2024-09-14T03:59:03.271 INFO:tasks.rados.rados.0.smithi070.stdout:3207: finishing rollback tid 1 to smithi07085154-4 2024-09-14T03:59:03.271 INFO:tasks.rados.rados.0.smithi070.stdout:3207: finishing rollback tid 2 to smithi07085154-4 2024-09-14T03:59:03.271 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 3195 (ObjNum 793 snap 275 seq_num 793) dirty exists 2024-09-14T03:59:03.274 INFO:tasks.rados.rados.0.smithi070.stdout:3206: finishing copy_from racing read to smithi07085154-44 2024-09-14T03:59:03.275 INFO:tasks.rados.rados.0.smithi070.stdout:3205: expect (ObjNum 984 snap 325 seq_num 984) 2024-09-14T03:59:03.328 INFO:tasks.rados.rados.0.smithi070.stdout:3206: finishing copy_from to smithi07085154-44 2024-09-14T03:59:03.328 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 3182 (ObjNum 961 snap 319 seq_num 961) dirty exists 2024-09-14T03:59:03.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:03 smithi094 ceph-mon[86597]: osdmap e707: 8 total, 8 up, 8 in 2024-09-14T03:59:03.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:03 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:59:03.535 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:59:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:59:03] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.51.0" 2024-09-14T03:59:03.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:03 smithi070 ceph-mon[96704]: osdmap e707: 8 total, 8 up, 8 in 2024-09-14T03:59:03.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:03 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:59:03.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:03 smithi070 ceph-mon[98640]: osdmap e707: 8 total, 8 up, 8 in 2024-09-14T03:59:03.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:03 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:59:04.208 INFO:tasks.rados.rados.0.smithi070.stdout:3205: done (3 left) 2024-09-14T03:59:04.208 INFO:tasks.rados.rados.0.smithi070.stdout:3206: done (2 left) 2024-09-14T03:59:04.208 INFO:tasks.rados.rados.0.smithi070.stdout:3207: done (1 left) 2024-09-14T03:59:04.209 INFO:tasks.rados.rados.0.smithi070.stdout:3208: done (0 left) 2024-09-14T03:59:04.209 INFO:tasks.rados.rados.0.smithi070.stdout:3209: snap_create 2024-09-14T03:59:04.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:04 smithi094 ceph-mon[86597]: osdmap e708: 8 total, 8 up, 8 in 2024-09-14T03:59:04.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:04 smithi094 ceph-mon[86597]: pgmap v661: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1.1 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 1.7 KiB/s rd, 3.2 MiB/s wr, 29 op/s 2024-09-14T03:59:04.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:04 smithi070 ceph-mon[98640]: osdmap e708: 8 total, 8 up, 8 in 2024-09-14T03:59:04.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:04 smithi070 ceph-mon[98640]: pgmap v661: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1.1 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 1.7 KiB/s rd, 3.2 MiB/s wr, 29 op/s 2024-09-14T03:59:04.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:04 smithi070 ceph-mon[96704]: osdmap e708: 8 total, 8 up, 8 in 2024-09-14T03:59:04.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:04 smithi070 ceph-mon[96704]: pgmap v661: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1.1 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 1.7 KiB/s rd, 3.2 MiB/s wr, 29 op/s 2024-09-14T03:59:05.212 INFO:tasks.rados.rados.0.smithi070.stdout:3209: done (0 left) 2024-09-14T03:59:05.212 INFO:tasks.rados.rados.0.smithi070.stdout:3210: delete oid 22 current snap is 327 2024-09-14T03:59:05.219 INFO:tasks.rados.rados.0.smithi070.stdout:3210: done (0 left) 2024-09-14T03:59:05.219 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 47 current snap is 327 2024-09-14T03:59:05.219 INFO:tasks.rados.rados.0.smithi070.stdout:3211: seq_num 990 ranges {0=589824,589824=753664,1343488=638976,1982464=212992} 2024-09-14T03:59:05.222 INFO:tasks.rados.rados.0.smithi070.stdout:3211: writing smithi07085154-47 from 0 to 589824 tid 1 2024-09-14T03:59:05.227 INFO:tasks.rados.rados.0.smithi070.stdout:3211: writing smithi07085154-47 from 589824 to 1343488 tid 2 2024-09-14T03:59:05.230 INFO:tasks.rados.rados.0.smithi070.stdout:3211: writing smithi07085154-47 from 1343488 to 1982464 tid 3 2024-09-14T03:59:05.231 INFO:tasks.rados.rados.0.smithi070.stdout:3211: writing smithi07085154-47 from 1982464 to 2195456 tid 4 2024-09-14T03:59:05.232 INFO:tasks.rados.rados.0.smithi070.stdout:3212: setattr oid 7 current snap is 327 2024-09-14T03:59:05.232 INFO:tasks.rados.rados.0.smithi070.stdout:3211: finishing write tid 1 to smithi07085154-47 2024-09-14T03:59:05.232 INFO:tasks.rados.rados.0.smithi070.stdout:3211: finishing write tid 2 to smithi07085154-47 2024-09-14T03:59:05.233 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 2 current snap is 327 2024-09-14T03:59:05.234 INFO:tasks.rados.rados.0.smithi070.stdout:3213: seq_num 991 ranges {8323072=606208,8929280=638976,9568256=794624,10362880=172032} 2024-09-14T03:59:05.280 INFO:tasks.rados.rados.0.smithi070.stdout:3213: writing smithi07085154-2 from 8323072 to 8929280 tid 1 2024-09-14T03:59:05.284 INFO:tasks.rados.rados.0.smithi070.stdout:3213: writing smithi07085154-2 from 8929280 to 9568256 tid 2 2024-09-14T03:59:05.288 INFO:tasks.rados.rados.0.smithi070.stdout:3213: writing smithi07085154-2 from 9568256 to 10362880 tid 3 2024-09-14T03:59:05.290 INFO:tasks.rados.rados.0.smithi070.stdout:3213: writing smithi07085154-2 from 10362880 to 10534912 tid 4 2024-09-14T03:59:05.290 INFO:tasks.rados.rados.0.smithi070.stdout:3214: rmattr oid 28 current snap is 327 2024-09-14T03:59:05.290 INFO:tasks.rados.rados.0.smithi070.stdout:3211: finishing write tid 3 to smithi07085154-47 2024-09-14T03:59:05.290 INFO:tasks.rados.rados.0.smithi070.stdout:3211: finishing write tid 4 to smithi07085154-47 2024-09-14T03:59:05.290 INFO:tasks.rados.rados.0.smithi070.stdout:3215: snap_create 2024-09-14T03:59:05.291 INFO:tasks.rados.rados.0.smithi070.stdout:3211: finishing write tid 5 to smithi07085154-47 2024-09-14T03:59:05.291 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 3196 (ObjNum 983 snap 324 seq_num 983) dirty exists 2024-09-14T03:59:05.291 INFO:tasks.rados.rados.0.smithi070.stdout:3211: finishing write tid 6 to smithi07085154-47 2024-09-14T03:59:05.291 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 47 v 3188 (ObjNum 990 snap 327 seq_num 990) dirty exists 2024-09-14T03:59:05.291 INFO:tasks.rados.rados.0.smithi070.stdout:3211: left oid 47 (ObjNum 990 snap 327 seq_num 990) 2024-09-14T03:59:05.291 INFO:tasks.rados.rados.0.smithi070.stdout:3213: finishing write tid 1 to smithi07085154-2 2024-09-14T03:59:05.291 INFO:tasks.rados.rados.0.smithi070.stdout:3213: finishing write tid 2 to smithi07085154-2 2024-09-14T03:59:05.292 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 28 v 3197 (ObjNum 639 snap 214 seq_num 639) dirty exists 2024-09-14T03:59:05.295 INFO:tasks.rados.rados.0.smithi070.stdout:3213: finishing write tid 3 to smithi07085154-2 2024-09-14T03:59:05.295 INFO:tasks.rados.rados.0.smithi070.stdout:3213: finishing write tid 4 to smithi07085154-2 2024-09-14T03:59:05.295 INFO:tasks.rados.rados.0.smithi070.stdout:3213: finishing write tid 5 to smithi07085154-2 2024-09-14T03:59:05.297 INFO:tasks.rados.rados.0.smithi070.stdout:3213: finishing write tid 6 to smithi07085154-2 2024-09-14T03:59:05.297 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 2 v 3803 (ObjNum 991 snap 327 seq_num 991) dirty exists 2024-09-14T03:59:05.297 INFO:tasks.rados.rados.0.smithi070.stdout:3213: left oid 2 (ObjNum 991 snap 327 seq_num 991) 2024-09-14T03:59:05.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:05 smithi094 ceph-mon[86597]: osdmap e709: 8 total, 8 up, 8 in 2024-09-14T03:59:05.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:05 smithi070 ceph-mon[96704]: osdmap e709: 8 total, 8 up, 8 in 2024-09-14T03:59:05.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:05 smithi070 ceph-mon[98640]: osdmap e709: 8 total, 8 up, 8 in 2024-09-14T03:59:06.220 INFO:tasks.rados.rados.0.smithi070.stdout:3211: done (4 left) 2024-09-14T03:59:06.221 INFO:tasks.rados.rados.0.smithi070.stdout:3212: done (3 left) 2024-09-14T03:59:06.221 INFO:tasks.rados.rados.0.smithi070.stdout:3213: done (2 left) 2024-09-14T03:59:06.221 INFO:tasks.rados.rados.0.smithi070.stdout:3214: done (1 left) 2024-09-14T03:59:06.221 INFO:tasks.rados.rados.0.smithi070.stdout:3215: done (0 left) 2024-09-14T03:59:06.221 INFO:tasks.rados.rados.0.smithi070.stdout:3216: copy_from oid 26 from oid 49 current snap is 328 2024-09-14T03:59:06.221 INFO:tasks.rados.rados.0.smithi070.stdout:3217: setattr oid 24 current snap is 328 2024-09-14T03:59:06.222 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 16 current snap is 328 2024-09-14T03:59:06.222 INFO:tasks.rados.rados.0.smithi070.stdout:3218: seq_num 992 ranges {0=622592,622592=524288,1146880=606208,1753088=475136} 2024-09-14T03:59:06.226 INFO:tasks.rados.rados.0.smithi070.stdout:3218: writing smithi07085154-16 from 0 to 622592 tid 1 2024-09-14T03:59:06.229 INFO:tasks.rados.rados.0.smithi070.stdout:3218: writing smithi07085154-16 from 622592 to 1146880 tid 2 2024-09-14T03:59:06.232 INFO:tasks.rados.rados.0.smithi070.stdout:3218: writing smithi07085154-16 from 1146880 to 1753088 tid 3 2024-09-14T03:59:06.235 INFO:tasks.rados.rados.0.smithi070.stdout:3218: writing smithi07085154-16 from 1753088 to 2228224 tid 4 2024-09-14T03:59:06.235 INFO:tasks.rados.rados.0.smithi070.stdout:3219: delete oid 41 current snap is 328 2024-09-14T03:59:06.235 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 3190 (ObjNum 161 snap 0 seq_num 0) dirty exists 2024-09-14T03:59:06.236 INFO:tasks.rados.rados.0.smithi070.stdout:3216: finishing copy_from racing read to smithi07085154-26 2024-09-14T03:59:06.237 INFO:tasks.rados.rados.0.smithi070.stdout:3218: finishing write tid 1 to smithi07085154-16 2024-09-14T03:59:06.237 INFO:tasks.rados.rados.0.smithi070.stdout:3216: finishing copy_from to smithi07085154-26 2024-09-14T03:59:06.237 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 3173 (ObjNum 898 snap 300 seq_num 898) dirty exists 2024-09-14T03:59:06.240 INFO:tasks.rados.rados.0.smithi070.stdout:3218: finishing write tid 2 to smithi07085154-16 2024-09-14T03:59:06.242 INFO:tasks.rados.rados.0.smithi070.stdout:3216: done (3 left) 2024-09-14T03:59:06.242 INFO:tasks.rados.rados.0.smithi070.stdout:3217: done (2 left) 2024-09-14T03:59:06.243 INFO:tasks.rados.rados.0.smithi070.stdout:3219: done (1 left) 2024-09-14T03:59:06.243 INFO:tasks.rados.rados.0.smithi070.stdout:3220: rmattr oid 10 current snap is 328 2024-09-14T03:59:06.243 INFO:tasks.rados.rados.0.smithi070.stdout:3220: done (1 left) 2024-09-14T03:59:06.243 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 5 current snap is 328 2024-09-14T03:59:06.243 INFO:tasks.rados.rados.0.smithi070.stdout:3221: seq_num 994 ranges {1925120=655360,2580480=688128,3268608=622592,3891200=294912} 2024-09-14T03:59:06.256 INFO:tasks.rados.rados.0.smithi070.stdout:3221: writing smithi07085154-5 from 1925120 to 2580480 tid 1 2024-09-14T03:59:06.260 INFO:tasks.rados.rados.0.smithi070.stdout:3221: writing smithi07085154-5 from 2580480 to 3268608 tid 2 2024-09-14T03:59:06.264 INFO:tasks.rados.rados.0.smithi070.stdout:3221: writing smithi07085154-5 from 3268608 to 3891200 tid 3 2024-09-14T03:59:06.266 INFO:tasks.rados.rados.0.smithi070.stdout:3221: writing smithi07085154-5 from 3891200 to 4186112 tid 4 2024-09-14T03:59:06.266 INFO:tasks.rados.rados.0.smithi070.stdout:3222: snap_create 2024-09-14T03:59:06.266 INFO:tasks.rados.rados.0.smithi070.stdout:3218: finishing write tid 3 to smithi07085154-16 2024-09-14T03:59:06.266 INFO:tasks.rados.rados.0.smithi070.stdout:3218: finishing write tid 4 to smithi07085154-16 2024-09-14T03:59:06.266 INFO:tasks.rados.rados.0.smithi070.stdout:3218: finishing write tid 5 to smithi07085154-16 2024-09-14T03:59:06.266 INFO:tasks.rados.rados.0.smithi070.stdout:3218: finishing write tid 6 to smithi07085154-16 2024-09-14T03:59:06.266 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 3810 (ObjNum 992 snap 328 seq_num 992) dirty exists 2024-09-14T03:59:06.267 INFO:tasks.rados.rados.0.smithi070.stdout:3218: left oid 16 (ObjNum 992 snap 328 seq_num 992) 2024-09-14T03:59:06.267 INFO:tasks.rados.rados.0.smithi070.stdout:3221: finishing write tid 1 to smithi07085154-5 2024-09-14T03:59:06.267 INFO:tasks.rados.rados.0.smithi070.stdout:3221: finishing write tid 2 to smithi07085154-5 2024-09-14T03:59:06.268 INFO:tasks.rados.rados.0.smithi070.stdout:3221: finishing write tid 3 to smithi07085154-5 2024-09-14T03:59:06.268 INFO:tasks.rados.rados.0.smithi070.stdout:3221: finishing write tid 4 to smithi07085154-5 2024-09-14T03:59:06.268 INFO:tasks.rados.rados.0.smithi070.stdout:3221: finishing write tid 5 to smithi07085154-5 2024-09-14T03:59:06.269 INFO:tasks.rados.rados.0.smithi070.stdout:3221: finishing write tid 6 to smithi07085154-5 2024-09-14T03:59:06.269 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 589 (ObjNum 994 snap 328 seq_num 994) dirty exists 2024-09-14T03:59:06.269 INFO:tasks.rados.rados.0.smithi070.stdout:3221: left oid 5 (ObjNum 994 snap 328 seq_num 994) 2024-09-14T03:59:06.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:06 smithi094 ceph-mon[86597]: osdmap e710: 8 total, 8 up, 8 in 2024-09-14T03:59:06.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:06 smithi094 ceph-mon[86597]: pgmap v664: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1.1 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 2.0 MiB/s rd, 984 KiB/s wr, 22 op/s 2024-09-14T03:59:06.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:06 smithi070 ceph-mon[96704]: osdmap e710: 8 total, 8 up, 8 in 2024-09-14T03:59:06.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:06 smithi070 ceph-mon[96704]: pgmap v664: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1.1 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 2.0 MiB/s rd, 984 KiB/s wr, 22 op/s 2024-09-14T03:59:06.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:06 smithi070 ceph-mon[98640]: osdmap e710: 8 total, 8 up, 8 in 2024-09-14T03:59:06.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:06 smithi070 ceph-mon[98640]: pgmap v664: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1.1 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 2.0 MiB/s rd, 984 KiB/s wr, 22 op/s 2024-09-14T03:59:07.228 INFO:tasks.rados.rados.0.smithi070.stdout:3218: done (2 left) 2024-09-14T03:59:07.228 INFO:tasks.rados.rados.0.smithi070.stdout:3221: done (1 left) 2024-09-14T03:59:07.228 INFO:tasks.rados.rados.0.smithi070.stdout:3222: done (0 left) 2024-09-14T03:59:07.228 INFO:tasks.rados.rados.0.smithi070.stdout:3223: delete oid 25 current snap is 329 2024-09-14T03:59:07.229 INFO:tasks.rados.rados.0.smithi070.stdout:3223: done (0 left) 2024-09-14T03:59:07.229 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 45 current snap is 329 2024-09-14T03:59:07.229 INFO:tasks.rados.rados.0.smithi070.stdout:3224: seq_num 996 ranges {2088960=688128,2777088=507904,3284992=581632,3866624=516096} 2024-09-14T03:59:07.244 INFO:tasks.rados.rados.0.smithi070.stdout:3224: writing smithi07085154-45 from 2088960 to 2777088 tid 1 2024-09-14T03:59:07.247 INFO:tasks.rados.rados.0.smithi070.stdout:3224: writing smithi07085154-45 from 2777088 to 3284992 tid 2 2024-09-14T03:59:07.251 INFO:tasks.rados.rados.0.smithi070.stdout:3224: writing smithi07085154-45 from 3284992 to 3866624 tid 3 2024-09-14T03:59:07.254 INFO:tasks.rados.rados.0.smithi070.stdout:3224: writing smithi07085154-45 from 3866624 to 4382720 tid 4 2024-09-14T03:59:07.255 INFO:tasks.rados.rados.0.smithi070.stdout:3225: read oid 17 snap -1 2024-09-14T03:59:07.255 INFO:tasks.rados.rados.0.smithi070.stdout:3225: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:59:07.255 INFO:tasks.rados.rados.0.smithi070.stdout:3226: rollback oid 5 current snap is 329 2024-09-14T03:59:07.255 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 5 to 315 2024-09-14T03:59:07.255 INFO:tasks.rados.rados.0.smithi070.stdout:3224: finishing write tid 1 to smithi07085154-45 2024-09-14T03:59:07.255 INFO:tasks.rados.rados.0.smithi070.stdout:3224: finishing write tid 2 to smithi07085154-45 2024-09-14T03:59:07.255 INFO:tasks.rados.rados.0.smithi070.stdout:3227: snap_create 2024-09-14T03:59:07.255 INFO:tasks.rados.rados.0.smithi070.stdout:3225: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:59:07.259 INFO:tasks.rados.rados.0.smithi070.stdout:3224: finishing write tid 3 to smithi07085154-45 2024-09-14T03:59:07.265 INFO:tasks.rados.rados.0.smithi070.stdout:3224: finishing write tid 4 to smithi07085154-45 2024-09-14T03:59:07.265 INFO:tasks.rados.rados.0.smithi070.stdout:3224: finishing write tid 5 to smithi07085154-45 2024-09-14T03:59:07.265 INFO:tasks.rados.rados.0.smithi070.stdout:3226: finishing rollback tid 0 to smithi07085154-5 2024-09-14T03:59:07.266 INFO:tasks.rados.rados.0.smithi070.stdout:3226: finishing rollback tid 1 to smithi07085154-5 2024-09-14T03:59:07.266 INFO:tasks.rados.rados.0.smithi070.stdout:3226: finishing rollback tid 2 to smithi07085154-5 2024-09-14T03:59:07.266 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 592 (ObjNum 914 snap 306 seq_num 914) dirty exists 2024-09-14T03:59:07.266 INFO:tasks.rados.rados.0.smithi070.stdout:3224: finishing write tid 6 to smithi07085154-45 2024-09-14T03:59:07.266 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 45 v 3190 (ObjNum 996 snap 329 seq_num 996) dirty exists 2024-09-14T03:59:07.266 INFO:tasks.rados.rados.0.smithi070.stdout:3224: left oid 45 (ObjNum 996 snap 329 seq_num 996) 2024-09-14T03:59:07.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:07 smithi094 ceph-mon[86597]: osdmap e711: 8 total, 8 up, 8 in 2024-09-14T03:59:07.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:07 smithi070 ceph-mon[96704]: osdmap e711: 8 total, 8 up, 8 in 2024-09-14T03:59:07.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:07 smithi070 ceph-mon[98640]: osdmap e711: 8 total, 8 up, 8 in 2024-09-14T03:59:08.236 INFO:tasks.rados.rados.0.smithi070.stdout:3224: done (3 left) 2024-09-14T03:59:08.236 INFO:tasks.rados.rados.0.smithi070.stdout:3225: done (2 left) 2024-09-14T03:59:08.236 INFO:tasks.rados.rados.0.smithi070.stdout:3226: done (1 left) 2024-09-14T03:59:08.236 INFO:tasks.rados.rados.0.smithi070.stdout:3227: done (0 left) 2024-09-14T03:59:08.236 INFO:tasks.rados.rados.0.smithi070.stdout:3228: read oid 15 snap -1 2024-09-14T03:59:08.236 INFO:tasks.rados.rados.0.smithi070.stdout:3228: expect (ObjNum 805 snap 281 seq_num 805) 2024-09-14T03:59:08.236 INFO:tasks.rados.rados.0.smithi070.stdout:3229: read oid 49 snap -1 2024-09-14T03:59:08.236 INFO:tasks.rados.rados.0.smithi070.stdout:3229: expect (ObjNum 898 snap 300 seq_num 898) 2024-09-14T03:59:08.236 INFO:tasks.rados.rados.0.smithi070.stdout:3230: read oid 12 snap -1 2024-09-14T03:59:08.236 INFO:tasks.rados.rados.0.smithi070.stdout:3230: expect deleted 2024-09-14T03:59:08.237 INFO:tasks.rados.rados.0.smithi070.stdout:3231: read oid 18 snap 327 2024-09-14T03:59:08.237 INFO:tasks.rados.rados.0.smithi070.stdout:3231: expect deleted 2024-09-14T03:59:08.237 INFO:tasks.rados.rados.0.smithi070.stdout:3232: copy_from oid 5 from oid 31 current snap is 330 2024-09-14T03:59:08.237 INFO:tasks.rados.rados.0.smithi070.stdout:3233: rollback oid 40 current snap is 330 2024-09-14T03:59:08.237 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 40 to 320 2024-09-14T03:59:08.237 INFO:tasks.rados.rados.0.smithi070.stdout:3234: read oid 17 snap -1 2024-09-14T03:59:08.237 INFO:tasks.rados.rados.0.smithi070.stdout:3234: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:59:08.237 INFO:tasks.rados.rados.0.smithi070.stdout:3235: copy_from oid 41 from oid 7 current snap is 330 2024-09-14T03:59:08.237 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 45 current snap is 330 2024-09-14T03:59:08.237 INFO:tasks.rados.rados.0.smithi070.stdout:3236: seq_num 997 ranges {4382720=704512,5087232=794624,5881856=794624,6676480=16384} 2024-09-14T03:59:08.264 INFO:tasks.rados.rados.0.smithi070.stdout:3236: writing smithi07085154-45 from 4382720 to 5087232 tid 1 2024-09-14T03:59:08.269 INFO:tasks.rados.rados.0.smithi070.stdout:3236: writing smithi07085154-45 from 5087232 to 5881856 tid 2 2024-09-14T03:59:08.274 INFO:tasks.rados.rados.0.smithi070.stdout:3236: writing smithi07085154-45 from 5881856 to 6676480 tid 3 2024-09-14T03:59:08.274 INFO:tasks.rados.rados.0.smithi070.stdout:3236: writing smithi07085154-45 from 6676480 to 6692864 tid 4 2024-09-14T03:59:08.274 INFO:tasks.rados.rados.0.smithi070.stdout:3234: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-14T03:59:08.277 INFO:tasks.rados.rados.0.smithi070.stdout:3229: expect (ObjNum 898 snap 300 seq_num 898) 2024-09-14T03:59:08.296 INFO:tasks.rados.rados.0.smithi070.stdout:3233: finishing rollback tid 0 to smithi07085154-40 2024-09-14T03:59:08.296 INFO:tasks.rados.rados.0.smithi070.stdout:3233: finishing rollback tid 1 to smithi07085154-40 2024-09-14T03:59:08.296 INFO:tasks.rados.rados.0.smithi070.stdout:3228: expect (ObjNum 805 snap 281 seq_num 805) 2024-09-14T03:59:08.334 INFO:tasks.rados.rados.0.smithi070.stdout:3233: finishing rollback tid 2 to smithi07085154-40 2024-09-14T03:59:08.334 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 3185 (ObjNum 912 snap 305 seq_num 912) dirty exists 2024-09-14T03:59:08.334 INFO:tasks.rados.rados.0.smithi070.stdout:3228: done (8 left) 2024-09-14T03:59:08.335 INFO:tasks.rados.rados.0.smithi070.stdout:3229: done (7 left) 2024-09-14T03:59:08.335 INFO:tasks.rados.rados.0.smithi070.stdout:3230: done (6 left) 2024-09-14T03:59:08.335 INFO:tasks.rados.rados.0.smithi070.stdout:3231: done (5 left) 2024-09-14T03:59:08.335 INFO:tasks.rados.rados.0.smithi070.stdout:3233: done (4 left) 2024-09-14T03:59:08.335 INFO:tasks.rados.rados.0.smithi070.stdout:3234: done (3 left) 2024-09-14T03:59:08.335 INFO:tasks.rados.rados.0.smithi070.stdout:3237: snap_remove snap 306 2024-09-14T03:59:08.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:08 smithi094 ceph-mon[86597]: osdmap e712: 8 total, 8 up, 8 in 2024-09-14T03:59:08.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:08 smithi094 ceph-mon[86597]: pgmap v667: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1.1 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s rd, 4.6 MiB/s wr, 33 op/s 2024-09-14T03:59:08.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:08 smithi070 ceph-mon[96704]: osdmap e712: 8 total, 8 up, 8 in 2024-09-14T03:59:08.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:08 smithi070 ceph-mon[96704]: pgmap v667: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1.1 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s rd, 4.6 MiB/s wr, 33 op/s 2024-09-14T03:59:08.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:08 smithi070 ceph-mon[98640]: osdmap e712: 8 total, 8 up, 8 in 2024-09-14T03:59:08.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:08 smithi070 ceph-mon[98640]: pgmap v667: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1.1 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s rd, 4.6 MiB/s wr, 33 op/s 2024-09-14T03:59:09.113 INFO:tasks.rados.rados.0.smithi070.stdout:3237: done (3 left) 2024-09-14T03:59:09.113 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 22 current snap is 330 2024-09-14T03:59:09.113 INFO:tasks.rados.rados.0.smithi070.stdout:3238: seq_num 998 ranges {0=720896,720896=679936,1400832=598016,1998848=327680} 2024-09-14T03:59:09.118 INFO:tasks.rados.rados.0.smithi070.stdout:3238: writing smithi07085154-22 from 0 to 720896 tid 1 2024-09-14T03:59:09.123 INFO:tasks.rados.rados.0.smithi070.stdout:3238: writing smithi07085154-22 from 720896 to 1400832 tid 2 2024-09-14T03:59:09.127 INFO:tasks.rados.rados.0.smithi070.stdout:3238: writing smithi07085154-22 from 1400832 to 1998848 tid 3 2024-09-14T03:59:09.130 INFO:tasks.rados.rados.0.smithi070.stdout:3238: writing smithi07085154-22 from 1998848 to 2326528 tid 4 2024-09-14T03:59:09.131 INFO:tasks.rados.rados.0.smithi070.stdout:3232: finishing copy_from racing read to smithi07085154-5 2024-09-14T03:59:09.131 INFO:tasks.rados.rados.0.smithi070.stdout:3232: finishing copy_from to smithi07085154-5 2024-09-14T03:59:09.131 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 3182 (ObjNum 961 snap 319 seq_num 961) dirty exists 2024-09-14T03:59:09.131 INFO:tasks.rados.rados.0.smithi070.stdout:3235: finishing copy_from racing read to smithi07085154-41 2024-09-14T03:59:09.132 INFO:tasks.rados.rados.0.smithi070.stdout:3235: finishing copy_from to smithi07085154-41 2024-09-14T03:59:09.132 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 3196 (ObjNum 983 snap 324 seq_num 983) dirty exists 2024-09-14T03:59:09.132 INFO:tasks.rados.rados.0.smithi070.stdout:3236: finishing write tid 1 to smithi07085154-45 2024-09-14T03:59:09.132 INFO:tasks.rados.rados.0.smithi070.stdout:3236: finishing write tid 2 to smithi07085154-45 2024-09-14T03:59:09.132 INFO:tasks.rados.rados.0.smithi070.stdout:3236: finishing write tid 3 to smithi07085154-45 2024-09-14T03:59:09.132 INFO:tasks.rados.rados.0.smithi070.stdout:3236: finishing write tid 4 to smithi07085154-45 2024-09-14T03:59:09.132 INFO:tasks.rados.rados.0.smithi070.stdout:3236: finishing write tid 5 to smithi07085154-45 2024-09-14T03:59:09.132 INFO:tasks.rados.rados.0.smithi070.stdout:3236: finishing write tid 6 to smithi07085154-45 2024-09-14T03:59:09.132 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 45 v 3195 (ObjNum 997 snap 330 seq_num 997) dirty exists 2024-09-14T03:59:09.132 INFO:tasks.rados.rados.0.smithi070.stdout:3236: left oid 45 (ObjNum 997 snap 330 seq_num 997) 2024-09-14T03:59:09.132 INFO:tasks.rados.rados.0.smithi070.stdout:3232: done (3 left) 2024-09-14T03:59:09.133 INFO:tasks.rados.rados.0.smithi070.stdout:3235: done (2 left) 2024-09-14T03:59:09.133 INFO:tasks.rados.rados.0.smithi070.stdout:3236: done (1 left) 2024-09-14T03:59:09.133 INFO:tasks.rados.rados.0.smithi070.stdout:3239: read oid 3 snap -1 2024-09-14T03:59:09.133 INFO:tasks.rados.rados.0.smithi070.stdout:3239: expect deleted 2024-09-14T03:59:09.133 INFO:tasks.rados.rados.0.smithi070.stdout:3240: rollback oid 30 current snap is 330 2024-09-14T03:59:09.133 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 30 to 325 2024-09-14T03:59:09.133 INFO:tasks.rados.rados.0.smithi070.stdout:3241: snap_create 2024-09-14T03:59:09.136 INFO:tasks.rados.rados.0.smithi070.stdout:3238: finishing write tid 1 to smithi07085154-22 2024-09-14T03:59:09.136 INFO:tasks.rados.rados.0.smithi070.stdout:3238: finishing write tid 2 to smithi07085154-22 2024-09-14T03:59:09.139 INFO:tasks.rados.rados.0.smithi070.stdout:3240: finishing rollback tid 0 to smithi07085154-30 2024-09-14T03:59:09.139 INFO:tasks.rados.rados.0.smithi070.stdout:3240: finishing rollback tid 1 to smithi07085154-30 2024-09-14T03:59:09.139 INFO:tasks.rados.rados.0.smithi070.stdout:3240: finishing rollback tid 2 to smithi07085154-30 2024-09-14T03:59:09.140 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 30 v 3191 (ObjNum 982 snap 324 seq_num 982) dirty exists 2024-09-14T03:59:09.141 INFO:tasks.rados.rados.0.smithi070.stdout:3238: finishing write tid 3 to smithi07085154-22 2024-09-14T03:59:09.141 INFO:tasks.rados.rados.0.smithi070.stdout:3238: finishing write tid 4 to smithi07085154-22 2024-09-14T03:59:09.142 INFO:tasks.rados.rados.0.smithi070.stdout:3238: finishing write tid 5 to smithi07085154-22 2024-09-14T03:59:09.143 INFO:tasks.rados.rados.0.smithi070.stdout:3238: finishing write tid 6 to smithi07085154-22 2024-09-14T03:59:09.143 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 3212 (ObjNum 998 snap 330 seq_num 998) dirty exists 2024-09-14T03:59:09.143 INFO:tasks.rados.rados.0.smithi070.stdout:3238: left oid 22 (ObjNum 998 snap 330 seq_num 998) 2024-09-14T03:59:09.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:09 smithi094 ceph-mon[86597]: osdmap e713: 8 total, 8 up, 8 in 2024-09-14T03:59:09.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:09 smithi094 ceph-mon[86597]: osdmap e714: 8 total, 8 up, 8 in 2024-09-14T03:59:09.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:09 smithi070 ceph-mon[96704]: osdmap e713: 8 total, 8 up, 8 in 2024-09-14T03:59:09.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:09 smithi070 ceph-mon[96704]: osdmap e714: 8 total, 8 up, 8 in 2024-09-14T03:59:09.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:09 smithi070 ceph-mon[98640]: osdmap e713: 8 total, 8 up, 8 in 2024-09-14T03:59:09.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:09 smithi070 ceph-mon[98640]: osdmap e714: 8 total, 8 up, 8 in 2024-09-14T03:59:10.114 INFO:tasks.rados.rados.0.smithi070.stdout:3238: done (3 left) 2024-09-14T03:59:10.114 INFO:tasks.rados.rados.0.smithi070.stdout:3239: done (2 left) 2024-09-14T03:59:10.114 INFO:tasks.rados.rados.0.smithi070.stdout:3240: done (1 left) 2024-09-14T03:59:10.114 INFO:tasks.rados.rados.0.smithi070.stdout:3241: done (0 left) 2024-09-14T03:59:10.114 INFO:tasks.rados.rados.0.smithi070.stdout:3242: read oid 11 snap -1 2024-09-14T03:59:10.114 INFO:tasks.rados.rados.0.smithi070.stdout:3242: expect deleted 2024-09-14T03:59:10.114 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 28 current snap is 331 2024-09-14T03:59:10.114 INFO:tasks.rados.rados.0.smithi070.stdout:3243: seq_num 999 ranges {1105920=737280,1843200=565248,2408448=761856,3170304=278528} 2024-09-14T03:59:10.123 INFO:tasks.rados.rados.0.smithi070.stdout:3243: writing smithi07085154-28 from 1105920 to 1843200 tid 1 2024-09-14T03:59:10.127 INFO:tasks.rados.rados.0.smithi070.stdout:3243: writing smithi07085154-28 from 1843200 to 2408448 tid 2 2024-09-14T03:59:10.132 INFO:tasks.rados.rados.0.smithi070.stdout:3243: writing smithi07085154-28 from 2408448 to 3170304 tid 3 2024-09-14T03:59:10.133 INFO:tasks.rados.rados.0.smithi070.stdout:3243: writing smithi07085154-28 from 3170304 to 3448832 tid 4 2024-09-14T03:59:10.134 INFO:tasks.rados.rados.0.smithi070.stdout:3244: delete oid 49 current snap is 331 2024-09-14T03:59:10.135 INFO:tasks.rados.rados.0.smithi070.stdout:3243: finishing write tid 1 to smithi07085154-28 2024-09-14T03:59:10.136 INFO:tasks.rados.rados.0.smithi070.stdout:3243: finishing write tid 2 to smithi07085154-28 2024-09-14T03:59:10.137 INFO:tasks.rados.rados.0.smithi070.stdout:3242: done (2 left) 2024-09-14T03:59:10.137 INFO:tasks.rados.rados.0.smithi070.stdout:3244: done (1 left) 2024-09-14T03:59:10.137 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 48 current snap is 331 2024-09-14T03:59:10.137 INFO:tasks.rados.rados.0.smithi070.stdout:3245: seq_num 1001 ranges {0=770048,770048=729088,1499136=770048,2269184=106496} 2024-09-14T03:59:10.141 INFO:tasks.rados.rados.0.smithi070.stdout:3245: writing smithi07085154-48 from 0 to 770048 tid 1 2024-09-14T03:59:10.145 INFO:tasks.rados.rados.0.smithi070.stdout:3245: writing smithi07085154-48 from 770048 to 1499136 tid 2 2024-09-14T03:59:10.150 INFO:tasks.rados.rados.0.smithi070.stdout:3245: writing smithi07085154-48 from 1499136 to 2269184 tid 3 2024-09-14T03:59:10.151 INFO:tasks.rados.rados.0.smithi070.stdout:3245: writing smithi07085154-48 from 2269184 to 2375680 tid 4 2024-09-14T03:59:10.152 INFO:tasks.rados.rados.0.smithi070.stdout:3246: setattr oid 21 current snap is 331 2024-09-14T03:59:10.152 INFO:tasks.rados.rados.0.smithi070.stdout:3243: finishing write tid 3 to smithi07085154-28 2024-09-14T03:59:10.152 INFO:tasks.rados.rados.0.smithi070.stdout:3243: finishing write tid 4 to smithi07085154-28 2024-09-14T03:59:10.152 INFO:tasks.rados.rados.0.smithi070.stdout:3243: finishing write tid 5 to smithi07085154-28 2024-09-14T03:59:10.152 INFO:tasks.rados.rados.0.smithi070.stdout:3243: finishing write tid 6 to smithi07085154-28 2024-09-14T03:59:10.152 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 28 v 3202 (ObjNum 999 snap 331 seq_num 999) dirty exists 2024-09-14T03:59:10.152 INFO:tasks.rados.rados.0.smithi070.stdout:3243: left oid 28 (ObjNum 999 snap 331 seq_num 999) 2024-09-14T03:59:10.152 INFO:tasks.rados.rados.0.smithi070.stdout:3245: finishing write tid 1 to smithi07085154-48 2024-09-14T03:59:10.152 INFO:tasks.rados.rados.0.smithi070.stdout:3245: finishing write tid 2 to smithi07085154-48 2024-09-14T03:59:10.153 INFO:tasks.rados.rados.0.smithi070.stdout:3243: done (2 left) 2024-09-14T03:59:10.153 INFO:tasks.rados.rados.0.smithi070.stdout:3247: rollback oid 44 current snap is 331 2024-09-14T03:59:10.153 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 44 to 309 2024-09-14T03:59:10.153 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 1 current snap is 331 2024-09-14T03:59:10.154 INFO:tasks.rados.rados.0.smithi070.stdout:3248: seq_num 1002 ranges {4005888=786432,4792320=614400,5406720=581632,5988352=409600} 2024-09-14T03:59:10.179 INFO:tasks.rados.rados.0.smithi070.stdout:3248: writing smithi07085154-1 from 4005888 to 4792320 tid 1 2024-09-14T03:59:10.182 INFO:tasks.rados.rados.0.smithi070.stdout:3248: writing smithi07085154-1 from 4792320 to 5406720 tid 2 2024-09-14T03:59:10.186 INFO:tasks.rados.rados.0.smithi070.stdout:3248: writing smithi07085154-1 from 5406720 to 5988352 tid 3 2024-09-14T03:59:10.189 INFO:tasks.rados.rados.0.smithi070.stdout:3248: writing smithi07085154-1 from 5988352 to 6397952 tid 4 2024-09-14T03:59:10.190 INFO:tasks.rados.rados.0.smithi070.stdout:3249: read oid 34 snap -1 2024-09-14T03:59:10.190 INFO:tasks.rados.rados.0.smithi070.stdout:3249: expect (ObjNum 938 snap 316 seq_num 938) 2024-09-14T03:59:10.190 INFO:tasks.rados.rados.0.smithi070.stdout:3250: rollback oid 24 current snap is 331 2024-09-14T03:59:10.190 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 24 to 324 2024-09-14T03:59:10.190 INFO:tasks.rados.rados.0.smithi070.stdout:3245: finishing write tid 3 to smithi07085154-48 2024-09-14T03:59:10.190 INFO:tasks.rados.rados.0.smithi070.stdout:3245: finishing write tid 4 to smithi07085154-48 2024-09-14T03:59:10.190 INFO:tasks.rados.rados.0.smithi070.stdout:3245: finishing write tid 5 to smithi07085154-48 2024-09-14T03:59:10.190 INFO:tasks.rados.rados.0.smithi070.stdout:3247: finishing rollback tid 0 to smithi07085154-44 2024-09-14T03:59:10.190 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 1721 (ObjNum 927 snap 310 seq_num 927) dirty exists 2024-09-14T03:59:10.190 INFO:tasks.rados.rados.0.smithi070.stdout:3245: finishing write tid 6 to smithi07085154-48 2024-09-14T03:59:10.190 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 2266 (ObjNum 1001 snap 331 seq_num 1001) dirty exists 2024-09-14T03:59:10.190 INFO:tasks.rados.rados.0.smithi070.stdout:3245: left oid 48 (ObjNum 1001 snap 331 seq_num 1001) 2024-09-14T03:59:10.190 INFO:tasks.rados.rados.0.smithi070.stdout:3247: finishing rollback tid 1 to smithi07085154-44 2024-09-14T03:59:10.191 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 3187 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-14T03:59:10.191 INFO:tasks.rados.rados.0.smithi070.stdout:3248: finishing write tid 1 to smithi07085154-1 2024-09-14T03:59:10.191 INFO:tasks.rados.rados.0.smithi070.stdout:3245: done (5 left) 2024-09-14T03:59:10.191 INFO:tasks.rados.rados.0.smithi070.stdout:3246: done (4 left) 2024-09-14T03:59:10.191 INFO:tasks.rados.rados.0.smithi070.stdout:3247: done (3 left) 2024-09-14T03:59:10.191 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 50 current snap is 331 2024-09-14T03:59:10.191 INFO:tasks.rados.rados.0.smithi070.stdout:3248: finishing write tid 2 to smithi07085154-1 2024-09-14T03:59:10.191 INFO:tasks.rados.rados.0.smithi070.stdout:3251: seq_num 1003 ranges {1097728=802816,1900544=499712,2400256=737280,3137536=368640} 2024-09-14T03:59:10.199 INFO:tasks.rados.rados.0.smithi070.stdout:3251: writing smithi07085154-50 from 1097728 to 1900544 tid 1 2024-09-14T03:59:10.202 INFO:tasks.rados.rados.0.smithi070.stdout:3251: writing smithi07085154-50 from 1900544 to 2400256 tid 2 2024-09-14T03:59:10.206 INFO:tasks.rados.rados.0.smithi070.stdout:3251: writing smithi07085154-50 from 2400256 to 3137536 tid 3 2024-09-14T03:59:10.208 INFO:tasks.rados.rados.0.smithi070.stdout:3251: writing smithi07085154-50 from 3137536 to 3506176 tid 4 2024-09-14T03:59:10.208 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 19 current snap is 331 2024-09-14T03:59:10.209 INFO:tasks.rados.rados.0.smithi070.stdout:3250: finishing rollback tid 0 to smithi07085154-24 2024-09-14T03:59:10.209 INFO:tasks.rados.rados.0.smithi070.stdout:3250: finishing rollback tid 1 to smithi07085154-24 2024-09-14T03:59:10.209 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 3192 (ObjNum 305 snap 0 seq_num 4214454195) dirty dne 2024-09-14T03:59:10.209 INFO:tasks.rados.rados.0.smithi070.stdout:3248: finishing write tid 3 to smithi07085154-1 2024-09-14T03:59:10.209 INFO:tasks.rados.rados.0.smithi070.stdout:3248: finishing write tid 4 to smithi07085154-1 2024-09-14T03:59:10.209 INFO:tasks.rados.rados.0.smithi070.stdout:3252: seq_num 1004 ranges {9912320=16384} 2024-09-14T03:59:10.261 INFO:tasks.rados.rados.0.smithi070.stdout:3252: writing smithi07085154-19 from 9912320 to 9928704 tid 1 2024-09-14T03:59:10.261 INFO:tasks.rados.rados.0.smithi070.stdout:3250: done (4 left) 2024-09-14T03:59:10.261 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 5 current snap is 331 2024-09-14T03:59:10.261 INFO:tasks.rados.rados.0.smithi070.stdout:3253: seq_num 1005 ranges {1703936=32768} 2024-09-14T03:59:10.270 INFO:tasks.rados.rados.0.smithi070.stdout:3253: writing smithi07085154-5 from 1703936 to 1736704 tid 1 2024-09-14T03:59:10.271 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 41 current snap is 331 2024-09-14T03:59:10.271 INFO:tasks.rados.rados.0.smithi070.stdout:3248: finishing write tid 5 to smithi07085154-1 2024-09-14T03:59:10.271 INFO:tasks.rados.rados.0.smithi070.stdout:3254: seq_num 1006 ranges {2072576=49152} 2024-09-14T03:59:10.282 INFO:tasks.rados.rados.0.smithi070.stdout:3254: writing smithi07085154-41 from 2072576 to 2121728 tid 1 2024-09-14T03:59:10.282 INFO:tasks.rados.rados.0.smithi070.stdout:3255: snap_remove snap 329 2024-09-14T03:59:10.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:59:10 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:59:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:59:11.121 INFO:tasks.rados.rados.0.smithi070.stdout:3248: finishing write tid 6 to smithi07085154-1 2024-09-14T03:59:11.121 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 1 v 3192 (ObjNum 1002 snap 331 seq_num 1002) dirty exists 2024-09-14T03:59:11.121 INFO:tasks.rados.rados.0.smithi070.stdout:3248: left oid 1 (ObjNum 1002 snap 331 seq_num 1002) 2024-09-14T03:59:11.121 INFO:tasks.rados.rados.0.smithi070.stdout:3249: expect (ObjNum 938 snap 316 seq_num 938) 2024-09-14T03:59:11.279 INFO:tasks.rados.rados.0.smithi070.stdout:3251: finishing write tid 1 to smithi07085154-50 2024-09-14T03:59:11.279 INFO:tasks.rados.rados.0.smithi070.stdout:3251: finishing write tid 2 to smithi07085154-50 2024-09-14T03:59:11.280 INFO:tasks.rados.rados.0.smithi070.stdout:3251: finishing write tid 3 to smithi07085154-50 2024-09-14T03:59:11.280 INFO:tasks.rados.rados.0.smithi070.stdout:3251: finishing write tid 4 to smithi07085154-50 2024-09-14T03:59:11.280 INFO:tasks.rados.rados.0.smithi070.stdout:3251: finishing write tid 5 to smithi07085154-50 2024-09-14T03:59:11.280 INFO:tasks.rados.rados.0.smithi070.stdout:3251: finishing write tid 6 to smithi07085154-50 2024-09-14T03:59:11.280 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 3217 (ObjNum 1003 snap 331 seq_num 1003) dirty exists 2024-09-14T03:59:11.280 INFO:tasks.rados.rados.0.smithi070.stdout:3251: left oid 50 (ObjNum 1003 snap 331 seq_num 1003) 2024-09-14T03:59:11.280 INFO:tasks.rados.rados.0.smithi070.stdout:3252: finishing write tid 1 to smithi07085154-19 2024-09-14T03:59:11.280 INFO:tasks.rados.rados.0.smithi070.stdout:3252: finishing write tid 2 to smithi07085154-19 2024-09-14T03:59:11.280 INFO:tasks.rados.rados.0.smithi070.stdout:3252: finishing write tid 3 to smithi07085154-19 2024-09-14T03:59:11.280 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 19 v 3219 (ObjNum 1004 snap 331 seq_num 1004) dirty exists 2024-09-14T03:59:11.280 INFO:tasks.rados.rados.0.smithi070.stdout:3252: left oid 19 (ObjNum 1004 snap 331 seq_num 1004) 2024-09-14T03:59:11.280 INFO:tasks.rados.rados.0.smithi070.stdout:3253: finishing write tid 1 to smithi07085154-5 2024-09-14T03:59:11.281 INFO:tasks.rados.rados.0.smithi070.stdout:3253: finishing write tid 2 to smithi07085154-5 2024-09-14T03:59:11.281 INFO:tasks.rados.rados.0.smithi070.stdout:3253: finishing write tid 3 to smithi07085154-5 2024-09-14T03:59:11.281 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 3184 (ObjNum 1005 snap 331 seq_num 1005) dirty exists 2024-09-14T03:59:11.281 INFO:tasks.rados.rados.0.smithi070.stdout:3253: left oid 5 (ObjNum 1005 snap 331 seq_num 1005) 2024-09-14T03:59:11.281 INFO:tasks.rados.rados.0.smithi070.stdout:3254: finishing write tid 1 to smithi07085154-41 2024-09-14T03:59:11.281 INFO:tasks.rados.rados.0.smithi070.stdout:3254: finishing write tid 2 to smithi07085154-41 2024-09-14T03:59:11.281 INFO:tasks.rados.rados.0.smithi070.stdout:3254: finishing write tid 3 to smithi07085154-41 2024-09-14T03:59:11.281 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 3198 (ObjNum 1006 snap 331 seq_num 1006) dirty exists 2024-09-14T03:59:11.281 INFO:tasks.rados.rados.0.smithi070.stdout:3254: left oid 41 (ObjNum 1006 snap 331 seq_num 1006) 2024-09-14T03:59:11.281 INFO:tasks.rados.rados.0.smithi070.stdout:3248: done (6 left) 2024-09-14T03:59:11.281 INFO:tasks.rados.rados.0.smithi070.stdout:3249: done (5 left) 2024-09-14T03:59:11.281 INFO:tasks.rados.rados.0.smithi070.stdout:3251: done (4 left) 2024-09-14T03:59:11.281 INFO:tasks.rados.rados.0.smithi070.stdout:3252: done (3 left) 2024-09-14T03:59:11.282 INFO:tasks.rados.rados.0.smithi070.stdout:3253: done (2 left) 2024-09-14T03:59:11.282 INFO:tasks.rados.rados.0.smithi070.stdout:3254: done (1 left) 2024-09-14T03:59:11.282 INFO:tasks.rados.rados.0.smithi070.stdout:3255: done (0 left) 2024-09-14T03:59:11.282 INFO:tasks.rados.rados.0.smithi070.stdout:3256: read oid 15 snap -1 2024-09-14T03:59:11.282 INFO:tasks.rados.rados.0.smithi070.stdout:3256: expect (ObjNum 805 snap 281 seq_num 805) 2024-09-14T03:59:11.282 INFO:tasks.rados.rados.0.smithi070.stdout:3257: snap_create 2024-09-14T03:59:11.282 INFO:tasks.rados.rados.0.smithi070.stdout:3256: expect (ObjNum 805 snap 281 seq_num 805) 2024-09-14T03:59:11.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:11 smithi094 ceph-mon[86597]: osdmap e715: 8 total, 8 up, 8 in 2024-09-14T03:59:11.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:11 smithi094 ceph-mon[86597]: pgmap v671: 121 pgs: 121 active+clean; 1.1 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 1.6 MiB/s rd, 4.7 MiB/s wr, 20 op/s 2024-09-14T03:59:11.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:11 smithi070 ceph-mon[96704]: osdmap e715: 8 total, 8 up, 8 in 2024-09-14T03:59:11.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:11 smithi070 ceph-mon[96704]: pgmap v671: 121 pgs: 121 active+clean; 1.1 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 1.6 MiB/s rd, 4.7 MiB/s wr, 20 op/s 2024-09-14T03:59:11.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:11 smithi070 ceph-mon[98640]: osdmap e715: 8 total, 8 up, 8 in 2024-09-14T03:59:11.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:11 smithi070 ceph-mon[98640]: pgmap v671: 121 pgs: 121 active+clean; 1.1 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 1.6 MiB/s rd, 4.7 MiB/s wr, 20 op/s 2024-09-14T03:59:12.121 INFO:tasks.rados.rados.0.smithi070.stdout:3256: done (1 left) 2024-09-14T03:59:12.121 INFO:tasks.rados.rados.0.smithi070.stdout:3257: done (0 left) 2024-09-14T03:59:12.121 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 18 current snap is 332 2024-09-14T03:59:12.121 INFO:tasks.rados.rados.0.smithi070.stdout:3258: seq_num 1007 ranges {0=65536} 2024-09-14T03:59:12.121 INFO:tasks.rados.rados.0.smithi070.stdout:3258: writing smithi07085154-18 from 0 to 65536 tid 1 2024-09-14T03:59:12.121 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 5 current snap is 332 2024-09-14T03:59:12.121 INFO:tasks.rados.rados.0.smithi070.stdout:3259: seq_num 1008 ranges {1736704=90112} 2024-09-14T03:59:12.131 INFO:tasks.rados.rados.0.smithi070.stdout:3259: writing smithi07085154-5 from 1736704 to 1826816 tid 1 2024-09-14T03:59:12.131 INFO:tasks.rados.rados.0.smithi070.stdout:3260: delete oid 35 current snap is 332 2024-09-14T03:59:12.131 INFO:tasks.rados.rados.0.smithi070.stdout:3258: finishing write tid 1 to smithi07085154-18 2024-09-14T03:59:12.131 INFO:tasks.rados.rados.0.smithi070.stdout:3258: finishing write tid 2 to smithi07085154-18 2024-09-14T03:59:12.132 INFO:tasks.rados.rados.0.smithi070.stdout:3258: finishing write tid 3 to smithi07085154-18 2024-09-14T03:59:12.132 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 3200 (ObjNum 1007 snap 332 seq_num 1007) dirty exists 2024-09-14T03:59:12.132 INFO:tasks.rados.rados.0.smithi070.stdout:3258: left oid 18 (ObjNum 1007 snap 332 seq_num 1007) 2024-09-14T03:59:12.134 INFO:tasks.rados.rados.0.smithi070.stdout:3258: done (2 left) 2024-09-14T03:59:12.135 INFO:tasks.rados.rados.0.smithi070.stdout:3260: done (1 left) 2024-09-14T03:59:12.135 INFO:tasks.rados.rados.0.smithi070.stdout:3261: delete oid 33 current snap is 332 2024-09-14T03:59:12.136 INFO:tasks.rados.rados.0.smithi070.stdout:3259: finishing write tid 1 to smithi07085154-5 2024-09-14T03:59:12.136 INFO:tasks.rados.rados.0.smithi070.stdout:3259: finishing write tid 2 to smithi07085154-5 2024-09-14T03:59:12.137 INFO:tasks.rados.rados.0.smithi070.stdout:3261: done (1 left) 2024-09-14T03:59:12.138 INFO:tasks.rados.rados.0.smithi070.stdout:3262: snap_create 2024-09-14T03:59:12.138 INFO:tasks.rados.rados.0.smithi070.stdout:3259: finishing write tid 3 to smithi07085154-5 2024-09-14T03:59:12.138 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 3186 (ObjNum 1008 snap 332 seq_num 1008) dirty exists 2024-09-14T03:59:12.138 INFO:tasks.rados.rados.0.smithi070.stdout:3259: left oid 5 (ObjNum 1008 snap 332 seq_num 1008) 2024-09-14T03:59:12.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:12 smithi094 ceph-mon[86597]: osdmap e716: 8 total, 8 up, 8 in 2024-09-14T03:59:12.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:12 smithi070 ceph-mon[96704]: osdmap e716: 8 total, 8 up, 8 in 2024-09-14T03:59:12.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:12 smithi070 ceph-mon[98640]: osdmap e716: 8 total, 8 up, 8 in 2024-09-14T03:59:13.131 INFO:tasks.rados.rados.0.smithi070.stdout:3259: done (1 left) 2024-09-14T03:59:13.131 INFO:tasks.rados.rados.0.smithi070.stdout:3262: done (0 left) 2024-09-14T03:59:13.131 INFO:tasks.rados.rados.0.smithi070.stdout:3263: copy_from oid 23 from oid 22 current snap is 333 2024-09-14T03:59:13.131 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 33 current snap is 333 2024-09-14T03:59:13.131 INFO:tasks.rados.rados.0.smithi070.stdout:3264: seq_num 1011 ranges {0=139264} 2024-09-14T03:59:13.131 INFO:tasks.rados.rados.0.smithi070.stdout:3264: writing smithi07085154-33 from 0 to 139264 tid 1 2024-09-14T03:59:13.131 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 13 current snap is 333 2024-09-14T03:59:13.131 INFO:tasks.rados.rados.0.smithi070.stdout:3265: seq_num 1012 ranges {1097728=155648} 2024-09-14T03:59:13.138 INFO:tasks.rados.rados.0.smithi070.stdout:3265: writing smithi07085154-13 from 1097728 to 1253376 tid 1 2024-09-14T03:59:13.138 INFO:tasks.rados.rados.0.smithi070.stdout:3266: rmattr oid 16 current snap is 333 2024-09-14T03:59:13.138 INFO:tasks.rados.rados.0.smithi070.stdout:3266: done (3 left) 2024-09-14T03:59:13.138 INFO:tasks.rados.rados.0.smithi070.stdout:3267: snap_create 2024-09-14T03:59:13.138 INFO:tasks.rados.rados.0.smithi070.stdout:3264: finishing write tid 1 to smithi07085154-33 2024-09-14T03:59:13.139 INFO:tasks.rados.rados.0.smithi070.stdout:3264: finishing write tid 2 to smithi07085154-33 2024-09-14T03:59:13.139 INFO:tasks.rados.rados.0.smithi070.stdout:3264: finishing write tid 3 to smithi07085154-33 2024-09-14T03:59:13.139 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 33 v 3198 (ObjNum 1011 snap 333 seq_num 1011) dirty exists 2024-09-14T03:59:13.139 INFO:tasks.rados.rados.0.smithi070.stdout:3264: left oid 33 (ObjNum 1011 snap 333 seq_num 1011) 2024-09-14T03:59:13.141 INFO:tasks.rados.rados.0.smithi070.stdout:3265: finishing write tid 1 to smithi07085154-13 2024-09-14T03:59:13.141 INFO:tasks.rados.rados.0.smithi070.stdout:3265: finishing write tid 2 to smithi07085154-13 2024-09-14T03:59:13.143 INFO:tasks.rados.rados.0.smithi070.stdout:3265: finishing write tid 3 to smithi07085154-13 2024-09-14T03:59:13.143 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 3174 (ObjNum 1012 snap 333 seq_num 1012) dirty exists 2024-09-14T03:59:13.143 INFO:tasks.rados.rados.0.smithi070.stdout:3265: left oid 13 (ObjNum 1012 snap 333 seq_num 1012) 2024-09-14T03:59:13.145 INFO:tasks.rados.rados.0.smithi070.stdout:3263: finishing copy_from racing read to smithi07085154-23 2024-09-14T03:59:13.146 INFO:tasks.rados.rados.0.smithi070.stdout:3263: finishing copy_from to smithi07085154-23 2024-09-14T03:59:13.146 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 3212 (ObjNum 998 snap 330 seq_num 998) dirty exists 2024-09-14T03:59:13.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:13 smithi070 ceph-mon[96704]: osdmap e717: 8 total, 8 up, 8 in 2024-09-14T03:59:13.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:13 smithi070 ceph-mon[96704]: pgmap v674: 121 pgs: 121 active+clean; 1.1 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.3 MiB/s rd, 3.8 MiB/s wr, 25 op/s 2024-09-14T03:59:13.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:13 smithi070 ceph-mon[98640]: osdmap e717: 8 total, 8 up, 8 in 2024-09-14T03:59:13.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:13 smithi070 ceph-mon[98640]: pgmap v674: 121 pgs: 121 active+clean; 1.1 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.3 MiB/s rd, 3.8 MiB/s wr, 25 op/s 2024-09-14T03:59:13.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:13 smithi094 ceph-mon[86597]: osdmap e717: 8 total, 8 up, 8 in 2024-09-14T03:59:13.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:13 smithi094 ceph-mon[86597]: pgmap v674: 121 pgs: 121 active+clean; 1.1 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.3 MiB/s rd, 3.8 MiB/s wr, 25 op/s 2024-09-14T03:59:13.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:59:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:59:13] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.51.0" 2024-09-14T03:59:14.113 INFO:tasks.rados.rados.0.smithi070.stdout:3263: done (3 left) 2024-09-14T03:59:14.113 INFO:tasks.rados.rados.0.smithi070.stdout:3264: done (2 left) 2024-09-14T03:59:14.113 INFO:tasks.rados.rados.0.smithi070.stdout:3265: done (1 left) 2024-09-14T03:59:14.113 INFO:tasks.rados.rados.0.smithi070.stdout:3267: done (0 left) 2024-09-14T03:59:14.113 INFO:tasks.rados.rados.0.smithi070.stdout:3268: delete oid 7 current snap is 334 2024-09-14T03:59:14.119 INFO:tasks.rados.rados.0.smithi070.stdout:3268: done (0 left) 2024-09-14T03:59:14.119 INFO:tasks.rados.rados.0.smithi070.stdout:3269: snap_remove snap 284 2024-09-14T03:59:14.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:14 smithi094 ceph-mon[86597]: osdmap e718: 8 total, 8 up, 8 in 2024-09-14T03:59:14.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:14 smithi094 ceph-mon[86597]: osdmap e719: 8 total, 8 up, 8 in 2024-09-14T03:59:14.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:14 smithi070 ceph-mon[96704]: osdmap e718: 8 total, 8 up, 8 in 2024-09-14T03:59:14.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:14 smithi070 ceph-mon[96704]: osdmap e719: 8 total, 8 up, 8 in 2024-09-14T03:59:14.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:14 smithi070 ceph-mon[98640]: osdmap e718: 8 total, 8 up, 8 in 2024-09-14T03:59:14.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:14 smithi070 ceph-mon[98640]: osdmap e719: 8 total, 8 up, 8 in 2024-09-14T03:59:15.115 INFO:tasks.rados.rados.0.smithi070.stdout:3269: done (0 left) 2024-09-14T03:59:15.116 INFO:tasks.rados.rados.0.smithi070.stdout:3270: read oid 20 snap -1 2024-09-14T03:59:15.116 INFO:tasks.rados.rados.0.smithi070.stdout:3270: expect (ObjNum 524 snap 174 seq_num 524) 2024-09-14T03:59:15.116 INFO:tasks.rados.rados.0.smithi070.stdout:3271: rollback oid 46 current snap is 334 2024-09-14T03:59:15.116 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 46 to 232 2024-09-14T03:59:15.116 INFO:tasks.rados.rados.0.smithi070.stdout:3272: read oid 21 snap -1 2024-09-14T03:59:15.116 INFO:tasks.rados.rados.0.smithi070.stdout:3272: expect (ObjNum 927 snap 310 seq_num 927) 2024-09-14T03:59:15.116 INFO:tasks.rados.rados.0.smithi070.stdout:3273: copy_from oid 26 from oid 48 current snap is 334 2024-09-14T03:59:15.116 INFO:tasks.rados.rados.0.smithi070.stdout:3274: snap_remove snap 327 2024-09-14T03:59:15.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:15 smithi094 ceph-mon[86597]: pgmap v677: 121 pgs: 121 active+clean; 1.1 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 2.6 MiB/s rd, 3.2 MiB/s wr, 23 op/s 2024-09-14T03:59:15.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:15 smithi094 ceph-mon[86597]: osdmap e720: 8 total, 8 up, 8 in 2024-09-14T03:59:15.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:15 smithi070 ceph-mon[98640]: pgmap v677: 121 pgs: 121 active+clean; 1.1 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 2.6 MiB/s rd, 3.2 MiB/s wr, 23 op/s 2024-09-14T03:59:15.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:15 smithi070 ceph-mon[98640]: osdmap e720: 8 total, 8 up, 8 in 2024-09-14T03:59:15.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:15 smithi070 ceph-mon[96704]: pgmap v677: 121 pgs: 121 active+clean; 1.1 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 2.6 MiB/s rd, 3.2 MiB/s wr, 23 op/s 2024-09-14T03:59:15.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:15 smithi070 ceph-mon[96704]: osdmap e720: 8 total, 8 up, 8 in 2024-09-14T03:59:16.116 INFO:tasks.rados.rados.0.smithi070.stdout:3274: done (4 left) 2024-09-14T03:59:16.116 INFO:tasks.rados.rados.0.smithi070.stdout:3275: read oid 36 snap 301 2024-09-14T03:59:16.117 INFO:tasks.rados.rados.0.smithi070.stdout:3275: expect (ObjNum 905 snap 301 seq_num 905) 2024-09-14T03:59:16.117 INFO:tasks.rados.rados.0.smithi070.stdout:3276: snap_remove snap 333 2024-09-14T03:59:17.122 INFO:tasks.rados.rados.0.smithi070.stdout:3276: done (5 left) 2024-09-14T03:59:17.123 INFO:tasks.rados.rados.0.smithi070.stdout:3277: snap_remove snap 316 2024-09-14T03:59:17.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:17 smithi094 ceph-mon[86597]: osdmap e721: 8 total, 8 up, 8 in 2024-09-14T03:59:17.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:17 smithi094 ceph-mon[86597]: pgmap v680: 121 pgs: 121 active+clean; 1.1 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-14T03:59:17.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:17 smithi070 ceph-mon[98640]: osdmap e721: 8 total, 8 up, 8 in 2024-09-14T03:59:17.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:17 smithi070 ceph-mon[98640]: pgmap v680: 121 pgs: 121 active+clean; 1.1 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-14T03:59:17.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:17 smithi070 ceph-mon[96704]: osdmap e721: 8 total, 8 up, 8 in 2024-09-14T03:59:17.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:17 smithi070 ceph-mon[96704]: pgmap v680: 121 pgs: 121 active+clean; 1.1 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-14T03:59:18.124 INFO:tasks.rados.rados.0.smithi070.stdout:3277: done (5 left) 2024-09-14T03:59:18.124 INFO:tasks.rados.rados.0.smithi070.stdout:3278: snap_remove snap 311 2024-09-14T03:59:18.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:18 smithi094 ceph-mon[86597]: osdmap e722: 8 total, 8 up, 8 in 2024-09-14T03:59:18.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:18 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:59:18.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:18 smithi070 ceph-mon[96704]: osdmap e722: 8 total, 8 up, 8 in 2024-09-14T03:59:18.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:18 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:59:18.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:18 smithi070 ceph-mon[98640]: osdmap e722: 8 total, 8 up, 8 in 2024-09-14T03:59:18.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:18 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:59:19.115 INFO:tasks.rados.rados.0.smithi070.stdout:3278: done (5 left) 2024-09-14T03:59:19.115 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 23 current snap is 334 2024-09-14T03:59:19.115 INFO:tasks.rados.rados.0.smithi070.stdout:3272: expect (ObjNum 927 snap 310 seq_num 927) 2024-09-14T03:59:19.192 INFO:tasks.rados.rados.0.smithi070.stdout:3271: finishing rollback tid 0 to smithi07085154-46 2024-09-14T03:59:19.192 INFO:tasks.rados.rados.0.smithi070.stdout:3271: finishing rollback tid 1 to smithi07085154-46 2024-09-14T03:59:19.192 INFO:tasks.rados.rados.0.smithi070.stdout:3271: finishing rollback tid 2 to smithi07085154-46 2024-09-14T03:59:19.192 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 46 v 3828 (ObjNum 649 snap 218 seq_num 649) dirty exists 2024-09-14T03:59:19.192 INFO:tasks.rados.rados.0.smithi070.stdout:3279: seq_num 1014 ranges {2326528=188416} 2024-09-14T03:59:19.205 INFO:tasks.rados.rados.0.smithi070.stdout:3279: writing smithi07085154-23 from 2326528 to 2514944 tid 1 2024-09-14T03:59:19.205 INFO:tasks.rados.rados.0.smithi070.stdout:3271: done (5 left) 2024-09-14T03:59:19.205 INFO:tasks.rados.rados.0.smithi070.stdout:3272: done (4 left) 2024-09-14T03:59:19.205 INFO:tasks.rados.rados.0.smithi070.stdout:3280: snap_create 2024-09-14T03:59:19.206 INFO:tasks.rados.rados.0.smithi070.stdout:3273: finishing copy_from racing read to smithi07085154-26 2024-09-14T03:59:19.206 INFO:tasks.rados.rados.0.smithi070.stdout:3270: expect (ObjNum 524 snap 174 seq_num 524) 2024-09-14T03:59:19.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:19 smithi094 ceph-mon[86597]: osdmap e723: 8 total, 8 up, 8 in 2024-09-14T03:59:19.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:19 smithi094 ceph-mon[86597]: pgmap v683: 121 pgs: 121 active+clean; 1.1 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.5 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-14T03:59:19.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:19 smithi094 ceph-mon[86597]: osdmap e724: 8 total, 8 up, 8 in 2024-09-14T03:59:19.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:19 smithi070 ceph-mon[96704]: osdmap e723: 8 total, 8 up, 8 in 2024-09-14T03:59:19.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:19 smithi070 ceph-mon[96704]: pgmap v683: 121 pgs: 121 active+clean; 1.1 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.5 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-14T03:59:19.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:19 smithi070 ceph-mon[96704]: osdmap e724: 8 total, 8 up, 8 in 2024-09-14T03:59:19.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:19 smithi070 ceph-mon[98640]: osdmap e723: 8 total, 8 up, 8 in 2024-09-14T03:59:19.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:19 smithi070 ceph-mon[98640]: pgmap v683: 121 pgs: 121 active+clean; 1.1 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.5 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-14T03:59:19.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:19 smithi070 ceph-mon[98640]: osdmap e724: 8 total, 8 up, 8 in 2024-09-14T03:59:19.648 INFO:tasks.rados.rados.0.smithi070.stdout:3273: finishing copy_from to smithi07085154-26 2024-09-14T03:59:19.649 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 2266 (ObjNum 1001 snap 331 seq_num 1001) dirty exists 2024-09-14T03:59:19.649 INFO:tasks.rados.rados.0.smithi070.stdout:3275: expect (ObjNum 905 snap 301 seq_num 905) 2024-09-14T03:59:19.714 INFO:tasks.rados.rados.0.smithi070.stdout:3279: finishing write tid 1 to smithi07085154-23 2024-09-14T03:59:19.714 INFO:tasks.rados.rados.0.smithi070.stdout:3279: finishing write tid 2 to smithi07085154-23 2024-09-14T03:59:19.715 INFO:tasks.rados.rados.0.smithi070.stdout:3279: finishing write tid 3 to smithi07085154-23 2024-09-14T03:59:19.715 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 3214 (ObjNum 1014 snap 334 seq_num 1014) dirty exists 2024-09-14T03:59:19.715 INFO:tasks.rados.rados.0.smithi070.stdout:3279: left oid 23 (ObjNum 1014 snap 334 seq_num 1014) 2024-09-14T03:59:20.143 INFO:tasks.rados.rados.0.smithi070.stdout:3270: done (4 left) 2024-09-14T03:59:20.144 INFO:tasks.rados.rados.0.smithi070.stdout:3273: done (3 left) 2024-09-14T03:59:20.144 INFO:tasks.rados.rados.0.smithi070.stdout:3275: done (2 left) 2024-09-14T03:59:20.144 INFO:tasks.rados.rados.0.smithi070.stdout:3279: done (1 left) 2024-09-14T03:59:20.144 INFO:tasks.rados.rados.0.smithi070.stdout:3280: done (0 left) 2024-09-14T03:59:20.144 INFO:tasks.rados.rados.0.smithi070.stdout:3281: rmattr oid 43 current snap is 335 2024-09-14T03:59:20.144 INFO:tasks.rados.rados.0.smithi070.stdout:3281: done (0 left) 2024-09-14T03:59:20.144 INFO:tasks.rados.rados.0.smithi070.stdout:3282: read oid 26 snap 320 2024-09-14T03:59:20.144 INFO:tasks.rados.rados.0.smithi070.stdout:3282: expect (ObjNum 952 snap 319 seq_num 952) 2024-09-14T03:59:20.144 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 30 current snap is 335 2024-09-14T03:59:20.144 INFO:tasks.rados.rados.0.smithi070.stdout:3283: seq_num 1015 ranges {2056192=204800} 2024-09-14T03:59:20.156 INFO:tasks.rados.rados.0.smithi070.stdout:3283: writing smithi07085154-30 from 2056192 to 2260992 tid 1 2024-09-14T03:59:20.156 INFO:tasks.rados.rados.0.smithi070.stdout:3284: read oid 32 snap -1 2024-09-14T03:59:20.156 INFO:tasks.rados.rados.0.smithi070.stdout:3284: expect deleted 2024-09-14T03:59:20.156 INFO:tasks.rados.rados.0.smithi070.stdout:3285: delete oid 16 current snap is 335 2024-09-14T03:59:20.161 INFO:tasks.rados.rados.0.smithi070.stdout:3283: finishing write tid 1 to smithi07085154-30 2024-09-14T03:59:20.161 INFO:tasks.rados.rados.0.smithi070.stdout:3283: finishing write tid 2 to smithi07085154-30 2024-09-14T03:59:20.163 INFO:tasks.rados.rados.0.smithi070.stdout:3283: finishing write tid 3 to smithi07085154-30 2024-09-14T03:59:20.163 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 30 v 3193 (ObjNum 1015 snap 335 seq_num 1015) dirty exists 2024-09-14T03:59:20.163 INFO:tasks.rados.rados.0.smithi070.stdout:3283: left oid 30 (ObjNum 1015 snap 335 seq_num 1015) 2024-09-14T03:59:20.166 INFO:tasks.rados.rados.0.smithi070.stdout:3282: expect (ObjNum 952 snap 319 seq_num 952) 2024-09-14T03:59:20.521 INFO:tasks.rados.rados.0.smithi070.stdout:3282: done (3 left) 2024-09-14T03:59:20.521 INFO:tasks.rados.rados.0.smithi070.stdout:3283: done (2 left) 2024-09-14T03:59:20.521 INFO:tasks.rados.rados.0.smithi070.stdout:3284: done (1 left) 2024-09-14T03:59:20.521 INFO:tasks.rados.rados.0.smithi070.stdout:3285: done (0 left) 2024-09-14T03:59:20.521 INFO:tasks.rados.rados.0.smithi070.stdout:3286: read oid 12 snap -1 2024-09-14T03:59:20.521 INFO:tasks.rados.rados.0.smithi070.stdout:3286: expect deleted 2024-09-14T03:59:20.521 INFO:tasks.rados.rados.0.smithi070.stdout:3287: rollback oid 5 current snap is 335 2024-09-14T03:59:20.521 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 5 to 232 2024-09-14T03:59:20.522 INFO:tasks.rados.rados.0.smithi070.stdout:3288: snap_create 2024-09-14T03:59:20.523 INFO:tasks.rados.rados.0.smithi070.stdout:3287: finishing rollback tid 0 to smithi07085154-5 2024-09-14T03:59:20.523 INFO:tasks.rados.rados.0.smithi070.stdout:3287: finishing rollback tid 1 to smithi07085154-5 2024-09-14T03:59:20.523 INFO:tasks.rados.rados.0.smithi070.stdout:3287: finishing rollback tid 2 to smithi07085154-5 2024-09-14T03:59:20.523 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 3189 (ObjNum 686 snap 231 seq_num 686) dirty exists 2024-09-14T03:59:20.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:59:20 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:59:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:59:21.148 INFO:tasks.rados.rados.0.smithi070.stdout:3286: done (2 left) 2024-09-14T03:59:21.148 INFO:tasks.rados.rados.0.smithi070.stdout:3287: done (1 left) 2024-09-14T03:59:21.148 INFO:tasks.rados.rados.0.smithi070.stdout:3288: done (0 left) 2024-09-14T03:59:21.149 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 4 current snap is 336 2024-09-14T03:59:21.149 INFO:tasks.rados.rados.0.smithi070.stdout:3289: seq_num 1017 ranges {1286144=237568} 2024-09-14T03:59:21.156 INFO:tasks.rados.rados.0.smithi070.stdout:3289: writing smithi07085154-4 from 1286144 to 1523712 tid 1 2024-09-14T03:59:21.157 INFO:tasks.rados.rados.0.smithi070.stdout:3290: copy_from oid 17 from oid 13 current snap is 336 2024-09-14T03:59:21.157 INFO:tasks.rados.rados.0.smithi070.stdout:3291: snap_remove snap 309 2024-09-14T03:59:21.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:21 smithi094 ceph-mon[86597]: pgmap v685: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.7 MiB/s rd, 1.2 MiB/s wr, 6 op/s 2024-09-14T03:59:21.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:21 smithi094 ceph-mon[86597]: osdmap e725: 8 total, 8 up, 8 in 2024-09-14T03:59:21.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:21 smithi070 ceph-mon[96704]: pgmap v685: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.7 MiB/s rd, 1.2 MiB/s wr, 6 op/s 2024-09-14T03:59:21.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:21 smithi070 ceph-mon[96704]: osdmap e725: 8 total, 8 up, 8 in 2024-09-14T03:59:21.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:21 smithi070 ceph-mon[98640]: pgmap v685: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.7 MiB/s rd, 1.2 MiB/s wr, 6 op/s 2024-09-14T03:59:21.538 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:21 smithi070 ceph-mon[98640]: osdmap e725: 8 total, 8 up, 8 in 2024-09-14T03:59:22.156 INFO:tasks.rados.rados.0.smithi070.stdout:3291: done (2 left) 2024-09-14T03:59:22.156 INFO:tasks.rados.rados.0.smithi070.stdout:3292: delete oid 42 current snap is 336 2024-09-14T03:59:22.157 INFO:tasks.rados.rados.0.smithi070.stdout:3289: finishing write tid 1 to smithi07085154-4 2024-09-14T03:59:22.157 INFO:tasks.rados.rados.0.smithi070.stdout:3289: finishing write tid 2 to smithi07085154-4 2024-09-14T03:59:22.157 INFO:tasks.rados.rados.0.smithi070.stdout:3289: finishing write tid 3 to smithi07085154-4 2024-09-14T03:59:22.157 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 3205 (ObjNum 1017 snap 336 seq_num 1017) dirty exists 2024-09-14T03:59:22.157 INFO:tasks.rados.rados.0.smithi070.stdout:3289: left oid 4 (ObjNum 1017 snap 336 seq_num 1017) 2024-09-14T03:59:22.157 INFO:tasks.rados.rados.0.smithi070.stdout:3290: finishing copy_from racing read to smithi07085154-17 2024-09-14T03:59:22.157 INFO:tasks.rados.rados.0.smithi070.stdout:3290: finishing copy_from to smithi07085154-17 2024-09-14T03:59:22.157 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 3174 (ObjNum 1012 snap 333 seq_num 1012) dirty exists 2024-09-14T03:59:22.174 INFO:tasks.rados.rados.0.smithi070.stdout:3289: done (2 left) 2024-09-14T03:59:22.174 INFO:tasks.rados.rados.0.smithi070.stdout:3290: done (1 left) 2024-09-14T03:59:22.174 INFO:tasks.rados.rados.0.smithi070.stdout:3292: done (0 left) 2024-09-14T03:59:22.174 INFO:tasks.rados.rados.0.smithi070.stdout:3293: setattr oid 12 current snap is 336 2024-09-14T03:59:22.176 INFO:tasks.rados.rados.0.smithi070.stdout:3294: read oid 5 snap -1 2024-09-14T03:59:22.176 INFO:tasks.rados.rados.0.smithi070.stdout:3294: expect (ObjNum 686 snap 231 seq_num 686) 2024-09-14T03:59:22.176 INFO:tasks.rados.rados.0.smithi070.stdout:3295: read oid 2 snap 324 2024-09-14T03:59:22.177 INFO:tasks.rados.rados.0.smithi070.stdout:3295: expect (ObjNum 841 snap 286 seq_num 841) 2024-09-14T03:59:22.177 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 25 current snap is 336 2024-09-14T03:59:22.177 INFO:tasks.rados.rados.0.smithi070.stdout:3296: seq_num 1019 ranges {0=270336} 2024-09-14T03:59:22.178 INFO:tasks.rados.rados.0.smithi070.stdout:3296: writing smithi07085154-25 from 0 to 270336 tid 1 2024-09-14T03:59:22.178 INFO:tasks.rados.rados.0.smithi070.stdout:3297: rollback oid 6 current snap is 336 2024-09-14T03:59:22.179 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 6 to 318 2024-09-14T03:59:22.179 INFO:tasks.rados.rados.0.smithi070.stdout:3298: setattr oid 3 current snap is 336 2024-09-14T03:59:22.181 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 39 current snap is 336 2024-09-14T03:59:22.181 INFO:tasks.rados.rados.0.smithi070.stdout:3299: seq_num 1020 ranges {1449984=286720} 2024-09-14T03:59:22.191 INFO:tasks.rados.rados.0.smithi070.stdout:3299: writing smithi07085154-39 from 1449984 to 1736704 tid 1 2024-09-14T03:59:22.191 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 24 current snap is 336 2024-09-14T03:59:22.192 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 2267 (ObjNum 609 snap 0 seq_num 0) dirty exists 2024-09-14T03:59:22.192 INFO:tasks.rados.rados.0.smithi070.stdout:3296: finishing write tid 1 to smithi07085154-25 2024-09-14T03:59:22.192 INFO:tasks.rados.rados.0.smithi070.stdout:3294: expect (ObjNum 686 snap 231 seq_num 686) 2024-09-14T03:59:22.249 INFO:tasks.rados.rados.0.smithi070.stdout:3296: finishing write tid 2 to smithi07085154-25 2024-09-14T03:59:22.249 INFO:tasks.rados.rados.0.smithi070.stdout:3296: finishing write tid 3 to smithi07085154-25 2024-09-14T03:59:22.249 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 3189 (ObjNum 1019 snap 336 seq_num 1019) dirty exists 2024-09-14T03:59:22.249 INFO:tasks.rados.rados.0.smithi070.stdout:3296: left oid 25 (ObjNum 1019 snap 336 seq_num 1019) 2024-09-14T03:59:22.249 INFO:tasks.rados.rados.0.smithi070.stdout:3297: finishing rollback tid 1 to smithi07085154-6 2024-09-14T03:59:22.249 INFO:tasks.rados.rados.0.smithi070.stdout:3297: finishing rollback tid 2 to smithi07085154-6 2024-09-14T03:59:22.249 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 3191 (ObjNum 934 snap 316 seq_num 934) dirty exists 2024-09-14T03:59:22.249 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 3 v 3194 (ObjNum 33 snap 0 seq_num 2930020960) dirty exists 2024-09-14T03:59:22.250 INFO:tasks.rados.rados.0.smithi070.stdout:3300: seq_num 1021 ranges {0=303104} 2024-09-14T03:59:22.251 INFO:tasks.rados.rados.0.smithi070.stdout:3300: writing smithi07085154-24 from 0 to 303104 tid 1 2024-09-14T03:59:22.251 INFO:tasks.rados.rados.0.smithi070.stdout:3293: done (7 left) 2024-09-14T03:59:22.251 INFO:tasks.rados.rados.0.smithi070.stdout:3294: done (6 left) 2024-09-14T03:59:22.251 INFO:tasks.rados.rados.0.smithi070.stdout:3296: done (5 left) 2024-09-14T03:59:22.251 INFO:tasks.rados.rados.0.smithi070.stdout:3297: done (4 left) 2024-09-14T03:59:22.251 INFO:tasks.rados.rados.0.smithi070.stdout:3298: done (3 left) 2024-09-14T03:59:22.251 INFO:tasks.rados.rados.0.smithi070.stdout:3301: delete oid 7 current snap is 336 2024-09-14T03:59:22.251 INFO:tasks.rados.rados.0.smithi070.stdout:3299: finishing write tid 1 to smithi07085154-39 2024-09-14T03:59:22.251 INFO:tasks.rados.rados.0.smithi070.stdout:3299: finishing write tid 2 to smithi07085154-39 2024-09-14T03:59:22.252 INFO:tasks.rados.rados.0.smithi070.stdout:3299: finishing write tid 3 to smithi07085154-39 2024-09-14T03:59:22.252 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 39 v 2642 (ObjNum 1020 snap 336 seq_num 1020) dirty exists 2024-09-14T03:59:22.252 INFO:tasks.rados.rados.0.smithi070.stdout:3299: left oid 39 (ObjNum 1020 snap 336 seq_num 1020) 2024-09-14T03:59:22.252 INFO:tasks.rados.rados.0.smithi070.stdout:3295: expect (ObjNum 841 snap 286 seq_num 841) 2024-09-14T03:59:22.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:22 smithi094 ceph-mon[86597]: osdmap e726: 8 total, 8 up, 8 in 2024-09-14T03:59:22.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:22 smithi070 ceph-mon[96704]: osdmap e726: 8 total, 8 up, 8 in 2024-09-14T03:59:22.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:22 smithi070 ceph-mon[98640]: osdmap e726: 8 total, 8 up, 8 in 2024-09-14T03:59:22.679 INFO:tasks.rados.rados.0.smithi070.stdout:3300: finishing write tid 1 to smithi07085154-24 2024-09-14T03:59:22.679 INFO:tasks.rados.rados.0.smithi070.stdout:3300: finishing write tid 2 to smithi07085154-24 2024-09-14T03:59:22.679 INFO:tasks.rados.rados.0.smithi070.stdout:3300: finishing write tid 3 to smithi07085154-24 2024-09-14T03:59:22.679 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 3216 (ObjNum 1021 snap 336 seq_num 1021) dirty exists 2024-09-14T03:59:22.679 INFO:tasks.rados.rados.0.smithi070.stdout:3300: left oid 24 (ObjNum 1021 snap 336 seq_num 1021) 2024-09-14T03:59:22.679 INFO:tasks.rados.rados.0.smithi070.stdout:3295: done (3 left) 2024-09-14T03:59:22.680 INFO:tasks.rados.rados.0.smithi070.stdout:3299: done (2 left) 2024-09-14T03:59:22.680 INFO:tasks.rados.rados.0.smithi070.stdout:3300: done (1 left) 2024-09-14T03:59:22.680 INFO:tasks.rados.rados.0.smithi070.stdout:3301: done (0 left) 2024-09-14T03:59:22.680 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 13 current snap is 336 2024-09-14T03:59:22.680 INFO:tasks.rados.rados.0.smithi070.stdout:3302: seq_num 1023 ranges {1253376=335872} 2024-09-14T03:59:22.687 INFO:tasks.rados.rados.0.smithi070.stdout:3302: writing smithi07085154-13 from 1253376 to 1589248 tid 1 2024-09-14T03:59:22.687 INFO:tasks.rados.rados.0.smithi070.stdout:3303: snap_create 2024-09-14T03:59:22.690 INFO:tasks.rados.rados.0.smithi070.stdout:3302: finishing write tid 1 to smithi07085154-13 2024-09-14T03:59:22.691 INFO:tasks.rados.rados.0.smithi070.stdout:3302: finishing write tid 2 to smithi07085154-13 2024-09-14T03:59:22.691 INFO:tasks.rados.rados.0.smithi070.stdout:3302: finishing write tid 3 to smithi07085154-13 2024-09-14T03:59:22.692 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 3176 (ObjNum 1023 snap 336 seq_num 1023) dirty exists 2024-09-14T03:59:22.692 INFO:tasks.rados.rados.0.smithi070.stdout:3302: left oid 13 (ObjNum 1023 snap 336 seq_num 1023) 2024-09-14T03:59:23.160 INFO:tasks.rados.rados.0.smithi070.stdout:3302: done (1 left) 2024-09-14T03:59:23.160 INFO:tasks.rados.rados.0.smithi070.stdout:3303: done (0 left) 2024-09-14T03:59:23.160 INFO:tasks.rados.rados.0.smithi070.stdout:3304: read oid 2 snap 335 2024-09-14T03:59:23.160 INFO:tasks.rados.rados.0.smithi070.stdout:3304: expect (ObjNum 991 snap 327 seq_num 991) 2024-09-14T03:59:23.160 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 12 current snap is 337 2024-09-14T03:59:23.160 INFO:tasks.rados.rados.0.smithi070.stdout:3305: seq_num 1024 ranges {0=352256} 2024-09-14T03:59:23.162 INFO:tasks.rados.rados.0.smithi070.stdout:3305: writing smithi07085154-12 from 0 to 352256 tid 1 2024-09-14T03:59:23.162 INFO:tasks.rados.rados.0.smithi070.stdout:3306: read oid 18 snap -1 2024-09-14T03:59:23.162 INFO:tasks.rados.rados.0.smithi070.stdout:3306: expect (ObjNum 1007 snap 332 seq_num 1007) 2024-09-14T03:59:23.162 INFO:tasks.rados.rados.0.smithi070.stdout:3307: read oid 14 snap -1 2024-09-14T03:59:23.162 INFO:tasks.rados.rados.0.smithi070.stdout:3307: expect (ObjNum 313 snap 105 seq_num 313) 2024-09-14T03:59:23.162 INFO:tasks.rados.rados.0.smithi070.stdout:3308: delete oid 50 current snap is 337 2024-09-14T03:59:23.168 INFO:tasks.rados.rados.0.smithi070.stdout:3306: expect (ObjNum 1007 snap 332 seq_num 1007) 2024-09-14T03:59:23.175 INFO:tasks.rados.rados.0.smithi070.stdout:3306: done (4 left) 2024-09-14T03:59:23.175 INFO:tasks.rados.rados.0.smithi070.stdout:3308: done (3 left) 2024-09-14T03:59:23.175 INFO:tasks.rados.rados.0.smithi070.stdout:3309: read oid 30 snap -1 2024-09-14T03:59:23.175 INFO:tasks.rados.rados.0.smithi070.stdout:3309: expect (ObjNum 1015 snap 335 seq_num 1015) 2024-09-14T03:59:23.175 INFO:tasks.rados.rados.0.smithi070.stdout:3310: read oid 10 snap 325 2024-09-14T03:59:23.176 INFO:tasks.rados.rados.0.smithi070.stdout:3310: expect (ObjNum 841 snap 286 seq_num 841) 2024-09-14T03:59:23.176 INFO:tasks.rados.rados.0.smithi070.stdout:3311: copy_from oid 8 from oid 42 current snap is 337 2024-09-14T03:59:23.176 INFO:tasks.rados.rados.0.smithi070.stdout:3312: read oid 7 snap 320 2024-09-14T03:59:23.176 INFO:tasks.rados.rados.0.smithi070.stdout:3312: expect (ObjNum 963 snap 319 seq_num 963) 2024-09-14T03:59:23.176 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 44 current snap is 337 2024-09-14T03:59:23.176 INFO:tasks.rados.rados.0.smithi070.stdout:3313: seq_num 1026 ranges {0=385024} 2024-09-14T03:59:23.176 INFO:tasks.rados.rados.0.smithi070.stdout:3313: writing smithi07085154-44 from 0 to 385024 tid 1 2024-09-14T03:59:23.176 INFO:tasks.rados.rados.0.smithi070.stdout:3314: snap_remove snap 328 2024-09-14T03:59:23.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:23 smithi070 ceph-mon[96704]: pgmap v688: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.9 MiB/s rd, 154 KiB/s wr, 4 op/s 2024-09-14T03:59:23.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:23 smithi070 ceph-mon[96704]: osdmap e727: 8 total, 8 up, 8 in 2024-09-14T03:59:23.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:23 smithi070 ceph-mon[98640]: pgmap v688: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.9 MiB/s rd, 154 KiB/s wr, 4 op/s 2024-09-14T03:59:23.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:23 smithi070 ceph-mon[98640]: osdmap e727: 8 total, 8 up, 8 in 2024-09-14T03:59:23.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:23 smithi094 ceph-mon[86597]: pgmap v688: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.9 MiB/s rd, 154 KiB/s wr, 4 op/s 2024-09-14T03:59:23.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:23 smithi094 ceph-mon[86597]: osdmap e727: 8 total, 8 up, 8 in 2024-09-14T03:59:23.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:59:23 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:59:23] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.51.0" 2024-09-14T03:59:24.116 INFO:tasks.rados.rados.0.smithi070.stdout:3314: done (8 left) 2024-09-14T03:59:24.116 INFO:tasks.rados.rados.0.smithi070.stdout:3315: read oid 21 snap 267 2024-09-14T03:59:24.116 INFO:tasks.rados.rados.0.smithi070.stdout:3315: expect deleted 2024-09-14T03:59:24.116 INFO:tasks.rados.rados.0.smithi070.stdout:3316: delete oid 17 current snap is 337 2024-09-14T03:59:24.117 INFO:tasks.rados.rados.0.smithi070.stdout:3305: finishing write tid 1 to smithi07085154-12 2024-09-14T03:59:24.117 INFO:tasks.rados.rados.0.smithi070.stdout:3307: expect (ObjNum 313 snap 105 seq_num 313) 2024-09-14T03:59:24.136 INFO:tasks.rados.rados.0.smithi070.stdout:3305: finishing write tid 2 to smithi07085154-12 2024-09-14T03:59:24.136 INFO:tasks.rados.rados.0.smithi070.stdout:3305: finishing write tid 3 to smithi07085154-12 2024-09-14T03:59:24.136 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 2269 (ObjNum 1024 snap 337 seq_num 1024) dirty exists 2024-09-14T03:59:24.136 INFO:tasks.rados.rados.0.smithi070.stdout:3305: left oid 12 (ObjNum 1024 snap 337 seq_num 1024) 2024-09-14T03:59:24.136 INFO:tasks.rados.rados.0.smithi070.stdout:3313: finishing write tid 1 to smithi07085154-44 2024-09-14T03:59:24.136 INFO:tasks.rados.rados.0.smithi070.stdout:3313: finishing write tid 2 to smithi07085154-44 2024-09-14T03:59:24.136 INFO:tasks.rados.rados.0.smithi070.stdout:3313: finishing write tid 3 to smithi07085154-44 2024-09-14T03:59:24.136 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 3194 (ObjNum 1026 snap 337 seq_num 1026) dirty exists 2024-09-14T03:59:24.136 INFO:tasks.rados.rados.0.smithi070.stdout:3313: left oid 44 (ObjNum 1026 snap 337 seq_num 1026) 2024-09-14T03:59:24.136 INFO:tasks.rados.rados.0.smithi070.stdout:3312: expect (ObjNum 963 snap 319 seq_num 963) 2024-09-14T03:59:24.237 INFO:tasks.rados.rados.0.smithi070.stdout:3309: expect (ObjNum 1015 snap 335 seq_num 1015) 2024-09-14T03:59:24.327 INFO:tasks.rados.rados.0.smithi070.stdout:3304: expect (ObjNum 991 snap 327 seq_num 991) 2024-09-14T03:59:24.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:24 smithi094 ceph-mon[86597]: osdmap e728: 8 total, 8 up, 8 in 2024-09-14T03:59:24.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:24 smithi094 ceph-mon[86597]: osdmap e729: 8 total, 8 up, 8 in 2024-09-14T03:59:24.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:24 smithi094 ceph-mon[86597]: pgmap v692: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.6 MiB/s rd, 777 KiB/s wr, 12 op/s 2024-09-14T03:59:24.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:24 smithi070 ceph-mon[96704]: osdmap e728: 8 total, 8 up, 8 in 2024-09-14T03:59:24.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:24 smithi070 ceph-mon[96704]: osdmap e729: 8 total, 8 up, 8 in 2024-09-14T03:59:24.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:24 smithi070 ceph-mon[96704]: pgmap v692: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.6 MiB/s rd, 777 KiB/s wr, 12 op/s 2024-09-14T03:59:24.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:24 smithi070 ceph-mon[98640]: osdmap e728: 8 total, 8 up, 8 in 2024-09-14T03:59:24.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:24 smithi070 ceph-mon[98640]: osdmap e729: 8 total, 8 up, 8 in 2024-09-14T03:59:24.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:24 smithi070 ceph-mon[98640]: pgmap v692: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.6 MiB/s rd, 777 KiB/s wr, 12 op/s 2024-09-14T03:59:24.942 INFO:tasks.rados.rados.0.smithi070.stdout:3311: finishing copy_from to smithi07085154-8 2024-09-14T03:59:24.942 INFO:tasks.rados.rados.0.smithi070.stdout:3311: got expected ENOENT (src dne) 2024-09-14T03:59:24.942 INFO:tasks.rados.rados.0.smithi070.stdout:3311: finishing copy_from racing read to smithi07085154-8 2024-09-14T03:59:24.942 INFO:tasks.rados.rados.0.smithi070.stdout:3310: expect (ObjNum 841 snap 286 seq_num 841) 2024-09-14T03:59:25.369 INFO:tasks.rados.rados.0.smithi070.stdout:3304: done (9 left) 2024-09-14T03:59:25.370 INFO:tasks.rados.rados.0.smithi070.stdout:3305: done (8 left) 2024-09-14T03:59:25.370 INFO:tasks.rados.rados.0.smithi070.stdout:3307: done (7 left) 2024-09-14T03:59:25.370 INFO:tasks.rados.rados.0.smithi070.stdout:3309: done (6 left) 2024-09-14T03:59:25.370 INFO:tasks.rados.rados.0.smithi070.stdout:3310: done (5 left) 2024-09-14T03:59:25.370 INFO:tasks.rados.rados.0.smithi070.stdout:3311: done (4 left) 2024-09-14T03:59:25.370 INFO:tasks.rados.rados.0.smithi070.stdout:3312: done (3 left) 2024-09-14T03:59:25.370 INFO:tasks.rados.rados.0.smithi070.stdout:3313: done (2 left) 2024-09-14T03:59:25.370 INFO:tasks.rados.rados.0.smithi070.stdout:3315: done (1 left) 2024-09-14T03:59:25.370 INFO:tasks.rados.rados.0.smithi070.stdout:3316: done (0 left) 2024-09-14T03:59:25.370 INFO:tasks.rados.rados.0.smithi070.stdout:3317: read oid 27 snap -1 2024-09-14T03:59:25.370 INFO:tasks.rados.rados.0.smithi070.stdout:3317: expect (ObjNum 829 snap 283 seq_num 829) 2024-09-14T03:59:25.370 INFO:tasks.rados.rados.0.smithi070.stdout:3318: snap_remove snap 310 2024-09-14T03:59:26.118 INFO:tasks.rados.rados.0.smithi070.stdout:3318: done (1 left) 2024-09-14T03:59:26.118 INFO:tasks.rados.rados.0.smithi070.stdout:3319: snap_remove snap 318 2024-09-14T03:59:27.122 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:27 smithi070 ceph-mon[96704]: osdmap e730: 8 total, 8 up, 8 in 2024-09-14T03:59:27.126 INFO:tasks.rados.rados.0.smithi070.stdout:3319: done (1 left) 2024-09-14T03:59:27.126 INFO:tasks.rados.rados.0.smithi070.stdout:3320: snap_create 2024-09-14T03:59:27.126 INFO:tasks.rados.rados.0.smithi070.stdout:3317: expect (ObjNum 829 snap 283 seq_num 829) 2024-09-14T03:59:27.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:27 smithi094 ceph-mon[86597]: osdmap e730: 8 total, 8 up, 8 in 2024-09-14T03:59:27.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:27 smithi094 ceph-mon[86597]: pgmap v694: 121 pgs: 121 active+clean; 1.1 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 49 KiB/s rd, 622 KiB/s wr, 9 op/s 2024-09-14T03:59:27.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:27 smithi070 ceph-mon[96704]: pgmap v694: 121 pgs: 121 active+clean; 1.1 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 49 KiB/s rd, 622 KiB/s wr, 9 op/s 2024-09-14T03:59:27.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:27 smithi070 ceph-mon[98640]: osdmap e730: 8 total, 8 up, 8 in 2024-09-14T03:59:27.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:27 smithi070 ceph-mon[98640]: pgmap v694: 121 pgs: 121 active+clean; 1.1 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 49 KiB/s rd, 622 KiB/s wr, 9 op/s 2024-09-14T03:59:28.128 INFO:tasks.rados.rados.0.smithi070.stdout:3317: done (1 left) 2024-09-14T03:59:28.128 INFO:tasks.rados.rados.0.smithi070.stdout:3320: done (0 left) 2024-09-14T03:59:28.129 INFO:tasks.rados.rados.0.smithi070.stdout:3321: delete oid 7 current snap is 338 2024-09-14T03:59:28.133 INFO:tasks.rados.rados.0.smithi070.stdout:3321: done (0 left) 2024-09-14T03:59:28.133 INFO:tasks.rados.rados.0.smithi070.stdout:3322: setattr oid 17 current snap is 338 2024-09-14T03:59:28.135 INFO:tasks.rados.rados.0.smithi070.stdout:3323: snap_remove snap 299 2024-09-14T03:59:28.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:28 smithi094 ceph-mon[86597]: osdmap e731: 8 total, 8 up, 8 in 2024-09-14T03:59:28.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:28 smithi070 ceph-mon[96704]: osdmap e731: 8 total, 8 up, 8 in 2024-09-14T03:59:28.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:28 smithi070 ceph-mon[98640]: osdmap e731: 8 total, 8 up, 8 in 2024-09-14T03:59:29.118 INFO:tasks.rados.rados.0.smithi070.stdout:3323: done (1 left) 2024-09-14T03:59:29.118 INFO:tasks.rados.rados.0.smithi070.stdout:3324: snap_create 2024-09-14T03:59:29.118 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 3193 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-14T03:59:29.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:29 smithi094 ceph-mon[86597]: osdmap e732: 8 total, 8 up, 8 in 2024-09-14T03:59:29.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:29 smithi094 ceph-mon[86597]: pgmap v697: 121 pgs: 121 active+clean; 1.1 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 155 KiB/s wr, 16 op/s 2024-09-14T03:59:29.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:29 smithi094 ceph-mon[86597]: osdmap e733: 8 total, 8 up, 8 in 2024-09-14T03:59:29.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:29 smithi070 ceph-mon[96704]: osdmap e732: 8 total, 8 up, 8 in 2024-09-14T03:59:29.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:29 smithi070 ceph-mon[96704]: pgmap v697: 121 pgs: 121 active+clean; 1.1 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 155 KiB/s wr, 16 op/s 2024-09-14T03:59:29.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:29 smithi070 ceph-mon[96704]: osdmap e733: 8 total, 8 up, 8 in 2024-09-14T03:59:29.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:29 smithi070 ceph-mon[98640]: osdmap e732: 8 total, 8 up, 8 in 2024-09-14T03:59:29.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:29 smithi070 ceph-mon[98640]: pgmap v697: 121 pgs: 121 active+clean; 1.1 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 155 KiB/s wr, 16 op/s 2024-09-14T03:59:29.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:29 smithi070 ceph-mon[98640]: osdmap e733: 8 total, 8 up, 8 in 2024-09-14T03:59:30.120 INFO:tasks.rados.rados.0.smithi070.stdout:3322: done (1 left) 2024-09-14T03:59:30.120 INFO:tasks.rados.rados.0.smithi070.stdout:3324: done (0 left) 2024-09-14T03:59:30.120 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 46 current snap is 339 2024-09-14T03:59:30.120 INFO:tasks.rados.rados.0.smithi070.stdout:3325: seq_num 1029 ranges {1785856=442368} 2024-09-14T03:59:30.132 INFO:tasks.rados.rados.0.smithi070.stdout:3325: writing smithi07085154-46 from 1785856 to 2228224 tid 1 2024-09-14T03:59:30.132 INFO:tasks.rados.rados.0.smithi070.stdout:3326: read oid 11 snap -1 2024-09-14T03:59:30.132 INFO:tasks.rados.rados.0.smithi070.stdout:3326: expect deleted 2024-09-14T03:59:30.132 INFO:tasks.rados.rados.0.smithi070.stdout:3327: setattr oid 38 current snap is 339 2024-09-14T03:59:30.133 INFO:tasks.rados.rados.0.smithi070.stdout:3326: done (2 left) 2024-09-14T03:59:30.133 INFO:tasks.rados.rados.0.smithi070.stdout:3328: snap_remove snap 321 2024-09-14T03:59:30.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:59:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:59:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:59:31.127 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 38 v 3195 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-14T03:59:31.127 INFO:tasks.rados.rados.0.smithi070.stdout:3325: finishing write tid 1 to smithi07085154-46 2024-09-14T03:59:31.127 INFO:tasks.rados.rados.0.smithi070.stdout:3325: finishing write tid 2 to smithi07085154-46 2024-09-14T03:59:31.127 INFO:tasks.rados.rados.0.smithi070.stdout:3325: finishing write tid 3 to smithi07085154-46 2024-09-14T03:59:31.127 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 46 v 3935 (ObjNum 1029 snap 339 seq_num 1029) dirty exists 2024-09-14T03:59:31.127 INFO:tasks.rados.rados.0.smithi070.stdout:3325: left oid 46 (ObjNum 1029 snap 339 seq_num 1029) 2024-09-14T03:59:31.127 INFO:tasks.rados.rados.0.smithi070.stdout:3325: done (2 left) 2024-09-14T03:59:31.128 INFO:tasks.rados.rados.0.smithi070.stdout:3327: done (1 left) 2024-09-14T03:59:31.128 INFO:tasks.rados.rados.0.smithi070.stdout:3328: done (0 left) 2024-09-14T03:59:31.128 INFO:tasks.rados.rados.0.smithi070.stdout:3329: snap_remove snap 313 2024-09-14T03:59:31.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:31 smithi094 ceph-mon[86597]: osdmap e734: 8 total, 8 up, 8 in 2024-09-14T03:59:31.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:31 smithi094 ceph-mon[86597]: pgmap v700: 121 pgs: 121 active+clean; 1.1 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 156 KiB/s wr, 20 op/s 2024-09-14T03:59:31.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:31 smithi070 ceph-mon[96704]: osdmap e734: 8 total, 8 up, 8 in 2024-09-14T03:59:31.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:31 smithi070 ceph-mon[96704]: pgmap v700: 121 pgs: 121 active+clean; 1.1 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 156 KiB/s wr, 20 op/s 2024-09-14T03:59:31.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:31 smithi070 ceph-mon[98640]: osdmap e734: 8 total, 8 up, 8 in 2024-09-14T03:59:31.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:31 smithi070 ceph-mon[98640]: pgmap v700: 121 pgs: 121 active+clean; 1.1 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 156 KiB/s wr, 20 op/s 2024-09-14T03:59:32.128 INFO:tasks.rados.rados.0.smithi070.stdout:3329: done (0 left) 2024-09-14T03:59:32.128 INFO:tasks.rados.rados.0.smithi070.stdout:3330: snap_create 2024-09-14T03:59:32.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:32 smithi094 ceph-mon[86597]: osdmap e735: 8 total, 8 up, 8 in 2024-09-14T03:59:32.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:32 smithi070 ceph-mon[96704]: osdmap e735: 8 total, 8 up, 8 in 2024-09-14T03:59:32.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:32 smithi070 ceph-mon[98640]: osdmap e735: 8 total, 8 up, 8 in 2024-09-14T03:59:33.140 INFO:tasks.rados.rados.0.smithi070.stdout:3330: done (0 left) 2024-09-14T03:59:33.141 INFO:tasks.rados.rados.0.smithi070.stdout:3331: rmattr oid 20 current snap is 340 2024-09-14T03:59:33.141 INFO:tasks.rados.rados.0.smithi070.stdout:3332: rollback oid 4 current snap is 340 2024-09-14T03:59:33.141 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 4 to 107 2024-09-14T03:59:33.141 INFO:tasks.rados.rados.0.smithi070.stdout:3333: rmattr oid 41 current snap is 340 2024-09-14T03:59:33.141 INFO:tasks.rados.rados.0.smithi070.stdout:3334: read oid 27 snap 334 2024-09-14T03:59:33.141 INFO:tasks.rados.rados.0.smithi070.stdout:3334: expect (ObjNum 829 snap 283 seq_num 829) 2024-09-14T03:59:33.141 INFO:tasks.rados.rados.0.smithi070.stdout:3335: read oid 21 snap 315 2024-09-14T03:59:33.141 INFO:tasks.rados.rados.0.smithi070.stdout:3335: expect (ObjNum 927 snap 310 seq_num 927) 2024-09-14T03:59:33.141 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 42 current snap is 340 2024-09-14T03:59:33.141 INFO:tasks.rados.rados.0.smithi070.stdout:3336: seq_num 1030 ranges {0=458752} 2024-09-14T03:59:33.143 INFO:tasks.rados.rados.0.smithi070.stdout:3336: writing smithi07085154-42 from 0 to 458752 tid 1 2024-09-14T03:59:33.144 INFO:tasks.rados.rados.0.smithi070.stdout:3337: delete oid 44 current snap is 340 2024-09-14T03:59:33.152 INFO:tasks.rados.rados.0.smithi070.stdout:3335: expect (ObjNum 927 snap 310 seq_num 927) 2024-09-14T03:59:33.216 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 3201 (ObjNum 1006 snap 331 seq_num 1006) dirty exists 2024-09-14T03:59:33.216 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 20 v 3206 (ObjNum 524 snap 174 seq_num 524) dirty exists 2024-09-14T03:59:33.216 INFO:tasks.rados.rados.0.smithi070.stdout:3331: done (6 left) 2024-09-14T03:59:33.216 INFO:tasks.rados.rados.0.smithi070.stdout:3333: done (5 left) 2024-09-14T03:59:33.216 INFO:tasks.rados.rados.0.smithi070.stdout:3335: done (4 left) 2024-09-14T03:59:33.216 INFO:tasks.rados.rados.0.smithi070.stdout:3337: done (3 left) 2024-09-14T03:59:33.216 INFO:tasks.rados.rados.0.smithi070.stdout:3338: rmattr oid 34 current snap is 340 2024-09-14T03:59:33.216 INFO:tasks.rados.rados.0.smithi070.stdout:3338: done (3 left) 2024-09-14T03:59:33.216 INFO:tasks.rados.rados.0.smithi070.stdout:3339: rollback oid 41 current snap is 340 2024-09-14T03:59:33.216 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 41 to 338 2024-09-14T03:59:33.216 INFO:tasks.rados.rados.0.smithi070.stdout:3332: finishing rollback tid 0 to smithi07085154-4 2024-09-14T03:59:33.216 INFO:tasks.rados.rados.0.smithi070.stdout:3332: finishing rollback tid 1 to smithi07085154-4 2024-09-14T03:59:33.216 INFO:tasks.rados.rados.0.smithi070.stdout:3332: finishing rollback tid 2 to smithi07085154-4 2024-09-14T03:59:33.217 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 3209 (ObjNum 243 snap 69 seq_num 243) dirty exists 2024-09-14T03:59:33.217 INFO:tasks.rados.rados.0.smithi070.stdout:3334: expect (ObjNum 829 snap 283 seq_num 829) 2024-09-14T03:59:33.325 INFO:tasks.rados.rados.0.smithi070.stdout:3336: finishing write tid 1 to smithi07085154-42 2024-09-14T03:59:33.325 INFO:tasks.rados.rados.0.smithi070.stdout:3336: finishing write tid 2 to smithi07085154-42 2024-09-14T03:59:33.325 INFO:tasks.rados.rados.0.smithi070.stdout:3336: finishing write tid 3 to smithi07085154-42 2024-09-14T03:59:33.325 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 3959 (ObjNum 1030 snap 340 seq_num 1030) dirty exists 2024-09-14T03:59:33.325 INFO:tasks.rados.rados.0.smithi070.stdout:3336: left oid 42 (ObjNum 1030 snap 340 seq_num 1030) 2024-09-14T03:59:33.325 INFO:tasks.rados.rados.0.smithi070.stdout:3332: done (3 left) 2024-09-14T03:59:33.325 INFO:tasks.rados.rados.0.smithi070.stdout:3334: done (2 left) 2024-09-14T03:59:33.325 INFO:tasks.rados.rados.0.smithi070.stdout:3336: done (1 left) 2024-09-14T03:59:33.325 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 26 current snap is 340 2024-09-14T03:59:33.325 INFO:tasks.rados.rados.0.smithi070.stdout:3339: finishing rollback tid 0 to smithi07085154-41 2024-09-14T03:59:33.325 INFO:tasks.rados.rados.0.smithi070.stdout:3340: seq_num 1032 ranges {2375680=491520} 2024-09-14T03:59:33.340 INFO:tasks.rados.rados.0.smithi070.stdout:3340: writing smithi07085154-26 from 2375680 to 2867200 tid 1 2024-09-14T03:59:33.340 INFO:tasks.rados.rados.0.smithi070.stdout:3341: read oid 8 snap 296 2024-09-14T03:59:33.340 INFO:tasks.rados.rados.0.smithi070.stdout:3341: expect (ObjNum -1362867904 snap 21849 seq_num 2930040128) 2024-09-14T03:59:33.340 INFO:tasks.rados.rados.0.smithi070.stdout:3342: copy_from oid 43 from oid 39 current snap is 340 2024-09-14T03:59:33.340 INFO:tasks.rados.rados.0.smithi070.stdout:3339: finishing rollback tid 1 to smithi07085154-41 2024-09-14T03:59:33.340 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 38 current snap is 340 2024-09-14T03:59:33.341 INFO:tasks.rados.rados.0.smithi070.stdout:3343: seq_num 1033 ranges {0=507904} 2024-09-14T03:59:33.343 INFO:tasks.rados.rados.0.smithi070.stdout:3343: writing smithi07085154-38 from 0 to 507904 tid 1 2024-09-14T03:59:33.343 INFO:tasks.rados.rados.0.smithi070.stdout:3344: rollback oid 24 current snap is 340 2024-09-14T03:59:33.343 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 24 to 254 2024-09-14T03:59:33.343 INFO:tasks.rados.rados.0.smithi070.stdout:3339: finishing rollback tid 2 to smithi07085154-41 2024-09-14T03:59:33.343 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 3204 (ObjNum 1006 snap 331 seq_num 1006) dirty exists 2024-09-14T03:59:33.343 INFO:tasks.rados.rados.0.smithi070.stdout:3341: expect (ObjNum -1362867904 snap 21849 seq_num 2930040128) 2024-09-14T03:59:33.346 INFO:tasks.rados.rados.0.smithi070.stdout:3339: done (5 left) 2024-09-14T03:59:33.346 INFO:tasks.rados.rados.0.smithi070.stdout:3341: done (4 left) 2024-09-14T03:59:33.346 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 14 current snap is 340 2024-09-14T03:59:33.346 INFO:tasks.rados.rados.0.smithi070.stdout:3345: seq_num 1034 ranges {450560=524288} 2024-09-14T03:59:33.353 INFO:tasks.rados.rados.0.smithi070.stdout:3345: writing smithi07085154-14 from 450560 to 974848 tid 1 2024-09-14T03:59:33.353 INFO:tasks.rados.rados.0.smithi070.stdout:3346: snap_create 2024-09-14T03:59:33.353 INFO:tasks.rados.rados.0.smithi070.stdout:3340: finishing write tid 1 to smithi07085154-26 2024-09-14T03:59:33.353 INFO:tasks.rados.rados.0.smithi070.stdout:3340: finishing write tid 2 to smithi07085154-26 2024-09-14T03:59:33.353 INFO:tasks.rados.rados.0.smithi070.stdout:3343: finishing write tid 1 to smithi07085154-38 2024-09-14T03:59:33.353 INFO:tasks.rados.rados.0.smithi070.stdout:3344: finishing rollback tid 0 to smithi07085154-24 2024-09-14T03:59:33.353 INFO:tasks.rados.rados.0.smithi070.stdout:3343: finishing write tid 2 to smithi07085154-38 2024-09-14T03:59:33.353 INFO:tasks.rados.rados.0.smithi070.stdout:3340: finishing write tid 3 to smithi07085154-26 2024-09-14T03:59:33.353 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 3962 (ObjNum 1032 snap 340 seq_num 1032) dirty exists 2024-09-14T03:59:33.353 INFO:tasks.rados.rados.0.smithi070.stdout:3340: left oid 26 (ObjNum 1032 snap 340 seq_num 1032) 2024-09-14T03:59:33.353 INFO:tasks.rados.rados.0.smithi070.stdout:3344: finishing rollback tid 1 to smithi07085154-24 2024-09-14T03:59:33.353 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 24 v 3218 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-14T03:59:33.353 INFO:tasks.rados.rados.0.smithi070.stdout:3342: finishing copy_from racing read to smithi07085154-43 2024-09-14T03:59:33.354 INFO:tasks.rados.rados.0.smithi070.stdout:3343: finishing write tid 3 to smithi07085154-38 2024-09-14T03:59:33.354 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 38 v 3197 (ObjNum 1033 snap 340 seq_num 1033) dirty exists 2024-09-14T03:59:33.354 INFO:tasks.rados.rados.0.smithi070.stdout:3343: left oid 38 (ObjNum 1033 snap 340 seq_num 1033) 2024-09-14T03:59:33.354 INFO:tasks.rados.rados.0.smithi070.stdout:3342: finishing copy_from to smithi07085154-43 2024-09-14T03:59:33.354 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 2642 (ObjNum 1020 snap 336 seq_num 1020) dirty exists 2024-09-14T03:59:33.356 INFO:tasks.rados.rados.0.smithi070.stdout:3345: finishing write tid 1 to smithi07085154-14 2024-09-14T03:59:33.356 INFO:tasks.rados.rados.0.smithi070.stdout:3345: finishing write tid 2 to smithi07085154-14 2024-09-14T03:59:33.357 INFO:tasks.rados.rados.0.smithi070.stdout:3345: finishing write tid 3 to smithi07085154-14 2024-09-14T03:59:33.357 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 3220 (ObjNum 1034 snap 340 seq_num 1034) dirty exists 2024-09-14T03:59:33.357 INFO:tasks.rados.rados.0.smithi070.stdout:3345: left oid 14 (ObjNum 1034 snap 340 seq_num 1034) 2024-09-14T03:59:33.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:33 smithi070 ceph-mon[96704]: osdmap e736: 8 total, 8 up, 8 in 2024-09-14T03:59:33.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:33 smithi070 ceph-mon[96704]: pgmap v703: 121 pgs: 121 active+clean; 1.1 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 108 KiB/s wr, 11 op/s 2024-09-14T03:59:33.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:33 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:59:33.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:33 smithi070 ceph-mon[98640]: osdmap e736: 8 total, 8 up, 8 in 2024-09-14T03:59:33.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:33 smithi070 ceph-mon[98640]: pgmap v703: 121 pgs: 121 active+clean; 1.1 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 108 KiB/s wr, 11 op/s 2024-09-14T03:59:33.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:33 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:59:33.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:33 smithi094 ceph-mon[86597]: osdmap e736: 8 total, 8 up, 8 in 2024-09-14T03:59:33.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:33 smithi094 ceph-mon[86597]: pgmap v703: 121 pgs: 121 active+clean; 1.1 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 108 KiB/s wr, 11 op/s 2024-09-14T03:59:33.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:33 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:59:33.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:59:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:59:33] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.51.0" 2024-09-14T03:59:34.119 INFO:tasks.rados.rados.0.smithi070.stdout:3340: done (5 left) 2024-09-14T03:59:34.120 INFO:tasks.rados.rados.0.smithi070.stdout:3342: done (4 left) 2024-09-14T03:59:34.120 INFO:tasks.rados.rados.0.smithi070.stdout:3343: done (3 left) 2024-09-14T03:59:34.120 INFO:tasks.rados.rados.0.smithi070.stdout:3344: done (2 left) 2024-09-14T03:59:34.120 INFO:tasks.rados.rados.0.smithi070.stdout:3345: done (1 left) 2024-09-14T03:59:34.120 INFO:tasks.rados.rados.0.smithi070.stdout:3346: done (0 left) 2024-09-14T03:59:34.120 INFO:tasks.rados.rados.0.smithi070.stdout:3347: rmattr oid 26 current snap is 341 2024-09-14T03:59:34.120 INFO:tasks.rados.rados.0.smithi070.stdout:3347: done (0 left) 2024-09-14T03:59:34.120 INFO:tasks.rados.rados.0.smithi070.stdout:3348: rollback oid 26 current snap is 341 2024-09-14T03:59:34.120 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 26 to 315 2024-09-14T03:59:34.120 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 9 current snap is 341 2024-09-14T03:59:34.120 INFO:tasks.rados.rados.0.smithi070.stdout:3349: seq_num 1035 ranges {3678208=540672} 2024-09-14T03:59:34.142 INFO:tasks.rados.rados.0.smithi070.stdout:3349: writing smithi07085154-9 from 3678208 to 4218880 tid 1 2024-09-14T03:59:34.143 INFO:tasks.rados.rados.0.smithi070.stdout:3350: snap_remove snap 336 2024-09-14T03:59:34.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:34 smithi094 ceph-mon[86597]: osdmap e737: 8 total, 8 up, 8 in 2024-09-14T03:59:34.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:34 smithi094 ceph-mon[86597]: osdmap e738: 8 total, 8 up, 8 in 2024-09-14T03:59:34.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:34 smithi070 ceph-mon[96704]: osdmap e737: 8 total, 8 up, 8 in 2024-09-14T03:59:34.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:34 smithi070 ceph-mon[96704]: osdmap e738: 8 total, 8 up, 8 in 2024-09-14T03:59:34.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:34 smithi070 ceph-mon[98640]: osdmap e737: 8 total, 8 up, 8 in 2024-09-14T03:59:34.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:34 smithi070 ceph-mon[98640]: osdmap e738: 8 total, 8 up, 8 in 2024-09-14T03:59:35.143 INFO:tasks.rados.rados.0.smithi070.stdout:3350: done (2 left) 2024-09-14T03:59:35.143 INFO:tasks.rados.rados.0.smithi070.stdout:3351: setattr oid 5 current snap is 341 2024-09-14T03:59:35.143 INFO:tasks.rados.rados.0.smithi070.stdout:3348: finishing rollback tid 0 to smithi07085154-26 2024-09-14T03:59:35.143 INFO:tasks.rados.rados.0.smithi070.stdout:3348: finishing rollback tid 1 to smithi07085154-26 2024-09-14T03:59:35.143 INFO:tasks.rados.rados.0.smithi070.stdout:3348: finishing rollback tid 2 to smithi07085154-26 2024-09-14T03:59:35.143 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 3966 (ObjNum 918 snap 308 seq_num 918) dirty exists 2024-09-14T03:59:35.143 INFO:tasks.rados.rados.0.smithi070.stdout:3349: finishing write tid 1 to smithi07085154-9 2024-09-14T03:59:35.143 INFO:tasks.rados.rados.0.smithi070.stdout:3349: finishing write tid 2 to smithi07085154-9 2024-09-14T03:59:35.144 INFO:tasks.rados.rados.0.smithi070.stdout:3349: finishing write tid 3 to smithi07085154-9 2024-09-14T03:59:35.144 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 3969 (ObjNum 1035 snap 341 seq_num 1035) dirty exists 2024-09-14T03:59:35.144 INFO:tasks.rados.rados.0.smithi070.stdout:3349: left oid 9 (ObjNum 1035 snap 341 seq_num 1035) 2024-09-14T03:59:35.145 INFO:tasks.rados.rados.0.smithi070.stdout:3348: done (2 left) 2024-09-14T03:59:35.145 INFO:tasks.rados.rados.0.smithi070.stdout:3349: done (1 left) 2024-09-14T03:59:35.145 INFO:tasks.rados.rados.0.smithi070.stdout:3352: read oid 43 snap -1 2024-09-14T03:59:35.145 INFO:tasks.rados.rados.0.smithi070.stdout:3352: expect (ObjNum 1020 snap 336 seq_num 1020) 2024-09-14T03:59:35.145 INFO:tasks.rados.rados.0.smithi070.stdout:3353: read oid 50 snap -1 2024-09-14T03:59:35.145 INFO:tasks.rados.rados.0.smithi070.stdout:3353: expect deleted 2024-09-14T03:59:35.145 INFO:tasks.rados.rados.0.smithi070.stdout:3354: copy_from oid 41 from oid 11 current snap is 341 2024-09-14T03:59:35.145 INFO:tasks.rados.rados.0.smithi070.stdout:3355: rmattr oid 19 current snap is 341 2024-09-14T03:59:35.146 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 8 current snap is 341 2024-09-14T03:59:35.146 INFO:tasks.rados.rados.0.smithi070.stdout:3356: seq_num 1036 ranges {1196032=557056} 2024-09-14T03:59:35.155 INFO:tasks.rados.rados.0.smithi070.stdout:3356: writing smithi07085154-8 from 1196032 to 1753088 tid 1 2024-09-14T03:59:35.156 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 28 current snap is 341 2024-09-14T03:59:35.156 INFO:tasks.rados.rados.0.smithi070.stdout:3357: seq_num 1037 ranges {3448832=573440} 2024-09-14T03:59:35.178 INFO:tasks.rados.rados.0.smithi070.stdout:3357: writing smithi07085154-28 from 3448832 to 4022272 tid 1 2024-09-14T03:59:35.178 INFO:tasks.rados.rados.0.smithi070.stdout:3358: snap_remove snap 254 2024-09-14T03:59:35.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:35 smithi094 ceph-mon[86597]: pgmap v706: 121 pgs: 121 active+clean; 1.1 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 108 KiB/s wr, 6 op/s 2024-09-14T03:59:35.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:35 smithi070 ceph-mon[96704]: pgmap v706: 121 pgs: 121 active+clean; 1.1 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 108 KiB/s wr, 6 op/s 2024-09-14T03:59:35.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:35 smithi070 ceph-mon[98640]: pgmap v706: 121 pgs: 121 active+clean; 1.1 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 108 KiB/s wr, 6 op/s 2024-09-14T03:59:36.154 INFO:tasks.rados.rados.0.smithi070.stdout:3358: done (7 left) 2024-09-14T03:59:36.155 INFO:tasks.rados.rados.0.smithi070.stdout:3359: snap_create 2024-09-14T03:59:36.155 INFO:tasks.rados.rados.0.smithi070.stdout:3354: finishing copy_from to smithi07085154-41 2024-09-14T03:59:36.155 INFO:tasks.rados.rados.0.smithi070.stdout:3354: got expected ENOENT (src dne) 2024-09-14T03:59:36.155 INFO:tasks.rados.rados.0.smithi070.stdout:3354: finishing copy_from racing read to smithi07085154-41 2024-09-14T03:59:36.155 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 3190 (ObjNum 686 snap 231 seq_num 686) dirty exists 2024-09-14T03:59:36.155 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 19 v 3221 (ObjNum 1004 snap 331 seq_num 1004) dirty exists 2024-09-14T03:59:36.155 INFO:tasks.rados.rados.0.smithi070.stdout:3356: finishing write tid 1 to smithi07085154-8 2024-09-14T03:59:36.155 INFO:tasks.rados.rados.0.smithi070.stdout:3356: finishing write tid 2 to smithi07085154-8 2024-09-14T03:59:36.155 INFO:tasks.rados.rados.0.smithi070.stdout:3352: expect (ObjNum 1020 snap 336 seq_num 1020) 2024-09-14T03:59:36.232 INFO:tasks.rados.rados.0.smithi070.stdout:3356: finishing write tid 3 to smithi07085154-8 2024-09-14T03:59:36.232 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 8 v 3206 (ObjNum 1036 snap 341 seq_num 1036) dirty exists 2024-09-14T03:59:36.232 INFO:tasks.rados.rados.0.smithi070.stdout:3356: left oid 8 (ObjNum 1036 snap 341 seq_num 1036) 2024-09-14T03:59:36.233 INFO:tasks.rados.rados.0.smithi070.stdout:3357: finishing write tid 1 to smithi07085154-28 2024-09-14T03:59:36.233 INFO:tasks.rados.rados.0.smithi070.stdout:3357: finishing write tid 2 to smithi07085154-28 2024-09-14T03:59:36.233 INFO:tasks.rados.rados.0.smithi070.stdout:3357: finishing write tid 3 to smithi07085154-28 2024-09-14T03:59:36.233 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 28 v 3211 (ObjNum 1037 snap 341 seq_num 1037) dirty exists 2024-09-14T03:59:36.233 INFO:tasks.rados.rados.0.smithi070.stdout:3357: left oid 28 (ObjNum 1037 snap 341 seq_num 1037) 2024-09-14T03:59:36.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:36 smithi094 ceph-mon[86597]: osdmap e739: 8 total, 8 up, 8 in 2024-09-14T03:59:36.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:36 smithi070 ceph-mon[96704]: osdmap e739: 8 total, 8 up, 8 in 2024-09-14T03:59:36.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:36 smithi070 ceph-mon[98640]: osdmap e739: 8 total, 8 up, 8 in 2024-09-14T03:59:37.154 INFO:tasks.rados.rados.0.smithi070.stdout:3351: done (7 left) 2024-09-14T03:59:37.154 INFO:tasks.rados.rados.0.smithi070.stdout:3352: done (6 left) 2024-09-14T03:59:37.154 INFO:tasks.rados.rados.0.smithi070.stdout:3353: done (5 left) 2024-09-14T03:59:37.155 INFO:tasks.rados.rados.0.smithi070.stdout:3354: done (4 left) 2024-09-14T03:59:37.155 INFO:tasks.rados.rados.0.smithi070.stdout:3355: done (3 left) 2024-09-14T03:59:37.155 INFO:tasks.rados.rados.0.smithi070.stdout:3356: done (2 left) 2024-09-14T03:59:37.155 INFO:tasks.rados.rados.0.smithi070.stdout:3357: done (1 left) 2024-09-14T03:59:37.155 INFO:tasks.rados.rados.0.smithi070.stdout:3359: done (0 left) 2024-09-14T03:59:37.155 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 17 current snap is 342 2024-09-14T03:59:37.155 INFO:tasks.rados.rados.0.smithi070.stdout:3360: seq_num 1038 ranges {0=589824} 2024-09-14T03:59:37.157 INFO:tasks.rados.rados.0.smithi070.stdout:3360: writing smithi07085154-17 from 0 to 589824 tid 1 2024-09-14T03:59:37.158 INFO:tasks.rados.rados.0.smithi070.stdout:3361: read oid 7 snap -1 2024-09-14T03:59:37.158 INFO:tasks.rados.rados.0.smithi070.stdout:3361: expect deleted 2024-09-14T03:59:37.158 INFO:tasks.rados.rados.0.smithi070.stdout:3362: snap_remove snap 322 2024-09-14T03:59:37.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:37 smithi094 ceph-mon[86597]: pgmap v708: 121 pgs: 121 active+clean; 1.1 GiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 22 KiB/s rd, 423 KiB/s wr, 4 op/s 2024-09-14T03:59:37.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:37 smithi094 ceph-mon[86597]: osdmap e740: 8 total, 8 up, 8 in 2024-09-14T03:59:37.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:37 smithi070 ceph-mon[96704]: pgmap v708: 121 pgs: 121 active+clean; 1.1 GiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 22 KiB/s rd, 423 KiB/s wr, 4 op/s 2024-09-14T03:59:37.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:37 smithi070 ceph-mon[96704]: osdmap e740: 8 total, 8 up, 8 in 2024-09-14T03:59:37.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:37 smithi070 ceph-mon[98640]: pgmap v708: 121 pgs: 121 active+clean; 1.1 GiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 22 KiB/s rd, 423 KiB/s wr, 4 op/s 2024-09-14T03:59:37.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:37 smithi070 ceph-mon[98640]: osdmap e740: 8 total, 8 up, 8 in 2024-09-14T03:59:38.165 INFO:tasks.rados.rados.0.smithi070.stdout:3362: done (2 left) 2024-09-14T03:59:38.165 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 16 current snap is 342 2024-09-14T03:59:38.165 INFO:tasks.rados.rados.0.smithi070.stdout:3363: seq_num 1039 ranges {0=606208} 2024-09-14T03:59:38.168 INFO:tasks.rados.rados.0.smithi070.stdout:3363: writing smithi07085154-16 from 0 to 606208 tid 1 2024-09-14T03:59:38.169 INFO:tasks.rados.rados.0.smithi070.stdout:3360: finishing write tid 1 to smithi07085154-17 2024-09-14T03:59:38.170 INFO:tasks.rados.rados.0.smithi070.stdout:3360: finishing write tid 2 to smithi07085154-17 2024-09-14T03:59:38.170 INFO:tasks.rados.rados.0.smithi070.stdout:3360: finishing write tid 3 to smithi07085154-17 2024-09-14T03:59:38.170 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 3195 (ObjNum 1038 snap 342 seq_num 1038) dirty exists 2024-09-14T03:59:38.170 INFO:tasks.rados.rados.0.smithi070.stdout:3360: left oid 17 (ObjNum 1038 snap 342 seq_num 1038) 2024-09-14T03:59:38.170 INFO:tasks.rados.rados.0.smithi070.stdout:3360: done (2 left) 2024-09-14T03:59:38.170 INFO:tasks.rados.rados.0.smithi070.stdout:3361: done (1 left) 2024-09-14T03:59:38.170 INFO:tasks.rados.rados.0.smithi070.stdout:3364: rmattr oid 4 current snap is 342 2024-09-14T03:59:38.170 INFO:tasks.rados.rados.0.smithi070.stdout:3364: done (1 left) 2024-09-14T03:59:38.170 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 33 current snap is 342 2024-09-14T03:59:38.170 INFO:tasks.rados.rados.0.smithi070.stdout:3365: seq_num 1040 ranges {139264=622592} 2024-09-14T03:59:38.174 INFO:tasks.rados.rados.0.smithi070.stdout:3365: writing smithi07085154-33 from 139264 to 761856 tid 1 2024-09-14T03:59:38.175 INFO:tasks.rados.rados.0.smithi070.stdout:3366: delete oid 50 current snap is 342 2024-09-14T03:59:38.184 INFO:tasks.rados.rados.0.smithi070.stdout:3365: finishing write tid 1 to smithi07085154-33 2024-09-14T03:59:38.184 INFO:tasks.rados.rados.0.smithi070.stdout:3365: finishing write tid 2 to smithi07085154-33 2024-09-14T03:59:38.187 INFO:tasks.rados.rados.0.smithi070.stdout:3365: finishing write tid 3 to smithi07085154-33 2024-09-14T03:59:38.187 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 33 v 3200 (ObjNum 1040 snap 342 seq_num 1040) dirty exists 2024-09-14T03:59:38.187 INFO:tasks.rados.rados.0.smithi070.stdout:3365: left oid 33 (ObjNum 1040 snap 342 seq_num 1040) 2024-09-14T03:59:38.188 INFO:tasks.rados.rados.0.smithi070.stdout:3365: done (2 left) 2024-09-14T03:59:38.188 INFO:tasks.rados.rados.0.smithi070.stdout:3366: done (1 left) 2024-09-14T03:59:38.188 INFO:tasks.rados.rados.0.smithi070.stdout:3367: copy_from oid 35 from oid 7 current snap is 342 2024-09-14T03:59:38.188 INFO:tasks.rados.rados.0.smithi070.stdout:3368: rmattr oid 43 current snap is 342 2024-09-14T03:59:38.188 INFO:tasks.rados.rados.0.smithi070.stdout:3368: done (2 left) 2024-09-14T03:59:38.188 INFO:tasks.rados.rados.0.smithi070.stdout:3369: snap_remove snap 335 2024-09-14T03:59:38.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:38 smithi094 ceph-mon[86597]: osdmap e741: 8 total, 8 up, 8 in 2024-09-14T03:59:38.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:38 smithi070 ceph-mon[96704]: osdmap e741: 8 total, 8 up, 8 in 2024-09-14T03:59:38.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:38 smithi070 ceph-mon[98640]: osdmap e741: 8 total, 8 up, 8 in 2024-09-14T03:59:39.121 INFO:tasks.rados.rados.0.smithi070.stdout:3369: done (2 left) 2024-09-14T03:59:39.121 INFO:tasks.rados.rados.0.smithi070.stdout:3370: setattr oid 27 current snap is 342 2024-09-14T03:59:39.121 INFO:tasks.rados.rados.0.smithi070.stdout:3367: finishing copy_from to smithi07085154-35 2024-09-14T03:59:39.121 INFO:tasks.rados.rados.0.smithi070.stdout:3367: got expected ENOENT (src dne) 2024-09-14T03:59:39.121 INFO:tasks.rados.rados.0.smithi070.stdout:3367: finishing copy_from racing read to smithi07085154-35 2024-09-14T03:59:39.121 INFO:tasks.rados.rados.0.smithi070.stdout:3363: finishing write tid 1 to smithi07085154-16 2024-09-14T03:59:39.121 INFO:tasks.rados.rados.0.smithi070.stdout:3363: finishing write tid 2 to smithi07085154-16 2024-09-14T03:59:39.121 INFO:tasks.rados.rados.0.smithi070.stdout:3363: finishing write tid 3 to smithi07085154-16 2024-09-14T03:59:39.121 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 3993 (ObjNum 1039 snap 342 seq_num 1039) dirty exists 2024-09-14T03:59:39.122 INFO:tasks.rados.rados.0.smithi070.stdout:3363: left oid 16 (ObjNum 1039 snap 342 seq_num 1039) 2024-09-14T03:59:39.123 INFO:tasks.rados.rados.0.smithi070.stdout:3363: done (2 left) 2024-09-14T03:59:39.123 INFO:tasks.rados.rados.0.smithi070.stdout:3367: done (1 left) 2024-09-14T03:59:39.124 INFO:tasks.rados.rados.0.smithi070.stdout:3371: copy_from oid 30 from oid 9 current snap is 342 2024-09-14T03:59:39.124 INFO:tasks.rados.rados.0.smithi070.stdout:3372: read oid 44 snap -1 2024-09-14T03:59:39.124 INFO:tasks.rados.rados.0.smithi070.stdout:3372: expect deleted 2024-09-14T03:59:39.124 INFO:tasks.rados.rados.0.smithi070.stdout:3373: read oid 49 snap 325 2024-09-14T03:59:39.124 INFO:tasks.rados.rados.0.smithi070.stdout:3373: expect (ObjNum 898 snap 300 seq_num 898) 2024-09-14T03:59:39.124 INFO:tasks.rados.rados.0.smithi070.stdout:3374: snap_remove snap 323 2024-09-14T03:59:39.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:39 smithi094 ceph-mon[86597]: pgmap v711: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3.8 MiB/s rd, 1.1 MiB/s wr, 22 op/s 2024-09-14T03:59:39.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:39 smithi094 ceph-mon[86597]: osdmap e742: 8 total, 8 up, 8 in 2024-09-14T03:59:39.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:39 smithi094 ceph-mon[86597]: osdmap e743: 8 total, 8 up, 8 in 2024-09-14T03:59:39.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:39 smithi070 ceph-mon[96704]: pgmap v711: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3.8 MiB/s rd, 1.1 MiB/s wr, 22 op/s 2024-09-14T03:59:39.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:39 smithi070 ceph-mon[96704]: osdmap e742: 8 total, 8 up, 8 in 2024-09-14T03:59:39.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:39 smithi070 ceph-mon[96704]: osdmap e743: 8 total, 8 up, 8 in 2024-09-14T03:59:39.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:39 smithi070 ceph-mon[98640]: pgmap v711: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.1 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3.8 MiB/s rd, 1.1 MiB/s wr, 22 op/s 2024-09-14T03:59:39.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:39 smithi070 ceph-mon[98640]: osdmap e742: 8 total, 8 up, 8 in 2024-09-14T03:59:39.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:39 smithi070 ceph-mon[98640]: osdmap e743: 8 total, 8 up, 8 in 2024-09-14T03:59:40.123 INFO:tasks.rados.rados.0.smithi070.stdout:3374: done (4 left) 2024-09-14T03:59:40.123 INFO:tasks.rados.rados.0.smithi070.stdout:3375: snap_remove snap 341 2024-09-14T03:59:40.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:59:40 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:59:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:59:41.128 INFO:tasks.rados.rados.0.smithi070.stdout:3375: done (4 left) 2024-09-14T03:59:41.128 INFO:tasks.rados.rados.0.smithi070.stdout:3376: snap_remove snap 290 2024-09-14T03:59:41.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:41 smithi094 ceph-mon[86597]: osdmap e744: 8 total, 8 up, 8 in 2024-09-14T03:59:41.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:41 smithi094 ceph-mon[86597]: pgmap v715: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.0 MiB/s wr, 21 op/s 2024-09-14T03:59:41.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:41 smithi070 ceph-mon[96704]: osdmap e744: 8 total, 8 up, 8 in 2024-09-14T03:59:41.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:41 smithi070 ceph-mon[96704]: pgmap v715: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.0 MiB/s wr, 21 op/s 2024-09-14T03:59:41.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:41 smithi070 ceph-mon[98640]: osdmap e744: 8 total, 8 up, 8 in 2024-09-14T03:59:41.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:41 smithi070 ceph-mon[98640]: pgmap v715: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.0 MiB/s wr, 21 op/s 2024-09-14T03:59:42.134 INFO:tasks.rados.rados.0.smithi070.stdout:3373: expect (ObjNum 898 snap 300 seq_num 898) 2024-09-14T03:59:42.163 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 4005 (ObjNum 829 snap 283 seq_num 829) dirty exists 2024-09-14T03:59:42.163 INFO:tasks.rados.rados.0.smithi070.stdout:3371: finishing copy_from racing read to smithi07085154-30 2024-09-14T03:59:42.163 INFO:tasks.rados.rados.0.smithi070.stdout:3371: finishing copy_from to smithi07085154-30 2024-09-14T03:59:42.163 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 30 v 3969 (ObjNum 1035 snap 341 seq_num 1035) dirty exists 2024-09-14T03:59:42.163 INFO:tasks.rados.rados.0.smithi070.stdout:3370: done (4 left) 2024-09-14T03:59:42.163 INFO:tasks.rados.rados.0.smithi070.stdout:3371: done (3 left) 2024-09-14T03:59:42.163 INFO:tasks.rados.rados.0.smithi070.stdout:3372: done (2 left) 2024-09-14T03:59:42.163 INFO:tasks.rados.rados.0.smithi070.stdout:3373: done (1 left) 2024-09-14T03:59:42.163 INFO:tasks.rados.rados.0.smithi070.stdout:3376: done (0 left) 2024-09-14T03:59:42.163 INFO:tasks.rados.rados.0.smithi070.stdout:3377: read oid 32 snap -1 2024-09-14T03:59:42.163 INFO:tasks.rados.rados.0.smithi070.stdout:3377: expect deleted 2024-09-14T03:59:42.164 INFO:tasks.rados.rados.0.smithi070.stdout:3378: read oid 35 snap -1 2024-09-14T03:59:42.164 INFO:tasks.rados.rados.0.smithi070.stdout:3378: expect deleted 2024-09-14T03:59:42.164 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 5 current snap is 342 2024-09-14T03:59:42.164 INFO:tasks.rados.rados.0.smithi070.stdout:3379: seq_num 1042 ranges {1900544=655360} 2024-09-14T03:59:42.176 INFO:tasks.rados.rados.0.smithi070.stdout:3379: writing smithi07085154-5 from 1900544 to 2555904 tid 1 2024-09-14T03:59:42.177 INFO:tasks.rados.rados.0.smithi070.stdout:3380: delete oid 50 current snap is 342 2024-09-14T03:59:42.180 INFO:tasks.rados.rados.0.smithi070.stdout:3377: done (3 left) 2024-09-14T03:59:42.180 INFO:tasks.rados.rados.0.smithi070.stdout:3378: done (2 left) 2024-09-14T03:59:42.180 INFO:tasks.rados.rados.0.smithi070.stdout:3380: done (1 left) 2024-09-14T03:59:42.180 INFO:tasks.rados.rados.0.smithi070.stdout:3381: read oid 26 snap 332 2024-09-14T03:59:42.180 INFO:tasks.rados.rados.0.smithi070.stdout:3381: expect (ObjNum 898 snap 300 seq_num 898) 2024-09-14T03:59:42.180 INFO:tasks.rados.rados.0.smithi070.stdout:3382: snap_create 2024-09-14T03:59:42.180 INFO:tasks.rados.rados.0.smithi070.stdout:3379: finishing write tid 1 to smithi07085154-5 2024-09-14T03:59:42.180 INFO:tasks.rados.rados.0.smithi070.stdout:3379: finishing write tid 2 to smithi07085154-5 2024-09-14T03:59:42.181 INFO:tasks.rados.rados.0.smithi070.stdout:3379: finishing write tid 3 to smithi07085154-5 2024-09-14T03:59:42.181 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 3192 (ObjNum 1042 snap 342 seq_num 1042) dirty exists 2024-09-14T03:59:42.181 INFO:tasks.rados.rados.0.smithi070.stdout:3379: left oid 5 (ObjNum 1042 snap 342 seq_num 1042) 2024-09-14T03:59:42.183 INFO:tasks.rados.rados.0.smithi070.stdout:3381: expect (ObjNum 898 snap 300 seq_num 898) 2024-09-14T03:59:42.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:42 smithi094 ceph-mon[86597]: osdmap e745: 8 total, 8 up, 8 in 2024-09-14T03:59:42.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:42 smithi070 ceph-mon[98640]: osdmap e745: 8 total, 8 up, 8 in 2024-09-14T03:59:42.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:42 smithi070 ceph-mon[96704]: osdmap e745: 8 total, 8 up, 8 in 2024-09-14T03:59:43.144 INFO:tasks.rados.rados.0.smithi070.stdout:3379: done (2 left) 2024-09-14T03:59:43.145 INFO:tasks.rados.rados.0.smithi070.stdout:3381: done (1 left) 2024-09-14T03:59:43.145 INFO:tasks.rados.rados.0.smithi070.stdout:3382: done (0 left) 2024-09-14T03:59:43.145 INFO:tasks.rados.rados.0.smithi070.stdout:3383: snap_remove snap 314 2024-09-14T03:59:43.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:43 smithi070 ceph-mon[96704]: osdmap e746: 8 total, 8 up, 8 in 2024-09-14T03:59:43.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:43 smithi070 ceph-mon[96704]: pgmap v718: 121 pgs: 121 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 1.7 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-14T03:59:43.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:43 smithi070 ceph-mon[98640]: osdmap e746: 8 total, 8 up, 8 in 2024-09-14T03:59:43.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:43 smithi070 ceph-mon[98640]: pgmap v718: 121 pgs: 121 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 1.7 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-14T03:59:43.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:43 smithi094 ceph-mon[86597]: osdmap e746: 8 total, 8 up, 8 in 2024-09-14T03:59:43.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:43 smithi094 ceph-mon[86597]: pgmap v718: 121 pgs: 121 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 1.7 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-14T03:59:43.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:59:43 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:59:43] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.51.0" 2024-09-14T03:59:44.122 INFO:tasks.rados.rados.0.smithi070.stdout:3383: done (0 left) 2024-09-14T03:59:44.122 INFO:tasks.rados.rados.0.smithi070.stdout:3384: snap_create 2024-09-14T03:59:44.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:44 smithi094 ceph-mon[86597]: osdmap e747: 8 total, 8 up, 8 in 2024-09-14T03:59:44.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:44 smithi094 ceph-mon[86597]: osdmap e748: 8 total, 8 up, 8 in 2024-09-14T03:59:44.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:44 smithi070 ceph-mon[98640]: osdmap e747: 8 total, 8 up, 8 in 2024-09-14T03:59:44.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:44 smithi070 ceph-mon[98640]: osdmap e748: 8 total, 8 up, 8 in 2024-09-14T03:59:44.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:44 smithi070 ceph-mon[96704]: osdmap e747: 8 total, 8 up, 8 in 2024-09-14T03:59:44.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:44 smithi070 ceph-mon[96704]: osdmap e748: 8 total, 8 up, 8 in 2024-09-14T03:59:45.127 INFO:tasks.rados.rados.0.smithi070.stdout:3384: done (0 left) 2024-09-14T03:59:45.127 INFO:tasks.rados.rados.0.smithi070.stdout:3385: read oid 11 snap -1 2024-09-14T03:59:45.127 INFO:tasks.rados.rados.0.smithi070.stdout:3385: expect deleted 2024-09-14T03:59:45.127 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 28 current snap is 344 2024-09-14T03:59:45.127 INFO:tasks.rados.rados.0.smithi070.stdout:3386: seq_num 1044 ranges {4022272=688128} 2024-09-14T03:59:45.152 INFO:tasks.rados.rados.0.smithi070.stdout:3386: writing smithi07085154-28 from 4022272 to 4710400 tid 1 2024-09-14T03:59:45.152 INFO:tasks.rados.rados.0.smithi070.stdout:3387: delete oid 46 current snap is 344 2024-09-14T03:59:45.160 INFO:tasks.rados.rados.0.smithi070.stdout:3385: done (2 left) 2024-09-14T03:59:45.160 INFO:tasks.rados.rados.0.smithi070.stdout:3387: done (1 left) 2024-09-14T03:59:45.160 INFO:tasks.rados.rados.0.smithi070.stdout:3388: read oid 44 snap -1 2024-09-14T03:59:45.160 INFO:tasks.rados.rados.0.smithi070.stdout:3388: expect deleted 2024-09-14T03:59:45.160 INFO:tasks.rados.rados.0.smithi070.stdout:3389: rollback oid 38 current snap is 344 2024-09-14T03:59:45.160 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 38 to 304 2024-09-14T03:59:45.160 INFO:tasks.rados.rados.0.smithi070.stdout:3390: rmattr oid 20 current snap is 344 2024-09-14T03:59:45.161 INFO:tasks.rados.rados.0.smithi070.stdout:3391: read oid 27 snap -1 2024-09-14T03:59:45.161 INFO:tasks.rados.rados.0.smithi070.stdout:3391: expect (ObjNum 829 snap 283 seq_num 829) 2024-09-14T03:59:45.161 INFO:tasks.rados.rados.0.smithi070.stdout:3392: delete oid 47 current snap is 344 2024-09-14T03:59:45.161 INFO:tasks.rados.rados.0.smithi070.stdout:3386: finishing write tid 1 to smithi07085154-28 2024-09-14T03:59:45.162 INFO:tasks.rados.rados.0.smithi070.stdout:3386: finishing write tid 2 to smithi07085154-28 2024-09-14T03:59:45.163 INFO:tasks.rados.rados.0.smithi070.stdout:3386: finishing write tid 3 to smithi07085154-28 2024-09-14T03:59:45.163 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 28 v 3213 (ObjNum 1044 snap 344 seq_num 1044) dirty exists 2024-09-14T03:59:45.163 INFO:tasks.rados.rados.0.smithi070.stdout:3386: left oid 28 (ObjNum 1044 snap 344 seq_num 1044) 2024-09-14T03:59:45.164 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 20 v 3214 (ObjNum 524 snap 174 seq_num 524) dirty exists 2024-09-14T03:59:45.165 INFO:tasks.rados.rados.0.smithi070.stdout:3389: finishing rollback tid 0 to smithi07085154-38 2024-09-14T03:59:45.165 INFO:tasks.rados.rados.0.smithi070.stdout:3389: finishing rollback tid 1 to smithi07085154-38 2024-09-14T03:59:45.165 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 38 v 3971 (ObjNum 129 snap 0 seq_num 2930441312) dirty dne 2024-09-14T03:59:45.170 INFO:tasks.rados.rados.0.smithi070.stdout:3386: done (5 left) 2024-09-14T03:59:45.170 INFO:tasks.rados.rados.0.smithi070.stdout:3388: done (4 left) 2024-09-14T03:59:45.170 INFO:tasks.rados.rados.0.smithi070.stdout:3389: done (3 left) 2024-09-14T03:59:45.170 INFO:tasks.rados.rados.0.smithi070.stdout:3390: done (2 left) 2024-09-14T03:59:45.170 INFO:tasks.rados.rados.0.smithi070.stdout:3392: done (1 left) 2024-09-14T03:59:45.170 INFO:tasks.rados.rados.0.smithi070.stdout:3393: read oid 44 snap -1 2024-09-14T03:59:45.170 INFO:tasks.rados.rados.0.smithi070.stdout:3393: expect deleted 2024-09-14T03:59:45.170 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 43 current snap is 344 2024-09-14T03:59:45.170 INFO:tasks.rados.rados.0.smithi070.stdout:3394: seq_num 1047 ranges {1736704=745472} 2024-09-14T03:59:45.183 INFO:tasks.rados.rados.0.smithi070.stdout:3394: writing smithi07085154-43 from 1736704 to 2482176 tid 1 2024-09-14T03:59:45.183 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 50 current snap is 344 2024-09-14T03:59:45.183 INFO:tasks.rados.rados.0.smithi070.stdout:3395: seq_num 1048 ranges {0=761856} 2024-09-14T03:59:45.188 INFO:tasks.rados.rados.0.smithi070.stdout:3395: writing smithi07085154-50 from 0 to 761856 tid 1 2024-09-14T03:59:45.188 INFO:tasks.rados.rados.0.smithi070.stdout:3396: read oid 41 snap 312 2024-09-14T03:59:45.188 INFO:tasks.rados.rados.0.smithi070.stdout:3396: expect (ObjNum 724 snap 248 seq_num 724) 2024-09-14T03:59:45.188 INFO:tasks.rados.rados.0.smithi070.stdout:3397: read oid 34 snap 273 2024-09-14T03:59:45.188 INFO:tasks.rados.rados.0.smithi070.stdout:3397: expect (ObjNum 762 snap 264 seq_num 762) 2024-09-14T03:59:45.188 INFO:tasks.rados.rados.0.smithi070.stdout:3398: copy_from oid 47 from oid 23 current snap is 344 2024-09-14T03:59:45.188 INFO:tasks.rados.rados.0.smithi070.stdout:3391: expect (ObjNum 829 snap 283 seq_num 829) 2024-09-14T03:59:45.302 INFO:tasks.rados.rados.0.smithi070.stdout:3394: finishing write tid 1 to smithi07085154-43 2024-09-14T03:59:45.302 INFO:tasks.rados.rados.0.smithi070.stdout:3394: finishing write tid 2 to smithi07085154-43 2024-09-14T03:59:45.302 INFO:tasks.rados.rados.0.smithi070.stdout:3394: finishing write tid 3 to smithi07085154-43 2024-09-14T03:59:45.302 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 3208 (ObjNum 1047 snap 344 seq_num 1047) dirty exists 2024-09-14T03:59:45.302 INFO:tasks.rados.rados.0.smithi070.stdout:3394: left oid 43 (ObjNum 1047 snap 344 seq_num 1047) 2024-09-14T03:59:45.302 INFO:tasks.rados.rados.0.smithi070.stdout:3395: finishing write tid 1 to smithi07085154-50 2024-09-14T03:59:45.302 INFO:tasks.rados.rados.0.smithi070.stdout:3396: expect (ObjNum 724 snap 248 seq_num 724) 2024-09-14T03:59:45.312 INFO:tasks.rados.rados.0.smithi070.stdout:3395: finishing write tid 2 to smithi07085154-50 2024-09-14T03:59:45.312 INFO:tasks.rados.rados.0.smithi070.stdout:3395: finishing write tid 3 to smithi07085154-50 2024-09-14T03:59:45.312 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 3223 (ObjNum 1048 snap 344 seq_num 1048) dirty exists 2024-09-14T03:59:45.313 INFO:tasks.rados.rados.0.smithi070.stdout:3395: left oid 50 (ObjNum 1048 snap 344 seq_num 1048) 2024-09-14T03:59:45.313 INFO:tasks.rados.rados.0.smithi070.stdout:3397: expect (ObjNum 762 snap 264 seq_num 762) 2024-09-14T03:59:45.333 INFO:tasks.rados.rados.0.smithi070.stdout:3398: finishing copy_from racing read to smithi07085154-47 2024-09-14T03:59:45.333 INFO:tasks.rados.rados.0.smithi070.stdout:3398: finishing copy_from to smithi07085154-47 2024-09-14T03:59:45.334 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 47 v 3214 (ObjNum 1014 snap 334 seq_num 1014) dirty exists 2024-09-14T03:59:45.334 INFO:tasks.rados.rados.0.smithi070.stdout:3391: done (6 left) 2024-09-14T03:59:45.334 INFO:tasks.rados.rados.0.smithi070.stdout:3393: done (5 left) 2024-09-14T03:59:45.334 INFO:tasks.rados.rados.0.smithi070.stdout:3394: done (4 left) 2024-09-14T03:59:45.334 INFO:tasks.rados.rados.0.smithi070.stdout:3395: done (3 left) 2024-09-14T03:59:45.334 INFO:tasks.rados.rados.0.smithi070.stdout:3396: done (2 left) 2024-09-14T03:59:45.334 INFO:tasks.rados.rados.0.smithi070.stdout:3397: done (1 left) 2024-09-14T03:59:45.334 INFO:tasks.rados.rados.0.smithi070.stdout:3398: done (0 left) 2024-09-14T03:59:45.334 INFO:tasks.rados.rados.0.smithi070.stdout:3399: copy_from oid 13 from oid 3 current snap is 344 2024-09-14T03:59:45.334 INFO:tasks.rados.rados.0.smithi070.stdout:3400: copy_from oid 14 from oid 35 current snap is 344 2024-09-14T03:59:45.334 INFO:tasks.rados.rados.0.smithi070.stdout:3401: snap_remove snap 338 2024-09-14T03:59:45.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:45 smithi094 ceph-mon[86597]: pgmap v721: 121 pgs: 121 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 1.2 MiB/s wr, 6 op/s 2024-09-14T03:59:45.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:45 smithi094 ceph-mon[86597]: osdmap e749: 8 total, 8 up, 8 in 2024-09-14T03:59:45.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:45 smithi070 ceph-mon[96704]: pgmap v721: 121 pgs: 121 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 1.2 MiB/s wr, 6 op/s 2024-09-14T03:59:45.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:45 smithi070 ceph-mon[96704]: osdmap e749: 8 total, 8 up, 8 in 2024-09-14T03:59:45.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:45 smithi070 ceph-mon[98640]: pgmap v721: 121 pgs: 121 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 1.2 MiB/s wr, 6 op/s 2024-09-14T03:59:45.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:45 smithi070 ceph-mon[98640]: osdmap e749: 8 total, 8 up, 8 in 2024-09-14T03:59:46.154 INFO:tasks.rados.rados.0.smithi070.stdout:3401: done (2 left) 2024-09-14T03:59:46.154 INFO:tasks.rados.rados.0.smithi070.stdout:3402: delete oid 4 current snap is 344 2024-09-14T03:59:46.154 INFO:tasks.rados.rados.0.smithi070.stdout:3400: finishing copy_from to smithi07085154-14 2024-09-14T03:59:46.154 INFO:tasks.rados.rados.0.smithi070.stdout:3400: got expected ENOENT (src dne) 2024-09-14T03:59:46.154 INFO:tasks.rados.rados.0.smithi070.stdout:3400: finishing copy_from racing read to smithi07085154-14 2024-09-14T03:59:46.154 INFO:tasks.rados.rados.0.smithi070.stdout:3399: finishing copy_from racing read to smithi07085154-13 2024-09-14T03:59:46.155 INFO:tasks.rados.rados.0.smithi070.stdout:3399: finishing copy_from to smithi07085154-13 2024-09-14T03:59:46.155 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 3194 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-14T03:59:46.167 INFO:tasks.rados.rados.0.smithi070.stdout:3399: done (2 left) 2024-09-14T03:59:46.167 INFO:tasks.rados.rados.0.smithi070.stdout:3400: done (1 left) 2024-09-14T03:59:46.168 INFO:tasks.rados.rados.0.smithi070.stdout:3402: done (0 left) 2024-09-14T03:59:46.168 INFO:tasks.rados.rados.0.smithi070.stdout:3403: snap_remove snap 312 2024-09-14T03:59:47.161 INFO:tasks.rados.rados.0.smithi070.stdout:3403: done (0 left) 2024-09-14T03:59:47.161 INFO:tasks.rados.rados.0.smithi070.stdout:3404: read oid 35 snap 315 2024-09-14T03:59:47.161 INFO:tasks.rados.rados.0.smithi070.stdout:3404: expect (ObjNum 932 snap 313 seq_num 932) 2024-09-14T03:59:47.161 INFO:tasks.rados.rados.0.smithi070.stdout:3405: snap_remove snap 107 2024-09-14T03:59:47.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:47 smithi094 ceph-mon[86597]: pgmap v723: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:59:47.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:47 smithi094 ceph-mon[86597]: osdmap e750: 8 total, 8 up, 8 in 2024-09-14T03:59:47.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:47 smithi070 ceph-mon[96704]: pgmap v723: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:59:47.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:47 smithi070 ceph-mon[96704]: osdmap e750: 8 total, 8 up, 8 in 2024-09-14T03:59:47.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:47 smithi070 ceph-mon[98640]: pgmap v723: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:59:47.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:47 smithi070 ceph-mon[98640]: osdmap e750: 8 total, 8 up, 8 in 2024-09-14T03:59:48.170 INFO:tasks.rados.rados.0.smithi070.stdout:3405: done (1 left) 2024-09-14T03:59:48.170 INFO:tasks.rados.rados.0.smithi070.stdout:3406: copy_from oid 40 from oid 7 current snap is 344 2024-09-14T03:59:48.170 INFO:tasks.rados.rados.0.smithi070.stdout:3404: expect (ObjNum 932 snap 313 seq_num 932) 2024-09-14T03:59:48.249 INFO:tasks.rados.rados.0.smithi070.stdout:3406: finishing copy_from to smithi07085154-40 2024-09-14T03:59:48.249 INFO:tasks.rados.rados.0.smithi070.stdout:3406: got expected ENOENT (src dne) 2024-09-14T03:59:48.249 INFO:tasks.rados.rados.0.smithi070.stdout:3406: finishing copy_from racing read to smithi07085154-40 2024-09-14T03:59:48.249 INFO:tasks.rados.rados.0.smithi070.stdout:3404: done (1 left) 2024-09-14T03:59:48.249 INFO:tasks.rados.rados.0.smithi070.stdout:3406: done (0 left) 2024-09-14T03:59:48.249 INFO:tasks.rados.rados.0.smithi070.stdout:3407: read oid 11 snap -1 2024-09-14T03:59:48.249 INFO:tasks.rados.rados.0.smithi070.stdout:3407: expect deleted 2024-09-14T03:59:48.249 INFO:tasks.rados.rados.0.smithi070.stdout:3408: rmattr oid 48 current snap is 344 2024-09-14T03:59:48.249 INFO:tasks.rados.rados.0.smithi070.stdout:3408: done (1 left) 2024-09-14T03:59:48.249 INFO:tasks.rados.rados.0.smithi070.stdout:3409: delete oid 36 current snap is 344 2024-09-14T03:59:48.252 INFO:tasks.rados.rados.0.smithi070.stdout:3407: done (1 left) 2024-09-14T03:59:48.252 INFO:tasks.rados.rados.0.smithi070.stdout:3409: done (0 left) 2024-09-14T03:59:48.252 INFO:tasks.rados.rados.0.smithi070.stdout:3410: delete oid 40 current snap is 344 2024-09-14T03:59:48.255 INFO:tasks.rados.rados.0.smithi070.stdout:3410: done (0 left) 2024-09-14T03:59:48.255 INFO:tasks.rados.rados.0.smithi070.stdout:3411: delete oid 37 current snap is 344 2024-09-14T03:59:48.257 INFO:tasks.rados.rados.0.smithi070.stdout:3411: done (0 left) 2024-09-14T03:59:48.258 INFO:tasks.rados.rados.0.smithi070.stdout:3412: rmattr oid 49 current snap is 344 2024-09-14T03:59:48.258 INFO:tasks.rados.rados.0.smithi070.stdout:3412: done (0 left) 2024-09-14T03:59:48.258 INFO:tasks.rados.rados.0.smithi070.stdout:3413: snap_remove snap 326 2024-09-14T03:59:48.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:48 smithi094 ceph-mon[86597]: osdmap e751: 8 total, 8 up, 8 in 2024-09-14T03:59:48.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:48 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:59:48.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:48 smithi070 ceph-mon[98640]: osdmap e751: 8 total, 8 up, 8 in 2024-09-14T03:59:48.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:48 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:59:48.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:48 smithi070 ceph-mon[96704]: osdmap e751: 8 total, 8 up, 8 in 2024-09-14T03:59:48.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:48 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T03:59:49.125 INFO:tasks.rados.rados.0.smithi070.stdout:3413: done (0 left) 2024-09-14T03:59:49.125 INFO:tasks.rados.rados.0.smithi070.stdout:3414: rmattr oid 50 current snap is 344 2024-09-14T03:59:49.125 INFO:tasks.rados.rados.0.smithi070.stdout:3414: done (0 left) 2024-09-14T03:59:49.126 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 44 current snap is 344 2024-09-14T03:59:49.126 INFO:tasks.rados.rados.0.smithi070.stdout:3415: seq_num 1053 ranges {0=442368,442368=401408} 2024-09-14T03:59:49.128 INFO:tasks.rados.rados.0.smithi070.stdout:3415: writing smithi07085154-44 from 0 to 442368 tid 1 2024-09-14T03:59:49.131 INFO:tasks.rados.rados.0.smithi070.stdout:3415: writing smithi07085154-44 from 442368 to 843776 tid 2 2024-09-14T03:59:49.132 INFO:tasks.rados.rados.0.smithi070.stdout:3416: rollback oid 4 current snap is 344 2024-09-14T03:59:49.132 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 4 to 267 2024-09-14T03:59:49.132 INFO:tasks.rados.rados.0.smithi070.stdout:3417: read oid 2 snap -1 2024-09-14T03:59:49.132 INFO:tasks.rados.rados.0.smithi070.stdout:3417: expect (ObjNum 991 snap 327 seq_num 991) 2024-09-14T03:59:49.132 INFO:tasks.rados.rados.0.smithi070.stdout:3418: delete oid 40 current snap is 344 2024-09-14T03:59:49.139 INFO:tasks.rados.rados.0.smithi070.stdout:3415: finishing write tid 1 to smithi07085154-44 2024-09-14T03:59:49.144 INFO:tasks.rados.rados.0.smithi070.stdout:3416: finishing rollback tid 1 to smithi07085154-4 2024-09-14T03:59:49.144 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 3215 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-14T03:59:49.145 INFO:tasks.rados.rados.0.smithi070.stdout:3415: finishing write tid 2 to smithi07085154-44 2024-09-14T03:59:49.146 INFO:tasks.rados.rados.0.smithi070.stdout:3415: finishing write tid 3 to smithi07085154-44 2024-09-14T03:59:49.148 INFO:tasks.rados.rados.0.smithi070.stdout:3416: done (3 left) 2024-09-14T03:59:49.148 INFO:tasks.rados.rados.0.smithi070.stdout:3418: done (2 left) 2024-09-14T03:59:49.148 INFO:tasks.rados.rados.0.smithi070.stdout:3419: rollback oid 33 current snap is 344 2024-09-14T03:59:49.148 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 33 to 213 2024-09-14T03:59:49.148 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 25 current snap is 344 2024-09-14T03:59:49.148 INFO:tasks.rados.rados.0.smithi070.stdout:3420: seq_num 1055 ranges {270336=475136,745472=401408} 2024-09-14T03:59:49.152 INFO:tasks.rados.rados.0.smithi070.stdout:3420: writing smithi07085154-25 from 270336 to 745472 tid 1 2024-09-14T03:59:49.156 INFO:tasks.rados.rados.0.smithi070.stdout:3420: writing smithi07085154-25 from 745472 to 1146880 tid 2 2024-09-14T03:59:49.159 INFO:tasks.rados.rados.0.smithi070.stdout:3421: copy_from oid 13 from oid 23 current snap is 344 2024-09-14T03:59:49.159 INFO:tasks.rados.rados.0.smithi070.stdout:3415: finishing write tid 4 to smithi07085154-44 2024-09-14T03:59:49.159 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 3200 (ObjNum 1053 snap 344 seq_num 1053) dirty exists 2024-09-14T03:59:49.159 INFO:tasks.rados.rados.0.smithi070.stdout:3415: left oid 44 (ObjNum 1053 snap 344 seq_num 1053) 2024-09-14T03:59:49.159 INFO:tasks.rados.rados.0.smithi070.stdout:3415: done (4 left) 2024-09-14T03:59:49.159 INFO:tasks.rados.rados.0.smithi070.stdout:3422: rmattr oid 28 current snap is 344 2024-09-14T03:59:49.159 INFO:tasks.rados.rados.0.smithi070.stdout:3419: finishing rollback tid 0 to smithi07085154-33 2024-09-14T03:59:49.160 INFO:tasks.rados.rados.0.smithi070.stdout:3419: finishing rollback tid 1 to smithi07085154-33 2024-09-14T03:59:49.160 INFO:tasks.rados.rados.0.smithi070.stdout:3419: finishing rollback tid 2 to smithi07085154-33 2024-09-14T03:59:49.160 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 33 v 3203 (ObjNum 590 snap 195 seq_num 590) dirty exists 2024-09-14T03:59:49.160 INFO:tasks.rados.rados.0.smithi070.stdout:3419: done (4 left) 2024-09-14T03:59:49.160 INFO:tasks.rados.rados.0.smithi070.stdout:3423: rmattr oid 29 current snap is 344 2024-09-14T03:59:49.160 INFO:tasks.rados.rados.0.smithi070.stdout:3424: read oid 39 snap -1 2024-09-14T03:59:49.160 INFO:tasks.rados.rados.0.smithi070.stdout:3424: expect (ObjNum 1020 snap 336 seq_num 1020) 2024-09-14T03:59:49.160 INFO:tasks.rados.rados.0.smithi070.stdout:3425: rollback oid 50 current snap is 344 2024-09-14T03:59:49.160 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 50 to 324 2024-09-14T03:59:49.160 INFO:tasks.rados.rados.0.smithi070.stdout:3426: read oid 38 snap -1 2024-09-14T03:59:49.160 INFO:tasks.rados.rados.0.smithi070.stdout:3426: expect deleted 2024-09-14T03:59:49.160 INFO:tasks.rados.rados.0.smithi070.stdout:3427: delete oid 8 current snap is 344 2024-09-14T03:59:49.161 INFO:tasks.rados.rados.0.smithi070.stdout:3420: finishing write tid 1 to smithi07085154-25 2024-09-14T03:59:49.164 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 28 v 3216 (ObjNum 1044 snap 344 seq_num 1044) dirty exists 2024-09-14T03:59:49.165 INFO:tasks.rados.rados.0.smithi070.stdout:3420: finishing write tid 2 to smithi07085154-25 2024-09-14T03:59:49.165 INFO:tasks.rados.rados.0.smithi070.stdout:3420: finishing write tid 3 to smithi07085154-25 2024-09-14T03:59:49.166 INFO:tasks.rados.rados.0.smithi070.stdout:3422: done (8 left) 2024-09-14T03:59:49.166 INFO:tasks.rados.rados.0.smithi070.stdout:3426: done (7 left) 2024-09-14T03:59:49.166 INFO:tasks.rados.rados.0.smithi070.stdout:3427: done (6 left) 2024-09-14T03:59:49.166 INFO:tasks.rados.rados.0.smithi070.stdout:3428: snap_remove snap 331 2024-09-14T03:59:49.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:49 smithi094 ceph-mon[86597]: pgmap v726: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.1 MiB/s wr, 11 op/s 2024-09-14T03:59:49.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:49 smithi094 ceph-mon[86597]: osdmap e752: 8 total, 8 up, 8 in 2024-09-14T03:59:49.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:49 smithi094 ceph-mon[86597]: osdmap e753: 8 total, 8 up, 8 in 2024-09-14T03:59:49.527 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 03:59:49 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4[92016]: 2024-09-14T03:59:49.148+0000 7f6d3c1e3640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap 198 after trimming was started 2024-09-14T03:59:49.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:49 smithi070 ceph-mon[96704]: pgmap v726: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.1 MiB/s wr, 11 op/s 2024-09-14T03:59:49.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:49 smithi070 ceph-mon[96704]: osdmap e752: 8 total, 8 up, 8 in 2024-09-14T03:59:49.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:49 smithi070 ceph-mon[96704]: osdmap e753: 8 total, 8 up, 8 in 2024-09-14T03:59:49.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:49 smithi070 ceph-mon[98640]: pgmap v726: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.1 MiB/s wr, 11 op/s 2024-09-14T03:59:49.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:49 smithi070 ceph-mon[98640]: osdmap e752: 8 total, 8 up, 8 in 2024-09-14T03:59:49.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:49 smithi070 ceph-mon[98640]: osdmap e753: 8 total, 8 up, 8 in 2024-09-14T03:59:50.176 INFO:tasks.rados.rados.0.smithi070.stdout:3428: done (6 left) 2024-09-14T03:59:50.176 INFO:tasks.rados.rados.0.smithi070.stdout:3429: read oid 4 snap -1 2024-09-14T03:59:50.176 INFO:tasks.rados.rados.0.smithi070.stdout:3429: expect deleted 2024-09-14T03:59:50.177 INFO:tasks.rados.rados.0.smithi070.stdout:3430: copy_from oid 8 from oid 33 current snap is 344 2024-09-14T03:59:50.177 INFO:tasks.rados.rados.0.smithi070.stdout:3431: rollback oid 36 current snap is 344 2024-09-14T03:59:50.177 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 36 to 296 2024-09-14T03:59:50.177 INFO:tasks.rados.rados.0.smithi070.stdout:3432: snap_create 2024-09-14T03:59:50.179 INFO:tasks.rados.rados.0.smithi070.stdout:3420: finishing write tid 4 to smithi07085154-25 2024-09-14T03:59:50.179 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 3198 (ObjNum 1055 snap 344 seq_num 1055) dirty exists 2024-09-14T03:59:50.179 INFO:tasks.rados.rados.0.smithi070.stdout:3420: left oid 25 (ObjNum 1055 snap 344 seq_num 1055) 2024-09-14T03:59:50.179 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 3224 (ObjNum 986 snap 325 seq_num 986) dirty exists 2024-09-14T03:59:50.179 INFO:tasks.rados.rados.0.smithi070.stdout:3425: finishing rollback tid 0 to smithi07085154-50 2024-09-14T03:59:50.179 INFO:tasks.rados.rados.0.smithi070.stdout:3425: finishing rollback tid 1 to smithi07085154-50 2024-09-14T03:59:50.179 INFO:tasks.rados.rados.0.smithi070.stdout:3425: finishing rollback tid 2 to smithi07085154-50 2024-09-14T03:59:50.179 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 3227 (ObjNum 940 snap 316 seq_num 940) dirty exists 2024-09-14T03:59:50.180 INFO:tasks.rados.rados.0.smithi070.stdout:3424: expect (ObjNum 1020 snap 336 seq_num 1020) 2024-09-14T03:59:50.264 INFO:tasks.rados.rados.0.smithi070.stdout:3421: finishing copy_from racing read to smithi07085154-13 2024-09-14T03:59:50.264 INFO:tasks.rados.rados.0.smithi070.stdout:3421: finishing copy_from to smithi07085154-13 2024-09-14T03:59:50.264 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 3214 (ObjNum 1014 snap 334 seq_num 1014) dirty exists 2024-09-14T03:59:50.265 INFO:tasks.rados.rados.0.smithi070.stdout:3417: expect (ObjNum 991 snap 327 seq_num 991) 2024-09-14T03:59:50.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:50 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:59:50.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:50 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:59:50.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:50 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:59:50.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:50 smithi094 ceph-mon[86597]: pgmap v729: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3.6 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-09-14T03:59:50.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 03:59:50 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T03:59:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T03:59:50.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:50 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:59:50.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:50 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:59:50.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:50 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:59:50.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:50 smithi070 ceph-mon[96704]: pgmap v729: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3.6 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-09-14T03:59:50.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:50 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T03:59:50.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:50 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:59:50.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:50 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:59:50.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:50 smithi070 ceph-mon[98640]: pgmap v729: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3.6 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-09-14T03:59:50.912 INFO:tasks.rados.rados.0.smithi070.stdout:3431: finishing rollback tid 1 to smithi07085154-36 2024-09-14T03:59:50.913 INFO:tasks.rados.rados.0.smithi070.stdout:3431: finishing rollback tid 2 to smithi07085154-36 2024-09-14T03:59:50.913 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 36 v 3202 (ObjNum 784 snap 274 seq_num 784) dirty exists 2024-09-14T03:59:50.913 INFO:tasks.rados.rados.0.smithi070.stdout:3430: finishing copy_from racing read to smithi07085154-8 2024-09-14T03:59:50.913 INFO:tasks.rados.rados.0.smithi070.stdout:3430: finishing copy_from to smithi07085154-8 2024-09-14T03:59:50.913 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 8 v 3203 (ObjNum 590 snap 195 seq_num 590) dirty exists 2024-09-14T03:59:51.177 INFO:tasks.rados.rados.0.smithi070.stdout:3417: done (9 left) 2024-09-14T03:59:51.177 INFO:tasks.rados.rados.0.smithi070.stdout:3420: done (8 left) 2024-09-14T03:59:51.177 INFO:tasks.rados.rados.0.smithi070.stdout:3421: done (7 left) 2024-09-14T03:59:51.177 INFO:tasks.rados.rados.0.smithi070.stdout:3423: done (6 left) 2024-09-14T03:59:51.177 INFO:tasks.rados.rados.0.smithi070.stdout:3424: done (5 left) 2024-09-14T03:59:51.178 INFO:tasks.rados.rados.0.smithi070.stdout:3425: done (4 left) 2024-09-14T03:59:51.178 INFO:tasks.rados.rados.0.smithi070.stdout:3429: done (3 left) 2024-09-14T03:59:51.178 INFO:tasks.rados.rados.0.smithi070.stdout:3430: done (2 left) 2024-09-14T03:59:51.178 INFO:tasks.rados.rados.0.smithi070.stdout:3431: done (1 left) 2024-09-14T03:59:51.178 INFO:tasks.rados.rados.0.smithi070.stdout:3432: done (0 left) 2024-09-14T03:59:51.178 INFO:tasks.rados.rados.0.smithi070.stdout:3433: rollback oid 13 current snap is 345 2024-09-14T03:59:51.178 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 13 to 301 2024-09-14T03:59:51.178 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 42 current snap is 345 2024-09-14T03:59:51.178 INFO:tasks.rados.rados.0.smithi070.stdout:3434: seq_num 1057 ranges {458752=507904,966656=401408} 2024-09-14T03:59:51.183 INFO:tasks.rados.rados.0.smithi070.stdout:3434: writing smithi07085154-42 from 458752 to 966656 tid 1 2024-09-14T03:59:51.186 INFO:tasks.rados.rados.0.smithi070.stdout:3434: writing smithi07085154-42 from 966656 to 1368064 tid 2 2024-09-14T03:59:51.186 INFO:tasks.rados.rados.0.smithi070.stdout:3435: copy_from oid 47 from oid 38 current snap is 345 2024-09-14T03:59:51.186 INFO:tasks.rados.rados.0.smithi070.stdout:3433: finishing rollback tid 0 to smithi07085154-13 2024-09-14T03:59:51.186 INFO:tasks.rados.rados.0.smithi070.stdout:3433: finishing rollback tid 1 to smithi07085154-13 2024-09-14T03:59:51.186 INFO:tasks.rados.rados.0.smithi070.stdout:3433: finishing rollback tid 2 to smithi07085154-13 2024-09-14T03:59:51.186 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 3217 (ObjNum 898 snap 300 seq_num 898) dirty exists 2024-09-14T03:59:51.186 INFO:tasks.rados.rados.0.smithi070.stdout:3433: done (2 left) 2024-09-14T03:59:51.186 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 6 current snap is 345 2024-09-14T03:59:51.187 INFO:tasks.rados.rados.0.smithi070.stdout:3436: seq_num 1058 ranges {1253376=524288,1777664=401408} 2024-09-14T03:59:51.195 INFO:tasks.rados.rados.0.smithi070.stdout:3436: writing smithi07085154-6 from 1253376 to 1777664 tid 1 2024-09-14T03:59:51.198 INFO:tasks.rados.rados.0.smithi070.stdout:3436: writing smithi07085154-6 from 1777664 to 2179072 tid 2 2024-09-14T03:59:51.198 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 12 current snap is 345 2024-09-14T03:59:51.198 INFO:tasks.rados.rados.0.smithi070.stdout:3435: finishing copy_from to smithi07085154-47 2024-09-14T03:59:51.198 INFO:tasks.rados.rados.0.smithi070.stdout:3435: got expected ENOENT (src dne) 2024-09-14T03:59:51.198 INFO:tasks.rados.rados.0.smithi070.stdout:3435: finishing copy_from racing read to smithi07085154-47 2024-09-14T03:59:51.198 INFO:tasks.rados.rados.0.smithi070.stdout:3434: finishing write tid 1 to smithi07085154-42 2024-09-14T03:59:51.199 INFO:tasks.rados.rados.0.smithi070.stdout:3434: finishing write tid 2 to smithi07085154-42 2024-09-14T03:59:51.199 INFO:tasks.rados.rados.0.smithi070.stdout:3434: finishing write tid 3 to smithi07085154-42 2024-09-14T03:59:51.199 INFO:tasks.rados.rados.0.smithi070.stdout:3434: finishing write tid 4 to smithi07085154-42 2024-09-14T03:59:51.199 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 4117 (ObjNum 1057 snap 345 seq_num 1057) dirty exists 2024-09-14T03:59:51.199 INFO:tasks.rados.rados.0.smithi070.stdout:3434: left oid 42 (ObjNum 1057 snap 345 seq_num 1057) 2024-09-14T03:59:51.199 INFO:tasks.rados.rados.0.smithi070.stdout:3437: seq_num 1059 ranges {352256=540672,892928=401408} 2024-09-14T03:59:51.203 INFO:tasks.rados.rados.0.smithi070.stdout:3437: writing smithi07085154-12 from 352256 to 892928 tid 1 2024-09-14T03:59:51.205 INFO:tasks.rados.rados.0.smithi070.stdout:3437: writing smithi07085154-12 from 892928 to 1294336 tid 2 2024-09-14T03:59:51.205 INFO:tasks.rados.rados.0.smithi070.stdout:3434: done (3 left) 2024-09-14T03:59:51.205 INFO:tasks.rados.rados.0.smithi070.stdout:3435: done (2 left) 2024-09-14T03:59:51.205 INFO:tasks.rados.rados.0.smithi070.stdout:3438: snap_create 2024-09-14T03:59:51.205 INFO:tasks.rados.rados.0.smithi070.stdout:3436: finishing write tid 1 to smithi07085154-6 2024-09-14T03:59:51.206 INFO:tasks.rados.rados.0.smithi070.stdout:3436: finishing write tid 2 to smithi07085154-6 2024-09-14T03:59:51.206 INFO:tasks.rados.rados.0.smithi070.stdout:3436: finishing write tid 3 to smithi07085154-6 2024-09-14T03:59:51.206 INFO:tasks.rados.rados.0.smithi070.stdout:3436: finishing write tid 4 to smithi07085154-6 2024-09-14T03:59:51.206 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 3201 (ObjNum 1058 snap 345 seq_num 1058) dirty exists 2024-09-14T03:59:51.206 INFO:tasks.rados.rados.0.smithi070.stdout:3436: left oid 6 (ObjNum 1058 snap 345 seq_num 1058) 2024-09-14T03:59:51.208 INFO:tasks.rados.rados.0.smithi070.stdout:3437: finishing write tid 1 to smithi07085154-12 2024-09-14T03:59:51.208 INFO:tasks.rados.rados.0.smithi070.stdout:3437: finishing write tid 2 to smithi07085154-12 2024-09-14T03:59:51.208 INFO:tasks.rados.rados.0.smithi070.stdout:3437: finishing write tid 3 to smithi07085154-12 2024-09-14T03:59:51.210 INFO:tasks.rados.rados.0.smithi070.stdout:3437: finishing write tid 4 to smithi07085154-12 2024-09-14T03:59:51.210 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 2272 (ObjNum 1059 snap 345 seq_num 1059) dirty exists 2024-09-14T03:59:51.210 INFO:tasks.rados.rados.0.smithi070.stdout:3437: left oid 12 (ObjNum 1059 snap 345 seq_num 1059) 2024-09-14T03:59:51.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:51 smithi094 ceph-mon[86597]: osdmap e754: 8 total, 8 up, 8 in 2024-09-14T03:59:51.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:51 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:59:51.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:51 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:59:51.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:51 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:59:51.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:51 smithi070 ceph-mon[96704]: osdmap e754: 8 total, 8 up, 8 in 2024-09-14T03:59:51.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:51 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:59:51.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:51 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:59:51.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:51 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:59:51.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:51 smithi070 ceph-mon[98640]: osdmap e754: 8 total, 8 up, 8 in 2024-09-14T03:59:51.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:51 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T03:59:51.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:51 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T03:59:51.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:51 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T03:59:52.184 INFO:tasks.rados.rados.0.smithi070.stdout:3436: done (2 left) 2024-09-14T03:59:52.184 INFO:tasks.rados.rados.0.smithi070.stdout:3437: done (1 left) 2024-09-14T03:59:52.184 INFO:tasks.rados.rados.0.smithi070.stdout:3438: done (0 left) 2024-09-14T03:59:52.184 INFO:tasks.rados.rados.0.smithi070.stdout:3439: read oid 35 snap 337 2024-09-14T03:59:52.184 INFO:tasks.rados.rados.0.smithi070.stdout:3439: expect deleted 2024-09-14T03:59:52.184 INFO:tasks.rados.rados.0.smithi070.stdout:3440: read oid 5 snap 232 2024-09-14T03:59:52.184 INFO:tasks.rados.rados.0.smithi070.stdout:3440: expect (ObjNum 686 snap 231 seq_num 686) 2024-09-14T03:59:52.184 INFO:tasks.rados.rados.0.smithi070.stdout:3441: rollback oid 29 current snap is 346 2024-09-14T03:59:52.184 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 29 to 296 2024-09-14T03:59:52.184 INFO:tasks.rados.rados.0.smithi070.stdout:3442: delete oid 14 current snap is 346 2024-09-14T03:59:52.188 INFO:tasks.rados.rados.0.smithi070.stdout:3439: done (3 left) 2024-09-14T03:59:52.188 INFO:tasks.rados.rados.0.smithi070.stdout:3442: done (2 left) 2024-09-14T03:59:52.188 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 42 current snap is 346 2024-09-14T03:59:52.188 INFO:tasks.rados.rados.0.smithi070.stdout:3443: seq_num 1061 ranges {1368064=573440,1941504=401408} 2024-09-14T03:59:52.199 INFO:tasks.rados.rados.0.smithi070.stdout:3443: writing smithi07085154-42 from 1368064 to 1941504 tid 1 2024-09-14T03:59:52.201 INFO:tasks.rados.rados.0.smithi070.stdout:3443: writing smithi07085154-42 from 1941504 to 2342912 tid 2 2024-09-14T03:59:52.201 INFO:tasks.rados.rados.0.smithi070.stdout:3444: read oid 6 snap -1 2024-09-14T03:59:52.202 INFO:tasks.rados.rados.0.smithi070.stdout:3444: expect (ObjNum 1058 snap 345 seq_num 1058) 2024-09-14T03:59:52.202 INFO:tasks.rados.rados.0.smithi070.stdout:3445: read oid 15 snap -1 2024-09-14T03:59:52.202 INFO:tasks.rados.rados.0.smithi070.stdout:3445: expect (ObjNum 805 snap 281 seq_num 805) 2024-09-14T03:59:52.202 INFO:tasks.rados.rados.0.smithi070.stdout:3446: snap_remove snap 344 2024-09-14T03:59:52.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:52 smithi094 ceph-mon[86597]: osdmap e755: 8 total, 8 up, 8 in 2024-09-14T03:59:52.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:52 smithi094 ceph-mon[86597]: pgmap v732: 121 pgs: 121 active+clean; 976 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.2 MiB/s rd, 1018 KiB/s wr, 11 op/s 2024-09-14T03:59:52.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:52 smithi070 ceph-mon[98640]: osdmap e755: 8 total, 8 up, 8 in 2024-09-14T03:59:52.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:52 smithi070 ceph-mon[98640]: pgmap v732: 121 pgs: 121 active+clean; 976 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.2 MiB/s rd, 1018 KiB/s wr, 11 op/s 2024-09-14T03:59:52.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:52 smithi070 ceph-mon[96704]: osdmap e755: 8 total, 8 up, 8 in 2024-09-14T03:59:52.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:52 smithi070 ceph-mon[96704]: pgmap v732: 121 pgs: 121 active+clean; 976 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.2 MiB/s rd, 1018 KiB/s wr, 11 op/s 2024-09-14T03:59:53.189 INFO:tasks.rados.rados.0.smithi070.stdout:3446: done (5 left) 2024-09-14T03:59:53.190 INFO:tasks.rados.rados.0.smithi070.stdout:3447: read oid 40 snap 236 2024-09-14T03:59:53.190 INFO:tasks.rados.rados.0.smithi070.stdout:3447: expect (ObjNum 678 snap 229 seq_num 678) 2024-09-14T03:59:53.190 INFO:tasks.rados.rados.0.smithi070.stdout:3448: rollback oid 34 current snap is 346 2024-09-14T03:59:53.190 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 34 to 334 2024-09-14T03:59:53.190 INFO:tasks.rados.rados.0.smithi070.stdout:3440: expect (ObjNum 686 snap 231 seq_num 686) 2024-09-14T03:59:53.243 INFO:tasks.rados.rados.0.smithi070.stdout:3441: finishing rollback tid 0 to smithi07085154-29 2024-09-14T03:59:53.243 INFO:tasks.rados.rados.0.smithi070.stdout:3441: finishing rollback tid 1 to smithi07085154-29 2024-09-14T03:59:53.243 INFO:tasks.rados.rados.0.smithi070.stdout:3441: finishing rollback tid 2 to smithi07085154-29 2024-09-14T03:59:53.243 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 3230 (ObjNum 872 snap 295 seq_num 872) dirty exists 2024-09-14T03:59:53.243 INFO:tasks.rados.rados.0.smithi070.stdout:3443: finishing write tid 1 to smithi07085154-42 2024-09-14T03:59:53.243 INFO:tasks.rados.rados.0.smithi070.stdout:3443: finishing write tid 2 to smithi07085154-42 2024-09-14T03:59:53.244 INFO:tasks.rados.rados.0.smithi070.stdout:3445: expect (ObjNum 805 snap 281 seq_num 805) 2024-09-14T03:59:53.280 INFO:tasks.rados.rados.0.smithi070.stdout:3443: finishing write tid 3 to smithi07085154-42 2024-09-14T03:59:53.281 INFO:tasks.rados.rados.0.smithi070.stdout:3443: finishing write tid 4 to smithi07085154-42 2024-09-14T03:59:53.281 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 4121 (ObjNum 1061 snap 346 seq_num 1061) dirty exists 2024-09-14T03:59:53.281 INFO:tasks.rados.rados.0.smithi070.stdout:3443: left oid 42 (ObjNum 1061 snap 346 seq_num 1061) 2024-09-14T03:59:53.281 INFO:tasks.rados.rados.0.smithi070.stdout:3444: expect (ObjNum 1058 snap 345 seq_num 1058) 2024-09-14T03:59:53.354 INFO:tasks.rados.rados.0.smithi070.stdout:3448: finishing rollback tid 0 to smithi07085154-34 2024-09-14T03:59:53.354 INFO:tasks.rados.rados.0.smithi070.stdout:3448: finishing rollback tid 1 to smithi07085154-34 2024-09-14T03:59:53.355 INFO:tasks.rados.rados.0.smithi070.stdout:3448: finishing rollback tid 2 to smithi07085154-34 2024-09-14T03:59:53.355 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 3233 (ObjNum 938 snap 316 seq_num 938) dirty exists 2024-09-14T03:59:53.355 INFO:tasks.rados.rados.0.smithi070.stdout:3447: expect (ObjNum 678 snap 229 seq_num 678) 2024-09-14T03:59:53.399 INFO:tasks.rados.rados.0.smithi070.stdout:3440: done (6 left) 2024-09-14T03:59:53.399 INFO:tasks.rados.rados.0.smithi070.stdout:3441: done (5 left) 2024-09-14T03:59:53.399 INFO:tasks.rados.rados.0.smithi070.stdout:3443: done (4 left) 2024-09-14T03:59:53.399 INFO:tasks.rados.rados.0.smithi070.stdout:3444: done (3 left) 2024-09-14T03:59:53.399 INFO:tasks.rados.rados.0.smithi070.stdout:3445: done (2 left) 2024-09-14T03:59:53.399 INFO:tasks.rados.rados.0.smithi070.stdout:3447: done (1 left) 2024-09-14T03:59:53.399 INFO:tasks.rados.rados.0.smithi070.stdout:3448: done (0 left) 2024-09-14T03:59:53.399 INFO:tasks.rados.rados.0.smithi070.stdout:3449: delete oid 11 current snap is 346 2024-09-14T03:59:53.401 INFO:tasks.rados.rados.0.smithi070.stdout:3449: done (0 left) 2024-09-14T03:59:53.401 INFO:tasks.rados.rados.0.smithi070.stdout:3450: delete oid 10 current snap is 346 2024-09-14T03:59:53.408 INFO:tasks.rados.rados.0.smithi070.stdout:3450: done (0 left) 2024-09-14T03:59:53.408 INFO:tasks.rados.rados.0.smithi070.stdout:3451: rollback oid 11 current snap is 346 2024-09-14T03:59:53.408 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 11 to 304 2024-09-14T03:59:53.409 INFO:tasks.rados.rados.0.smithi070.stdout:3452: rollback oid 20 current snap is 346 2024-09-14T03:59:53.409 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 20 to 320 2024-09-14T03:59:53.409 INFO:tasks.rados.rados.0.smithi070.stdout:3453: rollback oid 50 current snap is 346 2024-09-14T03:59:53.409 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 50 to 343 2024-09-14T03:59:53.409 INFO:tasks.rados.rados.0.smithi070.stdout:3454: delete oid 14 current snap is 346 2024-09-14T03:59:53.411 INFO:tasks.rados.rados.0.smithi070.stdout:3451: finishing rollback tid 1 to smithi07085154-11 2024-09-14T03:59:53.411 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 11 v 3171 (ObjNum 65 snap 0 seq_num 1314806895) dirty dne 2024-09-14T03:59:53.411 INFO:tasks.rados.rados.0.smithi070.stdout:3451: done (3 left) 2024-09-14T03:59:53.411 INFO:tasks.rados.rados.0.smithi070.stdout:3454: done (2 left) 2024-09-14T03:59:53.411 INFO:tasks.rados.rados.0.smithi070.stdout:3455: delete oid 32 current snap is 346 2024-09-14T03:59:53.413 INFO:tasks.rados.rados.0.smithi070.stdout:3453: finishing rollback tid 0 to smithi07085154-50 2024-09-14T03:59:53.413 INFO:tasks.rados.rados.0.smithi070.stdout:3453: finishing rollback tid 1 to smithi07085154-50 2024-09-14T03:59:53.413 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 3235 (ObjNum 65 snap 0 seq_num 1365078355) dirty dne 2024-09-14T03:59:53.414 INFO:tasks.rados.rados.0.smithi070.stdout:3453: done (2 left) 2024-09-14T03:59:53.414 INFO:tasks.rados.rados.0.smithi070.stdout:3455: done (1 left) 2024-09-14T03:59:53.414 INFO:tasks.rados.rados.0.smithi070.stdout:3456: read oid 50 snap 340 2024-09-14T03:59:53.414 INFO:tasks.rados.rados.0.smithi070.stdout:3456: expect deleted 2024-09-14T03:59:53.414 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 6 current snap is 346 2024-09-14T03:59:53.414 INFO:tasks.rados.rados.0.smithi070.stdout:3457: seq_num 1066 ranges {2179072=663552,2842624=401408} 2024-09-14T03:59:53.429 INFO:tasks.rados.rados.0.smithi070.stdout:3457: writing smithi07085154-6 from 2179072 to 2842624 tid 1 2024-09-14T03:59:53.432 INFO:tasks.rados.rados.0.smithi070.stdout:3457: writing smithi07085154-6 from 2842624 to 3244032 tid 2 2024-09-14T03:59:53.432 INFO:tasks.rados.rados.0.smithi070.stdout:3458: copy_from oid 37 from oid 19 current snap is 346 2024-09-14T03:59:53.432 INFO:tasks.rados.rados.0.smithi070.stdout:3452: finishing rollback tid 0 to smithi07085154-20 2024-09-14T03:59:53.432 INFO:tasks.rados.rados.0.smithi070.stdout:3452: finishing rollback tid 1 to smithi07085154-20 2024-09-14T03:59:53.432 INFO:tasks.rados.rados.0.smithi070.stdout:3452: finishing rollback tid 2 to smithi07085154-20 2024-09-14T03:59:53.433 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 20 v 3219 (ObjNum 524 snap 174 seq_num 524) dirty exists 2024-09-14T03:59:53.433 INFO:tasks.rados.rados.0.smithi070.stdout:3452: done (3 left) 2024-09-14T03:59:53.433 INFO:tasks.rados.rados.0.smithi070.stdout:3456: done (2 left) 2024-09-14T03:59:53.433 INFO:tasks.rados.rados.0.smithi070.stdout:3459: snap_create 2024-09-14T03:59:53.436 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 03:59:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:03:59:53] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.51.0" 2024-09-14T03:59:53.436 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:53 smithi070 ceph-mon[98640]: osdmap e756: 8 total, 8 up, 8 in 2024-09-14T03:59:53.436 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:53 smithi070 ceph-mon[96704]: osdmap e756: 8 total, 8 up, 8 in 2024-09-14T03:59:53.438 INFO:tasks.rados.rados.0.smithi070.stdout:3457: finishing write tid 1 to smithi07085154-6 2024-09-14T03:59:53.438 INFO:tasks.rados.rados.0.smithi070.stdout:3457: finishing write tid 2 to smithi07085154-6 2024-09-14T03:59:53.438 INFO:tasks.rados.rados.0.smithi070.stdout:3457: finishing write tid 3 to smithi07085154-6 2024-09-14T03:59:53.440 INFO:tasks.rados.rados.0.smithi070.stdout:3457: finishing write tid 4 to smithi07085154-6 2024-09-14T03:59:53.440 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 3204 (ObjNum 1066 snap 346 seq_num 1066) dirty exists 2024-09-14T03:59:53.440 INFO:tasks.rados.rados.0.smithi070.stdout:3457: left oid 6 (ObjNum 1066 snap 346 seq_num 1066) 2024-09-14T03:59:53.472 INFO:tasks.rados.rados.0.smithi070.stdout:3458: finishing copy_from racing read to smithi07085154-37 2024-09-14T03:59:53.473 INFO:tasks.rados.rados.0.smithi070.stdout:3458: finishing copy_from to smithi07085154-37 2024-09-14T03:59:53.473 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 37 v 3221 (ObjNum 1004 snap 331 seq_num 1004) dirty exists 2024-09-14T03:59:53.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:53 smithi094 ceph-mon[86597]: osdmap e756: 8 total, 8 up, 8 in 2024-09-14T03:59:54.125 INFO:tasks.rados.rados.0.smithi070.stdout:3457: done (2 left) 2024-09-14T03:59:54.125 INFO:tasks.rados.rados.0.smithi070.stdout:3458: done (1 left) 2024-09-14T03:59:54.125 INFO:tasks.rados.rados.0.smithi070.stdout:3459: done (0 left) 2024-09-14T03:59:54.125 INFO:tasks.rados.rados.0.smithi070.stdout:3460: delete oid 26 current snap is 347 2024-09-14T03:59:54.134 INFO:tasks.rados.rados.0.smithi070.stdout:3460: done (0 left) 2024-09-14T03:59:54.135 INFO:tasks.rados.rados.0.smithi070.stdout:3461: snap_remove snap 339 2024-09-14T03:59:54.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:54 smithi094 ceph-mon[86597]: osdmap e757: 8 total, 8 up, 8 in 2024-09-14T03:59:54.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:54 smithi094 ceph-mon[86597]: osdmap e758: 8 total, 8 up, 8 in 2024-09-14T03:59:54.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:54 smithi094 ceph-mon[86597]: pgmap v736: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 981 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 13 MiB/s rd, 1.7 MiB/s wr, 17 op/s 2024-09-14T03:59:54.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:54 smithi070 ceph-mon[96704]: osdmap e757: 8 total, 8 up, 8 in 2024-09-14T03:59:54.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:54 smithi070 ceph-mon[96704]: osdmap e758: 8 total, 8 up, 8 in 2024-09-14T03:59:54.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:54 smithi070 ceph-mon[96704]: pgmap v736: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 981 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 13 MiB/s rd, 1.7 MiB/s wr, 17 op/s 2024-09-14T03:59:54.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:54 smithi070 ceph-mon[98640]: osdmap e757: 8 total, 8 up, 8 in 2024-09-14T03:59:54.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:54 smithi070 ceph-mon[98640]: osdmap e758: 8 total, 8 up, 8 in 2024-09-14T03:59:54.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:54 smithi070 ceph-mon[98640]: pgmap v736: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 981 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 13 MiB/s rd, 1.7 MiB/s wr, 17 op/s 2024-09-14T03:59:55.137 INFO:tasks.rados.rados.0.smithi070.stdout:3461: done (0 left) 2024-09-14T03:59:55.137 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 1 current snap is 347 2024-09-14T03:59:55.137 INFO:tasks.rados.rados.0.smithi070.stdout:3462: seq_num 1068 ranges {6397952=696320,7094272=401408} 2024-09-14T03:59:55.176 INFO:tasks.rados.rados.0.smithi070.stdout:3462: writing smithi07085154-1 from 6397952 to 7094272 tid 1 2024-09-14T03:59:55.179 INFO:tasks.rados.rados.0.smithi070.stdout:3462: writing smithi07085154-1 from 7094272 to 7495680 tid 2 2024-09-14T03:59:55.179 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 31 current snap is 347 2024-09-14T03:59:55.179 INFO:tasks.rados.rados.0.smithi070.stdout:3463: seq_num 1069 ranges {1703936=712704,2416640=401408} 2024-09-14T03:59:55.192 INFO:tasks.rados.rados.0.smithi070.stdout:3463: writing smithi07085154-31 from 1703936 to 2416640 tid 1 2024-09-14T03:59:55.194 INFO:tasks.rados.rados.0.smithi070.stdout:3463: writing smithi07085154-31 from 2416640 to 2818048 tid 2 2024-09-14T03:59:55.194 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 38 current snap is 347 2024-09-14T03:59:55.194 INFO:tasks.rados.rados.0.smithi070.stdout:3462: finishing write tid 1 to smithi07085154-1 2024-09-14T03:59:55.194 INFO:tasks.rados.rados.0.smithi070.stdout:3462: finishing write tid 2 to smithi07085154-1 2024-09-14T03:59:55.195 INFO:tasks.rados.rados.0.smithi070.stdout:3462: finishing write tid 3 to smithi07085154-1 2024-09-14T03:59:55.195 INFO:tasks.rados.rados.0.smithi070.stdout:3462: finishing write tid 4 to smithi07085154-1 2024-09-14T03:59:55.195 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 1 v 3205 (ObjNum 1068 snap 347 seq_num 1068) dirty exists 2024-09-14T03:59:55.195 INFO:tasks.rados.rados.0.smithi070.stdout:3462: left oid 1 (ObjNum 1068 snap 347 seq_num 1068) 2024-09-14T03:59:55.195 INFO:tasks.rados.rados.0.smithi070.stdout:3464: seq_num 1070 ranges {0=729088,729088=401408} 2024-09-14T03:59:55.199 INFO:tasks.rados.rados.0.smithi070.stdout:3464: writing smithi07085154-38 from 0 to 729088 tid 1 2024-09-14T03:59:55.201 INFO:tasks.rados.rados.0.smithi070.stdout:3464: writing smithi07085154-38 from 729088 to 1130496 tid 2 2024-09-14T03:59:55.201 INFO:tasks.rados.rados.0.smithi070.stdout:3462: done (2 left) 2024-09-14T03:59:55.201 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 22 current snap is 347 2024-09-14T03:59:55.202 INFO:tasks.rados.rados.0.smithi070.stdout:3463: finishing write tid 1 to smithi07085154-31 2024-09-14T03:59:55.202 INFO:tasks.rados.rados.0.smithi070.stdout:3463: finishing write tid 2 to smithi07085154-31 2024-09-14T03:59:55.202 INFO:tasks.rados.rados.0.smithi070.stdout:3463: finishing write tid 3 to smithi07085154-31 2024-09-14T03:59:55.202 INFO:tasks.rados.rados.0.smithi070.stdout:3465: seq_num 1071 ranges {2326528=745472,3072000=401408} 2024-09-14T03:59:55.217 INFO:tasks.rados.rados.0.smithi070.stdout:3465: writing smithi07085154-22 from 2326528 to 3072000 tid 1 2024-09-14T03:59:55.220 INFO:tasks.rados.rados.0.smithi070.stdout:3465: writing smithi07085154-22 from 3072000 to 3473408 tid 2 2024-09-14T03:59:55.220 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 3 current snap is 347 2024-09-14T03:59:55.220 INFO:tasks.rados.rados.0.smithi070.stdout:3466: seq_num 1072 ranges {0=761856,761856=401408} 2024-09-14T03:59:55.224 INFO:tasks.rados.rados.0.smithi070.stdout:3466: writing smithi07085154-3 from 0 to 761856 tid 1 2024-09-14T03:59:55.228 INFO:tasks.rados.rados.0.smithi070.stdout:3466: writing smithi07085154-3 from 761856 to 1163264 tid 2 2024-09-14T03:59:55.228 INFO:tasks.rados.rados.0.smithi070.stdout:3467: rmattr oid 28 current snap is 347 2024-09-14T03:59:55.228 INFO:tasks.rados.rados.0.smithi070.stdout:3463: finishing write tid 4 to smithi07085154-31 2024-09-14T03:59:55.228 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 3207 (ObjNum 1069 snap 347 seq_num 1069) dirty exists 2024-09-14T03:59:55.228 INFO:tasks.rados.rados.0.smithi070.stdout:3463: left oid 31 (ObjNum 1069 snap 347 seq_num 1069) 2024-09-14T03:59:55.228 INFO:tasks.rados.rados.0.smithi070.stdout:3464: finishing write tid 1 to smithi07085154-38 2024-09-14T03:59:55.228 INFO:tasks.rados.rados.0.smithi070.stdout:3464: finishing write tid 2 to smithi07085154-38 2024-09-14T03:59:55.228 INFO:tasks.rados.rados.0.smithi070.stdout:3464: finishing write tid 3 to smithi07085154-38 2024-09-14T03:59:55.228 INFO:tasks.rados.rados.0.smithi070.stdout:3464: finishing write tid 4 to smithi07085154-38 2024-09-14T03:59:55.228 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 38 v 3975 (ObjNum 1070 snap 347 seq_num 1070) dirty exists 2024-09-14T03:59:55.228 INFO:tasks.rados.rados.0.smithi070.stdout:3464: left oid 38 (ObjNum 1070 snap 347 seq_num 1070) 2024-09-14T03:59:55.229 INFO:tasks.rados.rados.0.smithi070.stdout:3465: finishing write tid 1 to smithi07085154-22 2024-09-14T03:59:55.229 INFO:tasks.rados.rados.0.smithi070.stdout:3465: finishing write tid 2 to smithi07085154-22 2024-09-14T03:59:55.229 INFO:tasks.rados.rados.0.smithi070.stdout:3465: finishing write tid 3 to smithi07085154-22 2024-09-14T03:59:55.229 INFO:tasks.rados.rados.0.smithi070.stdout:3463: done (4 left) 2024-09-14T03:59:55.229 INFO:tasks.rados.rados.0.smithi070.stdout:3464: done (3 left) 2024-09-14T03:59:55.229 INFO:tasks.rados.rados.0.smithi070.stdout:3468: rollback oid 29 current snap is 347 2024-09-14T03:59:55.229 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 29 to 332 2024-09-14T03:59:55.229 INFO:tasks.rados.rados.0.smithi070.stdout:3465: finishing write tid 4 to smithi07085154-22 2024-09-14T03:59:55.229 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 3238 (ObjNum 1071 snap 347 seq_num 1071) dirty exists 2024-09-14T03:59:55.229 INFO:tasks.rados.rados.0.smithi070.stdout:3465: left oid 22 (ObjNum 1071 snap 347 seq_num 1071) 2024-09-14T03:59:55.229 INFO:tasks.rados.rados.0.smithi070.stdout:3465: done (3 left) 2024-09-14T03:59:55.229 INFO:tasks.rados.rados.0.smithi070.stdout:3469: rmattr oid 2 current snap is 347 2024-09-14T03:59:55.229 INFO:tasks.rados.rados.0.smithi070.stdout:3469: done (3 left) 2024-09-14T03:59:55.229 INFO:tasks.rados.rados.0.smithi070.stdout:3470: rmattr oid 42 current snap is 347 2024-09-14T03:59:55.230 INFO:tasks.rados.rados.0.smithi070.stdout:3470: done (3 left) 2024-09-14T03:59:55.230 INFO:tasks.rados.rados.0.smithi070.stdout:3471: read oid 23 snap -1 2024-09-14T03:59:55.230 INFO:tasks.rados.rados.0.smithi070.stdout:3471: expect (ObjNum 1014 snap 334 seq_num 1014) 2024-09-14T03:59:55.230 INFO:tasks.rados.rados.0.smithi070.stdout:3472: rollback oid 37 current snap is 347 2024-09-14T03:59:55.230 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 37 to 304 2024-09-14T03:59:55.230 INFO:tasks.rados.rados.0.smithi070.stdout:3473: read oid 33 snap -1 2024-09-14T03:59:55.230 INFO:tasks.rados.rados.0.smithi070.stdout:3473: expect (ObjNum 590 snap 195 seq_num 590) 2024-09-14T03:59:55.230 INFO:tasks.rados.rados.0.smithi070.stdout:3474: setattr oid 26 current snap is 347 2024-09-14T03:59:55.231 INFO:tasks.rados.rados.0.smithi070.stdout:3475: rmattr oid 42 current snap is 347 2024-09-14T03:59:55.231 INFO:tasks.rados.rados.0.smithi070.stdout:3475: done (7 left) 2024-09-14T03:59:55.231 INFO:tasks.rados.rados.0.smithi070.stdout:3476: read oid 20 snap -1 2024-09-14T03:59:55.231 INFO:tasks.rados.rados.0.smithi070.stdout:3476: expect (ObjNum 524 snap 174 seq_num 524) 2024-09-14T03:59:55.231 INFO:tasks.rados.rados.0.smithi070.stdout:3477: snap_remove snap 320 2024-09-14T03:59:56.141 INFO:tasks.rados.rados.0.smithi070.stdout:3477: done (8 left) 2024-09-14T03:59:56.142 INFO:tasks.rados.rados.0.smithi070.stdout:3478: read oid 31 snap -1 2024-09-14T03:59:56.142 INFO:tasks.rados.rados.0.smithi070.stdout:3478: expect (ObjNum 1069 snap 347 seq_num 1069) 2024-09-14T03:59:56.142 INFO:tasks.rados.rados.0.smithi070.stdout:3479: copy_from oid 44 from oid 15 current snap is 347 2024-09-14T03:59:56.142 INFO:tasks.rados.rados.0.smithi070.stdout:3466: finishing write tid 1 to smithi07085154-3 2024-09-14T03:59:56.142 INFO:tasks.rados.rados.0.smithi070.stdout:3480: copy_from oid 8 from oid 49 current snap is 347 2024-09-14T03:59:56.142 INFO:tasks.rados.rados.0.smithi070.stdout:3466: finishing write tid 2 to smithi07085154-3 2024-09-14T03:59:56.142 INFO:tasks.rados.rados.0.smithi070.stdout:3466: finishing write tid 3 to smithi07085154-3 2024-09-14T03:59:56.142 INFO:tasks.rados.rados.0.smithi070.stdout:3472: finishing rollback tid 0 to smithi07085154-37 2024-09-14T03:59:56.143 INFO:tasks.rados.rados.0.smithi070.stdout:3468: finishing rollback tid 0 to smithi07085154-29 2024-09-14T03:59:56.143 INFO:tasks.rados.rados.0.smithi070.stdout:3466: finishing write tid 4 to smithi07085154-3 2024-09-14T03:59:56.143 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 3 v 3978 (ObjNum 1072 snap 347 seq_num 1072) dirty exists 2024-09-14T03:59:56.143 INFO:tasks.rados.rados.0.smithi070.stdout:3466: left oid 3 (ObjNum 1072 snap 347 seq_num 1072) 2024-09-14T03:59:56.143 INFO:tasks.rados.rados.0.smithi070.stdout:3472: finishing rollback tid 1 to smithi07085154-37 2024-09-14T03:59:56.143 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 37 v 3223 (ObjNum 65 snap 0 seq_num 1466325345) dirty dne 2024-09-14T03:59:56.143 INFO:tasks.rados.rados.0.smithi070.stdout:3468: finishing rollback tid 1 to smithi07085154-29 2024-09-14T03:59:56.143 INFO:tasks.rados.rados.0.smithi070.stdout:3466: done (10 left) 2024-09-14T03:59:56.143 INFO:tasks.rados.rados.0.smithi070.stdout:3472: done (9 left) 2024-09-14T03:59:56.143 INFO:tasks.rados.rados.0.smithi070.stdout:3481: delete oid 18 current snap is 347 2024-09-14T03:59:56.143 INFO:tasks.rados.rados.0.smithi070.stdout:3468: finishing rollback tid 2 to smithi07085154-29 2024-09-14T03:59:56.143 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 3241 (ObjNum 986 snap 325 seq_num 986) dirty exists 2024-09-14T03:59:56.143 INFO:tasks.rados.rados.0.smithi070.stdout:3471: expect (ObjNum 1014 snap 334 seq_num 1014) 2024-09-14T03:59:56.257 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 28 v 3220 (ObjNum 1044 snap 344 seq_num 1044) dirty exists 2024-09-14T03:59:56.257 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 4138 (ObjNum 161 snap 0 seq_num 0) dirty exists 2024-09-14T03:59:56.257 INFO:tasks.rados.rados.0.smithi070.stdout:3476: expect (ObjNum 524 snap 174 seq_num 524) 2024-09-14T03:59:56.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:56 smithi094 ceph-mon[86597]: osdmap e759: 8 total, 8 up, 8 in 2024-09-14T03:59:56.556 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:56 smithi070 ceph-mon[96704]: osdmap e759: 8 total, 8 up, 8 in 2024-09-14T03:59:56.557 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:56 smithi070 ceph-mon[98640]: osdmap e759: 8 total, 8 up, 8 in 2024-09-14T03:59:56.702 INFO:tasks.rados.rados.0.smithi070.stdout:3473: expect (ObjNum 590 snap 195 seq_num 590) 2024-09-14T03:59:57.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:57 smithi094 ceph-mon[86597]: osdmap e760: 8 total, 8 up, 8 in 2024-09-14T03:59:57.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:57 smithi094 ceph-mon[86597]: pgmap v739: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1022 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.2 MiB/s rd, 7.2 MiB/s wr, 10 op/s 2024-09-14T03:59:57.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:57 smithi070 ceph-mon[96704]: osdmap e760: 8 total, 8 up, 8 in 2024-09-14T03:59:57.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:57 smithi070 ceph-mon[96704]: pgmap v739: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1022 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.2 MiB/s rd, 7.2 MiB/s wr, 10 op/s 2024-09-14T03:59:57.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:57 smithi070 ceph-mon[98640]: osdmap e760: 8 total, 8 up, 8 in 2024-09-14T03:59:57.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:57 smithi070 ceph-mon[98640]: pgmap v739: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1022 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.2 MiB/s rd, 7.2 MiB/s wr, 10 op/s 2024-09-14T03:59:58.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 03:59:58 smithi094 ceph-mon[86597]: pgmap v740: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1023 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 12 MiB/s rd, 6.6 MiB/s wr, 21 op/s 2024-09-14T03:59:58.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 03:59:58 smithi070 ceph-mon[96704]: pgmap v740: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1023 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 12 MiB/s rd, 6.6 MiB/s wr, 21 op/s 2024-09-14T03:59:58.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 03:59:58 smithi070 ceph-mon[98640]: pgmap v740: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 1023 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 12 MiB/s rd, 6.6 MiB/s wr, 21 op/s 2024-09-14T03:59:58.602 INFO:tasks.rados.rados.0.smithi070.stdout:3480: finishing copy_from to smithi07085154-8 2024-09-14T03:59:58.602 INFO:tasks.rados.rados.0.smithi070.stdout:3480: got expected ENOENT (src dne) 2024-09-14T03:59:58.602 INFO:tasks.rados.rados.0.smithi070.stdout:3480: finishing copy_from racing read to smithi07085154-8 2024-09-14T03:59:58.602 INFO:tasks.rados.rados.0.smithi070.stdout:3478: expect (ObjNum 1069 snap 347 seq_num 1069) 2024-09-14T03:59:58.699 INFO:tasks.rados.rados.0.smithi070.stdout:3479: finishing copy_from racing read to smithi07085154-44 2024-09-14T03:59:58.700 INFO:tasks.rados.rados.0.smithi070.stdout:3479: finishing copy_from to smithi07085154-44 2024-09-14T03:59:58.700 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 3185 (ObjNum 805 snap 281 seq_num 805) dirty exists 2024-09-14T03:59:58.700 INFO:tasks.rados.rados.0.smithi070.stdout:3467: done (9 left) 2024-09-14T03:59:58.700 INFO:tasks.rados.rados.0.smithi070.stdout:3468: done (8 left) 2024-09-14T03:59:58.700 INFO:tasks.rados.rados.0.smithi070.stdout:3471: done (7 left) 2024-09-14T03:59:58.700 INFO:tasks.rados.rados.0.smithi070.stdout:3473: done (6 left) 2024-09-14T03:59:58.700 INFO:tasks.rados.rados.0.smithi070.stdout:3474: done (5 left) 2024-09-14T03:59:58.700 INFO:tasks.rados.rados.0.smithi070.stdout:3476: done (4 left) 2024-09-14T03:59:58.700 INFO:tasks.rados.rados.0.smithi070.stdout:3478: done (3 left) 2024-09-14T03:59:58.700 INFO:tasks.rados.rados.0.smithi070.stdout:3479: done (2 left) 2024-09-14T03:59:58.701 INFO:tasks.rados.rados.0.smithi070.stdout:3480: done (1 left) 2024-09-14T03:59:58.701 INFO:tasks.rados.rados.0.smithi070.stdout:3481: done (0 left) 2024-09-14T03:59:58.701 INFO:tasks.rados.rados.0.smithi070.stdout:3482: rollback oid 44 current snap is 347 2024-09-14T03:59:58.701 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 44 to 296 2024-09-14T03:59:58.701 INFO:tasks.rados.rados.0.smithi070.stdout:3483: rollback oid 14 current snap is 347 2024-09-14T03:59:58.701 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 14 to 324 2024-09-14T03:59:58.701 INFO:tasks.rados.rados.0.smithi070.stdout:3484: delete oid 1 current snap is 347 2024-09-14T03:59:58.702 INFO:tasks.rados.rados.0.smithi070.stdout:3483: finishing rollback tid 1 to smithi07085154-14 2024-09-14T03:59:58.702 INFO:tasks.rados.rados.0.smithi070.stdout:3483: finishing rollback tid 2 to smithi07085154-14 2024-09-14T03:59:58.702 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 3223 (ObjNum 313 snap 105 seq_num 313) dirty exists 2024-09-14T03:59:58.705 INFO:tasks.rados.rados.0.smithi070.stdout:3482: finishing rollback tid 0 to smithi07085154-44 2024-09-14T03:59:58.705 INFO:tasks.rados.rados.0.smithi070.stdout:3482: finishing rollback tid 1 to smithi07085154-44 2024-09-14T03:59:58.706 INFO:tasks.rados.rados.0.smithi070.stdout:3482: finishing rollback tid 2 to smithi07085154-44 2024-09-14T03:59:58.706 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 3208 (ObjNum 865 snap 293 seq_num 865) dirty exists 2024-09-14T03:59:58.706 INFO:tasks.rados.rados.0.smithi070.stdout:3482: done (2 left) 2024-09-14T03:59:58.706 INFO:tasks.rados.rados.0.smithi070.stdout:3483: done (1 left) 2024-09-14T03:59:58.706 INFO:tasks.rados.rados.0.smithi070.stdout:3484: done (0 left) 2024-09-14T03:59:58.706 INFO:tasks.rados.rados.0.smithi070.stdout:3485: copy_from oid 49 from oid 18 current snap is 347 2024-09-14T03:59:58.706 INFO:tasks.rados.rados.0.smithi070.stdout:3486: snap_create 2024-09-14T03:59:58.707 INFO:tasks.rados.rados.0.smithi070.stdout:3485: finishing copy_from to smithi07085154-49 2024-09-14T03:59:58.707 INFO:tasks.rados.rados.0.smithi070.stdout:3485: got expected ENOENT (src dne) 2024-09-14T03:59:58.707 INFO:tasks.rados.rados.0.smithi070.stdout:3485: finishing copy_from racing read to smithi07085154-49 2024-09-14T03:59:59.127 INFO:tasks.rados.rados.0.smithi070.stdout:3485: done (1 left) 2024-09-14T03:59:59.127 INFO:tasks.rados.rados.0.smithi070.stdout:3486: done (0 left) 2024-09-14T03:59:59.127 INFO:tasks.rados.rados.0.smithi070.stdout:3487: delete oid 13 current snap is 348 2024-09-14T03:59:59.130 INFO:tasks.rados.rados.0.smithi070.stdout:3487: done (0 left) 2024-09-14T03:59:59.130 INFO:tasks.rados.rados.0.smithi070.stdout:3488: delete oid 2 current snap is 348 2024-09-14T03:59:59.138 INFO:tasks.rados.rados.0.smithi070.stdout:3488: done (0 left) 2024-09-14T03:59:59.138 INFO:tasks.rados.rados.0.smithi070.stdout:3489: snap_create 2024-09-14T04:00:00.133 INFO:tasks.rados.rados.0.smithi070.stdout:3489: done (0 left) 2024-09-14T04:00:00.133 INFO:tasks.rados.rados.0.smithi070.stdout:3490: read oid 49 snap -1 2024-09-14T04:00:00.133 INFO:tasks.rados.rados.0.smithi070.stdout:3490: expect deleted 2024-09-14T04:00:00.133 INFO:tasks.rados.rados.0.smithi070.stdout:3491: snap_remove snap 307 2024-09-14T04:00:00.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:00 smithi094 ceph-mon[86597]: osdmap e761: 8 total, 8 up, 8 in 2024-09-14T04:00:00.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:00 smithi094 ceph-mon[86597]: overall HEALTH_OK 2024-09-14T04:00:00.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:00:00 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:00:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:00:00.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:00 smithi070 ceph-mon[96704]: osdmap e761: 8 total, 8 up, 8 in 2024-09-14T04:00:00.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:00 smithi070 ceph-mon[96704]: overall HEALTH_OK 2024-09-14T04:00:00.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:00 smithi070 ceph-mon[98640]: osdmap e761: 8 total, 8 up, 8 in 2024-09-14T04:00:00.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:00 smithi070 ceph-mon[98640]: overall HEALTH_OK 2024-09-14T04:00:01.141 INFO:tasks.rados.rados.0.smithi070.stdout:3491: done (1 left) 2024-09-14T04:00:01.141 INFO:tasks.rados.rados.0.smithi070.stdout:3492: snap_remove snap 289 2024-09-14T04:00:01.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:01 smithi094 ceph-mon[86597]: osdmap e762: 8 total, 8 up, 8 in 2024-09-14T04:00:01.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:01 smithi094 ceph-mon[86597]: pgmap v743: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 18 MiB/s rd, 1.7 MiB/s wr, 18 op/s 2024-09-14T04:00:01.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:01 smithi070 ceph-mon[98640]: osdmap e762: 8 total, 8 up, 8 in 2024-09-14T04:00:01.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:01 smithi070 ceph-mon[98640]: pgmap v743: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 18 MiB/s rd, 1.7 MiB/s wr, 18 op/s 2024-09-14T04:00:01.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:01 smithi070 ceph-mon[96704]: osdmap e762: 8 total, 8 up, 8 in 2024-09-14T04:00:01.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:01 smithi070 ceph-mon[96704]: pgmap v743: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 18 MiB/s rd, 1.7 MiB/s wr, 18 op/s 2024-09-14T04:00:02.141 INFO:tasks.rados.rados.0.smithi070.stdout:3492: done (1 left) 2024-09-14T04:00:02.141 INFO:tasks.rados.rados.0.smithi070.stdout:3493: delete oid 14 current snap is 349 2024-09-14T04:00:02.153 INFO:tasks.rados.rados.0.smithi070.stdout:3490: done (1 left) 2024-09-14T04:00:02.153 INFO:tasks.rados.rados.0.smithi070.stdout:3493: done (0 left) 2024-09-14T04:00:02.154 INFO:tasks.rados.rados.0.smithi070.stdout:3494: rollback oid 19 current snap is 349 2024-09-14T04:00:02.154 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 19 to 296 2024-09-14T04:00:02.154 INFO:tasks.rados.rados.0.smithi070.stdout:3495: copy_from oid 18 from oid 42 current snap is 349 2024-09-14T04:00:02.154 INFO:tasks.rados.rados.0.smithi070.stdout:3496: read oid 30 snap 325 2024-09-14T04:00:02.154 INFO:tasks.rados.rados.0.smithi070.stdout:3496: expect (ObjNum 982 snap 324 seq_num 982) 2024-09-14T04:00:02.154 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 29 current snap is 349 2024-09-14T04:00:02.154 INFO:tasks.rados.rados.0.smithi070.stdout:3497: seq_num 1078 ranges {4055040=458752,4513792=720896,5234688=81920} 2024-09-14T04:00:02.181 INFO:tasks.rados.rados.0.smithi070.stdout:3497: writing smithi07085154-29 from 4055040 to 4513792 tid 1 2024-09-14T04:00:02.186 INFO:tasks.rados.rados.0.smithi070.stdout:3497: writing smithi07085154-29 from 4513792 to 5234688 tid 2 2024-09-14T04:00:02.187 INFO:tasks.rados.rados.0.smithi070.stdout:3497: writing smithi07085154-29 from 5234688 to 5316608 tid 3 2024-09-14T04:00:02.187 INFO:tasks.rados.rados.0.smithi070.stdout:3498: read oid 33 snap -1 2024-09-14T04:00:02.187 INFO:tasks.rados.rados.0.smithi070.stdout:3498: expect (ObjNum 590 snap 195 seq_num 590) 2024-09-14T04:00:02.187 INFO:tasks.rados.rados.0.smithi070.stdout:3499: copy_from oid 50 from oid 39 current snap is 349 2024-09-14T04:00:02.187 INFO:tasks.rados.rados.0.smithi070.stdout:3496: expect (ObjNum 982 snap 324 seq_num 982) 2024-09-14T04:00:02.260 INFO:tasks.rados.rados.0.smithi070.stdout:3494: finishing rollback tid 0 to smithi07085154-19 2024-09-14T04:00:02.260 INFO:tasks.rados.rados.0.smithi070.stdout:3494: finishing rollback tid 1 to smithi07085154-19 2024-09-14T04:00:02.260 INFO:tasks.rados.rados.0.smithi070.stdout:3494: finishing rollback tid 2 to smithi07085154-19 2024-09-14T04:00:02.260 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 19 v 3244 (ObjNum 659 snap 220 seq_num 659) dirty exists 2024-09-14T04:00:02.260 INFO:tasks.rados.rados.0.smithi070.stdout:3495: finishing copy_from racing read to smithi07085154-18 2024-09-14T04:00:02.260 INFO:tasks.rados.rados.0.smithi070.stdout:3495: finishing copy_from to smithi07085154-18 2024-09-14T04:00:02.260 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 4121 (ObjNum 1061 snap 346 seq_num 1061) dirty exists 2024-09-14T04:00:02.260 INFO:tasks.rados.rados.0.smithi070.stdout:3494: done (5 left) 2024-09-14T04:00:02.261 INFO:tasks.rados.rados.0.smithi070.stdout:3495: done (4 left) 2024-09-14T04:00:02.261 INFO:tasks.rados.rados.0.smithi070.stdout:3496: done (3 left) 2024-09-14T04:00:02.261 INFO:tasks.rados.rados.0.smithi070.stdout:3500: snap_create 2024-09-14T04:00:02.261 INFO:tasks.rados.rados.0.smithi070.stdout:3497: finishing write tid 1 to smithi07085154-29 2024-09-14T04:00:02.261 INFO:tasks.rados.rados.0.smithi070.stdout:3497: finishing write tid 2 to smithi07085154-29 2024-09-14T04:00:02.261 INFO:tasks.rados.rados.0.smithi070.stdout:3497: finishing write tid 3 to smithi07085154-29 2024-09-14T04:00:02.261 INFO:tasks.rados.rados.0.smithi070.stdout:3499: finishing copy_from racing read to smithi07085154-50 2024-09-14T04:00:02.261 INFO:tasks.rados.rados.0.smithi070.stdout:3497: finishing write tid 4 to smithi07085154-29 2024-09-14T04:00:02.261 INFO:tasks.rados.rados.0.smithi070.stdout:3497: finishing write tid 5 to smithi07085154-29 2024-09-14T04:00:02.261 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 3248 (ObjNum 1078 snap 349 seq_num 1078) dirty exists 2024-09-14T04:00:02.261 INFO:tasks.rados.rados.0.smithi070.stdout:3497: left oid 29 (ObjNum 1078 snap 349 seq_num 1078) 2024-09-14T04:00:02.262 INFO:tasks.rados.rados.0.smithi070.stdout:3499: finishing copy_from to smithi07085154-50 2024-09-14T04:00:02.262 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 2642 (ObjNum 1020 snap 336 seq_num 1020) dirty exists 2024-09-14T04:00:02.262 INFO:tasks.rados.rados.0.smithi070.stdout:3498: expect (ObjNum 590 snap 195 seq_num 590) 2024-09-14T04:00:02.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:02 smithi094 ceph-mon[86597]: osdmap e763: 8 total, 8 up, 8 in 2024-09-14T04:00:02.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:02 smithi070 ceph-mon[96704]: osdmap e763: 8 total, 8 up, 8 in 2024-09-14T04:00:02.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:02 smithi070 ceph-mon[98640]: osdmap e763: 8 total, 8 up, 8 in 2024-09-14T04:00:03.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:03 smithi070 ceph-mon[96704]: osdmap e764: 8 total, 8 up, 8 in 2024-09-14T04:00:03.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:03 smithi070 ceph-mon[96704]: pgmap v746: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1014 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-09-14T04:00:03.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:03 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:00:03.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:03 smithi070 ceph-mon[98640]: osdmap e764: 8 total, 8 up, 8 in 2024-09-14T04:00:03.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:03 smithi070 ceph-mon[98640]: pgmap v746: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1014 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-09-14T04:00:03.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:03 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:00:03.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:03 smithi094 ceph-mon[86597]: osdmap e764: 8 total, 8 up, 8 in 2024-09-14T04:00:03.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:03 smithi094 ceph-mon[86597]: pgmap v746: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1014 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-09-14T04:00:03.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:03 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:00:03.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:00:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:00:03] "GET /metrics HTTP/1.1" 200 36502 "" "Prometheus/2.51.0" 2024-09-14T04:00:04.171 INFO:tasks.rados.rados.0.smithi070.stdout:3497: done (3 left) 2024-09-14T04:00:04.172 INFO:tasks.rados.rados.0.smithi070.stdout:3498: done (2 left) 2024-09-14T04:00:04.172 INFO:tasks.rados.rados.0.smithi070.stdout:3499: done (1 left) 2024-09-14T04:00:04.172 INFO:tasks.rados.rados.0.smithi070.stdout:3500: done (0 left) 2024-09-14T04:00:04.172 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 26 current snap is 350 2024-09-14T04:00:04.172 INFO:tasks.rados.rados.0.smithi070.stdout:3501: seq_num 1079 ranges {0=475136,475136=606208,1081344=196608} 2024-09-14T04:00:04.174 INFO:tasks.rados.rados.0.smithi070.stdout:3501: writing smithi07085154-26 from 0 to 475136 tid 1 2024-09-14T04:00:04.177 INFO:tasks.rados.rados.0.smithi070.stdout:3501: writing smithi07085154-26 from 475136 to 1081344 tid 2 2024-09-14T04:00:04.179 INFO:tasks.rados.rados.0.smithi070.stdout:3501: writing smithi07085154-26 from 1081344 to 1277952 tid 3 2024-09-14T04:00:04.179 INFO:tasks.rados.rados.0.smithi070.stdout:3502: rollback oid 10 current snap is 350 2024-09-14T04:00:04.179 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 10 to 315 2024-09-14T04:00:04.179 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 49 current snap is 350 2024-09-14T04:00:04.179 INFO:tasks.rados.rados.0.smithi070.stdout:3503: seq_num 1080 ranges {0=491520,491520=540672,1032192=262144} 2024-09-14T04:00:04.181 INFO:tasks.rados.rados.0.smithi070.stdout:3503: writing smithi07085154-49 from 0 to 491520 tid 1 2024-09-14T04:00:04.185 INFO:tasks.rados.rados.0.smithi070.stdout:3503: writing smithi07085154-49 from 491520 to 1032192 tid 2 2024-09-14T04:00:04.187 INFO:tasks.rados.rados.0.smithi070.stdout:3503: writing smithi07085154-49 from 1032192 to 1294336 tid 3 2024-09-14T04:00:04.188 INFO:tasks.rados.rados.0.smithi070.stdout:3504: read oid 25 snap -1 2024-09-14T04:00:04.188 INFO:tasks.rados.rados.0.smithi070.stdout:3504: expect (ObjNum 1055 snap 344 seq_num 1055) 2024-09-14T04:00:04.188 INFO:tasks.rados.rados.0.smithi070.stdout:3505: read oid 24 snap -1 2024-09-14T04:00:04.188 INFO:tasks.rados.rados.0.smithi070.stdout:3505: expect deleted 2024-09-14T04:00:04.188 INFO:tasks.rados.rados.0.smithi070.stdout:3506: snap_create 2024-09-14T04:00:04.188 INFO:tasks.rados.rados.0.smithi070.stdout:3501: finishing write tid 1 to smithi07085154-26 2024-09-14T04:00:04.188 INFO:tasks.rados.rados.0.smithi070.stdout:3501: finishing write tid 2 to smithi07085154-26 2024-09-14T04:00:04.188 INFO:tasks.rados.rados.0.smithi070.stdout:3501: finishing write tid 3 to smithi07085154-26 2024-09-14T04:00:04.188 INFO:tasks.rados.rados.0.smithi070.stdout:3503: finishing write tid 1 to smithi07085154-49 2024-09-14T04:00:04.189 INFO:tasks.rados.rados.0.smithi070.stdout:3501: finishing write tid 4 to smithi07085154-26 2024-09-14T04:00:04.189 INFO:tasks.rados.rados.0.smithi070.stdout:3503: finishing write tid 2 to smithi07085154-49 2024-09-14T04:00:04.190 INFO:tasks.rados.rados.0.smithi070.stdout:3502: finishing rollback tid 1 to smithi07085154-10 2024-09-14T04:00:04.190 INFO:tasks.rados.rados.0.smithi070.stdout:3501: finishing write tid 5 to smithi07085154-26 2024-09-14T04:00:04.191 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 4191 (ObjNum 1079 snap 350 seq_num 1079) dirty exists 2024-09-14T04:00:04.191 INFO:tasks.rados.rados.0.smithi070.stdout:3501: left oid 26 (ObjNum 1079 snap 350 seq_num 1079) 2024-09-14T04:00:04.191 INFO:tasks.rados.rados.0.smithi070.stdout:3503: finishing write tid 3 to smithi07085154-49 2024-09-14T04:00:04.191 INFO:tasks.rados.rados.0.smithi070.stdout:3503: finishing write tid 4 to smithi07085154-49 2024-09-14T04:00:04.191 INFO:tasks.rados.rados.0.smithi070.stdout:3503: finishing write tid 5 to smithi07085154-49 2024-09-14T04:00:04.191 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 49 v 3207 (ObjNum 1080 snap 350 seq_num 1080) dirty exists 2024-09-14T04:00:04.191 INFO:tasks.rados.rados.0.smithi070.stdout:3503: left oid 49 (ObjNum 1080 snap 350 seq_num 1080) 2024-09-14T04:00:04.192 INFO:tasks.rados.rados.0.smithi070.stdout:3502: finishing rollback tid 2 to smithi07085154-10 2024-09-14T04:00:04.193 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 4193 (ObjNum 841 snap 286 seq_num 841) dirty exists 2024-09-14T04:00:04.195 INFO:tasks.rados.rados.0.smithi070.stdout:3504: expect (ObjNum 1055 snap 344 seq_num 1055) 2024-09-14T04:00:04.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:04 smithi094 ceph-mon[86597]: osdmap e765: 8 total, 8 up, 8 in 2024-09-14T04:00:04.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:04 smithi094 ceph-mon[86597]: osdmap e766: 8 total, 8 up, 8 in 2024-09-14T04:00:04.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:04 smithi070 ceph-mon[96704]: osdmap e765: 8 total, 8 up, 8 in 2024-09-14T04:00:04.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:04 smithi070 ceph-mon[96704]: osdmap e766: 8 total, 8 up, 8 in 2024-09-14T04:00:04.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:04 smithi070 ceph-mon[98640]: osdmap e765: 8 total, 8 up, 8 in 2024-09-14T04:00:04.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:04 smithi070 ceph-mon[98640]: osdmap e766: 8 total, 8 up, 8 in 2024-09-14T04:00:05.157 INFO:tasks.rados.rados.0.smithi070.stdout:3501: done (5 left) 2024-09-14T04:00:05.157 INFO:tasks.rados.rados.0.smithi070.stdout:3502: done (4 left) 2024-09-14T04:00:05.157 INFO:tasks.rados.rados.0.smithi070.stdout:3503: done (3 left) 2024-09-14T04:00:05.157 INFO:tasks.rados.rados.0.smithi070.stdout:3504: done (2 left) 2024-09-14T04:00:05.157 INFO:tasks.rados.rados.0.smithi070.stdout:3505: done (1 left) 2024-09-14T04:00:05.157 INFO:tasks.rados.rados.0.smithi070.stdout:3506: done (0 left) 2024-09-14T04:00:05.157 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 13 current snap is 351 2024-09-14T04:00:05.157 INFO:tasks.rados.rados.0.smithi070.stdout:3507: seq_num 1081 ranges {0=507904,507904=425984,933888=376832} 2024-09-14T04:00:05.159 INFO:tasks.rados.rados.0.smithi070.stdout:3507: writing smithi07085154-13 from 0 to 507904 tid 1 2024-09-14T04:00:05.163 INFO:tasks.rados.rados.0.smithi070.stdout:3507: writing smithi07085154-13 from 507904 to 933888 tid 2 2024-09-14T04:00:05.165 INFO:tasks.rados.rados.0.smithi070.stdout:3507: writing smithi07085154-13 from 933888 to 1310720 tid 3 2024-09-14T04:00:05.166 INFO:tasks.rados.rados.0.smithi070.stdout:3508: read oid 26 snap -1 2024-09-14T04:00:05.166 INFO:tasks.rados.rados.0.smithi070.stdout:3508: expect (ObjNum 1079 snap 350 seq_num 1079) 2024-09-14T04:00:05.166 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 36 current snap is 351 2024-09-14T04:00:05.166 INFO:tasks.rados.rados.0.smithi070.stdout:3509: seq_num 1082 ranges {1138688=524288,1662976=704512,2367488=98304} 2024-09-14T04:00:05.175 INFO:tasks.rados.rados.0.smithi070.stdout:3509: writing smithi07085154-36 from 1138688 to 1662976 tid 1 2024-09-14T04:00:05.179 INFO:tasks.rados.rados.0.smithi070.stdout:3509: writing smithi07085154-36 from 1662976 to 2367488 tid 2 2024-09-14T04:00:05.180 INFO:tasks.rados.rados.0.smithi070.stdout:3509: writing smithi07085154-36 from 2367488 to 2465792 tid 3 2024-09-14T04:00:05.180 INFO:tasks.rados.rados.0.smithi070.stdout:3510: rmattr oid 17 current snap is 351 2024-09-14T04:00:05.180 INFO:tasks.rados.rados.0.smithi070.stdout:3507: finishing write tid 1 to smithi07085154-13 2024-09-14T04:00:05.181 INFO:tasks.rados.rados.0.smithi070.stdout:3507: finishing write tid 2 to smithi07085154-13 2024-09-14T04:00:05.181 INFO:tasks.rados.rados.0.smithi070.stdout:3507: finishing write tid 3 to smithi07085154-13 2024-09-14T04:00:05.181 INFO:tasks.rados.rados.0.smithi070.stdout:3507: finishing write tid 4 to smithi07085154-13 2024-09-14T04:00:05.181 INFO:tasks.rados.rados.0.smithi070.stdout:3507: finishing write tid 5 to smithi07085154-13 2024-09-14T04:00:05.181 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 3222 (ObjNum 1081 snap 351 seq_num 1081) dirty exists 2024-09-14T04:00:05.181 INFO:tasks.rados.rados.0.smithi070.stdout:3507: left oid 13 (ObjNum 1081 snap 351 seq_num 1081) 2024-09-14T04:00:05.181 INFO:tasks.rados.rados.0.smithi070.stdout:3508: expect (ObjNum 1079 snap 350 seq_num 1079) 2024-09-14T04:00:05.217 INFO:tasks.rados.rados.0.smithi070.stdout:3509: finishing write tid 1 to smithi07085154-36 2024-09-14T04:00:05.217 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 3208 (ObjNum 1038 snap 342 seq_num 1038) dirty exists 2024-09-14T04:00:05.217 INFO:tasks.rados.rados.0.smithi070.stdout:3509: finishing write tid 2 to smithi07085154-36 2024-09-14T04:00:05.217 INFO:tasks.rados.rados.0.smithi070.stdout:3509: finishing write tid 3 to smithi07085154-36 2024-09-14T04:00:05.217 INFO:tasks.rados.rados.0.smithi070.stdout:3509: finishing write tid 4 to smithi07085154-36 2024-09-14T04:00:05.217 INFO:tasks.rados.rados.0.smithi070.stdout:3509: finishing write tid 5 to smithi07085154-36 2024-09-14T04:00:05.217 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 36 v 3213 (ObjNum 1082 snap 351 seq_num 1082) dirty exists 2024-09-14T04:00:05.217 INFO:tasks.rados.rados.0.smithi070.stdout:3509: left oid 36 (ObjNum 1082 snap 351 seq_num 1082) 2024-09-14T04:00:05.217 INFO:tasks.rados.rados.0.smithi070.stdout:3507: done (3 left) 2024-09-14T04:00:05.217 INFO:tasks.rados.rados.0.smithi070.stdout:3508: done (2 left) 2024-09-14T04:00:05.218 INFO:tasks.rados.rados.0.smithi070.stdout:3509: done (1 left) 2024-09-14T04:00:05.218 INFO:tasks.rados.rados.0.smithi070.stdout:3510: done (0 left) 2024-09-14T04:00:05.218 INFO:tasks.rados.rados.0.smithi070.stdout:3511: read oid 37 snap -1 2024-09-14T04:00:05.218 INFO:tasks.rados.rados.0.smithi070.stdout:3511: expect deleted 2024-09-14T04:00:05.218 INFO:tasks.rados.rados.0.smithi070.stdout:3512: read oid 49 snap -1 2024-09-14T04:00:05.218 INFO:tasks.rados.rados.0.smithi070.stdout:3512: expect (ObjNum 1080 snap 350 seq_num 1080) 2024-09-14T04:00:05.218 INFO:tasks.rados.rados.0.smithi070.stdout:3513: copy_from oid 34 from oid 10 current snap is 351 2024-09-14T04:00:05.218 INFO:tasks.rados.rados.0.smithi070.stdout:3514: snap_remove snap 347 2024-09-14T04:00:05.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:05 smithi094 ceph-mon[86597]: pgmap v749: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1016 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 1 op/s 2024-09-14T04:00:05.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:05 smithi070 ceph-mon[96704]: pgmap v749: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1016 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 1 op/s 2024-09-14T04:00:05.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:05 smithi070 ceph-mon[98640]: pgmap v749: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1016 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 1 op/s 2024-09-14T04:00:06.164 INFO:tasks.rados.rados.0.smithi070.stdout:3514: done (3 left) 2024-09-14T04:00:06.164 INFO:tasks.rados.rados.0.smithi070.stdout:3515: snap_create 2024-09-14T04:00:06.165 INFO:tasks.rados.rados.0.smithi070.stdout:3512: expect (ObjNum 1080 snap 350 seq_num 1080) 2024-09-14T04:00:06.203 INFO:tasks.rados.rados.0.smithi070.stdout:3513: finishing copy_from racing read to smithi07085154-34 2024-09-14T04:00:06.203 INFO:tasks.rados.rados.0.smithi070.stdout:3513: finishing copy_from to smithi07085154-34 2024-09-14T04:00:06.203 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 4193 (ObjNum 841 snap 286 seq_num 841) dirty exists 2024-09-14T04:00:06.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:06 smithi094 ceph-mon[86597]: osdmap e767: 8 total, 8 up, 8 in 2024-09-14T04:00:06.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:06 smithi070 ceph-mon[96704]: osdmap e767: 8 total, 8 up, 8 in 2024-09-14T04:00:06.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:06 smithi070 ceph-mon[98640]: osdmap e767: 8 total, 8 up, 8 in 2024-09-14T04:00:07.165 INFO:tasks.rados.rados.0.smithi070.stdout:3511: done (3 left) 2024-09-14T04:00:07.165 INFO:tasks.rados.rados.0.smithi070.stdout:3512: done (2 left) 2024-09-14T04:00:07.165 INFO:tasks.rados.rados.0.smithi070.stdout:3513: done (1 left) 2024-09-14T04:00:07.165 INFO:tasks.rados.rados.0.smithi070.stdout:3515: done (0 left) 2024-09-14T04:00:07.165 INFO:tasks.rados.rados.0.smithi070.stdout:3516: snap_create 2024-09-14T04:00:07.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:07 smithi094 ceph-mon[86597]: pgmap v751: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1018 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 12 MiB/s rd, 571 KiB/s wr, 2 op/s 2024-09-14T04:00:07.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:07 smithi094 ceph-mon[86597]: osdmap e768: 8 total, 8 up, 8 in 2024-09-14T04:00:07.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:07 smithi070 ceph-mon[96704]: pgmap v751: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1018 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 12 MiB/s rd, 571 KiB/s wr, 2 op/s 2024-09-14T04:00:07.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:07 smithi070 ceph-mon[96704]: osdmap e768: 8 total, 8 up, 8 in 2024-09-14T04:00:07.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:07 smithi070 ceph-mon[98640]: pgmap v751: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 1018 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 12 MiB/s rd, 571 KiB/s wr, 2 op/s 2024-09-14T04:00:07.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:07 smithi070 ceph-mon[98640]: osdmap e768: 8 total, 8 up, 8 in 2024-09-14T04:00:08.176 INFO:tasks.rados.rados.0.smithi070.stdout:3516: done (0 left) 2024-09-14T04:00:08.176 INFO:tasks.rados.rados.0.smithi070.stdout:3517: copy_from oid 28 from oid 26 current snap is 353 2024-09-14T04:00:08.176 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 20 current snap is 353 2024-09-14T04:00:08.176 INFO:tasks.rados.rados.0.smithi070.stdout:3518: seq_num 1083 ranges {8445952=540672,8986624=589824,9576448=212992} 2024-09-14T04:00:08.224 INFO:tasks.rados.rados.0.smithi070.stdout:3518: writing smithi07085154-20 from 8445952 to 8986624 tid 1 2024-09-14T04:00:08.228 INFO:tasks.rados.rados.0.smithi070.stdout:3518: writing smithi07085154-20 from 8986624 to 9576448 tid 2 2024-09-14T04:00:08.229 INFO:tasks.rados.rados.0.smithi070.stdout:3518: writing smithi07085154-20 from 9576448 to 9789440 tid 3 2024-09-14T04:00:08.230 INFO:tasks.rados.rados.0.smithi070.stdout:3519: snap_create 2024-09-14T04:00:08.230 INFO:tasks.rados.rados.0.smithi070.stdout:3517: finishing copy_from racing read to smithi07085154-28 2024-09-14T04:00:08.230 INFO:tasks.rados.rados.0.smithi070.stdout:3517: finishing copy_from to smithi07085154-28 2024-09-14T04:00:08.230 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 28 v 4191 (ObjNum 1079 snap 350 seq_num 1079) dirty exists 2024-09-14T04:00:08.231 INFO:tasks.rados.rados.0.smithi070.stdout:3518: finishing write tid 1 to smithi07085154-20 2024-09-14T04:00:08.234 INFO:tasks.rados.rados.0.smithi070.stdout:3518: finishing write tid 2 to smithi07085154-20 2024-09-14T04:00:08.235 INFO:tasks.rados.rados.0.smithi070.stdout:3518: finishing write tid 3 to smithi07085154-20 2024-09-14T04:00:08.235 INFO:tasks.rados.rados.0.smithi070.stdout:3518: finishing write tid 4 to smithi07085154-20 2024-09-14T04:00:08.236 INFO:tasks.rados.rados.0.smithi070.stdout:3518: finishing write tid 5 to smithi07085154-20 2024-09-14T04:00:08.236 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 20 v 4195 (ObjNum 1083 snap 353 seq_num 1083) dirty exists 2024-09-14T04:00:08.236 INFO:tasks.rados.rados.0.smithi070.stdout:3518: left oid 20 (ObjNum 1083 snap 353 seq_num 1083) 2024-09-14T04:00:08.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:08 smithi094 ceph-mon[86597]: osdmap e769: 8 total, 8 up, 8 in 2024-09-14T04:00:08.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:08 smithi094 ceph-mon[86597]: pgmap v754: 121 pgs: 121 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 14 MiB/s rd, 3.9 MiB/s wr, 9 op/s 2024-09-14T04:00:08.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:08 smithi070 ceph-mon[98640]: osdmap e769: 8 total, 8 up, 8 in 2024-09-14T04:00:08.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:08 smithi070 ceph-mon[98640]: pgmap v754: 121 pgs: 121 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 14 MiB/s rd, 3.9 MiB/s wr, 9 op/s 2024-09-14T04:00:08.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:08 smithi070 ceph-mon[96704]: osdmap e769: 8 total, 8 up, 8 in 2024-09-14T04:00:08.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:08 smithi070 ceph-mon[96704]: pgmap v754: 121 pgs: 121 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 14 MiB/s rd, 3.9 MiB/s wr, 9 op/s 2024-09-14T04:00:09.130 INFO:tasks.rados.rados.0.smithi070.stdout:3517: done (2 left) 2024-09-14T04:00:09.131 INFO:tasks.rados.rados.0.smithi070.stdout:3518: done (1 left) 2024-09-14T04:00:09.131 INFO:tasks.rados.rados.0.smithi070.stdout:3519: done (0 left) 2024-09-14T04:00:09.131 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 5 current snap is 354 2024-09-14T04:00:09.131 INFO:tasks.rados.rados.0.smithi070.stdout:3520: seq_num 1084 ranges {2555904=557056,3112960=475136,3588096=327680} 2024-09-14T04:00:09.147 INFO:tasks.rados.rados.0.smithi070.stdout:3520: writing smithi07085154-5 from 2555904 to 3112960 tid 1 2024-09-14T04:00:09.150 INFO:tasks.rados.rados.0.smithi070.stdout:3520: writing smithi07085154-5 from 3112960 to 3588096 tid 2 2024-09-14T04:00:09.152 INFO:tasks.rados.rados.0.smithi070.stdout:3520: writing smithi07085154-5 from 3588096 to 3915776 tid 3 2024-09-14T04:00:09.152 INFO:tasks.rados.rados.0.smithi070.stdout:3521: delete oid 35 current snap is 354 2024-09-14T04:00:09.153 INFO:tasks.rados.rados.0.smithi070.stdout:3520: finishing write tid 1 to smithi07085154-5 2024-09-14T04:00:09.155 INFO:tasks.rados.rados.0.smithi070.stdout:3521: done (1 left) 2024-09-14T04:00:09.155 INFO:tasks.rados.rados.0.smithi070.stdout:3522: rmattr oid 45 current snap is 354 2024-09-14T04:00:09.155 INFO:tasks.rados.rados.0.smithi070.stdout:3522: done (1 left) 2024-09-14T04:00:09.155 INFO:tasks.rados.rados.0.smithi070.stdout:3523: read oid 42 snap -1 2024-09-14T04:00:09.155 INFO:tasks.rados.rados.0.smithi070.stdout:3523: expect (ObjNum 1061 snap 346 seq_num 1061) 2024-09-14T04:00:09.155 INFO:tasks.rados.rados.0.smithi070.stdout:3524: read oid 46 snap -1 2024-09-14T04:00:09.155 INFO:tasks.rados.rados.0.smithi070.stdout:3524: expect deleted 2024-09-14T04:00:09.155 INFO:tasks.rados.rados.0.smithi070.stdout:3525: copy_from oid 25 from oid 1 current snap is 354 2024-09-14T04:00:09.155 INFO:tasks.rados.rados.0.smithi070.stdout:3526: rmattr oid 30 current snap is 354 2024-09-14T04:00:09.155 INFO:tasks.rados.rados.0.smithi070.stdout:3526: done (4 left) 2024-09-14T04:00:09.156 INFO:tasks.rados.rados.0.smithi070.stdout:3527: read oid 14 snap 267 2024-09-14T04:00:09.156 INFO:tasks.rados.rados.0.smithi070.stdout:3527: expect (ObjNum 398 snap 138 seq_num 398) 2024-09-14T04:00:09.156 INFO:tasks.rados.rados.0.smithi070.stdout:3528: delete oid 13 current snap is 354 2024-09-14T04:00:09.156 INFO:tasks.rados.rados.0.smithi070.stdout:3525: finishing copy_from to smithi07085154-25 2024-09-14T04:00:09.156 INFO:tasks.rados.rados.0.smithi070.stdout:3525: got expected ENOENT (src dne) 2024-09-14T04:00:09.157 INFO:tasks.rados.rados.0.smithi070.stdout:3525: finishing copy_from racing read to smithi07085154-25 2024-09-14T04:00:09.157 INFO:tasks.rados.rados.0.smithi070.stdout:3520: finishing write tid 2 to smithi07085154-5 2024-09-14T04:00:09.157 INFO:tasks.rados.rados.0.smithi070.stdout:3520: finishing write tid 3 to smithi07085154-5 2024-09-14T04:00:09.158 INFO:tasks.rados.rados.0.smithi070.stdout:3520: finishing write tid 4 to smithi07085154-5 2024-09-14T04:00:09.159 INFO:tasks.rados.rados.0.smithi070.stdout:3520: finishing write tid 5 to smithi07085154-5 2024-09-14T04:00:09.160 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 3196 (ObjNum 1084 snap 354 seq_num 1084) dirty exists 2024-09-14T04:00:09.160 INFO:tasks.rados.rados.0.smithi070.stdout:3520: left oid 5 (ObjNum 1084 snap 354 seq_num 1084) 2024-09-14T04:00:09.160 INFO:tasks.rados.rados.0.smithi070.stdout:3520: done (5 left) 2024-09-14T04:00:09.160 INFO:tasks.rados.rados.0.smithi070.stdout:3524: done (4 left) 2024-09-14T04:00:09.160 INFO:tasks.rados.rados.0.smithi070.stdout:3525: done (3 left) 2024-09-14T04:00:09.160 INFO:tasks.rados.rados.0.smithi070.stdout:3528: done (2 left) 2024-09-14T04:00:09.160 INFO:tasks.rados.rados.0.smithi070.stdout:3529: rollback oid 18 current snap is 354 2024-09-14T04:00:09.160 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 18 to 213 2024-09-14T04:00:09.160 INFO:tasks.rados.rados.0.smithi070.stdout:3530: snap_remove snap 353 2024-09-14T04:00:09.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:09 smithi094 ceph-mon[86597]: osdmap e770: 8 total, 8 up, 8 in 2024-09-14T04:00:09.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:09 smithi094 ceph-mon[86597]: osdmap e771: 8 total, 8 up, 8 in 2024-09-14T04:00:09.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:09 smithi070 ceph-mon[96704]: osdmap e770: 8 total, 8 up, 8 in 2024-09-14T04:00:09.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:09 smithi070 ceph-mon[96704]: osdmap e771: 8 total, 8 up, 8 in 2024-09-14T04:00:09.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:09 smithi070 ceph-mon[98640]: osdmap e770: 8 total, 8 up, 8 in 2024-09-14T04:00:09.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:09 smithi070 ceph-mon[98640]: osdmap e771: 8 total, 8 up, 8 in 2024-09-14T04:00:10.132 INFO:tasks.rados.rados.0.smithi070.stdout:3530: done (3 left) 2024-09-14T04:00:10.133 INFO:tasks.rados.rados.0.smithi070.stdout:3531: copy_from oid 6 from oid 16 current snap is 354 2024-09-14T04:00:10.133 INFO:tasks.rados.rados.0.smithi070.stdout:3523: expect (ObjNum 1061 snap 346 seq_num 1061) 2024-09-14T04:00:10.224 INFO:tasks.rados.rados.0.smithi070.stdout:3527: expect (ObjNum 398 snap 138 seq_num 398) 2024-09-14T04:00:10.290 INFO:tasks.rados.rados.0.smithi070.stdout:3529: finishing rollback tid 0 to smithi07085154-18 2024-09-14T04:00:10.291 INFO:tasks.rados.rados.0.smithi070.stdout:3529: finishing rollback tid 1 to smithi07085154-18 2024-09-14T04:00:10.291 INFO:tasks.rados.rados.0.smithi070.stdout:3523: done (3 left) 2024-09-14T04:00:10.291 INFO:tasks.rados.rados.0.smithi070.stdout:3527: done (2 left) 2024-09-14T04:00:10.291 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 22 current snap is 354 2024-09-14T04:00:10.291 INFO:tasks.rados.rados.0.smithi070.stdout:3529: finishing rollback tid 2 to smithi07085154-18 2024-09-14T04:00:10.291 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 4124 (ObjNum 596 snap 199 seq_num 596) dirty exists 2024-09-14T04:00:10.291 INFO:tasks.rados.rados.0.smithi070.stdout:3531: finishing copy_from racing read to smithi07085154-6 2024-09-14T04:00:10.291 INFO:tasks.rados.rados.0.smithi070.stdout:3531: finishing copy_from to smithi07085154-6 2024-09-14T04:00:10.291 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 3993 (ObjNum 1039 snap 342 seq_num 1039) dirty exists 2024-09-14T04:00:10.291 INFO:tasks.rados.rados.0.smithi070.stdout:3532: seq_num 1087 ranges {3473408=614400,4087808=532480,4620288=270336} 2024-09-14T04:00:10.311 INFO:tasks.rados.rados.0.smithi070.stdout:3532: writing smithi07085154-22 from 3473408 to 4087808 tid 1 2024-09-14T04:00:10.314 INFO:tasks.rados.rados.0.smithi070.stdout:3532: writing smithi07085154-22 from 4087808 to 4620288 tid 2 2024-09-14T04:00:10.316 INFO:tasks.rados.rados.0.smithi070.stdout:3532: writing smithi07085154-22 from 4620288 to 4890624 tid 3 2024-09-14T04:00:10.316 INFO:tasks.rados.rados.0.smithi070.stdout:3529: done (2 left) 2024-09-14T04:00:10.316 INFO:tasks.rados.rados.0.smithi070.stdout:3531: done (1 left) 2024-09-14T04:00:10.316 INFO:tasks.rados.rados.0.smithi070.stdout:3533: snap_remove snap 349 2024-09-14T04:00:10.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:00:10 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:00:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:00:11.138 INFO:tasks.rados.rados.0.smithi070.stdout:3533: done (1 left) 2024-09-14T04:00:11.139 INFO:tasks.rados.rados.0.smithi070.stdout:3534: rollback oid 42 current snap is 354 2024-09-14T04:00:11.139 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 42 to 352 2024-09-14T04:00:11.139 INFO:tasks.rados.rados.0.smithi070.stdout:3532: finishing write tid 1 to smithi07085154-22 2024-09-14T04:00:11.139 INFO:tasks.rados.rados.0.smithi070.stdout:3532: finishing write tid 2 to smithi07085154-22 2024-09-14T04:00:11.139 INFO:tasks.rados.rados.0.smithi070.stdout:3532: finishing write tid 3 to smithi07085154-22 2024-09-14T04:00:11.139 INFO:tasks.rados.rados.0.smithi070.stdout:3532: finishing write tid 4 to smithi07085154-22 2024-09-14T04:00:11.139 INFO:tasks.rados.rados.0.smithi070.stdout:3532: finishing write tid 5 to smithi07085154-22 2024-09-14T04:00:11.139 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 4197 (ObjNum 1087 snap 354 seq_num 1087) dirty exists 2024-09-14T04:00:11.139 INFO:tasks.rados.rados.0.smithi070.stdout:3532: left oid 22 (ObjNum 1087 snap 354 seq_num 1087) 2024-09-14T04:00:11.139 INFO:tasks.rados.rados.0.smithi070.stdout:3532: done (1 left) 2024-09-14T04:00:11.139 INFO:tasks.rados.rados.0.smithi070.stdout:3535: snap_remove snap 232 2024-09-14T04:00:11.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:11 smithi094 ceph-mon[86597]: osdmap e772: 8 total, 8 up, 8 in 2024-09-14T04:00:11.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:11 smithi094 ceph-mon[86597]: pgmap v758: 121 pgs: 121 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.2 MiB/s rd, 4.0 MiB/s wr, 13 op/s 2024-09-14T04:00:11.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:11 smithi070 ceph-mon[96704]: osdmap e772: 8 total, 8 up, 8 in 2024-09-14T04:00:11.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:11 smithi070 ceph-mon[96704]: pgmap v758: 121 pgs: 121 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.2 MiB/s rd, 4.0 MiB/s wr, 13 op/s 2024-09-14T04:00:11.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:11 smithi070 ceph-mon[98640]: osdmap e772: 8 total, 8 up, 8 in 2024-09-14T04:00:11.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:11 smithi070 ceph-mon[98640]: pgmap v758: 121 pgs: 121 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.2 MiB/s rd, 4.0 MiB/s wr, 13 op/s 2024-09-14T04:00:12.140 INFO:tasks.rados.rados.0.smithi070.stdout:3535: done (1 left) 2024-09-14T04:00:12.140 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 9 current snap is 354 2024-09-14T04:00:12.140 INFO:tasks.rados.rados.0.smithi070.stdout:3536: seq_num 1088 ranges {4218880=630784,4849664=466944,5316608=335872} 2024-09-14T04:00:12.168 INFO:tasks.rados.rados.0.smithi070.stdout:3536: writing smithi07085154-9 from 4218880 to 4849664 tid 1 2024-09-14T04:00:12.171 INFO:tasks.rados.rados.0.smithi070.stdout:3536: writing smithi07085154-9 from 4849664 to 5316608 tid 2 2024-09-14T04:00:12.173 INFO:tasks.rados.rados.0.smithi070.stdout:3536: writing smithi07085154-9 from 5316608 to 5652480 tid 3 2024-09-14T04:00:12.174 INFO:tasks.rados.rados.0.smithi070.stdout:3537: snap_create 2024-09-14T04:00:12.174 INFO:tasks.rados.rados.0.smithi070.stdout:3534: finishing rollback tid 0 to smithi07085154-42 2024-09-14T04:00:12.174 INFO:tasks.rados.rados.0.smithi070.stdout:3534: finishing rollback tid 1 to smithi07085154-42 2024-09-14T04:00:12.174 INFO:tasks.rados.rados.0.smithi070.stdout:3534: finishing rollback tid 2 to smithi07085154-42 2024-09-14T04:00:12.174 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 4201 (ObjNum 1061 snap 346 seq_num 1061) dirty exists 2024-09-14T04:00:12.180 INFO:tasks.rados.rados.0.smithi070.stdout:3536: finishing write tid 1 to smithi07085154-9 2024-09-14T04:00:12.180 INFO:tasks.rados.rados.0.smithi070.stdout:3536: finishing write tid 2 to smithi07085154-9 2024-09-14T04:00:12.184 INFO:tasks.rados.rados.0.smithi070.stdout:3536: finishing write tid 3 to smithi07085154-9 2024-09-14T04:00:12.184 INFO:tasks.rados.rados.0.smithi070.stdout:3536: finishing write tid 4 to smithi07085154-9 2024-09-14T04:00:12.187 INFO:tasks.rados.rados.0.smithi070.stdout:3536: finishing write tid 5 to smithi07085154-9 2024-09-14T04:00:12.187 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 4220 (ObjNum 1088 snap 354 seq_num 1088) dirty exists 2024-09-14T04:00:12.187 INFO:tasks.rados.rados.0.smithi070.stdout:3536: left oid 9 (ObjNum 1088 snap 354 seq_num 1088) 2024-09-14T04:00:12.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:12 smithi094 ceph-mon[86597]: osdmap e773: 8 total, 8 up, 8 in 2024-09-14T04:00:12.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:12 smithi070 ceph-mon[96704]: osdmap e773: 8 total, 8 up, 8 in 2024-09-14T04:00:12.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:12 smithi070 ceph-mon[98640]: osdmap e773: 8 total, 8 up, 8 in 2024-09-14T04:00:13.151 INFO:tasks.rados.rados.0.smithi070.stdout:3534: done (2 left) 2024-09-14T04:00:13.151 INFO:tasks.rados.rados.0.smithi070.stdout:3536: done (1 left) 2024-09-14T04:00:13.151 INFO:tasks.rados.rados.0.smithi070.stdout:3537: done (0 left) 2024-09-14T04:00:13.151 INFO:tasks.rados.rados.0.smithi070.stdout:3538: read oid 31 snap -1 2024-09-14T04:00:13.151 INFO:tasks.rados.rados.0.smithi070.stdout:3538: expect (ObjNum 1069 snap 347 seq_num 1069) 2024-09-14T04:00:13.152 INFO:tasks.rados.rados.0.smithi070.stdout:3539: read oid 45 snap -1 2024-09-14T04:00:13.152 INFO:tasks.rados.rados.0.smithi070.stdout:3539: expect (ObjNum 997 snap 330 seq_num 997) 2024-09-14T04:00:13.152 INFO:tasks.rados.rados.0.smithi070.stdout:3540: read oid 38 snap 332 2024-09-14T04:00:13.152 INFO:tasks.rados.rados.0.smithi070.stdout:3540: expect deleted 2024-09-14T04:00:13.152 INFO:tasks.rados.rados.0.smithi070.stdout:3541: delete oid 32 current snap is 355 2024-09-14T04:00:13.157 INFO:tasks.rados.rados.0.smithi070.stdout:3540: done (3 left) 2024-09-14T04:00:13.157 INFO:tasks.rados.rados.0.smithi070.stdout:3541: done (2 left) 2024-09-14T04:00:13.157 INFO:tasks.rados.rados.0.smithi070.stdout:3542: copy_from oid 40 from oid 26 current snap is 355 2024-09-14T04:00:13.157 INFO:tasks.rados.rados.0.smithi070.stdout:3543: snap_remove snap 236 2024-09-14T04:00:13.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:13 smithi070 ceph-mon[96704]: osdmap e774: 8 total, 8 up, 8 in 2024-09-14T04:00:13.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:13 smithi070 ceph-mon[96704]: pgmap v761: 121 pgs: 121 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.9 MiB/s wr, 13 op/s 2024-09-14T04:00:13.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:13 smithi070 ceph-mon[98640]: osdmap e774: 8 total, 8 up, 8 in 2024-09-14T04:00:13.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:13 smithi070 ceph-mon[98640]: pgmap v761: 121 pgs: 121 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.9 MiB/s wr, 13 op/s 2024-09-14T04:00:13.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:13 smithi094 ceph-mon[86597]: osdmap e774: 8 total, 8 up, 8 in 2024-09-14T04:00:13.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:13 smithi094 ceph-mon[86597]: pgmap v761: 121 pgs: 121 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.9 MiB/s wr, 13 op/s 2024-09-14T04:00:13.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:00:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:00:13] "GET /metrics HTTP/1.1" 200 36502 "" "Prometheus/2.51.0" 2024-09-14T04:00:14.131 INFO:tasks.rados.rados.0.smithi070.stdout:3543: done (3 left) 2024-09-14T04:00:14.131 INFO:tasks.rados.rados.0.smithi070.stdout:3544: snap_remove snap 325 2024-09-14T04:00:14.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:14 smithi094 ceph-mon[86597]: osdmap e775: 8 total, 8 up, 8 in 2024-09-14T04:00:14.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:14 smithi094 ceph-mon[86597]: osdmap e776: 8 total, 8 up, 8 in 2024-09-14T04:00:14.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:14 smithi070 ceph-mon[96704]: osdmap e775: 8 total, 8 up, 8 in 2024-09-14T04:00:14.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:14 smithi070 ceph-mon[96704]: osdmap e776: 8 total, 8 up, 8 in 2024-09-14T04:00:14.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:14 smithi070 ceph-mon[98640]: osdmap e775: 8 total, 8 up, 8 in 2024-09-14T04:00:14.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:14 smithi070 ceph-mon[98640]: osdmap e776: 8 total, 8 up, 8 in 2024-09-14T04:00:15.135 INFO:tasks.rados.rados.0.smithi070.stdout:3538: expect (ObjNum 1069 snap 347 seq_num 1069) 2024-09-14T04:00:15.246 INFO:tasks.rados.rados.0.smithi070.stdout:3542: finishing copy_from racing read to smithi07085154-40 2024-09-14T04:00:15.246 INFO:tasks.rados.rados.0.smithi070.stdout:3542: finishing copy_from to smithi07085154-40 2024-09-14T04:00:15.246 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 4191 (ObjNum 1079 snap 350 seq_num 1079) dirty exists 2024-09-14T04:00:15.246 INFO:tasks.rados.rados.0.smithi070.stdout:3539: expect (ObjNum 997 snap 330 seq_num 997) 2024-09-14T04:00:15.515 INFO:tasks.rados.rados.0.smithi070.stdout:3538: done (3 left) 2024-09-14T04:00:15.515 INFO:tasks.rados.rados.0.smithi070.stdout:3539: done (2 left) 2024-09-14T04:00:15.515 INFO:tasks.rados.rados.0.smithi070.stdout:3542: done (1 left) 2024-09-14T04:00:15.515 INFO:tasks.rados.rados.0.smithi070.stdout:3544: done (0 left) 2024-09-14T04:00:15.516 INFO:tasks.rados.rados.0.smithi070.stdout:3545: rollback oid 42 current snap is 355 2024-09-14T04:00:15.516 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 42 to 350 2024-09-14T04:00:15.516 INFO:tasks.rados.rados.0.smithi070.stdout:3546: copy_from oid 23 from oid 29 current snap is 355 2024-09-14T04:00:15.516 INFO:tasks.rados.rados.0.smithi070.stdout:3547: read oid 17 snap 351 2024-09-14T04:00:15.516 INFO:tasks.rados.rados.0.smithi070.stdout:3547: expect (ObjNum 1038 snap 342 seq_num 1038) 2024-09-14T04:00:15.516 INFO:tasks.rados.rados.0.smithi070.stdout:3548: read oid 11 snap -1 2024-09-14T04:00:15.516 INFO:tasks.rados.rados.0.smithi070.stdout:3548: expect deleted 2024-09-14T04:00:15.516 INFO:tasks.rados.rados.0.smithi070.stdout:3549: delete oid 35 current snap is 355 2024-09-14T04:00:15.517 INFO:tasks.rados.rados.0.smithi070.stdout:3548: done (4 left) 2024-09-14T04:00:15.517 INFO:tasks.rados.rados.0.smithi070.stdout:3549: done (3 left) 2024-09-14T04:00:15.517 INFO:tasks.rados.rados.0.smithi070.stdout:3550: copy_from oid 38 from oid 7 current snap is 355 2024-09-14T04:00:15.517 INFO:tasks.rados.rados.0.smithi070.stdout:3551: setattr oid 43 current snap is 355 2024-09-14T04:00:15.518 INFO:tasks.rados.rados.0.smithi070.stdout:3550: finishing copy_from to smithi07085154-38 2024-09-14T04:00:15.518 INFO:tasks.rados.rados.0.smithi070.stdout:3550: got expected ENOENT (src dne) 2024-09-14T04:00:15.518 INFO:tasks.rados.rados.0.smithi070.stdout:3552: copy_from oid 1 from oid 15 current snap is 355 2024-09-14T04:00:15.518 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 21 current snap is 355 2024-09-14T04:00:15.518 INFO:tasks.rados.rados.0.smithi070.stdout:3553: seq_num 1091 ranges {1859584=679936,2539520=516096,3055616=286720} 2024-09-14T04:00:15.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:15 smithi094 ceph-mon[86597]: pgmap v764: 121 pgs: 121 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.4 MiB/s rd, 1.7 MiB/s wr, 9 op/s 2024-09-14T04:00:15.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:15 smithi094 ceph-mon[86597]: osdmap e777: 8 total, 8 up, 8 in 2024-09-14T04:00:15.532 INFO:tasks.rados.rados.0.smithi070.stdout:3553: writing smithi07085154-21 from 1859584 to 2539520 tid 1 2024-09-14T04:00:15.535 INFO:tasks.rados.rados.0.smithi070.stdout:3553: writing smithi07085154-21 from 2539520 to 3055616 tid 2 2024-09-14T04:00:15.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:15 smithi070 ceph-mon[96704]: pgmap v764: 121 pgs: 121 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.4 MiB/s rd, 1.7 MiB/s wr, 9 op/s 2024-09-14T04:00:15.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:15 smithi070 ceph-mon[96704]: osdmap e777: 8 total, 8 up, 8 in 2024-09-14T04:00:15.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:15 smithi070 ceph-mon[98640]: pgmap v764: 121 pgs: 121 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.4 MiB/s rd, 1.7 MiB/s wr, 9 op/s 2024-09-14T04:00:15.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:15 smithi070 ceph-mon[98640]: osdmap e777: 8 total, 8 up, 8 in 2024-09-14T04:00:15.537 INFO:tasks.rados.rados.0.smithi070.stdout:3553: writing smithi07085154-21 from 3055616 to 3342336 tid 3 2024-09-14T04:00:15.537 INFO:tasks.rados.rados.0.smithi070.stdout:3554: snap_create 2024-09-14T04:00:15.537 INFO:tasks.rados.rados.0.smithi070.stdout:3550: finishing copy_from racing read to smithi07085154-38 2024-09-14T04:00:15.537 INFO:tasks.rados.rados.0.smithi070.stdout:3547: expect (ObjNum 1038 snap 342 seq_num 1038) 2024-09-14T04:00:15.556 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 4125 (ObjNum 1047 snap 344 seq_num 1047) dirty exists 2024-09-14T04:00:15.556 INFO:tasks.rados.rados.0.smithi070.stdout:3545: finishing rollback tid 0 to smithi07085154-42 2024-09-14T04:00:15.556 INFO:tasks.rados.rados.0.smithi070.stdout:3545: finishing rollback tid 1 to smithi07085154-42 2024-09-14T04:00:15.557 INFO:tasks.rados.rados.0.smithi070.stdout:3545: finishing rollback tid 2 to smithi07085154-42 2024-09-14T04:00:15.557 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 4254 (ObjNum 1061 snap 346 seq_num 1061) dirty exists 2024-09-14T04:00:15.557 INFO:tasks.rados.rados.0.smithi070.stdout:3552: finishing copy_from racing read to smithi07085154-1 2024-09-14T04:00:15.557 INFO:tasks.rados.rados.0.smithi070.stdout:3552: finishing copy_from to smithi07085154-1 2024-09-14T04:00:15.557 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 1 v 3185 (ObjNum 805 snap 281 seq_num 805) dirty exists 2024-09-14T04:00:15.557 INFO:tasks.rados.rados.0.smithi070.stdout:3553: finishing write tid 1 to smithi07085154-21 2024-09-14T04:00:15.557 INFO:tasks.rados.rados.0.smithi070.stdout:3553: finishing write tid 2 to smithi07085154-21 2024-09-14T04:00:15.557 INFO:tasks.rados.rados.0.smithi070.stdout:3546: finishing copy_from racing read to smithi07085154-23 2024-09-14T04:00:15.557 INFO:tasks.rados.rados.0.smithi070.stdout:3553: finishing write tid 3 to smithi07085154-21 2024-09-14T04:00:15.557 INFO:tasks.rados.rados.0.smithi070.stdout:3553: finishing write tid 4 to smithi07085154-21 2024-09-14T04:00:15.557 INFO:tasks.rados.rados.0.smithi070.stdout:3546: finishing copy_from to smithi07085154-23 2024-09-14T04:00:15.557 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 3248 (ObjNum 1078 snap 349 seq_num 1078) dirty exists 2024-09-14T04:00:15.557 INFO:tasks.rados.rados.0.smithi070.stdout:3553: finishing write tid 5 to smithi07085154-21 2024-09-14T04:00:15.557 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 1725 (ObjNum 1091 snap 355 seq_num 1091) dirty exists 2024-09-14T04:00:15.558 INFO:tasks.rados.rados.0.smithi070.stdout:3553: left oid 21 (ObjNum 1091 snap 355 seq_num 1091) 2024-09-14T04:00:16.160 INFO:tasks.rados.rados.0.smithi070.stdout:3545: done (7 left) 2024-09-14T04:00:16.160 INFO:tasks.rados.rados.0.smithi070.stdout:3546: done (6 left) 2024-09-14T04:00:16.160 INFO:tasks.rados.rados.0.smithi070.stdout:3547: done (5 left) 2024-09-14T04:00:16.160 INFO:tasks.rados.rados.0.smithi070.stdout:3550: done (4 left) 2024-09-14T04:00:16.160 INFO:tasks.rados.rados.0.smithi070.stdout:3551: done (3 left) 2024-09-14T04:00:16.161 INFO:tasks.rados.rados.0.smithi070.stdout:3552: done (2 left) 2024-09-14T04:00:16.161 INFO:tasks.rados.rados.0.smithi070.stdout:3553: done (1 left) 2024-09-14T04:00:16.161 INFO:tasks.rados.rados.0.smithi070.stdout:3554: done (0 left) 2024-09-14T04:00:16.161 INFO:tasks.rados.rados.0.smithi070.stdout:3555: read oid 10 snap -1 2024-09-14T04:00:16.161 INFO:tasks.rados.rados.0.smithi070.stdout:3555: expect (ObjNum 841 snap 286 seq_num 841) 2024-09-14T04:00:16.161 INFO:tasks.rados.rados.0.smithi070.stdout:3556: snap_create 2024-09-14T04:00:16.190 INFO:tasks.rados.rados.0.smithi070.stdout:3555: expect (ObjNum 841 snap 286 seq_num 841) 2024-09-14T04:00:17.161 INFO:tasks.rados.rados.0.smithi070.stdout:3555: done (1 left) 2024-09-14T04:00:17.162 INFO:tasks.rados.rados.0.smithi070.stdout:3556: done (0 left) 2024-09-14T04:00:17.162 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 42 current snap is 357 2024-09-14T04:00:17.162 INFO:tasks.rados.rados.0.smithi070.stdout:3557: seq_num 1092 ranges {2342912=696320,3039232=802816} 2024-09-14T04:00:17.177 INFO:tasks.rados.rados.0.smithi070.stdout:3557: writing smithi07085154-42 from 2342912 to 3039232 tid 1 2024-09-14T04:00:17.182 INFO:tasks.rados.rados.0.smithi070.stdout:3557: writing smithi07085154-42 from 3039232 to 3842048 tid 2 2024-09-14T04:00:17.182 INFO:tasks.rados.rados.0.smithi070.stdout:3558: setattr oid 12 current snap is 357 2024-09-14T04:00:17.184 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 25 current snap is 357 2024-09-14T04:00:17.184 INFO:tasks.rados.rados.0.smithi070.stdout:3559: seq_num 1093 ranges {1146880=712704,1859584=688128,2547712=114688} 2024-09-14T04:00:17.194 INFO:tasks.rados.rados.0.smithi070.stdout:3559: writing smithi07085154-25 from 1146880 to 1859584 tid 1 2024-09-14T04:00:17.198 INFO:tasks.rados.rados.0.smithi070.stdout:3559: writing smithi07085154-25 from 1859584 to 2547712 tid 2 2024-09-14T04:00:17.199 INFO:tasks.rados.rados.0.smithi070.stdout:3559: writing smithi07085154-25 from 2547712 to 2662400 tid 3 2024-09-14T04:00:17.199 INFO:tasks.rados.rados.0.smithi070.stdout:3560: read oid 23 snap -1 2024-09-14T04:00:17.199 INFO:tasks.rados.rados.0.smithi070.stdout:3560: expect (ObjNum 1078 snap 349 seq_num 1078) 2024-09-14T04:00:17.199 INFO:tasks.rados.rados.0.smithi070.stdout:3561: snap_create 2024-09-14T04:00:17.199 INFO:tasks.rados.rados.0.smithi070.stdout:3557: finishing write tid 1 to smithi07085154-42 2024-09-14T04:00:17.199 INFO:tasks.rados.rados.0.smithi070.stdout:3557: finishing write tid 2 to smithi07085154-42 2024-09-14T04:00:17.200 INFO:tasks.rados.rados.0.smithi070.stdout:3557: finishing write tid 3 to smithi07085154-42 2024-09-14T04:00:17.200 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 2273 (ObjNum 1059 snap 345 seq_num 1059) dirty exists 2024-09-14T04:00:17.200 INFO:tasks.rados.rados.0.smithi070.stdout:3557: finishing write tid 4 to smithi07085154-42 2024-09-14T04:00:17.200 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 4258 (ObjNum 1092 snap 357 seq_num 1092) dirty exists 2024-09-14T04:00:17.200 INFO:tasks.rados.rados.0.smithi070.stdout:3557: left oid 42 (ObjNum 1092 snap 357 seq_num 1092) 2024-09-14T04:00:17.206 INFO:tasks.rados.rados.0.smithi070.stdout:3559: finishing write tid 1 to smithi07085154-25 2024-09-14T04:00:17.208 INFO:tasks.rados.rados.0.smithi070.stdout:3559: finishing write tid 2 to smithi07085154-25 2024-09-14T04:00:17.208 INFO:tasks.rados.rados.0.smithi070.stdout:3559: finishing write tid 3 to smithi07085154-25 2024-09-14T04:00:17.208 INFO:tasks.rados.rados.0.smithi070.stdout:3559: finishing write tid 4 to smithi07085154-25 2024-09-14T04:00:17.211 INFO:tasks.rados.rados.0.smithi070.stdout:3559: finishing write tid 5 to smithi07085154-25 2024-09-14T04:00:17.211 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 3997 (ObjNum 1093 snap 357 seq_num 1093) dirty exists 2024-09-14T04:00:17.211 INFO:tasks.rados.rados.0.smithi070.stdout:3559: left oid 25 (ObjNum 1093 snap 357 seq_num 1093) 2024-09-14T04:00:17.226 INFO:tasks.rados.rados.0.smithi070.stdout:3560: expect (ObjNum 1078 snap 349 seq_num 1078) 2024-09-14T04:00:17.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:17 smithi094 ceph-mon[86597]: pgmap v766: 121 pgs: 121 active+clean; 1007 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.8 MiB/s rd, 459 KiB/s wr, 3 op/s 2024-09-14T04:00:17.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:17 smithi094 ceph-mon[86597]: osdmap e778: 8 total, 8 up, 8 in 2024-09-14T04:00:17.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:17 smithi070 ceph-mon[96704]: pgmap v766: 121 pgs: 121 active+clean; 1007 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.8 MiB/s rd, 459 KiB/s wr, 3 op/s 2024-09-14T04:00:17.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:17 smithi070 ceph-mon[96704]: osdmap e778: 8 total, 8 up, 8 in 2024-09-14T04:00:17.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:17 smithi070 ceph-mon[98640]: pgmap v766: 121 pgs: 121 active+clean; 1007 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.8 MiB/s rd, 459 KiB/s wr, 3 op/s 2024-09-14T04:00:17.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:17 smithi070 ceph-mon[98640]: osdmap e778: 8 total, 8 up, 8 in 2024-09-14T04:00:18.171 INFO:tasks.rados.rados.0.smithi070.stdout:3557: done (4 left) 2024-09-14T04:00:18.171 INFO:tasks.rados.rados.0.smithi070.stdout:3558: done (3 left) 2024-09-14T04:00:18.171 INFO:tasks.rados.rados.0.smithi070.stdout:3559: done (2 left) 2024-09-14T04:00:18.171 INFO:tasks.rados.rados.0.smithi070.stdout:3560: done (1 left) 2024-09-14T04:00:18.171 INFO:tasks.rados.rados.0.smithi070.stdout:3561: done (0 left) 2024-09-14T04:00:18.171 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 34 current snap is 358 2024-09-14T04:00:18.171 INFO:tasks.rados.rados.0.smithi070.stdout:3562: seq_num 1094 ranges {8323072=729088,9052160=573440,9625600=229376} 2024-09-14T04:00:18.218 INFO:tasks.rados.rados.0.smithi070.stdout:3562: writing smithi07085154-34 from 8323072 to 9052160 tid 1 2024-09-14T04:00:18.221 INFO:tasks.rados.rados.0.smithi070.stdout:3562: writing smithi07085154-34 from 9052160 to 9625600 tid 2 2024-09-14T04:00:18.223 INFO:tasks.rados.rados.0.smithi070.stdout:3562: writing smithi07085154-34 from 9625600 to 9854976 tid 3 2024-09-14T04:00:18.223 INFO:tasks.rados.rados.0.smithi070.stdout:3563: read oid 46 snap -1 2024-09-14T04:00:18.223 INFO:tasks.rados.rados.0.smithi070.stdout:3563: expect deleted 2024-09-14T04:00:18.223 INFO:tasks.rados.rados.0.smithi070.stdout:3564: read oid 22 snap -1 2024-09-14T04:00:18.223 INFO:tasks.rados.rados.0.smithi070.stdout:3564: expect (ObjNum 1087 snap 354 seq_num 1087) 2024-09-14T04:00:18.223 INFO:tasks.rados.rados.0.smithi070.stdout:3565: copy_from oid 44 from oid 9 current snap is 358 2024-09-14T04:00:18.224 INFO:tasks.rados.rados.0.smithi070.stdout:3566: rmattr oid 5 current snap is 358 2024-09-14T04:00:18.224 INFO:tasks.rados.rados.0.smithi070.stdout:3567: delete oid 28 current snap is 358 2024-09-14T04:00:18.229 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 3197 (ObjNum 1084 snap 354 seq_num 1084) dirty exists 2024-09-14T04:00:18.229 INFO:tasks.rados.rados.0.smithi070.stdout:3566: done (5 left) 2024-09-14T04:00:18.229 INFO:tasks.rados.rados.0.smithi070.stdout:3567: done (4 left) 2024-09-14T04:00:18.229 INFO:tasks.rados.rados.0.smithi070.stdout:3568: delete oid 12 current snap is 358 2024-09-14T04:00:18.230 INFO:tasks.rados.rados.0.smithi070.stdout:3562: finishing write tid 1 to smithi07085154-34 2024-09-14T04:00:18.231 INFO:tasks.rados.rados.0.smithi070.stdout:3562: finishing write tid 2 to smithi07085154-34 2024-09-14T04:00:18.233 INFO:tasks.rados.rados.0.smithi070.stdout:3562: finishing write tid 3 to smithi07085154-34 2024-09-14T04:00:18.234 INFO:tasks.rados.rados.0.smithi070.stdout:3563: done (4 left) 2024-09-14T04:00:18.234 INFO:tasks.rados.rados.0.smithi070.stdout:3568: done (3 left) 2024-09-14T04:00:18.234 INFO:tasks.rados.rados.0.smithi070.stdout:3569: snap_create 2024-09-14T04:00:18.237 INFO:tasks.rados.rados.0.smithi070.stdout:3562: finishing write tid 4 to smithi07085154-34 2024-09-14T04:00:18.255 INFO:tasks.rados.rados.0.smithi070.stdout:3565: finishing copy_from racing read to smithi07085154-44 2024-09-14T04:00:18.259 INFO:tasks.rados.rados.0.smithi070.stdout:3564: expect (ObjNum 1087 snap 354 seq_num 1087) 2024-09-14T04:00:18.473 INFO:tasks.rados.rados.0.smithi070.stdout:3562: finishing write tid 5 to smithi07085154-34 2024-09-14T04:00:18.473 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 4201 (ObjNum 1094 snap 358 seq_num 1094) dirty exists 2024-09-14T04:00:18.473 INFO:tasks.rados.rados.0.smithi070.stdout:3562: left oid 34 (ObjNum 1094 snap 358 seq_num 1094) 2024-09-14T04:00:18.473 INFO:tasks.rados.rados.0.smithi070.stdout:3565: finishing copy_from to smithi07085154-44 2024-09-14T04:00:18.473 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 4220 (ObjNum 1088 snap 354 seq_num 1088) dirty exists 2024-09-14T04:00:18.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:18 smithi094 ceph-mon[86597]: osdmap e779: 8 total, 8 up, 8 in 2024-09-14T04:00:18.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:18 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:00:18.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:18 smithi070 ceph-mon[98640]: osdmap e779: 8 total, 8 up, 8 in 2024-09-14T04:00:18.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:18 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:00:18.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:18 smithi070 ceph-mon[96704]: osdmap e779: 8 total, 8 up, 8 in 2024-09-14T04:00:18.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:18 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:00:19.133 INFO:tasks.rados.rados.0.smithi070.stdout:3562: done (3 left) 2024-09-14T04:00:19.133 INFO:tasks.rados.rados.0.smithi070.stdout:3564: done (2 left) 2024-09-14T04:00:19.133 INFO:tasks.rados.rados.0.smithi070.stdout:3565: done (1 left) 2024-09-14T04:00:19.133 INFO:tasks.rados.rados.0.smithi070.stdout:3569: done (0 left) 2024-09-14T04:00:19.133 INFO:tasks.rados.rados.0.smithi070.stdout:3570: snap_remove snap 345 2024-09-14T04:00:19.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:19 smithi094 ceph-mon[86597]: pgmap v769: 121 pgs: 121 active+clean; 935 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-14T04:00:19.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:19 smithi094 ceph-mon[86597]: osdmap e780: 8 total, 8 up, 8 in 2024-09-14T04:00:19.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:19 smithi094 ceph-mon[86597]: osdmap e781: 8 total, 8 up, 8 in 2024-09-14T04:00:19.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:19 smithi070 ceph-mon[96704]: pgmap v769: 121 pgs: 121 active+clean; 935 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-14T04:00:19.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:19 smithi070 ceph-mon[96704]: osdmap e780: 8 total, 8 up, 8 in 2024-09-14T04:00:19.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:19 smithi070 ceph-mon[96704]: osdmap e781: 8 total, 8 up, 8 in 2024-09-14T04:00:19.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:19 smithi070 ceph-mon[98640]: pgmap v769: 121 pgs: 121 active+clean; 935 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2024-09-14T04:00:19.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:19 smithi070 ceph-mon[98640]: osdmap e780: 8 total, 8 up, 8 in 2024-09-14T04:00:19.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:19 smithi070 ceph-mon[98640]: osdmap e781: 8 total, 8 up, 8 in 2024-09-14T04:00:20.136 INFO:tasks.rados.rados.0.smithi070.stdout:3570: done (0 left) 2024-09-14T04:00:20.136 INFO:tasks.rados.rados.0.smithi070.stdout:3571: snap_create 2024-09-14T04:00:20.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:00:20 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:00:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:00:21.140 INFO:tasks.rados.rados.0.smithi070.stdout:3571: done (0 left) 2024-09-14T04:00:21.140 INFO:tasks.rados.rados.0.smithi070.stdout:3572: copy_from oid 13 from oid 23 current snap is 360 2024-09-14T04:00:21.140 INFO:tasks.rados.rados.0.smithi070.stdout:3573: delete oid 7 current snap is 360 2024-09-14T04:00:21.144 INFO:tasks.rados.rados.0.smithi070.stdout:3573: done (1 left) 2024-09-14T04:00:21.144 INFO:tasks.rados.rados.0.smithi070.stdout:3574: snap_create 2024-09-14T04:00:21.168 INFO:tasks.rados.rados.0.smithi070.stdout:3572: finishing copy_from racing read to smithi07085154-13 2024-09-14T04:00:21.171 INFO:tasks.rados.rados.0.smithi070.stdout:3572: finishing copy_from to smithi07085154-13 2024-09-14T04:00:21.171 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 3248 (ObjNum 1078 snap 349 seq_num 1078) dirty exists 2024-09-14T04:00:21.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:21 smithi094 ceph-mon[86597]: osdmap e782: 8 total, 8 up, 8 in 2024-09-14T04:00:21.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:21 smithi094 ceph-mon[86597]: pgmap v773: 121 pgs: 121 active+clean; 887 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.4 MiB/s wr, 17 op/s 2024-09-14T04:00:21.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:21 smithi070 ceph-mon[96704]: osdmap e782: 8 total, 8 up, 8 in 2024-09-14T04:00:21.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:21 smithi070 ceph-mon[96704]: pgmap v773: 121 pgs: 121 active+clean; 887 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.4 MiB/s wr, 17 op/s 2024-09-14T04:00:21.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:21 smithi070 ceph-mon[98640]: osdmap e782: 8 total, 8 up, 8 in 2024-09-14T04:00:21.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:21 smithi070 ceph-mon[98640]: pgmap v773: 121 pgs: 121 active+clean; 887 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.4 MiB/s wr, 17 op/s 2024-09-14T04:00:22.142 INFO:tasks.rados.rados.0.smithi070.stdout:3572: done (1 left) 2024-09-14T04:00:22.142 INFO:tasks.rados.rados.0.smithi070.stdout:3574: done (0 left) 2024-09-14T04:00:22.142 INFO:tasks.rados.rados.0.smithi070.stdout:3575: snap_create 2024-09-14T04:00:22.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:22 smithi094 ceph-mon[86597]: osdmap e783: 8 total, 8 up, 8 in 2024-09-14T04:00:22.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:22 smithi070 ceph-mon[96704]: osdmap e783: 8 total, 8 up, 8 in 2024-09-14T04:00:22.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:22 smithi070 ceph-mon[98640]: osdmap e783: 8 total, 8 up, 8 in 2024-09-14T04:00:23.151 INFO:tasks.rados.rados.0.smithi070.stdout:3575: done (0 left) 2024-09-14T04:00:23.152 INFO:tasks.rados.rados.0.smithi070.stdout:3576: snap_create 2024-09-14T04:00:23.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:23 smithi070 ceph-mon[96704]: osdmap e784: 8 total, 8 up, 8 in 2024-09-14T04:00:23.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:23 smithi070 ceph-mon[96704]: pgmap v776: 121 pgs: 121 active+clean; 893 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.8 MiB/s rd, 2.7 MiB/s wr, 19 op/s 2024-09-14T04:00:23.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:23 smithi070 ceph-mon[98640]: osdmap e784: 8 total, 8 up, 8 in 2024-09-14T04:00:23.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:23 smithi070 ceph-mon[98640]: pgmap v776: 121 pgs: 121 active+clean; 893 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.8 MiB/s rd, 2.7 MiB/s wr, 19 op/s 2024-09-14T04:00:23.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:23 smithi094 ceph-mon[86597]: osdmap e784: 8 total, 8 up, 8 in 2024-09-14T04:00:23.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:23 smithi094 ceph-mon[86597]: pgmap v776: 121 pgs: 121 active+clean; 893 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.8 MiB/s rd, 2.7 MiB/s wr, 19 op/s 2024-09-14T04:00:23.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:00:23 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:00:23] "GET /metrics HTTP/1.1" 200 36502 "" "Prometheus/2.51.0" 2024-09-14T04:00:24.134 INFO:tasks.rados.rados.0.smithi070.stdout:3576: done (0 left) 2024-09-14T04:00:24.134 INFO:tasks.rados.rados.0.smithi070.stdout:3577: snap_create 2024-09-14T04:00:24.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:24 smithi094 ceph-mon[86597]: osdmap e785: 8 total, 8 up, 8 in 2024-09-14T04:00:24.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:24 smithi094 ceph-mon[86597]: osdmap e786: 8 total, 8 up, 8 in 2024-09-14T04:00:24.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:24 smithi070 ceph-mon[98640]: osdmap e785: 8 total, 8 up, 8 in 2024-09-14T04:00:24.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:24 smithi070 ceph-mon[98640]: osdmap e786: 8 total, 8 up, 8 in 2024-09-14T04:00:24.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:24 smithi070 ceph-mon[96704]: osdmap e785: 8 total, 8 up, 8 in 2024-09-14T04:00:24.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:24 smithi070 ceph-mon[96704]: osdmap e786: 8 total, 8 up, 8 in 2024-09-14T04:00:25.139 INFO:tasks.rados.rados.0.smithi070.stdout:3577: done (0 left) 2024-09-14T04:00:25.139 INFO:tasks.rados.rados.0.smithi070.stdout:3578: read oid 3 snap 348 2024-09-14T04:00:25.139 INFO:tasks.rados.rados.0.smithi070.stdout:3578: expect (ObjNum 1072 snap 347 seq_num 1072) 2024-09-14T04:00:25.139 INFO:tasks.rados.rados.0.smithi070.stdout:3579: snap_create 2024-09-14T04:00:25.149 INFO:tasks.rados.rados.0.smithi070.stdout:3578: expect (ObjNum 1072 snap 347 seq_num 1072) 2024-09-14T04:00:25.529 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:25 smithi094 ceph-mon[86597]: pgmap v779: 121 pgs: 121 active+clean; 898 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.3 MiB/s rd, 3.3 MiB/s wr, 7 op/s 2024-09-14T04:00:25.529 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:25 smithi094 ceph-mon[86597]: osdmap e787: 8 total, 8 up, 8 in 2024-09-14T04:00:25.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:25 smithi070 ceph-mon[98640]: pgmap v779: 121 pgs: 121 active+clean; 898 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.3 MiB/s rd, 3.3 MiB/s wr, 7 op/s 2024-09-14T04:00:25.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:25 smithi070 ceph-mon[98640]: osdmap e787: 8 total, 8 up, 8 in 2024-09-14T04:00:25.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:25 smithi070 ceph-mon[96704]: pgmap v779: 121 pgs: 121 active+clean; 898 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.3 MiB/s rd, 3.3 MiB/s wr, 7 op/s 2024-09-14T04:00:25.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:25 smithi070 ceph-mon[96704]: osdmap e787: 8 total, 8 up, 8 in 2024-09-14T04:00:26.141 INFO:tasks.rados.rados.0.smithi070.stdout:3578: done (1 left) 2024-09-14T04:00:26.141 INFO:tasks.rados.rados.0.smithi070.stdout:3579: done (0 left) 2024-09-14T04:00:26.141 INFO:tasks.rados.rados.0.smithi070.stdout:3580: read oid 48 snap 337 2024-09-14T04:00:26.142 INFO:tasks.rados.rados.0.smithi070.stdout:3580: expect (ObjNum 1001 snap 331 seq_num 1001) 2024-09-14T04:00:26.142 INFO:tasks.rados.rados.0.smithi070.stdout:3581: read oid 39 snap 355 2024-09-14T04:00:26.142 INFO:tasks.rados.rados.0.smithi070.stdout:3581: expect (ObjNum 1020 snap 336 seq_num 1020) 2024-09-14T04:00:26.142 INFO:tasks.rados.rados.0.smithi070.stdout:3582: delete oid 36 current snap is 365 2024-09-14T04:00:26.145 INFO:tasks.rados.rados.0.smithi070.stdout:3582: done (2 left) 2024-09-14T04:00:26.145 INFO:tasks.rados.rados.0.smithi070.stdout:3583: snap_remove snap 334 2024-09-14T04:00:27.149 INFO:tasks.rados.rados.0.smithi070.stdout:3583: done (2 left) 2024-09-14T04:00:27.149 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 38 current snap is 365 2024-09-14T04:00:27.149 INFO:tasks.rados.rados.0.smithi070.stdout:3584: seq_num 1099 ranges {1130496=409600,1540096=442368,1982464=720896,2703360=40960} 2024-09-14T04:00:27.157 INFO:tasks.rados.rados.0.smithi070.stdout:3584: writing smithi07085154-38 from 1130496 to 1540096 tid 1 2024-09-14T04:00:27.160 INFO:tasks.rados.rados.0.smithi070.stdout:3584: writing smithi07085154-38 from 1540096 to 1982464 tid 2 2024-09-14T04:00:27.165 INFO:tasks.rados.rados.0.smithi070.stdout:3584: writing smithi07085154-38 from 1982464 to 2703360 tid 3 2024-09-14T04:00:27.166 INFO:tasks.rados.rados.0.smithi070.stdout:3584: writing smithi07085154-38 from 2703360 to 2744320 tid 4 2024-09-14T04:00:27.166 INFO:tasks.rados.rados.0.smithi070.stdout:3585: snap_create 2024-09-14T04:00:27.166 INFO:tasks.rados.rados.0.smithi070.stdout:3581: expect (ObjNum 1020 snap 336 seq_num 1020) 2024-09-14T04:00:27.239 INFO:tasks.rados.rados.0.smithi070.stdout:3580: expect (ObjNum 1001 snap 331 seq_num 1001) 2024-09-14T04:00:27.343 INFO:tasks.rados.rados.0.smithi070.stdout:3584: finishing write tid 1 to smithi07085154-38 2024-09-14T04:00:27.343 INFO:tasks.rados.rados.0.smithi070.stdout:3584: finishing write tid 2 to smithi07085154-38 2024-09-14T04:00:27.343 INFO:tasks.rados.rados.0.smithi070.stdout:3584: finishing write tid 3 to smithi07085154-38 2024-09-14T04:00:27.343 INFO:tasks.rados.rados.0.smithi070.stdout:3584: finishing write tid 4 to smithi07085154-38 2024-09-14T04:00:27.343 INFO:tasks.rados.rados.0.smithi070.stdout:3584: finishing write tid 5 to smithi07085154-38 2024-09-14T04:00:27.344 INFO:tasks.rados.rados.0.smithi070.stdout:3584: finishing write tid 6 to smithi07085154-38 2024-09-14T04:00:27.344 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 38 v 3983 (ObjNum 1099 snap 365 seq_num 1099) dirty exists 2024-09-14T04:00:27.344 INFO:tasks.rados.rados.0.smithi070.stdout:3584: left oid 38 (ObjNum 1099 snap 365 seq_num 1099) 2024-09-14T04:00:27.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:27 smithi094 ceph-mon[86597]: osdmap e788: 8 total, 8 up, 8 in 2024-09-14T04:00:27.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:27 smithi094 ceph-mon[86597]: pgmap v782: 121 pgs: 121 active+clean; 898 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 1.3 MiB/s wr, 0 op/s 2024-09-14T04:00:27.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:27 smithi070 ceph-mon[98640]: osdmap e788: 8 total, 8 up, 8 in 2024-09-14T04:00:27.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:27 smithi070 ceph-mon[98640]: pgmap v782: 121 pgs: 121 active+clean; 898 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 1.3 MiB/s wr, 0 op/s 2024-09-14T04:00:27.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:27 smithi070 ceph-mon[96704]: osdmap e788: 8 total, 8 up, 8 in 2024-09-14T04:00:27.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:27 smithi070 ceph-mon[96704]: pgmap v782: 121 pgs: 121 active+clean; 898 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 1.3 MiB/s wr, 0 op/s 2024-09-14T04:00:28.151 INFO:tasks.rados.rados.0.smithi070.stdout:3580: done (3 left) 2024-09-14T04:00:28.151 INFO:tasks.rados.rados.0.smithi070.stdout:3581: done (2 left) 2024-09-14T04:00:28.151 INFO:tasks.rados.rados.0.smithi070.stdout:3584: done (1 left) 2024-09-14T04:00:28.151 INFO:tasks.rados.rados.0.smithi070.stdout:3585: done (0 left) 2024-09-14T04:00:28.151 INFO:tasks.rados.rados.0.smithi070.stdout:3586: read oid 48 snap -1 2024-09-14T04:00:28.152 INFO:tasks.rados.rados.0.smithi070.stdout:3586: expect (ObjNum 1001 snap 331 seq_num 1001) 2024-09-14T04:00:28.152 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 31 current snap is 366 2024-09-14T04:00:28.152 INFO:tasks.rados.rados.0.smithi070.stdout:3587: seq_num 1100 ranges {2818048=425984,3244032=729088,3973120=475136} 2024-09-14T04:00:28.168 INFO:tasks.rados.rados.0.smithi070.stdout:3587: writing smithi07085154-31 from 2818048 to 3244032 tid 1 2024-09-14T04:00:28.173 INFO:tasks.rados.rados.0.smithi070.stdout:3587: writing smithi07085154-31 from 3244032 to 3973120 tid 2 2024-09-14T04:00:28.176 INFO:tasks.rados.rados.0.smithi070.stdout:3587: writing smithi07085154-31 from 3973120 to 4448256 tid 3 2024-09-14T04:00:28.176 INFO:tasks.rados.rados.0.smithi070.stdout:3588: read oid 49 snap -1 2024-09-14T04:00:28.176 INFO:tasks.rados.rados.0.smithi070.stdout:3588: expect (ObjNum 1080 snap 350 seq_num 1080) 2024-09-14T04:00:28.176 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 10 current snap is 366 2024-09-14T04:00:28.176 INFO:tasks.rados.rados.0.smithi070.stdout:3589: seq_num 1101 ranges {8323072=442368,8765440=606208,9371648=598016} 2024-09-14T04:00:28.222 INFO:tasks.rados.rados.0.smithi070.stdout:3589: writing smithi07085154-10 from 8323072 to 8765440 tid 1 2024-09-14T04:00:28.226 INFO:tasks.rados.rados.0.smithi070.stdout:3589: writing smithi07085154-10 from 8765440 to 9371648 tid 2 2024-09-14T04:00:28.231 INFO:tasks.rados.rados.0.smithi070.stdout:3589: writing smithi07085154-10 from 9371648 to 9969664 tid 3 2024-09-14T04:00:28.231 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 7 current snap is 366 2024-09-14T04:00:28.231 INFO:tasks.rados.rados.0.smithi070.stdout:3590: seq_num 1102 ranges {0=458752,458752=491520,950272=491520,1441792=221184} 2024-09-14T04:00:28.234 INFO:tasks.rados.rados.0.smithi070.stdout:3590: writing smithi07085154-7 from 0 to 458752 tid 1 2024-09-14T04:00:28.237 INFO:tasks.rados.rados.0.smithi070.stdout:3590: writing smithi07085154-7 from 458752 to 950272 tid 2 2024-09-14T04:00:28.241 INFO:tasks.rados.rados.0.smithi070.stdout:3590: writing smithi07085154-7 from 950272 to 1441792 tid 3 2024-09-14T04:00:28.242 INFO:tasks.rados.rados.0.smithi070.stdout:3590: writing smithi07085154-7 from 1441792 to 1662976 tid 4 2024-09-14T04:00:28.242 INFO:tasks.rados.rados.0.smithi070.stdout:3591: read oid 23 snap -1 2024-09-14T04:00:28.242 INFO:tasks.rados.rados.0.smithi070.stdout:3591: expect (ObjNum 1078 snap 349 seq_num 1078) 2024-09-14T04:00:28.243 INFO:tasks.rados.rados.0.smithi070.stdout:3592: copy_from oid 22 from oid 26 current snap is 366 2024-09-14T04:00:28.243 INFO:tasks.rados.rados.0.smithi070.stdout:3586: expect (ObjNum 1001 snap 331 seq_num 1001) 2024-09-14T04:00:28.308 INFO:tasks.rados.rados.0.smithi070.stdout:3587: finishing write tid 1 to smithi07085154-31 2024-09-14T04:00:28.308 INFO:tasks.rados.rados.0.smithi070.stdout:3587: finishing write tid 2 to smithi07085154-31 2024-09-14T04:00:28.308 INFO:tasks.rados.rados.0.smithi070.stdout:3587: finishing write tid 3 to smithi07085154-31 2024-09-14T04:00:28.308 INFO:tasks.rados.rados.0.smithi070.stdout:3587: finishing write tid 4 to smithi07085154-31 2024-09-14T04:00:28.308 INFO:tasks.rados.rados.0.smithi070.stdout:3588: expect (ObjNum 1080 snap 350 seq_num 1080) 2024-09-14T04:00:28.340 INFO:tasks.rados.rados.0.smithi070.stdout:3587: finishing write tid 5 to smithi07085154-31 2024-09-14T04:00:28.341 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 4001 (ObjNum 1100 snap 366 seq_num 1100) dirty exists 2024-09-14T04:00:28.341 INFO:tasks.rados.rados.0.smithi070.stdout:3587: left oid 31 (ObjNum 1100 snap 366 seq_num 1100) 2024-09-14T04:00:28.341 INFO:tasks.rados.rados.0.smithi070.stdout:3589: finishing write tid 1 to smithi07085154-10 2024-09-14T04:00:28.341 INFO:tasks.rados.rados.0.smithi070.stdout:3589: finishing write tid 2 to smithi07085154-10 2024-09-14T04:00:28.341 INFO:tasks.rados.rados.0.smithi070.stdout:3590: finishing write tid 1 to smithi07085154-7 2024-09-14T04:00:28.341 INFO:tasks.rados.rados.0.smithi070.stdout:3589: finishing write tid 3 to smithi07085154-10 2024-09-14T04:00:28.341 INFO:tasks.rados.rados.0.smithi070.stdout:3589: finishing write tid 4 to smithi07085154-10 2024-09-14T04:00:28.341 INFO:tasks.rados.rados.0.smithi070.stdout:3589: finishing write tid 5 to smithi07085154-10 2024-09-14T04:00:28.341 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 4289 (ObjNum 1101 snap 366 seq_num 1101) dirty exists 2024-09-14T04:00:28.341 INFO:tasks.rados.rados.0.smithi070.stdout:3589: left oid 10 (ObjNum 1101 snap 366 seq_num 1101) 2024-09-14T04:00:28.341 INFO:tasks.rados.rados.0.smithi070.stdout:3590: finishing write tid 2 to smithi07085154-7 2024-09-14T04:00:28.342 INFO:tasks.rados.rados.0.smithi070.stdout:3586: done (6 left) 2024-09-14T04:00:28.342 INFO:tasks.rados.rados.0.smithi070.stdout:3587: done (5 left) 2024-09-14T04:00:28.342 INFO:tasks.rados.rados.0.smithi070.stdout:3588: done (4 left) 2024-09-14T04:00:28.342 INFO:tasks.rados.rados.0.smithi070.stdout:3589: done (3 left) 2024-09-14T04:00:28.342 INFO:tasks.rados.rados.0.smithi070.stdout:3593: delete oid 33 current snap is 366 2024-09-14T04:00:28.342 INFO:tasks.rados.rados.0.smithi070.stdout:3590: finishing write tid 3 to smithi07085154-7 2024-09-14T04:00:28.342 INFO:tasks.rados.rados.0.smithi070.stdout:3590: finishing write tid 4 to smithi07085154-7 2024-09-14T04:00:28.342 INFO:tasks.rados.rados.0.smithi070.stdout:3590: finishing write tid 5 to smithi07085154-7 2024-09-14T04:00:28.342 INFO:tasks.rados.rados.0.smithi070.stdout:3590: finishing write tid 6 to smithi07085154-7 2024-09-14T04:00:28.342 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 4201 (ObjNum 1102 snap 366 seq_num 1102) dirty exists 2024-09-14T04:00:28.342 INFO:tasks.rados.rados.0.smithi070.stdout:3590: left oid 7 (ObjNum 1102 snap 366 seq_num 1102) 2024-09-14T04:00:28.342 INFO:tasks.rados.rados.0.smithi070.stdout:3592: finishing copy_from racing read to smithi07085154-22 2024-09-14T04:00:28.343 INFO:tasks.rados.rados.0.smithi070.stdout:3592: finishing copy_from to smithi07085154-22 2024-09-14T04:00:28.343 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 4191 (ObjNum 1079 snap 350 seq_num 1079) dirty exists 2024-09-14T04:00:28.343 INFO:tasks.rados.rados.0.smithi070.stdout:3591: expect (ObjNum 1078 snap 349 seq_num 1078) 2024-09-14T04:00:28.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:28 smithi094 ceph-mon[86597]: osdmap e789: 8 total, 8 up, 8 in 2024-09-14T04:00:28.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:28 smithi070 ceph-mon[96704]: osdmap e789: 8 total, 8 up, 8 in 2024-09-14T04:00:28.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:28 smithi070 ceph-mon[98640]: osdmap e789: 8 total, 8 up, 8 in 2024-09-14T04:00:28.579 INFO:tasks.rados.rados.0.smithi070.stdout:3590: done (3 left) 2024-09-14T04:00:28.579 INFO:tasks.rados.rados.0.smithi070.stdout:3591: done (2 left) 2024-09-14T04:00:28.579 INFO:tasks.rados.rados.0.smithi070.stdout:3592: done (1 left) 2024-09-14T04:00:28.579 INFO:tasks.rados.rados.0.smithi070.stdout:3593: done (0 left) 2024-09-14T04:00:28.579 INFO:tasks.rados.rados.0.smithi070.stdout:3594: snap_remove snap 359 2024-09-14T04:00:29.136 INFO:tasks.rados.rados.0.smithi070.stdout:3594: done (0 left) 2024-09-14T04:00:29.136 INFO:tasks.rados.rados.0.smithi070.stdout:3595: copy_from oid 22 from oid 44 current snap is 366 2024-09-14T04:00:29.136 INFO:tasks.rados.rados.0.smithi070.stdout:3596: copy_from oid 12 from oid 2 current snap is 366 2024-09-14T04:00:29.136 INFO:tasks.rados.rados.0.smithi070.stdout:3597: read oid 43 snap -1 2024-09-14T04:00:29.136 INFO:tasks.rados.rados.0.smithi070.stdout:3597: expect (ObjNum 1047 snap 344 seq_num 1047) 2024-09-14T04:00:29.136 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 17 current snap is 366 2024-09-14T04:00:29.136 INFO:tasks.rados.rados.0.smithi070.stdout:3598: seq_num 1104 ranges {589824=499712,1089536=712704,1802240=491520} 2024-09-14T04:00:29.142 INFO:tasks.rados.rados.0.smithi070.stdout:3598: writing smithi07085154-17 from 589824 to 1089536 tid 1 2024-09-14T04:00:29.147 INFO:tasks.rados.rados.0.smithi070.stdout:3598: writing smithi07085154-17 from 1089536 to 1802240 tid 2 2024-09-14T04:00:29.150 INFO:tasks.rados.rados.0.smithi070.stdout:3598: writing smithi07085154-17 from 1802240 to 2293760 tid 3 2024-09-14T04:00:29.150 INFO:tasks.rados.rados.0.smithi070.stdout:3599: setattr oid 42 current snap is 366 2024-09-14T04:00:29.151 INFO:tasks.rados.rados.0.smithi070.stdout:3596: finishing copy_from to smithi07085154-12 2024-09-14T04:00:29.151 INFO:tasks.rados.rados.0.smithi070.stdout:3596: got expected ENOENT (src dne) 2024-09-14T04:00:29.151 INFO:tasks.rados.rados.0.smithi070.stdout:3596: finishing copy_from racing read to smithi07085154-12 2024-09-14T04:00:29.152 INFO:tasks.rados.rados.0.smithi070.stdout:3597: expect (ObjNum 1047 snap 344 seq_num 1047) 2024-09-14T04:00:29.269 INFO:tasks.rados.rados.0.smithi070.stdout:3598: finishing write tid 1 to smithi07085154-17 2024-09-14T04:00:29.269 INFO:tasks.rados.rados.0.smithi070.stdout:3598: finishing write tid 2 to smithi07085154-17 2024-09-14T04:00:29.269 INFO:tasks.rados.rados.0.smithi070.stdout:3598: finishing write tid 3 to smithi07085154-17 2024-09-14T04:00:29.269 INFO:tasks.rados.rados.0.smithi070.stdout:3598: finishing write tid 4 to smithi07085154-17 2024-09-14T04:00:29.269 INFO:tasks.rados.rados.0.smithi070.stdout:3598: finishing write tid 5 to smithi07085154-17 2024-09-14T04:00:29.269 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 4005 (ObjNum 1104 snap 366 seq_num 1104) dirty exists 2024-09-14T04:00:29.269 INFO:tasks.rados.rados.0.smithi070.stdout:3598: left oid 17 (ObjNum 1104 snap 366 seq_num 1104) 2024-09-14T04:00:29.269 INFO:tasks.rados.rados.0.smithi070.stdout:3595: finishing copy_from racing read to smithi07085154-22 2024-09-14T04:00:29.269 INFO:tasks.rados.rados.0.smithi070.stdout:3595: finishing copy_from to smithi07085154-22 2024-09-14T04:00:29.269 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 4220 (ObjNum 1088 snap 354 seq_num 1088) dirty exists 2024-09-14T04:00:29.270 INFO:tasks.rados.rados.0.smithi070.stdout:3595: done (4 left) 2024-09-14T04:00:29.270 INFO:tasks.rados.rados.0.smithi070.stdout:3596: done (3 left) 2024-09-14T04:00:29.270 INFO:tasks.rados.rados.0.smithi070.stdout:3597: done (2 left) 2024-09-14T04:00:29.270 INFO:tasks.rados.rados.0.smithi070.stdout:3598: done (1 left) 2024-09-14T04:00:29.270 INFO:tasks.rados.rados.0.smithi070.stdout:3600: snap_create 2024-09-14T04:00:29.272 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 4293 (ObjNum 1092 snap 357 seq_num 1092) dirty exists 2024-09-14T04:00:29.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:29 smithi094 ceph-mon[86597]: pgmap v785: 121 pgs: 121 active+clean; 898 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 866 KiB/s rd, 0 op/s 2024-09-14T04:00:29.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:29 smithi094 ceph-mon[86597]: osdmap e790: 8 total, 8 up, 8 in 2024-09-14T04:00:29.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:29 smithi094 ceph-mon[86597]: osdmap e791: 8 total, 8 up, 8 in 2024-09-14T04:00:29.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:29 smithi070 ceph-mon[96704]: pgmap v785: 121 pgs: 121 active+clean; 898 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 866 KiB/s rd, 0 op/s 2024-09-14T04:00:29.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:29 smithi070 ceph-mon[96704]: osdmap e790: 8 total, 8 up, 8 in 2024-09-14T04:00:29.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:29 smithi070 ceph-mon[96704]: osdmap e791: 8 total, 8 up, 8 in 2024-09-14T04:00:29.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:29 smithi070 ceph-mon[98640]: pgmap v785: 121 pgs: 121 active+clean; 898 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 866 KiB/s rd, 0 op/s 2024-09-14T04:00:29.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:29 smithi070 ceph-mon[98640]: osdmap e790: 8 total, 8 up, 8 in 2024-09-14T04:00:29.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:29 smithi070 ceph-mon[98640]: osdmap e791: 8 total, 8 up, 8 in 2024-09-14T04:00:30.163 INFO:tasks.rados.rados.0.smithi070.stdout:3599: done (1 left) 2024-09-14T04:00:30.164 INFO:tasks.rados.rados.0.smithi070.stdout:3600: done (0 left) 2024-09-14T04:00:30.164 INFO:tasks.rados.rados.0.smithi070.stdout:3601: snap_create 2024-09-14T04:00:30.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:00:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:00:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:00:31.166 INFO:tasks.rados.rados.0.smithi070.stdout:3601: done (0 left) 2024-09-14T04:00:31.166 INFO:tasks.rados.rados.0.smithi070.stdout:3602: delete oid 2 current snap is 368 2024-09-14T04:00:31.177 INFO:tasks.rados.rados.0.smithi070.stdout:3602: done (0 left) 2024-09-14T04:00:31.177 INFO:tasks.rados.rados.0.smithi070.stdout:3603: snap_remove snap 364 2024-09-14T04:00:31.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:31 smithi094 ceph-mon[86597]: pgmap v787: 121 pgs: 121 active+clean; 900 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.2 MiB/s rd, 397 KiB/s wr, 7 op/s 2024-09-14T04:00:31.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:31 smithi094 ceph-mon[86597]: osdmap e792: 8 total, 8 up, 8 in 2024-09-14T04:00:31.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:31 smithi070 ceph-mon[96704]: pgmap v787: 121 pgs: 121 active+clean; 900 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.2 MiB/s rd, 397 KiB/s wr, 7 op/s 2024-09-14T04:00:31.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:31 smithi070 ceph-mon[96704]: osdmap e792: 8 total, 8 up, 8 in 2024-09-14T04:00:31.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:31 smithi070 ceph-mon[98640]: pgmap v787: 121 pgs: 121 active+clean; 900 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.2 MiB/s rd, 397 KiB/s wr, 7 op/s 2024-09-14T04:00:31.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:31 smithi070 ceph-mon[98640]: osdmap e792: 8 total, 8 up, 8 in 2024-09-14T04:00:32.176 INFO:tasks.rados.rados.0.smithi070.stdout:3603: done (0 left) 2024-09-14T04:00:32.176 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 15 current snap is 368 2024-09-14T04:00:32.177 INFO:tasks.rados.rados.0.smithi070.stdout:3604: seq_num 1106 ranges {1490944=532480,2023424=483328,2506752=475136,2981888=245760} 2024-09-14T04:00:32.188 INFO:tasks.rados.rados.0.smithi070.stdout:3604: writing smithi07085154-15 from 1490944 to 2023424 tid 1 2024-09-14T04:00:32.191 INFO:tasks.rados.rados.0.smithi070.stdout:3604: writing smithi07085154-15 from 2023424 to 2506752 tid 2 2024-09-14T04:00:32.194 INFO:tasks.rados.rados.0.smithi070.stdout:3604: writing smithi07085154-15 from 2506752 to 2981888 tid 3 2024-09-14T04:00:32.196 INFO:tasks.rados.rados.0.smithi070.stdout:3604: writing smithi07085154-15 from 2981888 to 3227648 tid 4 2024-09-14T04:00:32.196 INFO:tasks.rados.rados.0.smithi070.stdout:3605: read oid 39 snap -1 2024-09-14T04:00:32.196 INFO:tasks.rados.rados.0.smithi070.stdout:3605: expect (ObjNum 1020 snap 336 seq_num 1020) 2024-09-14T04:00:32.196 INFO:tasks.rados.rados.0.smithi070.stdout:3606: rollback oid 17 current snap is 368 2024-09-14T04:00:32.196 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 17 to 354 2024-09-14T04:00:32.197 INFO:tasks.rados.rados.0.smithi070.stdout:3604: finishing write tid 1 to smithi07085154-15 2024-09-14T04:00:32.197 INFO:tasks.rados.rados.0.smithi070.stdout:3607: read oid 19 snap -1 2024-09-14T04:00:32.197 INFO:tasks.rados.rados.0.smithi070.stdout:3607: expect (ObjNum 659 snap 220 seq_num 659) 2024-09-14T04:00:32.197 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 2 current snap is 368 2024-09-14T04:00:32.197 INFO:tasks.rados.rados.0.smithi070.stdout:3608: seq_num 1107 ranges {0=548864,548864=770048,1318912=434176} 2024-09-14T04:00:32.199 INFO:tasks.rados.rados.0.smithi070.stdout:3608: writing smithi07085154-2 from 0 to 548864 tid 1 2024-09-14T04:00:32.204 INFO:tasks.rados.rados.0.smithi070.stdout:3608: writing smithi07085154-2 from 548864 to 1318912 tid 2 2024-09-14T04:00:32.207 INFO:tasks.rados.rados.0.smithi070.stdout:3608: writing smithi07085154-2 from 1318912 to 1753088 tid 3 2024-09-14T04:00:32.208 INFO:tasks.rados.rados.0.smithi070.stdout:3609: rmattr oid 5 current snap is 368 2024-09-14T04:00:32.208 INFO:tasks.rados.rados.0.smithi070.stdout:3604: finishing write tid 2 to smithi07085154-15 2024-09-14T04:00:32.208 INFO:tasks.rados.rados.0.smithi070.stdout:3610: rmattr oid 40 current snap is 368 2024-09-14T04:00:32.208 INFO:tasks.rados.rados.0.smithi070.stdout:3604: finishing write tid 3 to smithi07085154-15 2024-09-14T04:00:32.208 INFO:tasks.rados.rados.0.smithi070.stdout:3604: finishing write tid 4 to smithi07085154-15 2024-09-14T04:00:32.208 INFO:tasks.rados.rados.0.smithi070.stdout:3604: finishing write tid 5 to smithi07085154-15 2024-09-14T04:00:32.208 INFO:tasks.rados.rados.0.smithi070.stdout:3606: finishing rollback tid 0 to smithi07085154-17 2024-09-14T04:00:32.208 INFO:tasks.rados.rados.0.smithi070.stdout:3606: finishing rollback tid 1 to smithi07085154-17 2024-09-14T04:00:32.208 INFO:tasks.rados.rados.0.smithi070.stdout:3606: finishing rollback tid 2 to smithi07085154-17 2024-09-14T04:00:32.208 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 4008 (ObjNum 1038 snap 342 seq_num 1038) dirty exists 2024-09-14T04:00:32.208 INFO:tasks.rados.rados.0.smithi070.stdout:3605: expect (ObjNum 1020 snap 336 seq_num 1020) 2024-09-14T04:00:32.278 INFO:tasks.rados.rados.0.smithi070.stdout:3604: finishing write tid 6 to smithi07085154-15 2024-09-14T04:00:32.278 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 15 v 4130 (ObjNum 1106 snap 368 seq_num 1106) dirty exists 2024-09-14T04:00:32.278 INFO:tasks.rados.rados.0.smithi070.stdout:3604: left oid 15 (ObjNum 1106 snap 368 seq_num 1106) 2024-09-14T04:00:32.278 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 3198 (ObjNum 1084 snap 354 seq_num 1084) dirty exists 2024-09-14T04:00:32.278 INFO:tasks.rados.rados.0.smithi070.stdout:3608: finishing write tid 1 to smithi07085154-2 2024-09-14T04:00:32.278 INFO:tasks.rados.rados.0.smithi070.stdout:3608: finishing write tid 2 to smithi07085154-2 2024-09-14T04:00:32.278 INFO:tasks.rados.rados.0.smithi070.stdout:3608: finishing write tid 3 to smithi07085154-2 2024-09-14T04:00:32.278 INFO:tasks.rados.rados.0.smithi070.stdout:3604: done (6 left) 2024-09-14T04:00:32.279 INFO:tasks.rados.rados.0.smithi070.stdout:3605: done (5 left) 2024-09-14T04:00:32.279 INFO:tasks.rados.rados.0.smithi070.stdout:3606: done (4 left) 2024-09-14T04:00:32.279 INFO:tasks.rados.rados.0.smithi070.stdout:3609: done (3 left) 2024-09-14T04:00:32.279 INFO:tasks.rados.rados.0.smithi070.stdout:3611: snap_create 2024-09-14T04:00:32.279 INFO:tasks.rados.rados.0.smithi070.stdout:3608: finishing write tid 4 to smithi07085154-2 2024-09-14T04:00:32.279 INFO:tasks.rados.rados.0.smithi070.stdout:3608: finishing write tid 5 to smithi07085154-2 2024-09-14T04:00:32.279 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 2 v 4302 (ObjNum 1107 snap 368 seq_num 1107) dirty exists 2024-09-14T04:00:32.279 INFO:tasks.rados.rados.0.smithi070.stdout:3608: left oid 2 (ObjNum 1107 snap 368 seq_num 1107) 2024-09-14T04:00:32.279 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 4222 (ObjNum 1079 snap 350 seq_num 1079) dirty exists 2024-09-14T04:00:32.279 INFO:tasks.rados.rados.0.smithi070.stdout:3607: expect (ObjNum 659 snap 220 seq_num 659) 2024-09-14T04:00:32.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:32 smithi070 ceph-mon[96704]: osdmap e793: 8 total, 8 up, 8 in 2024-09-14T04:00:32.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:32 smithi070 ceph-mon[96704]: pgmap v790: 121 pgs: 121 active+clean; 910 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 3.2 MiB/s wr, 17 op/s 2024-09-14T04:00:32.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:32 smithi070 ceph-mon[98640]: osdmap e793: 8 total, 8 up, 8 in 2024-09-14T04:00:32.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:32 smithi070 ceph-mon[98640]: pgmap v790: 121 pgs: 121 active+clean; 910 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 3.2 MiB/s wr, 17 op/s 2024-09-14T04:00:32.538 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:32 smithi094 ceph-mon[86597]: osdmap e793: 8 total, 8 up, 8 in 2024-09-14T04:00:32.538 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:32 smithi094 ceph-mon[86597]: pgmap v790: 121 pgs: 121 active+clean; 910 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 3.2 MiB/s wr, 17 op/s 2024-09-14T04:00:33.179 INFO:tasks.rados.rados.0.smithi070.stdout:3607: done (3 left) 2024-09-14T04:00:33.180 INFO:tasks.rados.rados.0.smithi070.stdout:3608: done (2 left) 2024-09-14T04:00:33.180 INFO:tasks.rados.rados.0.smithi070.stdout:3610: done (1 left) 2024-09-14T04:00:33.180 INFO:tasks.rados.rados.0.smithi070.stdout:3611: done (0 left) 2024-09-14T04:00:33.180 INFO:tasks.rados.rados.0.smithi070.stdout:3612: read oid 49 snap -1 2024-09-14T04:00:33.180 INFO:tasks.rados.rados.0.smithi070.stdout:3612: expect (ObjNum 1080 snap 350 seq_num 1080) 2024-09-14T04:00:33.180 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 4 current snap is 369 2024-09-14T04:00:33.180 INFO:tasks.rados.rados.0.smithi070.stdout:3613: seq_num 1108 ranges {0=565248,565248=647168,1212416=483328,1695744=73728} 2024-09-14T04:00:33.182 INFO:tasks.rados.rados.0.smithi070.stdout:3613: writing smithi07085154-4 from 0 to 565248 tid 1 2024-09-14T04:00:33.186 INFO:tasks.rados.rados.0.smithi070.stdout:3613: writing smithi07085154-4 from 565248 to 1212416 tid 2 2024-09-14T04:00:33.190 INFO:tasks.rados.rados.0.smithi070.stdout:3613: writing smithi07085154-4 from 1212416 to 1695744 tid 3 2024-09-14T04:00:33.191 INFO:tasks.rados.rados.0.smithi070.stdout:3613: writing smithi07085154-4 from 1695744 to 1769472 tid 4 2024-09-14T04:00:33.191 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 16 current snap is 369 2024-09-14T04:00:33.191 INFO:tasks.rados.rados.0.smithi070.stdout:3614: seq_num 1109 ranges {606208=581632,1187840=532480,1720320=647168,2367488=24576} 2024-09-14T04:00:33.197 INFO:tasks.rados.rados.0.smithi070.stdout:3614: writing smithi07085154-16 from 606208 to 1187840 tid 1 2024-09-14T04:00:33.200 INFO:tasks.rados.rados.0.smithi070.stdout:3614: writing smithi07085154-16 from 1187840 to 1720320 tid 2 2024-09-14T04:00:33.204 INFO:tasks.rados.rados.0.smithi070.stdout:3614: writing smithi07085154-16 from 1720320 to 2367488 tid 3 2024-09-14T04:00:33.205 INFO:tasks.rados.rados.0.smithi070.stdout:3614: writing smithi07085154-16 from 2367488 to 2392064 tid 4 2024-09-14T04:00:33.205 INFO:tasks.rados.rados.0.smithi070.stdout:3615: read oid 37 snap 350 2024-09-14T04:00:33.205 INFO:tasks.rados.rados.0.smithi070.stdout:3615: expect deleted 2024-09-14T04:00:33.205 INFO:tasks.rados.rados.0.smithi070.stdout:3616: snap_create 2024-09-14T04:00:33.205 INFO:tasks.rados.rados.0.smithi070.stdout:3612: expect (ObjNum 1080 snap 350 seq_num 1080) 2024-09-14T04:00:33.239 INFO:tasks.rados.rados.0.smithi070.stdout:3613: finishing write tid 1 to smithi07085154-4 2024-09-14T04:00:33.239 INFO:tasks.rados.rados.0.smithi070.stdout:3613: finishing write tid 2 to smithi07085154-4 2024-09-14T04:00:33.239 INFO:tasks.rados.rados.0.smithi070.stdout:3613: finishing write tid 3 to smithi07085154-4 2024-09-14T04:00:33.239 INFO:tasks.rados.rados.0.smithi070.stdout:3613: finishing write tid 4 to smithi07085154-4 2024-09-14T04:00:33.239 INFO:tasks.rados.rados.0.smithi070.stdout:3613: finishing write tid 5 to smithi07085154-4 2024-09-14T04:00:33.239 INFO:tasks.rados.rados.0.smithi070.stdout:3613: finishing write tid 6 to smithi07085154-4 2024-09-14T04:00:33.239 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 4206 (ObjNum 1108 snap 369 seq_num 1108) dirty exists 2024-09-14T04:00:33.240 INFO:tasks.rados.rados.0.smithi070.stdout:3613: left oid 4 (ObjNum 1108 snap 369 seq_num 1108) 2024-09-14T04:00:33.240 INFO:tasks.rados.rados.0.smithi070.stdout:3614: finishing write tid 1 to smithi07085154-16 2024-09-14T04:00:33.240 INFO:tasks.rados.rados.0.smithi070.stdout:3614: finishing write tid 2 to smithi07085154-16 2024-09-14T04:00:33.240 INFO:tasks.rados.rados.0.smithi070.stdout:3614: finishing write tid 3 to smithi07085154-16 2024-09-14T04:00:33.240 INFO:tasks.rados.rados.0.smithi070.stdout:3614: finishing write tid 4 to smithi07085154-16 2024-09-14T04:00:33.240 INFO:tasks.rados.rados.0.smithi070.stdout:3614: finishing write tid 5 to smithi07085154-16 2024-09-14T04:00:33.240 INFO:tasks.rados.rados.0.smithi070.stdout:3614: finishing write tid 6 to smithi07085154-16 2024-09-14T04:00:33.240 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 4308 (ObjNum 1109 snap 369 seq_num 1109) dirty exists 2024-09-14T04:00:33.240 INFO:tasks.rados.rados.0.smithi070.stdout:3614: left oid 16 (ObjNum 1109 snap 369 seq_num 1109) 2024-09-14T04:00:33.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:33 smithi094 ceph-mon[86597]: osdmap e794: 8 total, 8 up, 8 in 2024-09-14T04:00:33.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:33 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:00:33.535 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:00:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:00:33] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.51.0" 2024-09-14T04:00:33.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:33 smithi070 ceph-mon[98640]: osdmap e794: 8 total, 8 up, 8 in 2024-09-14T04:00:33.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:33 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:00:33.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:33 smithi070 ceph-mon[96704]: osdmap e794: 8 total, 8 up, 8 in 2024-09-14T04:00:33.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:33 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:00:34.137 INFO:tasks.rados.rados.0.smithi070.stdout:3612: done (4 left) 2024-09-14T04:00:34.137 INFO:tasks.rados.rados.0.smithi070.stdout:3613: done (3 left) 2024-09-14T04:00:34.137 INFO:tasks.rados.rados.0.smithi070.stdout:3614: done (2 left) 2024-09-14T04:00:34.137 INFO:tasks.rados.rados.0.smithi070.stdout:3615: done (1 left) 2024-09-14T04:00:34.137 INFO:tasks.rados.rados.0.smithi070.stdout:3616: done (0 left) 2024-09-14T04:00:34.137 INFO:tasks.rados.rados.0.smithi070.stdout:3617: rollback oid 22 current snap is 370 2024-09-14T04:00:34.138 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 22 to 348 2024-09-14T04:00:34.138 INFO:tasks.rados.rados.0.smithi070.stdout:3618: snap_remove snap 365 2024-09-14T04:00:34.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:34 smithi094 ceph-mon[86597]: osdmap e795: 8 total, 8 up, 8 in 2024-09-14T04:00:34.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:34 smithi094 ceph-mon[86597]: osdmap e796: 8 total, 8 up, 8 in 2024-09-14T04:00:34.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:34 smithi094 ceph-mon[86597]: pgmap v794: 121 pgs: 121 active+clean; 911 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.1 MiB/s rd, 3.2 MiB/s wr, 15 op/s 2024-09-14T04:00:34.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:34 smithi070 ceph-mon[96704]: osdmap e795: 8 total, 8 up, 8 in 2024-09-14T04:00:34.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:34 smithi070 ceph-mon[96704]: osdmap e796: 8 total, 8 up, 8 in 2024-09-14T04:00:34.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:34 smithi070 ceph-mon[96704]: pgmap v794: 121 pgs: 121 active+clean; 911 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.1 MiB/s rd, 3.2 MiB/s wr, 15 op/s 2024-09-14T04:00:34.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:34 smithi070 ceph-mon[98640]: osdmap e795: 8 total, 8 up, 8 in 2024-09-14T04:00:34.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:34 smithi070 ceph-mon[98640]: osdmap e796: 8 total, 8 up, 8 in 2024-09-14T04:00:34.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:34 smithi070 ceph-mon[98640]: pgmap v794: 121 pgs: 121 active+clean; 911 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.1 MiB/s rd, 3.2 MiB/s wr, 15 op/s 2024-09-14T04:00:35.147 INFO:tasks.rados.rados.0.smithi070.stdout:3618: done (1 left) 2024-09-14T04:00:35.147 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 50 current snap is 370 2024-09-14T04:00:35.147 INFO:tasks.rados.rados.0.smithi070.stdout:3619: seq_num 1110 ranges {1736704=598016,2334720=466944,2801664=450560,3252224=286720} 2024-09-14T04:00:35.160 INFO:tasks.rados.rados.0.smithi070.stdout:3619: writing smithi07085154-50 from 1736704 to 2334720 tid 1 2024-09-14T04:00:35.163 INFO:tasks.rados.rados.0.smithi070.stdout:3619: writing smithi07085154-50 from 2334720 to 2801664 tid 2 2024-09-14T04:00:35.165 INFO:tasks.rados.rados.0.smithi070.stdout:3619: writing smithi07085154-50 from 2801664 to 3252224 tid 3 2024-09-14T04:00:35.167 INFO:tasks.rados.rados.0.smithi070.stdout:3619: writing smithi07085154-50 from 3252224 to 3538944 tid 4 2024-09-14T04:00:35.168 INFO:tasks.rados.rados.0.smithi070.stdout:3620: delete oid 17 current snap is 370 2024-09-14T04:00:35.168 INFO:tasks.rados.rados.0.smithi070.stdout:3617: finishing rollback tid 0 to smithi07085154-22 2024-09-14T04:00:35.168 INFO:tasks.rados.rados.0.smithi070.stdout:3617: finishing rollback tid 1 to smithi07085154-22 2024-09-14T04:00:35.168 INFO:tasks.rados.rados.0.smithi070.stdout:3617: finishing rollback tid 2 to smithi07085154-22 2024-09-14T04:00:35.168 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 4223 (ObjNum 1071 snap 347 seq_num 1071) dirty exists 2024-09-14T04:00:35.168 INFO:tasks.rados.rados.0.smithi070.stdout:3619: finishing write tid 1 to smithi07085154-50 2024-09-14T04:00:35.168 INFO:tasks.rados.rados.0.smithi070.stdout:3619: finishing write tid 2 to smithi07085154-50 2024-09-14T04:00:35.171 INFO:tasks.rados.rados.0.smithi070.stdout:3619: finishing write tid 3 to smithi07085154-50 2024-09-14T04:00:35.171 INFO:tasks.rados.rados.0.smithi070.stdout:3617: done (2 left) 2024-09-14T04:00:35.171 INFO:tasks.rados.rados.0.smithi070.stdout:3620: done (1 left) 2024-09-14T04:00:35.171 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 14 current snap is 370 2024-09-14T04:00:35.171 INFO:tasks.rados.rados.0.smithi070.stdout:3621: seq_num 1112 ranges {0=630784,630784=638976,1269760=466944,1736704=98304} 2024-09-14T04:00:35.175 INFO:tasks.rados.rados.0.smithi070.stdout:3621: writing smithi07085154-14 from 0 to 630784 tid 1 2024-09-14T04:00:35.178 INFO:tasks.rados.rados.0.smithi070.stdout:3621: writing smithi07085154-14 from 630784 to 1269760 tid 2 2024-09-14T04:00:35.181 INFO:tasks.rados.rados.0.smithi070.stdout:3621: writing smithi07085154-14 from 1269760 to 1736704 tid 3 2024-09-14T04:00:35.182 INFO:tasks.rados.rados.0.smithi070.stdout:3621: writing smithi07085154-14 from 1736704 to 1835008 tid 4 2024-09-14T04:00:35.182 INFO:tasks.rados.rados.0.smithi070.stdout:3622: read oid 44 snap -1 2024-09-14T04:00:35.183 INFO:tasks.rados.rados.0.smithi070.stdout:3622: expect (ObjNum 1088 snap 354 seq_num 1088) 2024-09-14T04:00:35.183 INFO:tasks.rados.rados.0.smithi070.stdout:3623: rollback oid 11 current snap is 370 2024-09-14T04:00:35.183 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 11 to 363 2024-09-14T04:00:35.183 INFO:tasks.rados.rados.0.smithi070.stdout:3619: finishing write tid 4 to smithi07085154-50 2024-09-14T04:00:35.183 INFO:tasks.rados.rados.0.smithi070.stdout:3619: finishing write tid 5 to smithi07085154-50 2024-09-14T04:00:35.183 INFO:tasks.rados.rados.0.smithi070.stdout:3619: finishing write tid 6 to smithi07085154-50 2024-09-14T04:00:35.183 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 4228 (ObjNum 1110 snap 370 seq_num 1110) dirty exists 2024-09-14T04:00:35.183 INFO:tasks.rados.rados.0.smithi070.stdout:3619: left oid 50 (ObjNum 1110 snap 370 seq_num 1110) 2024-09-14T04:00:35.183 INFO:tasks.rados.rados.0.smithi070.stdout:3621: finishing write tid 1 to smithi07085154-14 2024-09-14T04:00:35.183 INFO:tasks.rados.rados.0.smithi070.stdout:3621: finishing write tid 2 to smithi07085154-14 2024-09-14T04:00:35.183 INFO:tasks.rados.rados.0.smithi070.stdout:3619: done (3 left) 2024-09-14T04:00:35.183 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 13 current snap is 370 2024-09-14T04:00:35.183 INFO:tasks.rados.rados.0.smithi070.stdout:3624: seq_num 1113 ranges {5316608=647168,5963776=524288,6488064=622592,7110656=57344} 2024-09-14T04:00:35.214 INFO:tasks.rados.rados.0.smithi070.stdout:3624: writing smithi07085154-13 from 5316608 to 5963776 tid 1 2024-09-14T04:00:35.217 INFO:tasks.rados.rados.0.smithi070.stdout:3624: writing smithi07085154-13 from 5963776 to 6488064 tid 2 2024-09-14T04:00:35.221 INFO:tasks.rados.rados.0.smithi070.stdout:3624: writing smithi07085154-13 from 6488064 to 7110656 tid 3 2024-09-14T04:00:35.222 INFO:tasks.rados.rados.0.smithi070.stdout:3624: writing smithi07085154-13 from 7110656 to 7168000 tid 4 2024-09-14T04:00:35.222 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 16 current snap is 370 2024-09-14T04:00:35.222 INFO:tasks.rados.rados.0.smithi070.stdout:3623: finishing rollback tid 1 to smithi07085154-11 2024-09-14T04:00:35.222 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 11 v 3171 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-14T04:00:35.222 INFO:tasks.rados.rados.0.smithi070.stdout:3621: finishing write tid 3 to smithi07085154-14 2024-09-14T04:00:35.222 INFO:tasks.rados.rados.0.smithi070.stdout:3621: finishing write tid 4 to smithi07085154-14 2024-09-14T04:00:35.222 INFO:tasks.rados.rados.0.smithi070.stdout:3621: finishing write tid 5 to smithi07085154-14 2024-09-14T04:00:35.223 INFO:tasks.rados.rados.0.smithi070.stdout:3621: finishing write tid 6 to smithi07085154-14 2024-09-14T04:00:35.223 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 3253 (ObjNum 1112 snap 370 seq_num 1112) dirty exists 2024-09-14T04:00:35.223 INFO:tasks.rados.rados.0.smithi070.stdout:3621: left oid 14 (ObjNum 1112 snap 370 seq_num 1112) 2024-09-14T04:00:35.223 INFO:tasks.rados.rados.0.smithi070.stdout:3622: expect (ObjNum 1088 snap 354 seq_num 1088) 2024-09-14T04:00:35.551 INFO:tasks.rados.rados.0.smithi070.stdout:3624: finishing write tid 1 to smithi07085154-13 2024-09-14T04:00:35.551 INFO:tasks.rados.rados.0.smithi070.stdout:3624: finishing write tid 2 to smithi07085154-13 2024-09-14T04:00:35.551 INFO:tasks.rados.rados.0.smithi070.stdout:3624: finishing write tid 3 to smithi07085154-13 2024-09-14T04:00:35.551 INFO:tasks.rados.rados.0.smithi070.stdout:3624: finishing write tid 4 to smithi07085154-13 2024-09-14T04:00:35.552 INFO:tasks.rados.rados.0.smithi070.stdout:3624: finishing write tid 5 to smithi07085154-13 2024-09-14T04:00:35.552 INFO:tasks.rados.rados.0.smithi070.stdout:3624: finishing write tid 6 to smithi07085154-13 2024-09-14T04:00:35.552 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 3253 (ObjNum 1113 snap 370 seq_num 1113) dirty exists 2024-09-14T04:00:35.552 INFO:tasks.rados.rados.0.smithi070.stdout:3624: left oid 13 (ObjNum 1113 snap 370 seq_num 1113) 2024-09-14T04:00:35.552 INFO:tasks.rados.rados.0.smithi070.stdout:3625: seq_num 1114 ranges {2392064=663552,3055616=802816,3858432=401408} 2024-09-14T04:00:35.566 INFO:tasks.rados.rados.0.smithi070.stdout:3625: writing smithi07085154-16 from 2392064 to 3055616 tid 1 2024-09-14T04:00:35.572 INFO:tasks.rados.rados.0.smithi070.stdout:3625: writing smithi07085154-16 from 3055616 to 3858432 tid 2 2024-09-14T04:00:35.575 INFO:tasks.rados.rados.0.smithi070.stdout:3625: writing smithi07085154-16 from 3858432 to 4259840 tid 3 2024-09-14T04:00:35.575 INFO:tasks.rados.rados.0.smithi070.stdout:3621: done (4 left) 2024-09-14T04:00:35.575 INFO:tasks.rados.rados.0.smithi070.stdout:3622: done (3 left) 2024-09-14T04:00:35.575 INFO:tasks.rados.rados.0.smithi070.stdout:3623: done (2 left) 2024-09-14T04:00:35.575 INFO:tasks.rados.rados.0.smithi070.stdout:3624: done (1 left) 2024-09-14T04:00:35.576 INFO:tasks.rados.rados.0.smithi070.stdout:3626: rollback oid 33 current snap is 370 2024-09-14T04:00:35.576 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 33 to 213 2024-09-14T04:00:35.576 INFO:tasks.rados.rados.0.smithi070.stdout:3625: finishing write tid 1 to smithi07085154-16 2024-09-14T04:00:35.576 INFO:tasks.rados.rados.0.smithi070.stdout:3627: snap_create 2024-09-14T04:00:35.578 INFO:tasks.rados.rados.0.smithi070.stdout:3626: finishing rollback tid 1 to smithi07085154-33 2024-09-14T04:00:35.578 INFO:tasks.rados.rados.0.smithi070.stdout:3626: finishing rollback tid 2 to smithi07085154-33 2024-09-14T04:00:35.578 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 33 v 3210 (ObjNum 590 snap 195 seq_num 590) dirty exists 2024-09-14T04:00:35.580 INFO:tasks.rados.rados.0.smithi070.stdout:3625: finishing write tid 2 to smithi07085154-16 2024-09-14T04:00:35.580 INFO:tasks.rados.rados.0.smithi070.stdout:3625: finishing write tid 3 to smithi07085154-16 2024-09-14T04:00:35.580 INFO:tasks.rados.rados.0.smithi070.stdout:3625: finishing write tid 4 to smithi07085154-16 2024-09-14T04:00:35.582 INFO:tasks.rados.rados.0.smithi070.stdout:3625: finishing write tid 5 to smithi07085154-16 2024-09-14T04:00:35.582 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 4319 (ObjNum 1114 snap 370 seq_num 1114) dirty exists 2024-09-14T04:00:35.582 INFO:tasks.rados.rados.0.smithi070.stdout:3625: left oid 16 (ObjNum 1114 snap 370 seq_num 1114) 2024-09-14T04:00:36.148 INFO:tasks.rados.rados.0.smithi070.stdout:3625: done (2 left) 2024-09-14T04:00:36.148 INFO:tasks.rados.rados.0.smithi070.stdout:3626: done (1 left) 2024-09-14T04:00:36.148 INFO:tasks.rados.rados.0.smithi070.stdout:3627: done (0 left) 2024-09-14T04:00:36.148 INFO:tasks.rados.rados.0.smithi070.stdout:3628: rollback oid 6 current snap is 371 2024-09-14T04:00:36.148 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 6 to 366 2024-09-14T04:00:36.149 INFO:tasks.rados.rados.0.smithi070.stdout:3629: read oid 16 snap -1 2024-09-14T04:00:36.149 INFO:tasks.rados.rados.0.smithi070.stdout:3629: expect (ObjNum 1114 snap 370 seq_num 1114) 2024-09-14T04:00:36.149 INFO:tasks.rados.rados.0.smithi070.stdout:3630: snap_remove snap 369 2024-09-14T04:00:36.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:36 smithi094 ceph-mon[86597]: osdmap e797: 8 total, 8 up, 8 in 2024-09-14T04:00:36.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:36 smithi070 ceph-mon[96704]: osdmap e797: 8 total, 8 up, 8 in 2024-09-14T04:00:36.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:36 smithi070 ceph-mon[98640]: osdmap e797: 8 total, 8 up, 8 in 2024-09-14T04:00:37.155 INFO:tasks.rados.rados.0.smithi070.stdout:3628: finishing rollback tid 0 to smithi07085154-6 2024-09-14T04:00:37.156 INFO:tasks.rados.rados.0.smithi070.stdout:3628: finishing rollback tid 1 to smithi07085154-6 2024-09-14T04:00:37.156 INFO:tasks.rados.rados.0.smithi070.stdout:3628: finishing rollback tid 2 to smithi07085154-6 2024-09-14T04:00:37.156 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 4012 (ObjNum 1039 snap 342 seq_num 1039) dirty exists 2024-09-14T04:00:37.156 INFO:tasks.rados.rados.0.smithi070.stdout:3629: expect (ObjNum 1114 snap 370 seq_num 1114) 2024-09-14T04:00:37.311 INFO:tasks.rados.rados.0.smithi070.stdout:3628: done (2 left) 2024-09-14T04:00:37.311 INFO:tasks.rados.rados.0.smithi070.stdout:3629: done (1 left) 2024-09-14T04:00:37.311 INFO:tasks.rados.rados.0.smithi070.stdout:3630: done (0 left) 2024-09-14T04:00:37.312 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 38 current snap is 371 2024-09-14T04:00:37.312 INFO:tasks.rados.rados.0.smithi070.stdout:3631: seq_num 1115 ranges {2744320=679936,3424256=688128,4112384=516096} 2024-09-14T04:00:37.329 INFO:tasks.rados.rados.0.smithi070.stdout:3631: writing smithi07085154-38 from 2744320 to 3424256 tid 1 2024-09-14T04:00:37.333 INFO:tasks.rados.rados.0.smithi070.stdout:3631: writing smithi07085154-38 from 3424256 to 4112384 tid 2 2024-09-14T04:00:37.336 INFO:tasks.rados.rados.0.smithi070.stdout:3631: writing smithi07085154-38 from 4112384 to 4628480 tid 3 2024-09-14T04:00:37.336 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 50 current snap is 371 2024-09-14T04:00:37.336 INFO:tasks.rados.rados.0.smithi070.stdout:3632: seq_num 1116 ranges {3538944=696320,4235264=573440,4808704=450560,5259264=180224} 2024-09-14T04:00:37.359 INFO:tasks.rados.rados.0.smithi070.stdout:3632: writing smithi07085154-50 from 3538944 to 4235264 tid 1 2024-09-14T04:00:37.362 INFO:tasks.rados.rados.0.smithi070.stdout:3632: writing smithi07085154-50 from 4235264 to 4808704 tid 2 2024-09-14T04:00:37.365 INFO:tasks.rados.rados.0.smithi070.stdout:3632: writing smithi07085154-50 from 4808704 to 5259264 tid 3 2024-09-14T04:00:37.366 INFO:tasks.rados.rados.0.smithi070.stdout:3632: writing smithi07085154-50 from 5259264 to 5439488 tid 4 2024-09-14T04:00:37.366 INFO:tasks.rados.rados.0.smithi070.stdout:3633: rollback oid 22 current snap is 371 2024-09-14T04:00:37.366 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 22 to 370 2024-09-14T04:00:37.366 INFO:tasks.rados.rados.0.smithi070.stdout:3631: finishing write tid 1 to smithi07085154-38 2024-09-14T04:00:37.366 INFO:tasks.rados.rados.0.smithi070.stdout:3631: finishing write tid 2 to smithi07085154-38 2024-09-14T04:00:37.367 INFO:tasks.rados.rados.0.smithi070.stdout:3631: finishing write tid 3 to smithi07085154-38 2024-09-14T04:00:37.367 INFO:tasks.rados.rados.0.smithi070.stdout:3631: finishing write tid 4 to smithi07085154-38 2024-09-14T04:00:37.367 INFO:tasks.rados.rados.0.smithi070.stdout:3631: finishing write tid 5 to smithi07085154-38 2024-09-14T04:00:37.367 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 38 v 3987 (ObjNum 1115 snap 371 seq_num 1115) dirty exists 2024-09-14T04:00:37.367 INFO:tasks.rados.rados.0.smithi070.stdout:3631: left oid 38 (ObjNum 1115 snap 371 seq_num 1115) 2024-09-14T04:00:37.367 INFO:tasks.rados.rados.0.smithi070.stdout:3632: finishing write tid 1 to smithi07085154-50 2024-09-14T04:00:37.367 INFO:tasks.rados.rados.0.smithi070.stdout:3632: finishing write tid 2 to smithi07085154-50 2024-09-14T04:00:37.367 INFO:tasks.rados.rados.0.smithi070.stdout:3631: done (2 left) 2024-09-14T04:00:37.367 INFO:tasks.rados.rados.0.smithi070.stdout:3634: snap_create 2024-09-14T04:00:37.369 INFO:tasks.rados.rados.0.smithi070.stdout:3632: finishing write tid 3 to smithi07085154-50 2024-09-14T04:00:37.370 INFO:tasks.rados.rados.0.smithi070.stdout:3632: finishing write tid 4 to smithi07085154-50 2024-09-14T04:00:37.373 INFO:tasks.rados.rados.0.smithi070.stdout:3632: finishing write tid 5 to smithi07085154-50 2024-09-14T04:00:37.373 INFO:tasks.rados.rados.0.smithi070.stdout:3633: finishing rollback tid 0 to smithi07085154-22 2024-09-14T04:00:37.373 INFO:tasks.rados.rados.0.smithi070.stdout:3633: finishing rollback tid 1 to smithi07085154-22 2024-09-14T04:00:37.375 INFO:tasks.rados.rados.0.smithi070.stdout:3632: finishing write tid 6 to smithi07085154-50 2024-09-14T04:00:37.375 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 4233 (ObjNum 1116 snap 371 seq_num 1116) dirty exists 2024-09-14T04:00:37.376 INFO:tasks.rados.rados.0.smithi070.stdout:3632: left oid 50 (ObjNum 1116 snap 371 seq_num 1116) 2024-09-14T04:00:37.376 INFO:tasks.rados.rados.0.smithi070.stdout:3633: finishing rollback tid 2 to smithi07085154-22 2024-09-14T04:00:37.376 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 4236 (ObjNum 1071 snap 347 seq_num 1071) dirty exists 2024-09-14T04:00:37.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:37 smithi094 ceph-mon[86597]: osdmap e798: 8 total, 8 up, 8 in 2024-09-14T04:00:37.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:37 smithi094 ceph-mon[86597]: pgmap v797: 121 pgs: 121 active+clean; 913 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 2.2 MiB/s rd, 844 KiB/s wr, 7 op/s 2024-09-14T04:00:37.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:37 smithi070 ceph-mon[96704]: osdmap e798: 8 total, 8 up, 8 in 2024-09-14T04:00:37.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:37 smithi070 ceph-mon[96704]: pgmap v797: 121 pgs: 121 active+clean; 913 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 2.2 MiB/s rd, 844 KiB/s wr, 7 op/s 2024-09-14T04:00:37.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:37 smithi070 ceph-mon[98640]: osdmap e798: 8 total, 8 up, 8 in 2024-09-14T04:00:37.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:37 smithi070 ceph-mon[98640]: pgmap v797: 121 pgs: 121 active+clean; 913 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 2.2 MiB/s rd, 844 KiB/s wr, 7 op/s 2024-09-14T04:00:38.166 INFO:tasks.rados.rados.0.smithi070.stdout:3632: done (2 left) 2024-09-14T04:00:38.167 INFO:tasks.rados.rados.0.smithi070.stdout:3633: done (1 left) 2024-09-14T04:00:38.167 INFO:tasks.rados.rados.0.smithi070.stdout:3634: done (0 left) 2024-09-14T04:00:38.167 INFO:tasks.rados.rados.0.smithi070.stdout:3635: read oid 5 snap -1 2024-09-14T04:00:38.167 INFO:tasks.rados.rados.0.smithi070.stdout:3635: expect (ObjNum 1084 snap 354 seq_num 1084) 2024-09-14T04:00:38.167 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 41 current snap is 372 2024-09-14T04:00:38.167 INFO:tasks.rados.rados.0.smithi070.stdout:3636: seq_num 1117 ranges {2121728=712704,2834432=458752,3293184=655360,3948544=90112} 2024-09-14T04:00:38.182 INFO:tasks.rados.rados.0.smithi070.stdout:3636: writing smithi07085154-41 from 2121728 to 2834432 tid 1 2024-09-14T04:00:38.185 INFO:tasks.rados.rados.0.smithi070.stdout:3636: writing smithi07085154-41 from 2834432 to 3293184 tid 2 2024-09-14T04:00:38.189 INFO:tasks.rados.rados.0.smithi070.stdout:3636: writing smithi07085154-41 from 3293184 to 3948544 tid 3 2024-09-14T04:00:38.190 INFO:tasks.rados.rados.0.smithi070.stdout:3636: writing smithi07085154-41 from 3948544 to 4038656 tid 4 2024-09-14T04:00:38.190 INFO:tasks.rados.rados.0.smithi070.stdout:3637: read oid 43 snap 352 2024-09-14T04:00:38.190 INFO:tasks.rados.rados.0.smithi070.stdout:3637: expect (ObjNum 1047 snap 344 seq_num 1047) 2024-09-14T04:00:38.190 INFO:tasks.rados.rados.0.smithi070.stdout:3638: snap_create 2024-09-14T04:00:38.190 INFO:tasks.rados.rados.0.smithi070.stdout:3635: expect (ObjNum 1084 snap 354 seq_num 1084) 2024-09-14T04:00:38.361 INFO:tasks.rados.rados.0.smithi070.stdout:3636: finishing write tid 1 to smithi07085154-41 2024-09-14T04:00:38.362 INFO:tasks.rados.rados.0.smithi070.stdout:3636: finishing write tid 2 to smithi07085154-41 2024-09-14T04:00:38.362 INFO:tasks.rados.rados.0.smithi070.stdout:3636: finishing write tid 3 to smithi07085154-41 2024-09-14T04:00:38.362 INFO:tasks.rados.rados.0.smithi070.stdout:3636: finishing write tid 4 to smithi07085154-41 2024-09-14T04:00:38.362 INFO:tasks.rados.rados.0.smithi070.stdout:3636: finishing write tid 5 to smithi07085154-41 2024-09-14T04:00:38.362 INFO:tasks.rados.rados.0.smithi070.stdout:3637: expect (ObjNum 1047 snap 344 seq_num 1047) 2024-09-14T04:00:38.473 INFO:tasks.rados.rados.0.smithi070.stdout:3636: finishing write tid 6 to smithi07085154-41 2024-09-14T04:00:38.473 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 41 v 4135 (ObjNum 1117 snap 372 seq_num 1117) dirty exists 2024-09-14T04:00:38.473 INFO:tasks.rados.rados.0.smithi070.stdout:3636: left oid 41 (ObjNum 1117 snap 372 seq_num 1117) 2024-09-14T04:00:38.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:38 smithi094 ceph-mon[86597]: osdmap e799: 8 total, 8 up, 8 in 2024-09-14T04:00:38.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:38 smithi070 ceph-mon[96704]: osdmap e799: 8 total, 8 up, 8 in 2024-09-14T04:00:38.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:38 smithi070 ceph-mon[98640]: osdmap e799: 8 total, 8 up, 8 in 2024-09-14T04:00:39.138 INFO:tasks.rados.rados.0.smithi070.stdout:3635: done (3 left) 2024-09-14T04:00:39.138 INFO:tasks.rados.rados.0.smithi070.stdout:3636: done (2 left) 2024-09-14T04:00:39.138 INFO:tasks.rados.rados.0.smithi070.stdout:3637: done (1 left) 2024-09-14T04:00:39.139 INFO:tasks.rados.rados.0.smithi070.stdout:3638: done (0 left) 2024-09-14T04:00:39.139 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 44 current snap is 373 2024-09-14T04:00:39.139 INFO:tasks.rados.rados.0.smithi070.stdout:3639: seq_num 1118 ranges {5652480=729088,6381568=794624,7176192=409600} 2024-09-14T04:00:39.173 INFO:tasks.rados.rados.0.smithi070.stdout:3639: writing smithi07085154-44 from 5652480 to 6381568 tid 1 2024-09-14T04:00:39.177 INFO:tasks.rados.rados.0.smithi070.stdout:3639: writing smithi07085154-44 from 6381568 to 7176192 tid 2 2024-09-14T04:00:39.180 INFO:tasks.rados.rados.0.smithi070.stdout:3639: writing smithi07085154-44 from 7176192 to 7585792 tid 3 2024-09-14T04:00:39.180 INFO:tasks.rados.rados.0.smithi070.stdout:3640: snap_remove snap 355 2024-09-14T04:00:39.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:39 smithi094 ceph-mon[86597]: pgmap v799: 121 pgs: 121 active+clean; 928 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 3.4 MiB/s wr, 16 op/s 2024-09-14T04:00:39.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:39 smithi094 ceph-mon[86597]: osdmap e800: 8 total, 8 up, 8 in 2024-09-14T04:00:39.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:39 smithi094 ceph-mon[86597]: osdmap e801: 8 total, 8 up, 8 in 2024-09-14T04:00:39.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:39 smithi070 ceph-mon[96704]: pgmap v799: 121 pgs: 121 active+clean; 928 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 3.4 MiB/s wr, 16 op/s 2024-09-14T04:00:39.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:39 smithi070 ceph-mon[96704]: osdmap e800: 8 total, 8 up, 8 in 2024-09-14T04:00:39.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:39 smithi070 ceph-mon[96704]: osdmap e801: 8 total, 8 up, 8 in 2024-09-14T04:00:39.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:39 smithi070 ceph-mon[98640]: pgmap v799: 121 pgs: 121 active+clean; 928 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 3.4 MiB/s wr, 16 op/s 2024-09-14T04:00:39.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:39 smithi070 ceph-mon[98640]: osdmap e800: 8 total, 8 up, 8 in 2024-09-14T04:00:39.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:39 smithi070 ceph-mon[98640]: osdmap e801: 8 total, 8 up, 8 in 2024-09-14T04:00:40.174 INFO:tasks.rados.rados.0.smithi070.stdout:3640: done (1 left) 2024-09-14T04:00:40.174 INFO:tasks.rados.rados.0.smithi070.stdout:3641: delete oid 42 current snap is 373 2024-09-14T04:00:40.174 INFO:tasks.rados.rados.0.smithi070.stdout:3639: finishing write tid 1 to smithi07085154-44 2024-09-14T04:00:40.174 INFO:tasks.rados.rados.0.smithi070.stdout:3639: finishing write tid 2 to smithi07085154-44 2024-09-14T04:00:40.174 INFO:tasks.rados.rados.0.smithi070.stdout:3639: finishing write tid 3 to smithi07085154-44 2024-09-14T04:00:40.174 INFO:tasks.rados.rados.0.smithi070.stdout:3639: finishing write tid 4 to smithi07085154-44 2024-09-14T04:00:40.175 INFO:tasks.rados.rados.0.smithi070.stdout:3639: finishing write tid 5 to smithi07085154-44 2024-09-14T04:00:40.175 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 4226 (ObjNum 1118 snap 373 seq_num 1118) dirty exists 2024-09-14T04:00:40.175 INFO:tasks.rados.rados.0.smithi070.stdout:3639: left oid 44 (ObjNum 1118 snap 373 seq_num 1118) 2024-09-14T04:00:40.188 INFO:tasks.rados.rados.0.smithi070.stdout:3639: done (1 left) 2024-09-14T04:00:40.188 INFO:tasks.rados.rados.0.smithi070.stdout:3641: done (0 left) 2024-09-14T04:00:40.188 INFO:tasks.rados.rados.0.smithi070.stdout:3642: setattr oid 19 current snap is 373 2024-09-14T04:00:40.190 INFO:tasks.rados.rados.0.smithi070.stdout:3643: snap_create 2024-09-14T04:00:40.195 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 19 v 4237 (ObjNum 659 snap 220 seq_num 659) dirty exists 2024-09-14T04:00:40.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:00:40 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:00:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:00:41.174 INFO:tasks.rados.rados.0.smithi070.stdout:3642: done (1 left) 2024-09-14T04:00:41.174 INFO:tasks.rados.rados.0.smithi070.stdout:3643: done (0 left) 2024-09-14T04:00:41.175 INFO:tasks.rados.rados.0.smithi070.stdout:3644: rollback oid 1 current snap is 374 2024-09-14T04:00:41.175 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 1 to 372 2024-09-14T04:00:41.175 INFO:tasks.rados.rados.0.smithi070.stdout:3645: rollback oid 18 current snap is 374 2024-09-14T04:00:41.175 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 18 to 367 2024-09-14T04:00:41.175 INFO:tasks.rados.rados.0.smithi070.stdout:3646: copy_from oid 6 from oid 4 current snap is 374 2024-09-14T04:00:41.175 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 15 current snap is 374 2024-09-14T04:00:41.175 INFO:tasks.rados.rados.0.smithi070.stdout:3647: seq_num 1120 ranges {3227648=761856,3989504=557056,4546560=425984,4972544=221184} 2024-09-14T04:00:41.196 INFO:tasks.rados.rados.0.smithi070.stdout:3647: writing smithi07085154-15 from 3227648 to 3989504 tid 1 2024-09-14T04:00:41.200 INFO:tasks.rados.rados.0.smithi070.stdout:3647: writing smithi07085154-15 from 3989504 to 4546560 tid 2 2024-09-14T04:00:41.202 INFO:tasks.rados.rados.0.smithi070.stdout:3647: writing smithi07085154-15 from 4546560 to 4972544 tid 3 2024-09-14T04:00:41.204 INFO:tasks.rados.rados.0.smithi070.stdout:3647: writing smithi07085154-15 from 4972544 to 5193728 tid 4 2024-09-14T04:00:41.204 INFO:tasks.rados.rados.0.smithi070.stdout:3648: read oid 39 snap 348 2024-09-14T04:00:41.204 INFO:tasks.rados.rados.0.smithi070.stdout:3648: expect (ObjNum 1020 snap 336 seq_num 1020) 2024-09-14T04:00:41.204 INFO:tasks.rados.rados.0.smithi070.stdout:3649: snap_create 2024-09-14T04:00:41.204 INFO:tasks.rados.rados.0.smithi070.stdout:3644: finishing rollback tid 0 to smithi07085154-1 2024-09-14T04:00:41.205 INFO:tasks.rados.rados.0.smithi070.stdout:3644: finishing rollback tid 1 to smithi07085154-1 2024-09-14T04:00:41.205 INFO:tasks.rados.rados.0.smithi070.stdout:3644: finishing rollback tid 2 to smithi07085154-1 2024-09-14T04:00:41.205 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 1 v 4229 (ObjNum 805 snap 281 seq_num 805) dirty exists 2024-09-14T04:00:41.205 INFO:tasks.rados.rados.0.smithi070.stdout:3645: finishing rollback tid 0 to smithi07085154-18 2024-09-14T04:00:41.205 INFO:tasks.rados.rados.0.smithi070.stdout:3645: finishing rollback tid 1 to smithi07085154-18 2024-09-14T04:00:41.205 INFO:tasks.rados.rados.0.smithi070.stdout:3645: finishing rollback tid 2 to smithi07085154-18 2024-09-14T04:00:41.205 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 4138 (ObjNum 596 snap 199 seq_num 596) dirty exists 2024-09-14T04:00:41.205 INFO:tasks.rados.rados.0.smithi070.stdout:3646: finishing copy_from racing read to smithi07085154-6 2024-09-14T04:00:41.205 INFO:tasks.rados.rados.0.smithi070.stdout:3646: finishing copy_from to smithi07085154-6 2024-09-14T04:00:41.205 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 4206 (ObjNum 1108 snap 369 seq_num 1108) dirty exists 2024-09-14T04:00:41.205 INFO:tasks.rados.rados.0.smithi070.stdout:3647: finishing write tid 1 to smithi07085154-15 2024-09-14T04:00:41.206 INFO:tasks.rados.rados.0.smithi070.stdout:3647: finishing write tid 2 to smithi07085154-15 2024-09-14T04:00:41.208 INFO:tasks.rados.rados.0.smithi070.stdout:3647: finishing write tid 3 to smithi07085154-15 2024-09-14T04:00:41.209 INFO:tasks.rados.rados.0.smithi070.stdout:3647: finishing write tid 4 to smithi07085154-15 2024-09-14T04:00:41.209 INFO:tasks.rados.rados.0.smithi070.stdout:3647: finishing write tid 5 to smithi07085154-15 2024-09-14T04:00:41.210 INFO:tasks.rados.rados.0.smithi070.stdout:3647: finishing write tid 6 to smithi07085154-15 2024-09-14T04:00:41.211 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 15 v 4143 (ObjNum 1120 snap 374 seq_num 1120) dirty exists 2024-09-14T04:00:41.211 INFO:tasks.rados.rados.0.smithi070.stdout:3647: left oid 15 (ObjNum 1120 snap 374 seq_num 1120) 2024-09-14T04:00:41.212 INFO:tasks.rados.rados.0.smithi070.stdout:3648: expect (ObjNum 1020 snap 336 seq_num 1020) 2024-09-14T04:00:41.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:41 smithi094 ceph-mon[86597]: pgmap v802: 121 pgs: 121 active+clean; 946 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 15 MiB/s rd, 3.5 MiB/s wr, 18 op/s 2024-09-14T04:00:41.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:41 smithi094 ceph-mon[86597]: osdmap e802: 8 total, 8 up, 8 in 2024-09-14T04:00:41.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:41 smithi070 ceph-mon[96704]: pgmap v802: 121 pgs: 121 active+clean; 946 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 15 MiB/s rd, 3.5 MiB/s wr, 18 op/s 2024-09-14T04:00:41.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:41 smithi070 ceph-mon[96704]: osdmap e802: 8 total, 8 up, 8 in 2024-09-14T04:00:41.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:41 smithi070 ceph-mon[98640]: pgmap v802: 121 pgs: 121 active+clean; 946 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 15 MiB/s rd, 3.5 MiB/s wr, 18 op/s 2024-09-14T04:00:41.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:41 smithi070 ceph-mon[98640]: osdmap e802: 8 total, 8 up, 8 in 2024-09-14T04:00:42.180 INFO:tasks.rados.rados.0.smithi070.stdout:3644: done (5 left) 2024-09-14T04:00:42.180 INFO:tasks.rados.rados.0.smithi070.stdout:3645: done (4 left) 2024-09-14T04:00:42.181 INFO:tasks.rados.rados.0.smithi070.stdout:3646: done (3 left) 2024-09-14T04:00:42.181 INFO:tasks.rados.rados.0.smithi070.stdout:3647: done (2 left) 2024-09-14T04:00:42.181 INFO:tasks.rados.rados.0.smithi070.stdout:3648: done (1 left) 2024-09-14T04:00:42.181 INFO:tasks.rados.rados.0.smithi070.stdout:3649: done (0 left) 2024-09-14T04:00:42.181 INFO:tasks.rados.rados.0.smithi070.stdout:3650: rmattr oid 27 current snap is 375 2024-09-14T04:00:42.181 INFO:tasks.rados.rados.0.smithi070.stdout:3651: copy_from oid 28 from oid 23 current snap is 375 2024-09-14T04:00:42.181 INFO:tasks.rados.rados.0.smithi070.stdout:3652: setattr oid 29 current snap is 375 2024-09-14T04:00:42.182 INFO:tasks.rados.rados.0.smithi070.stdout:3653: read oid 46 snap -1 2024-09-14T04:00:42.182 INFO:tasks.rados.rados.0.smithi070.stdout:3653: expect deleted 2024-09-14T04:00:42.182 INFO:tasks.rados.rados.0.smithi070.stdout:3654: rollback oid 4 current snap is 375 2024-09-14T04:00:42.182 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 4 to 351 2024-09-14T04:00:42.182 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 37 current snap is 375 2024-09-14T04:00:42.182 INFO:tasks.rados.rados.0.smithi070.stdout:3655: seq_num 1121 ranges {0=778240,778240=442368,1220608=638976,1859584=122880} 2024-09-14T04:00:42.187 INFO:tasks.rados.rados.0.smithi070.stdout:3655: writing smithi07085154-37 from 0 to 778240 tid 1 2024-09-14T04:00:42.190 INFO:tasks.rados.rados.0.smithi070.stdout:3655: writing smithi07085154-37 from 778240 to 1220608 tid 2 2024-09-14T04:00:42.194 INFO:tasks.rados.rados.0.smithi070.stdout:3655: writing smithi07085154-37 from 1220608 to 1859584 tid 3 2024-09-14T04:00:42.196 INFO:tasks.rados.rados.0.smithi070.stdout:3655: writing smithi07085154-37 from 1859584 to 1982464 tid 4 2024-09-14T04:00:42.196 INFO:tasks.rados.rados.0.smithi070.stdout:3656: rollback oid 22 current snap is 375 2024-09-14T04:00:42.196 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 22 to 351 2024-09-14T04:00:42.196 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 4331 (ObjNum 829 snap 283 seq_num 829) dirty exists 2024-09-14T04:00:42.196 INFO:tasks.rados.rados.0.smithi070.stdout:3654: finishing rollback tid 0 to smithi07085154-4 2024-09-14T04:00:42.196 INFO:tasks.rados.rados.0.smithi070.stdout:3654: finishing rollback tid 1 to smithi07085154-4 2024-09-14T04:00:42.196 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 4208 (ObjNum 65 snap 0 seq_num 1466658151) dirty dne 2024-09-14T04:00:42.196 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 4238 (ObjNum 1078 snap 349 seq_num 1078) dirty exists 2024-09-14T04:00:42.196 INFO:tasks.rados.rados.0.smithi070.stdout:3650: done (6 left) 2024-09-14T04:00:42.197 INFO:tasks.rados.rados.0.smithi070.stdout:3652: done (5 left) 2024-09-14T04:00:42.197 INFO:tasks.rados.rados.0.smithi070.stdout:3653: done (4 left) 2024-09-14T04:00:42.197 INFO:tasks.rados.rados.0.smithi070.stdout:3654: done (3 left) 2024-09-14T04:00:42.197 INFO:tasks.rados.rados.0.smithi070.stdout:3657: setattr oid 18 current snap is 375 2024-09-14T04:00:42.197 INFO:tasks.rados.rados.0.smithi070.stdout:3658: snap_create 2024-09-14T04:00:42.200 INFO:tasks.rados.rados.0.smithi070.stdout:3655: finishing write tid 1 to smithi07085154-37 2024-09-14T04:00:42.201 INFO:tasks.rados.rados.0.smithi070.stdout:3655: finishing write tid 2 to smithi07085154-37 2024-09-14T04:00:42.201 INFO:tasks.rados.rados.0.smithi070.stdout:3655: finishing write tid 3 to smithi07085154-37 2024-09-14T04:00:42.201 INFO:tasks.rados.rados.0.smithi070.stdout:3656: finishing rollback tid 0 to smithi07085154-22 2024-09-14T04:00:42.201 INFO:tasks.rados.rados.0.smithi070.stdout:3655: finishing write tid 4 to smithi07085154-37 2024-09-14T04:00:42.201 INFO:tasks.rados.rados.0.smithi070.stdout:3656: finishing rollback tid 1 to smithi07085154-22 2024-09-14T04:00:42.201 INFO:tasks.rados.rados.0.smithi070.stdout:3656: finishing rollback tid 2 to smithi07085154-22 2024-09-14T04:00:42.201 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 4241 (ObjNum 1071 snap 347 seq_num 1071) dirty exists 2024-09-14T04:00:42.204 INFO:tasks.rados.rados.0.smithi070.stdout:3655: finishing write tid 5 to smithi07085154-37 2024-09-14T04:00:42.209 INFO:tasks.rados.rados.0.smithi070.stdout:3655: finishing write tid 6 to smithi07085154-37 2024-09-14T04:00:42.210 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 37 v 4148 (ObjNum 1121 snap 375 seq_num 1121) dirty exists 2024-09-14T04:00:42.210 INFO:tasks.rados.rados.0.smithi070.stdout:3655: left oid 37 (ObjNum 1121 snap 375 seq_num 1121) 2024-09-14T04:00:42.211 INFO:tasks.rados.rados.0.smithi070.stdout:3651: finishing copy_from racing read to smithi07085154-28 2024-09-14T04:00:42.211 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 4149 (ObjNum 596 snap 199 seq_num 596) dirty exists 2024-09-14T04:00:42.214 INFO:tasks.rados.rados.0.smithi070.stdout:3651: finishing copy_from to smithi07085154-28 2024-09-14T04:00:42.214 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 28 v 3248 (ObjNum 1078 snap 349 seq_num 1078) dirty exists 2024-09-14T04:00:42.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:42 smithi094 ceph-mon[86597]: osdmap e803: 8 total, 8 up, 8 in 2024-09-14T04:00:42.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:42 smithi094 ceph-mon[86597]: pgmap v805: 121 pgs: 121 active+clean; 950 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2024-09-14T04:00:42.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:42 smithi070 ceph-mon[96704]: osdmap e803: 8 total, 8 up, 8 in 2024-09-14T04:00:42.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:42 smithi070 ceph-mon[96704]: pgmap v805: 121 pgs: 121 active+clean; 950 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2024-09-14T04:00:42.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:42 smithi070 ceph-mon[98640]: osdmap e803: 8 total, 8 up, 8 in 2024-09-14T04:00:42.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:42 smithi070 ceph-mon[98640]: pgmap v805: 121 pgs: 121 active+clean; 950 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2024-09-14T04:00:43.185 INFO:tasks.rados.rados.0.smithi070.stdout:3651: done (4 left) 2024-09-14T04:00:43.186 INFO:tasks.rados.rados.0.smithi070.stdout:3655: done (3 left) 2024-09-14T04:00:43.186 INFO:tasks.rados.rados.0.smithi070.stdout:3656: done (2 left) 2024-09-14T04:00:43.186 INFO:tasks.rados.rados.0.smithi070.stdout:3657: done (1 left) 2024-09-14T04:00:43.186 INFO:tasks.rados.rados.0.smithi070.stdout:3658: done (0 left) 2024-09-14T04:00:43.186 INFO:tasks.rados.rados.0.smithi070.stdout:3659: read oid 20 snap -1 2024-09-14T04:00:43.186 INFO:tasks.rados.rados.0.smithi070.stdout:3659: expect (ObjNum 1083 snap 353 seq_num 1083) 2024-09-14T04:00:43.186 INFO:tasks.rados.rados.0.smithi070.stdout:3660: snap_remove snap 342 2024-09-14T04:00:43.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:43 smithi094 ceph-mon[86597]: osdmap e804: 8 total, 8 up, 8 in 2024-09-14T04:00:43.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:43 smithi070 ceph-mon[96704]: osdmap e804: 8 total, 8 up, 8 in 2024-09-14T04:00:43.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:43 smithi070 ceph-mon[98640]: osdmap e804: 8 total, 8 up, 8 in 2024-09-14T04:00:43.536 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:00:43 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:00:43] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.51.0" 2024-09-14T04:00:44.141 INFO:tasks.rados.rados.0.smithi070.stdout:3660: done (1 left) 2024-09-14T04:00:44.141 INFO:tasks.rados.rados.0.smithi070.stdout:3661: copy_from oid 9 from oid 15 current snap is 376 2024-09-14T04:00:44.141 INFO:tasks.rados.rados.0.smithi070.stdout:3659: expect (ObjNum 1083 snap 353 seq_num 1083) 2024-09-14T04:00:44.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:44 smithi094 ceph-mon[86597]: osdmap e805: 8 total, 8 up, 8 in 2024-09-14T04:00:44.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:44 smithi094 ceph-mon[86597]: osdmap e806: 8 total, 8 up, 8 in 2024-09-14T04:00:44.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:44 smithi094 ceph-mon[86597]: pgmap v809: 121 pgs: 121 active+clean; 953 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.8 MiB/s wr, 13 op/s 2024-09-14T04:00:44.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:44 smithi070 ceph-mon[96704]: osdmap e805: 8 total, 8 up, 8 in 2024-09-14T04:00:44.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:44 smithi070 ceph-mon[96704]: osdmap e806: 8 total, 8 up, 8 in 2024-09-14T04:00:44.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:44 smithi070 ceph-mon[96704]: pgmap v809: 121 pgs: 121 active+clean; 953 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.8 MiB/s wr, 13 op/s 2024-09-14T04:00:44.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:44 smithi070 ceph-mon[98640]: osdmap e805: 8 total, 8 up, 8 in 2024-09-14T04:00:44.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:44 smithi070 ceph-mon[98640]: osdmap e806: 8 total, 8 up, 8 in 2024-09-14T04:00:44.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:44 smithi070 ceph-mon[98640]: pgmap v809: 121 pgs: 121 active+clean; 953 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.8 MiB/s wr, 13 op/s 2024-09-14T04:00:44.766 INFO:tasks.rados.rados.0.smithi070.stdout:3661: finishing copy_from racing read to smithi07085154-9 2024-09-14T04:00:44.766 INFO:tasks.rados.rados.0.smithi070.stdout:3661: finishing copy_from to smithi07085154-9 2024-09-14T04:00:44.766 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 4143 (ObjNum 1120 snap 374 seq_num 1120) dirty exists 2024-09-14T04:00:44.766 INFO:tasks.rados.rados.0.smithi070.stdout:3659: done (1 left) 2024-09-14T04:00:44.767 INFO:tasks.rados.rados.0.smithi070.stdout:3661: done (0 left) 2024-09-14T04:00:44.767 INFO:tasks.rados.rados.0.smithi070.stdout:3662: rmattr oid 30 current snap is 376 2024-09-14T04:00:44.767 INFO:tasks.rados.rados.0.smithi070.stdout:3662: done (0 left) 2024-09-14T04:00:44.767 INFO:tasks.rados.rados.0.smithi070.stdout:3663: read oid 31 snap 371 2024-09-14T04:00:44.767 INFO:tasks.rados.rados.0.smithi070.stdout:3663: expect (ObjNum 1100 snap 366 seq_num 1100) 2024-09-14T04:00:44.767 INFO:tasks.rados.rados.0.smithi070.stdout:3664: setattr oid 44 current snap is 376 2024-09-14T04:00:44.768 INFO:tasks.rados.rados.0.smithi070.stdout:3665: read oid 27 snap -1 2024-09-14T04:00:44.768 INFO:tasks.rados.rados.0.smithi070.stdout:3665: expect (ObjNum 829 snap 283 seq_num 829) 2024-09-14T04:00:44.768 INFO:tasks.rados.rados.0.smithi070.stdout:3666: read oid 18 snap -1 2024-09-14T04:00:44.768 INFO:tasks.rados.rados.0.smithi070.stdout:3666: expect (ObjNum 596 snap 199 seq_num 596) 2024-09-14T04:00:44.768 INFO:tasks.rados.rados.0.smithi070.stdout:3667: setattr oid 50 current snap is 376 2024-09-14T04:00:44.768 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 36 current snap is 376 2024-09-14T04:00:44.768 INFO:tasks.rados.rados.0.smithi070.stdout:3668: seq_num 1122 ranges {0=794624,794624=729088,1523712=475136} 2024-09-14T04:00:44.773 INFO:tasks.rados.rados.0.smithi070.stdout:3668: writing smithi07085154-36 from 0 to 794624 tid 1 2024-09-14T04:00:44.778 INFO:tasks.rados.rados.0.smithi070.stdout:3668: writing smithi07085154-36 from 794624 to 1523712 tid 2 2024-09-14T04:00:44.782 INFO:tasks.rados.rados.0.smithi070.stdout:3668: writing smithi07085154-36 from 1523712 to 1998848 tid 3 2024-09-14T04:00:44.782 INFO:tasks.rados.rados.0.smithi070.stdout:3669: read oid 41 snap 361 2024-09-14T04:00:44.783 INFO:tasks.rados.rados.0.smithi070.stdout:3669: expect (ObjNum 1006 snap 331 seq_num 1006) 2024-09-14T04:00:44.783 INFO:tasks.rados.rados.0.smithi070.stdout:3670: snap_remove snap 357 2024-09-14T04:00:45.188 INFO:tasks.rados.rados.0.smithi070.stdout:3670: done (7 left) 2024-09-14T04:00:45.188 INFO:tasks.rados.rados.0.smithi070.stdout:3671: read oid 48 snap 273 2024-09-14T04:00:45.188 INFO:tasks.rados.rados.0.smithi070.stdout:3671: expect (ObjNum 757 snap 263 seq_num 757) 2024-09-14T04:00:45.188 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 23 current snap is 376 2024-09-14T04:00:45.188 INFO:tasks.rados.rados.0.smithi070.stdout:3672: seq_num 1123 ranges {5316608=409600,5726208=614400,6340608=598016,6938624=393216} 2024-09-14T04:00:45.221 INFO:tasks.rados.rados.0.smithi070.stdout:3672: writing smithi07085154-23 from 5316608 to 5726208 tid 1 2024-09-14T04:00:45.224 INFO:tasks.rados.rados.0.smithi070.stdout:3672: writing smithi07085154-23 from 5726208 to 6340608 tid 2 2024-09-14T04:00:45.227 INFO:tasks.rados.rados.0.smithi070.stdout:3672: writing smithi07085154-23 from 6340608 to 6938624 tid 3 2024-09-14T04:00:45.230 INFO:tasks.rados.rados.0.smithi070.stdout:3672: writing smithi07085154-23 from 6938624 to 7331840 tid 4 2024-09-14T04:00:45.230 INFO:tasks.rados.rados.0.smithi070.stdout:3673: snap_create 2024-09-14T04:00:45.230 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 4242 (ObjNum 1116 snap 371 seq_num 1116) dirty exists 2024-09-14T04:00:45.230 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 4230 (ObjNum 1118 snap 373 seq_num 1118) dirty exists 2024-09-14T04:00:45.230 INFO:tasks.rados.rados.0.smithi070.stdout:3665: expect (ObjNum 829 snap 283 seq_num 829) 2024-09-14T04:00:45.340 INFO:tasks.rados.rados.0.smithi070.stdout:3666: expect (ObjNum 596 snap 199 seq_num 596) 2024-09-14T04:00:45.618 INFO:tasks.rados.rados.0.smithi070.stdout:3663: expect (ObjNum 1100 snap 366 seq_num 1100) 2024-09-14T04:00:45.800 INFO:tasks.rados.rados.0.smithi070.stdout:3668: finishing write tid 1 to smithi07085154-36 2024-09-14T04:00:45.800 INFO:tasks.rados.rados.0.smithi070.stdout:3668: finishing write tid 2 to smithi07085154-36 2024-09-14T04:00:45.800 INFO:tasks.rados.rados.0.smithi070.stdout:3668: finishing write tid 3 to smithi07085154-36 2024-09-14T04:00:45.800 INFO:tasks.rados.rados.0.smithi070.stdout:3668: finishing write tid 4 to smithi07085154-36 2024-09-14T04:00:45.800 INFO:tasks.rados.rados.0.smithi070.stdout:3668: finishing write tid 5 to smithi07085154-36 2024-09-14T04:00:45.800 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 36 v 4234 (ObjNum 1122 snap 376 seq_num 1122) dirty exists 2024-09-14T04:00:45.800 INFO:tasks.rados.rados.0.smithi070.stdout:3668: left oid 36 (ObjNum 1122 snap 376 seq_num 1122) 2024-09-14T04:00:45.800 INFO:tasks.rados.rados.0.smithi070.stdout:3669: expect (ObjNum 1006 snap 331 seq_num 1006) 2024-09-14T04:00:45.887 INFO:tasks.rados.rados.0.smithi070.stdout:3671: expect (ObjNum 757 snap 263 seq_num 757) 2024-09-14T04:00:46.098 INFO:tasks.rados.rados.0.smithi070.stdout:3672: finishing write tid 1 to smithi07085154-23 2024-09-14T04:00:46.098 INFO:tasks.rados.rados.0.smithi070.stdout:3672: finishing write tid 2 to smithi07085154-23 2024-09-14T04:00:46.098 INFO:tasks.rados.rados.0.smithi070.stdout:3672: finishing write tid 3 to smithi07085154-23 2024-09-14T04:00:46.098 INFO:tasks.rados.rados.0.smithi070.stdout:3672: finishing write tid 4 to smithi07085154-23 2024-09-14T04:00:46.098 INFO:tasks.rados.rados.0.smithi070.stdout:3672: finishing write tid 5 to smithi07085154-23 2024-09-14T04:00:46.098 INFO:tasks.rados.rados.0.smithi070.stdout:3672: finishing write tid 6 to smithi07085154-23 2024-09-14T04:00:46.098 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 3258 (ObjNum 1123 snap 376 seq_num 1123) dirty exists 2024-09-14T04:00:46.098 INFO:tasks.rados.rados.0.smithi070.stdout:3672: left oid 23 (ObjNum 1123 snap 376 seq_num 1123) 2024-09-14T04:00:46.194 INFO:tasks.rados.rados.0.smithi070.stdout:3663: done (9 left) 2024-09-14T04:00:46.194 INFO:tasks.rados.rados.0.smithi070.stdout:3664: done (8 left) 2024-09-14T04:00:46.194 INFO:tasks.rados.rados.0.smithi070.stdout:3665: done (7 left) 2024-09-14T04:00:46.194 INFO:tasks.rados.rados.0.smithi070.stdout:3666: done (6 left) 2024-09-14T04:00:46.194 INFO:tasks.rados.rados.0.smithi070.stdout:3667: done (5 left) 2024-09-14T04:00:46.194 INFO:tasks.rados.rados.0.smithi070.stdout:3668: done (4 left) 2024-09-14T04:00:46.194 INFO:tasks.rados.rados.0.smithi070.stdout:3669: done (3 left) 2024-09-14T04:00:46.194 INFO:tasks.rados.rados.0.smithi070.stdout:3671: done (2 left) 2024-09-14T04:00:46.194 INFO:tasks.rados.rados.0.smithi070.stdout:3672: done (1 left) 2024-09-14T04:00:46.194 INFO:tasks.rados.rados.0.smithi070.stdout:3673: done (0 left) 2024-09-14T04:00:46.194 INFO:tasks.rados.rados.0.smithi070.stdout:3674: copy_from oid 10 from oid 36 current snap is 377 2024-09-14T04:00:46.195 INFO:tasks.rados.rados.0.smithi070.stdout:3675: read oid 26 snap -1 2024-09-14T04:00:46.195 INFO:tasks.rados.rados.0.smithi070.stdout:3675: expect (ObjNum 1079 snap 350 seq_num 1079) 2024-09-14T04:00:46.195 INFO:tasks.rados.rados.0.smithi070.stdout:3676: read oid 37 snap 363 2024-09-14T04:00:46.195 INFO:tasks.rados.rados.0.smithi070.stdout:3676: expect deleted 2024-09-14T04:00:46.195 INFO:tasks.rados.rados.0.smithi070.stdout:3677: snap_remove snap 374 2024-09-14T04:00:46.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:46 smithi094 ceph-mon[86597]: osdmap e807: 8 total, 8 up, 8 in 2024-09-14T04:00:46.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:46 smithi094 ceph-mon[86597]: pgmap v811: 121 pgs: 121 active+clean; 957 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.8 MiB/s wr, 13 op/s 2024-09-14T04:00:46.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:46 smithi070 ceph-mon[96704]: osdmap e807: 8 total, 8 up, 8 in 2024-09-14T04:00:46.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:46 smithi070 ceph-mon[96704]: pgmap v811: 121 pgs: 121 active+clean; 957 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.8 MiB/s wr, 13 op/s 2024-09-14T04:00:46.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:46 smithi070 ceph-mon[98640]: osdmap e807: 8 total, 8 up, 8 in 2024-09-14T04:00:46.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:46 smithi070 ceph-mon[98640]: pgmap v811: 121 pgs: 121 active+clean; 957 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.8 MiB/s wr, 13 op/s 2024-09-14T04:00:47.195 INFO:tasks.rados.rados.0.smithi070.stdout:3677: done (3 left) 2024-09-14T04:00:47.196 INFO:tasks.rados.rados.0.smithi070.stdout:3678: read oid 5 snap -1 2024-09-14T04:00:47.196 INFO:tasks.rados.rados.0.smithi070.stdout:3678: expect (ObjNum 1084 snap 354 seq_num 1084) 2024-09-14T04:00:47.196 INFO:tasks.rados.rados.0.smithi070.stdout:3679: rmattr oid 32 current snap is 377 2024-09-14T04:00:47.196 INFO:tasks.rados.rados.0.smithi070.stdout:3679: done (4 left) 2024-09-14T04:00:47.196 INFO:tasks.rados.rados.0.smithi070.stdout:3680: rollback oid 28 current snap is 377 2024-09-14T04:00:47.196 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 28 to 348 2024-09-14T04:00:47.196 INFO:tasks.rados.rados.0.smithi070.stdout:3675: expect (ObjNum 1079 snap 350 seq_num 1079) 2024-09-14T04:00:47.241 INFO:tasks.rados.rados.0.smithi070.stdout:3674: finishing copy_from racing read to smithi07085154-10 2024-09-14T04:00:47.241 INFO:tasks.rados.rados.0.smithi070.stdout:3674: finishing copy_from to smithi07085154-10 2024-09-14T04:00:47.241 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 4234 (ObjNum 1122 snap 376 seq_num 1122) dirty exists 2024-09-14T04:00:47.241 INFO:tasks.rados.rados.0.smithi070.stdout:3674: done (4 left) 2024-09-14T04:00:47.241 INFO:tasks.rados.rados.0.smithi070.stdout:3675: done (3 left) 2024-09-14T04:00:47.242 INFO:tasks.rados.rados.0.smithi070.stdout:3676: done (2 left) 2024-09-14T04:00:47.242 INFO:tasks.rados.rados.0.smithi070.stdout:3681: read oid 45 snap -1 2024-09-14T04:00:47.242 INFO:tasks.rados.rados.0.smithi070.stdout:3681: expect (ObjNum 997 snap 330 seq_num 997) 2024-09-14T04:00:47.242 INFO:tasks.rados.rados.0.smithi070.stdout:3682: snap_create 2024-09-14T04:00:47.242 INFO:tasks.rados.rados.0.smithi070.stdout:3678: expect (ObjNum 1084 snap 354 seq_num 1084) 2024-09-14T04:00:47.420 INFO:tasks.rados.rados.0.smithi070.stdout:3680: finishing rollback tid 0 to smithi07085154-28 2024-09-14T04:00:47.421 INFO:tasks.rados.rados.0.smithi070.stdout:3680: finishing rollback tid 1 to smithi07085154-28 2024-09-14T04:00:47.421 INFO:tasks.rados.rados.0.smithi070.stdout:3680: finishing rollback tid 2 to smithi07085154-28 2024-09-14T04:00:47.421 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 28 v 4211 (ObjNum 1044 snap 344 seq_num 1044) dirty exists 2024-09-14T04:00:47.421 INFO:tasks.rados.rados.0.smithi070.stdout:3681: expect (ObjNum 997 snap 330 seq_num 997) 2024-09-14T04:00:47.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:47 smithi094 ceph-mon[86597]: osdmap e808: 8 total, 8 up, 8 in 2024-09-14T04:00:47.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:47 smithi070 ceph-mon[96704]: osdmap e808: 8 total, 8 up, 8 in 2024-09-14T04:00:47.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:47 smithi070 ceph-mon[98640]: osdmap e808: 8 total, 8 up, 8 in 2024-09-14T04:00:48.204 INFO:tasks.rados.rados.0.smithi070.stdout:3678: done (3 left) 2024-09-14T04:00:48.204 INFO:tasks.rados.rados.0.smithi070.stdout:3680: done (2 left) 2024-09-14T04:00:48.205 INFO:tasks.rados.rados.0.smithi070.stdout:3681: done (1 left) 2024-09-14T04:00:48.205 INFO:tasks.rados.rados.0.smithi070.stdout:3682: done (0 left) 2024-09-14T04:00:48.205 INFO:tasks.rados.rados.0.smithi070.stdout:3683: read oid 49 snap 373 2024-09-14T04:00:48.205 INFO:tasks.rados.rados.0.smithi070.stdout:3683: expect (ObjNum 1080 snap 350 seq_num 1080) 2024-09-14T04:00:48.205 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 26 current snap is 378 2024-09-14T04:00:48.205 INFO:tasks.rados.rados.0.smithi070.stdout:3684: seq_num 1124 ranges {1277952=434176,1712128=499712,2211840=409600,2621440=663552,3284992=32768} 2024-09-14T04:00:48.213 INFO:tasks.rados.rados.0.smithi070.stdout:3684: writing smithi07085154-26 from 1277952 to 1712128 tid 1 2024-09-14T04:00:48.216 INFO:tasks.rados.rados.0.smithi070.stdout:3684: writing smithi07085154-26 from 1712128 to 2211840 tid 2 2024-09-14T04:00:48.219 INFO:tasks.rados.rados.0.smithi070.stdout:3684: writing smithi07085154-26 from 2211840 to 2621440 tid 3 2024-09-14T04:00:48.223 INFO:tasks.rados.rados.0.smithi070.stdout:3684: writing smithi07085154-26 from 2621440 to 3284992 tid 4 2024-09-14T04:00:48.223 INFO:tasks.rados.rados.0.smithi070.stdout:3684: writing smithi07085154-26 from 3284992 to 3317760 tid 5 2024-09-14T04:00:48.223 INFO:tasks.rados.rados.0.smithi070.stdout:3685: rollback oid 34 current snap is 378 2024-09-14T04:00:48.223 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 34 to 363 2024-09-14T04:00:48.223 INFO:tasks.rados.rados.0.smithi070.stdout:3683: expect (ObjNum 1080 snap 350 seq_num 1080) 2024-09-14T04:00:48.258 INFO:tasks.rados.rados.0.smithi070.stdout:3684: finishing write tid 1 to smithi07085154-26 2024-09-14T04:00:48.259 INFO:tasks.rados.rados.0.smithi070.stdout:3684: finishing write tid 2 to smithi07085154-26 2024-09-14T04:00:48.259 INFO:tasks.rados.rados.0.smithi070.stdout:3684: finishing write tid 3 to smithi07085154-26 2024-09-14T04:00:48.259 INFO:tasks.rados.rados.0.smithi070.stdout:3685: finishing rollback tid 0 to smithi07085154-34 2024-09-14T04:00:48.259 INFO:tasks.rados.rados.0.smithi070.stdout:3685: finishing rollback tid 1 to smithi07085154-34 2024-09-14T04:00:48.259 INFO:tasks.rados.rados.0.smithi070.stdout:3685: finishing rollback tid 2 to smithi07085154-34 2024-09-14T04:00:48.259 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 34 v 4245 (ObjNum 1094 snap 358 seq_num 1094) dirty exists 2024-09-14T04:00:48.259 INFO:tasks.rados.rados.0.smithi070.stdout:3684: finishing write tid 4 to smithi07085154-26 2024-09-14T04:00:48.259 INFO:tasks.rados.rados.0.smithi070.stdout:3684: finishing write tid 5 to smithi07085154-26 2024-09-14T04:00:48.259 INFO:tasks.rados.rados.0.smithi070.stdout:3684: finishing write tid 6 to smithi07085154-26 2024-09-14T04:00:48.259 INFO:tasks.rados.rados.0.smithi070.stdout:3684: finishing write tid 7 to smithi07085154-26 2024-09-14T04:00:48.259 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 4374 (ObjNum 1124 snap 378 seq_num 1124) dirty exists 2024-09-14T04:00:48.259 INFO:tasks.rados.rados.0.smithi070.stdout:3684: left oid 26 (ObjNum 1124 snap 378 seq_num 1124) 2024-09-14T04:00:48.260 INFO:tasks.rados.rados.0.smithi070.stdout:3683: done (2 left) 2024-09-14T04:00:48.260 INFO:tasks.rados.rados.0.smithi070.stdout:3684: done (1 left) 2024-09-14T04:00:48.260 INFO:tasks.rados.rados.0.smithi070.stdout:3685: done (0 left) 2024-09-14T04:00:48.260 INFO:tasks.rados.rados.0.smithi070.stdout:3686: delete oid 26 current snap is 378 2024-09-14T04:00:48.262 INFO:tasks.rados.rados.0.smithi070.stdout:3686: done (0 left) 2024-09-14T04:00:48.262 INFO:tasks.rados.rados.0.smithi070.stdout:3687: read oid 4 snap -1 2024-09-14T04:00:48.262 INFO:tasks.rados.rados.0.smithi070.stdout:3687: expect deleted 2024-09-14T04:00:48.262 INFO:tasks.rados.rados.0.smithi070.stdout:3688: copy_from oid 3 from oid 32 current snap is 378 2024-09-14T04:00:48.262 INFO:tasks.rados.rados.0.smithi070.stdout:3689: snap_remove snap 273 2024-09-14T04:00:48.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:48 smithi094 ceph-mon[86597]: osdmap e809: 8 total, 8 up, 8 in 2024-09-14T04:00:48.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:48 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:00:48.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:48 smithi094 ceph-mon[86597]: pgmap v814: 121 pgs: 121 active+clean; 974 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 16 MiB/s rd, 4.4 MiB/s wr, 29 op/s 2024-09-14T04:00:48.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:48 smithi070 ceph-mon[96704]: osdmap e809: 8 total, 8 up, 8 in 2024-09-14T04:00:48.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:48 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:00:48.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:48 smithi070 ceph-mon[96704]: pgmap v814: 121 pgs: 121 active+clean; 974 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 16 MiB/s rd, 4.4 MiB/s wr, 29 op/s 2024-09-14T04:00:48.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:48 smithi070 ceph-mon[98640]: osdmap e809: 8 total, 8 up, 8 in 2024-09-14T04:00:48.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:48 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:00:48.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:48 smithi070 ceph-mon[98640]: pgmap v814: 121 pgs: 121 active+clean; 974 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 16 MiB/s rd, 4.4 MiB/s wr, 29 op/s 2024-09-14T04:00:49.142 INFO:tasks.rados.rados.0.smithi070.stdout:3689: done (2 left) 2024-09-14T04:00:49.142 INFO:tasks.rados.rados.0.smithi070.stdout:3690: snap_remove snap 267 2024-09-14T04:00:49.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:49 smithi094 ceph-mon[86597]: osdmap e810: 8 total, 8 up, 8 in 2024-09-14T04:00:49.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:49 smithi094 ceph-mon[86597]: osdmap e811: 8 total, 8 up, 8 in 2024-09-14T04:00:49.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:49 smithi070 ceph-mon[96704]: osdmap e810: 8 total, 8 up, 8 in 2024-09-14T04:00:49.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:49 smithi070 ceph-mon[96704]: osdmap e811: 8 total, 8 up, 8 in 2024-09-14T04:00:49.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:49 smithi070 ceph-mon[98640]: osdmap e810: 8 total, 8 up, 8 in 2024-09-14T04:00:49.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:49 smithi070 ceph-mon[98640]: osdmap e811: 8 total, 8 up, 8 in 2024-09-14T04:00:50.149 INFO:tasks.rados.rados.0.smithi070.stdout:3688: finishing copy_from to smithi07085154-3 2024-09-14T04:00:50.149 INFO:tasks.rados.rados.0.smithi070.stdout:3688: got expected ENOENT (src dne) 2024-09-14T04:00:50.149 INFO:tasks.rados.rados.0.smithi070.stdout:3688: finishing copy_from racing read to smithi07085154-3 2024-09-14T04:00:50.149 INFO:tasks.rados.rados.0.smithi070.stdout:3687: done (2 left) 2024-09-14T04:00:50.149 INFO:tasks.rados.rados.0.smithi070.stdout:3688: done (1 left) 2024-09-14T04:00:50.149 INFO:tasks.rados.rados.0.smithi070.stdout:3690: done (0 left) 2024-09-14T04:00:50.149 INFO:tasks.rados.rados.0.smithi070.stdout:3691: rollback oid 19 current snap is 378 2024-09-14T04:00:50.149 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 19 to 354 2024-09-14T04:00:50.150 INFO:tasks.rados.rados.0.smithi070.stdout:3692: delete oid 22 current snap is 378 2024-09-14T04:00:50.176 INFO:tasks.rados.rados.0.smithi070.stdout:3691: finishing rollback tid 0 to smithi07085154-19 2024-09-14T04:00:50.176 INFO:tasks.rados.rados.0.smithi070.stdout:3691: finishing rollback tid 1 to smithi07085154-19 2024-09-14T04:00:50.176 INFO:tasks.rados.rados.0.smithi070.stdout:3691: finishing rollback tid 2 to smithi07085154-19 2024-09-14T04:00:50.176 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 19 v 4248 (ObjNum 659 snap 220 seq_num 659) dirty exists 2024-09-14T04:00:50.176 INFO:tasks.rados.rados.0.smithi070.stdout:3691: done (1 left) 2024-09-14T04:00:50.176 INFO:tasks.rados.rados.0.smithi070.stdout:3692: done (0 left) 2024-09-14T04:00:50.176 INFO:tasks.rados.rados.0.smithi070.stdout:3693: snap_create 2024-09-14T04:00:50.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:00:50 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:00:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:00:51.147 INFO:tasks.rados.rados.0.smithi070.stdout:3693: done (0 left) 2024-09-14T04:00:51.147 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 31 current snap is 379 2024-09-14T04:00:51.147 INFO:tasks.rados.rados.0.smithi070.stdout:3694: seq_num 1127 ranges {4448256=483328,4931584=598016,5529600=581632,6111232=425984} 2024-09-14T04:00:51.173 INFO:tasks.rados.rados.0.smithi070.stdout:3694: writing smithi07085154-31 from 4448256 to 4931584 tid 1 2024-09-14T04:00:51.176 INFO:tasks.rados.rados.0.smithi070.stdout:3694: writing smithi07085154-31 from 4931584 to 5529600 tid 2 2024-09-14T04:00:51.180 INFO:tasks.rados.rados.0.smithi070.stdout:3694: writing smithi07085154-31 from 5529600 to 6111232 tid 3 2024-09-14T04:00:51.182 INFO:tasks.rados.rados.0.smithi070.stdout:3694: writing smithi07085154-31 from 6111232 to 6537216 tid 4 2024-09-14T04:00:51.183 INFO:tasks.rados.rados.0.smithi070.stdout:3695: delete oid 30 current snap is 379 2024-09-14T04:00:51.183 INFO:tasks.rados.rados.0.smithi070.stdout:3694: finishing write tid 1 to smithi07085154-31 2024-09-14T04:00:51.183 INFO:tasks.rados.rados.0.smithi070.stdout:3694: finishing write tid 2 to smithi07085154-31 2024-09-14T04:00:51.183 INFO:tasks.rados.rados.0.smithi070.stdout:3694: finishing write tid 3 to smithi07085154-31 2024-09-14T04:00:51.187 INFO:tasks.rados.rados.0.smithi070.stdout:3694: finishing write tid 4 to smithi07085154-31 2024-09-14T04:00:51.187 INFO:tasks.rados.rados.0.smithi070.stdout:3694: finishing write tid 5 to smithi07085154-31 2024-09-14T04:00:51.187 INFO:tasks.rados.rados.0.smithi070.stdout:3695: done (1 left) 2024-09-14T04:00:51.187 INFO:tasks.rados.rados.0.smithi070.stdout:3696: delete oid 22 current snap is 379 2024-09-14T04:00:51.188 INFO:tasks.rados.rados.0.smithi070.stdout:3694: finishing write tid 6 to smithi07085154-31 2024-09-14T04:00:51.188 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 4211 (ObjNum 1127 snap 379 seq_num 1127) dirty exists 2024-09-14T04:00:51.188 INFO:tasks.rados.rados.0.smithi070.stdout:3694: left oid 31 (ObjNum 1127 snap 379 seq_num 1127) 2024-09-14T04:00:51.190 INFO:tasks.rados.rados.0.smithi070.stdout:3694: done (1 left) 2024-09-14T04:00:51.191 INFO:tasks.rados.rados.0.smithi070.stdout:3696: done (0 left) 2024-09-14T04:00:51.191 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 44 current snap is 379 2024-09-14T04:00:51.191 INFO:tasks.rados.rados.0.smithi070.stdout:3697: seq_num 1130 ranges {7585792=532480,8118272=655360,8773632=753664,9527296=196608} 2024-09-14T04:00:51.233 INFO:tasks.rados.rados.0.smithi070.stdout:3697: writing smithi07085154-44 from 7585792 to 8118272 tid 1 2024-09-14T04:00:51.237 INFO:tasks.rados.rados.0.smithi070.stdout:3697: writing smithi07085154-44 from 8118272 to 8773632 tid 2 2024-09-14T04:00:51.241 INFO:tasks.rados.rados.0.smithi070.stdout:3697: writing smithi07085154-44 from 8773632 to 9527296 tid 3 2024-09-14T04:00:51.242 INFO:tasks.rados.rados.0.smithi070.stdout:3697: writing smithi07085154-44 from 9527296 to 9723904 tid 4 2024-09-14T04:00:51.243 INFO:tasks.rados.rados.0.smithi070.stdout:3698: snap_remove snap 367 2024-09-14T04:00:51.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:51 smithi094 ceph-mon[86597]: osdmap e812: 8 total, 8 up, 8 in 2024-09-14T04:00:51.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:51 smithi094 ceph-mon[86597]: pgmap v818: 121 pgs: 121 active+clean; 973 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 23 MiB/s rd, 3.5 MiB/s wr, 27 op/s 2024-09-14T04:00:51.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:51 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:00:51.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:51 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:00:51.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:51 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:00:51.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:51 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:00:51.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:51 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:00:51.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:51 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:00:51.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:51 smithi070 ceph-mon[96704]: osdmap e812: 8 total, 8 up, 8 in 2024-09-14T04:00:51.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:51 smithi070 ceph-mon[96704]: pgmap v818: 121 pgs: 121 active+clean; 973 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 23 MiB/s rd, 3.5 MiB/s wr, 27 op/s 2024-09-14T04:00:51.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:51 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:00:51.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:51 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:00:51.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:51 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:00:51.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:51 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:00:51.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:51 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:00:51.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:51 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:00:51.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:51 smithi070 ceph-mon[98640]: osdmap e812: 8 total, 8 up, 8 in 2024-09-14T04:00:51.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:51 smithi070 ceph-mon[98640]: pgmap v818: 121 pgs: 121 active+clean; 973 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 23 MiB/s rd, 3.5 MiB/s wr, 27 op/s 2024-09-14T04:00:51.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:51 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:00:51.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:51 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:00:51.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:51 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:00:51.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:51 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:00:51.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:51 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:00:51.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:51 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:00:52.155 INFO:tasks.rados.rados.0.smithi070.stdout:3697: finishing write tid 1 to smithi07085154-44 2024-09-14T04:00:52.155 INFO:tasks.rados.rados.0.smithi070.stdout:3697: finishing write tid 2 to smithi07085154-44 2024-09-14T04:00:52.155 INFO:tasks.rados.rados.0.smithi070.stdout:3697: finishing write tid 3 to smithi07085154-44 2024-09-14T04:00:52.155 INFO:tasks.rados.rados.0.smithi070.stdout:3697: finishing write tid 4 to smithi07085154-44 2024-09-14T04:00:52.155 INFO:tasks.rados.rados.0.smithi070.stdout:3697: finishing write tid 5 to smithi07085154-44 2024-09-14T04:00:52.155 INFO:tasks.rados.rados.0.smithi070.stdout:3697: finishing write tid 6 to smithi07085154-44 2024-09-14T04:00:52.155 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 4239 (ObjNum 1130 snap 379 seq_num 1130) dirty exists 2024-09-14T04:00:52.155 INFO:tasks.rados.rados.0.smithi070.stdout:3697: left oid 44 (ObjNum 1130 snap 379 seq_num 1130) 2024-09-14T04:00:52.155 INFO:tasks.rados.rados.0.smithi070.stdout:3697: done (1 left) 2024-09-14T04:00:52.156 INFO:tasks.rados.rados.0.smithi070.stdout:3698: done (0 left) 2024-09-14T04:00:52.156 INFO:tasks.rados.rados.0.smithi070.stdout:3699: snap_create 2024-09-14T04:00:52.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:52 smithi094 ceph-mon[86597]: osdmap e813: 8 total, 8 up, 8 in 2024-09-14T04:00:52.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:52 smithi070 ceph-mon[96704]: osdmap e813: 8 total, 8 up, 8 in 2024-09-14T04:00:52.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:52 smithi070 ceph-mon[98640]: osdmap e813: 8 total, 8 up, 8 in 2024-09-14T04:00:53.160 INFO:tasks.rados.rados.0.smithi070.stdout:3699: done (0 left) 2024-09-14T04:00:53.160 INFO:tasks.rados.rados.0.smithi070.stdout:3700: rollback oid 21 current snap is 380 2024-09-14T04:00:53.160 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 21 to 354 2024-09-14T04:00:53.160 INFO:tasks.rados.rados.0.smithi070.stdout:3701: rollback oid 38 current snap is 380 2024-09-14T04:00:53.160 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 38 to 363 2024-09-14T04:00:53.160 INFO:tasks.rados.rados.0.smithi070.stdout:3702: snap_create 2024-09-14T04:00:53.164 INFO:tasks.rados.rados.0.smithi070.stdout:3700: finishing rollback tid 0 to smithi07085154-21 2024-09-14T04:00:53.165 INFO:tasks.rados.rados.0.smithi070.stdout:3700: finishing rollback tid 1 to smithi07085154-21 2024-09-14T04:00:53.165 INFO:tasks.rados.rados.0.smithi070.stdout:3700: finishing rollback tid 2 to smithi07085154-21 2024-09-14T04:00:53.165 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 1728 (ObjNum 927 snap 310 seq_num 927) dirty exists 2024-09-14T04:00:53.168 INFO:tasks.rados.rados.0.smithi070.stdout:3701: finishing rollback tid 0 to smithi07085154-38 2024-09-14T04:00:53.168 INFO:tasks.rados.rados.0.smithi070.stdout:3701: finishing rollback tid 1 to smithi07085154-38 2024-09-14T04:00:53.168 INFO:tasks.rados.rados.0.smithi070.stdout:3701: finishing rollback tid 2 to smithi07085154-38 2024-09-14T04:00:53.169 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 38 v 3991 (ObjNum 1070 snap 347 seq_num 1070) dirty exists 2024-09-14T04:00:53.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:53 smithi070 ceph-mon[96704]: osdmap e814: 8 total, 8 up, 8 in 2024-09-14T04:00:53.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:53 smithi070 ceph-mon[96704]: pgmap v821: 121 pgs: 121 active+clean; 891 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12 MiB/s rd, 998 KiB/s wr, 15 op/s 2024-09-14T04:00:53.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:53 smithi070 ceph-mon[98640]: osdmap e814: 8 total, 8 up, 8 in 2024-09-14T04:00:53.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:53 smithi070 ceph-mon[98640]: pgmap v821: 121 pgs: 121 active+clean; 891 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12 MiB/s rd, 998 KiB/s wr, 15 op/s 2024-09-14T04:00:53.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:53 smithi094 ceph-mon[86597]: osdmap e814: 8 total, 8 up, 8 in 2024-09-14T04:00:53.529 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:53 smithi094 ceph-mon[86597]: pgmap v821: 121 pgs: 121 active+clean; 891 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12 MiB/s rd, 998 KiB/s wr, 15 op/s 2024-09-14T04:00:53.678 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:00:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:00:53] "GET /metrics HTTP/1.1" 200 36505 "" "Prometheus/2.51.0" 2024-09-14T04:00:54.144 INFO:tasks.rados.rados.0.smithi070.stdout:3700: done (2 left) 2024-09-14T04:00:54.144 INFO:tasks.rados.rados.0.smithi070.stdout:3701: done (1 left) 2024-09-14T04:00:54.144 INFO:tasks.rados.rados.0.smithi070.stdout:3702: done (0 left) 2024-09-14T04:00:54.144 INFO:tasks.rados.rados.0.smithi070.stdout:3703: read oid 9 snap 363 2024-09-14T04:00:54.144 INFO:tasks.rados.rados.0.smithi070.stdout:3703: expect (ObjNum 1088 snap 354 seq_num 1088) 2024-09-14T04:00:54.144 INFO:tasks.rados.rados.0.smithi070.stdout:3704: delete oid 37 current snap is 381 2024-09-14T04:00:54.150 INFO:tasks.rados.rados.0.smithi070.stdout:3704: done (1 left) 2024-09-14T04:00:54.150 INFO:tasks.rados.rados.0.smithi070.stdout:3705: rollback oid 8 current snap is 381 2024-09-14T04:00:54.151 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 8 to 330 2024-09-14T04:00:54.151 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 32 current snap is 381 2024-09-14T04:00:54.151 INFO:tasks.rados.rados.0.smithi070.stdout:3706: seq_num 1132 ranges {0=565248,565248=425984,991232=770048,1761280=409600} 2024-09-14T04:00:54.154 INFO:tasks.rados.rados.0.smithi070.stdout:3706: writing smithi07085154-32 from 0 to 565248 tid 1 2024-09-14T04:00:54.158 INFO:tasks.rados.rados.0.smithi070.stdout:3706: writing smithi07085154-32 from 565248 to 991232 tid 2 2024-09-14T04:00:54.163 INFO:tasks.rados.rados.0.smithi070.stdout:3706: writing smithi07085154-32 from 991232 to 1761280 tid 3 2024-09-14T04:00:54.166 INFO:tasks.rados.rados.0.smithi070.stdout:3706: writing smithi07085154-32 from 1761280 to 2170880 tid 4 2024-09-14T04:00:54.167 INFO:tasks.rados.rados.0.smithi070.stdout:3707: rmattr oid 7 current snap is 381 2024-09-14T04:00:54.167 INFO:tasks.rados.rados.0.smithi070.stdout:3707: done (3 left) 2024-09-14T04:00:54.167 INFO:tasks.rados.rados.0.smithi070.stdout:3708: copy_from oid 6 from oid 34 current snap is 381 2024-09-14T04:00:54.167 INFO:tasks.rados.rados.0.smithi070.stdout:3705: finishing rollback tid 0 to smithi07085154-8 2024-09-14T04:00:54.167 INFO:tasks.rados.rados.0.smithi070.stdout:3705: finishing rollback tid 1 to smithi07085154-8 2024-09-14T04:00:54.167 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 49 current snap is 381 2024-09-14T04:00:54.167 INFO:tasks.rados.rados.0.smithi070.stdout:3709: seq_num 1133 ranges {1294336=581632,1875968=753664,2629632=573440,3203072=278528} 2024-09-14T04:00:54.177 INFO:tasks.rados.rados.0.smithi070.stdout:3709: writing smithi07085154-49 from 1294336 to 1875968 tid 1 2024-09-14T04:00:54.182 INFO:tasks.rados.rados.0.smithi070.stdout:3709: writing smithi07085154-49 from 1875968 to 2629632 tid 2 2024-09-14T04:00:54.185 INFO:tasks.rados.rados.0.smithi070.stdout:3709: writing smithi07085154-49 from 2629632 to 3203072 tid 3 2024-09-14T04:00:54.187 INFO:tasks.rados.rados.0.smithi070.stdout:3709: writing smithi07085154-49 from 3203072 to 3481600 tid 4 2024-09-14T04:00:54.187 INFO:tasks.rados.rados.0.smithi070.stdout:3710: delete oid 5 current snap is 381 2024-09-14T04:00:54.187 INFO:tasks.rados.rados.0.smithi070.stdout:3705: finishing rollback tid 2 to smithi07085154-8 2024-09-14T04:00:54.187 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 8 v 4153 (ObjNum 923 snap 309 seq_num 923) dirty exists 2024-09-14T04:00:54.187 INFO:tasks.rados.rados.0.smithi070.stdout:3703: expect (ObjNum 1088 snap 354 seq_num 1088) 2024-09-14T04:00:54.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:54 smithi094 ceph-mon[86597]: osdmap e815: 8 total, 8 up, 8 in 2024-09-14T04:00:54.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:54 smithi094 ceph-mon[86597]: osdmap e816: 8 total, 8 up, 8 in 2024-09-14T04:00:54.529 INFO:tasks.rados.rados.0.smithi070.stdout:3706: finishing write tid 1 to smithi07085154-32 2024-09-14T04:00:54.529 INFO:tasks.rados.rados.0.smithi070.stdout:3706: finishing write tid 2 to smithi07085154-32 2024-09-14T04:00:54.529 INFO:tasks.rados.rados.0.smithi070.stdout:3703: done (5 left) 2024-09-14T04:00:54.529 INFO:tasks.rados.rados.0.smithi070.stdout:3705: done (4 left) 2024-09-14T04:00:54.529 INFO:tasks.rados.rados.0.smithi070.stdout:3710: done (3 left) 2024-09-14T04:00:54.529 INFO:tasks.rados.rados.0.smithi070.stdout:3711: rollback oid 48 current snap is 381 2024-09-14T04:00:54.529 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 48 to 351 2024-09-14T04:00:54.529 INFO:tasks.rados.rados.0.smithi070.stdout:3706: finishing write tid 3 to smithi07085154-32 2024-09-14T04:00:54.530 INFO:tasks.rados.rados.0.smithi070.stdout:3706: finishing write tid 4 to smithi07085154-32 2024-09-14T04:00:54.530 INFO:tasks.rados.rados.0.smithi070.stdout:3706: finishing write tid 5 to smithi07085154-32 2024-09-14T04:00:54.530 INFO:tasks.rados.rados.0.smithi070.stdout:3706: finishing write tid 6 to smithi07085154-32 2024-09-14T04:00:54.530 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 3258 (ObjNum 1132 snap 381 seq_num 1132) dirty exists 2024-09-14T04:00:54.530 INFO:tasks.rados.rados.0.smithi070.stdout:3706: left oid 32 (ObjNum 1132 snap 381 seq_num 1132) 2024-09-14T04:00:54.530 INFO:tasks.rados.rados.0.smithi070.stdout:3709: finishing write tid 1 to smithi07085154-49 2024-09-14T04:00:54.530 INFO:tasks.rados.rados.0.smithi070.stdout:3709: finishing write tid 2 to smithi07085154-49 2024-09-14T04:00:54.530 INFO:tasks.rados.rados.0.smithi070.stdout:3709: finishing write tid 3 to smithi07085154-49 2024-09-14T04:00:54.530 INFO:tasks.rados.rados.0.smithi070.stdout:3709: finishing write tid 4 to smithi07085154-49 2024-09-14T04:00:54.530 INFO:tasks.rados.rados.0.smithi070.stdout:3709: finishing write tid 5 to smithi07085154-49 2024-09-14T04:00:54.530 INFO:tasks.rados.rados.0.smithi070.stdout:3709: finishing write tid 6 to smithi07085154-49 2024-09-14T04:00:54.530 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 49 v 3215 (ObjNum 1133 snap 381 seq_num 1133) dirty exists 2024-09-14T04:00:54.530 INFO:tasks.rados.rados.0.smithi070.stdout:3709: left oid 49 (ObjNum 1133 snap 381 seq_num 1133) 2024-09-14T04:00:54.530 INFO:tasks.rados.rados.0.smithi070.stdout:3708: finishing copy_from racing read to smithi07085154-6 2024-09-14T04:00:54.530 INFO:tasks.rados.rados.0.smithi070.stdout:3708: finishing copy_from to smithi07085154-6 2024-09-14T04:00:54.531 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 4245 (ObjNum 1094 snap 358 seq_num 1094) dirty exists 2024-09-14T04:00:54.531 INFO:tasks.rados.rados.0.smithi070.stdout:3706: done (3 left) 2024-09-14T04:00:54.531 INFO:tasks.rados.rados.0.smithi070.stdout:3708: done (2 left) 2024-09-14T04:00:54.531 INFO:tasks.rados.rados.0.smithi070.stdout:3709: done (1 left) 2024-09-14T04:00:54.531 INFO:tasks.rados.rados.0.smithi070.stdout:3712: setattr oid 36 current snap is 381 2024-09-14T04:00:54.531 INFO:tasks.rados.rados.0.smithi070.stdout:3713: read oid 20 snap -1 2024-09-14T04:00:54.531 INFO:tasks.rados.rados.0.smithi070.stdout:3713: expect (ObjNum 1083 snap 353 seq_num 1083) 2024-09-14T04:00:54.532 INFO:tasks.rados.rados.0.smithi070.stdout:3714: rmattr oid 28 current snap is 381 2024-09-14T04:00:54.532 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 4 current snap is 381 2024-09-14T04:00:54.532 INFO:tasks.rados.rados.0.smithi070.stdout:3715: seq_num 1135 ranges {0=614400,614400=524288,1138688=540672,1679360=483328,2162688=57344} 2024-09-14T04:00:54.533 INFO:tasks.rados.rados.0.smithi070.stdout:3715: writing smithi07085154-4 from 0 to 614400 tid 1 2024-09-14T04:00:54.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:54 smithi070 ceph-mon[96704]: osdmap e815: 8 total, 8 up, 8 in 2024-09-14T04:00:54.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:54 smithi070 ceph-mon[96704]: osdmap e816: 8 total, 8 up, 8 in 2024-09-14T04:00:54.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:54 smithi070 ceph-mon[98640]: osdmap e815: 8 total, 8 up, 8 in 2024-09-14T04:00:54.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:54 smithi070 ceph-mon[98640]: osdmap e816: 8 total, 8 up, 8 in 2024-09-14T04:00:54.537 INFO:tasks.rados.rados.0.smithi070.stdout:3715: writing smithi07085154-4 from 614400 to 1138688 tid 2 2024-09-14T04:00:54.540 INFO:tasks.rados.rados.0.smithi070.stdout:3715: writing smithi07085154-4 from 1138688 to 1679360 tid 3 2024-09-14T04:00:54.542 INFO:tasks.rados.rados.0.smithi070.stdout:3715: writing smithi07085154-4 from 1679360 to 2162688 tid 4 2024-09-14T04:00:54.543 INFO:tasks.rados.rados.0.smithi070.stdout:3715: writing smithi07085154-4 from 2162688 to 2220032 tid 5 2024-09-14T04:00:54.543 INFO:tasks.rados.rados.0.smithi070.stdout:3716: snap_remove snap 324 2024-09-14T04:00:55.172 INFO:tasks.rados.rados.0.smithi070.stdout:3716: done (5 left) 2024-09-14T04:00:55.172 INFO:tasks.rados.rados.0.smithi070.stdout:3717: rollback oid 25 current snap is 381 2024-09-14T04:00:55.172 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 36 v 4240 (ObjNum 1122 snap 376 seq_num 1122) dirty exists 2024-09-14T04:00:55.172 INFO:tasks.rados.rados.0.smithi070.stdout:3711: finishing rollback tid 0 to smithi07085154-48 2024-09-14T04:00:55.172 INFO:tasks.rados.rados.0.smithi070.stdout:3711: finishing rollback tid 1 to smithi07085154-48 2024-09-14T04:00:55.172 INFO:tasks.rados.rados.0.smithi070.stdout:3711: finishing rollback tid 2 to smithi07085154-48 2024-09-14T04:00:55.173 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 2277 (ObjNum 1001 snap 331 seq_num 1001) dirty exists 2024-09-14T04:00:55.173 INFO:tasks.rados.rados.0.smithi070.stdout:3713: expect (ObjNum 1083 snap 353 seq_num 1083) 2024-09-14T04:00:55.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:55 smithi094 ceph-mon[86597]: pgmap v824: 121 pgs: 121 active+clean; 868 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.5 MiB/s wr, 8 op/s 2024-09-14T04:00:55.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:55 smithi070 ceph-mon[96704]: pgmap v824: 121 pgs: 121 active+clean; 868 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.5 MiB/s wr, 8 op/s 2024-09-14T04:00:55.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:55 smithi070 ceph-mon[98640]: pgmap v824: 121 pgs: 121 active+clean; 868 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.5 MiB/s wr, 8 op/s 2024-09-14T04:00:55.793 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 28 v 4212 (ObjNum 1044 snap 344 seq_num 1044) dirty exists 2024-09-14T04:00:55.793 INFO:tasks.rados.rados.0.smithi070.stdout:3715: finishing write tid 1 to smithi07085154-4 2024-09-14T04:00:55.793 INFO:tasks.rados.rados.0.smithi070.stdout:3715: finishing write tid 2 to smithi07085154-4 2024-09-14T04:00:55.793 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 25 to 340 2024-09-14T04:00:55.793 INFO:tasks.rados.rados.0.smithi070.stdout:3715: finishing write tid 3 to smithi07085154-4 2024-09-14T04:00:55.793 INFO:tasks.rados.rados.0.smithi070.stdout:3715: finishing write tid 4 to smithi07085154-4 2024-09-14T04:00:55.793 INFO:tasks.rados.rados.0.smithi070.stdout:3715: finishing write tid 5 to smithi07085154-4 2024-09-14T04:00:55.793 INFO:tasks.rados.rados.0.smithi070.stdout:3715: finishing write tid 6 to smithi07085154-4 2024-09-14T04:00:55.793 INFO:tasks.rados.rados.0.smithi070.stdout:3715: finishing write tid 7 to smithi07085154-4 2024-09-14T04:00:55.794 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 4218 (ObjNum 1135 snap 381 seq_num 1135) dirty exists 2024-09-14T04:00:55.794 INFO:tasks.rados.rados.0.smithi070.stdout:3715: left oid 4 (ObjNum 1135 snap 381 seq_num 1135) 2024-09-14T04:00:55.794 INFO:tasks.rados.rados.0.smithi070.stdout:3711: done (5 left) 2024-09-14T04:00:55.794 INFO:tasks.rados.rados.0.smithi070.stdout:3712: done (4 left) 2024-09-14T04:00:55.794 INFO:tasks.rados.rados.0.smithi070.stdout:3713: done (3 left) 2024-09-14T04:00:55.794 INFO:tasks.rados.rados.0.smithi070.stdout:3714: done (2 left) 2024-09-14T04:00:55.794 INFO:tasks.rados.rados.0.smithi070.stdout:3715: done (1 left) 2024-09-14T04:00:55.794 INFO:tasks.rados.rados.0.smithi070.stdout:3718: read oid 28 snap -1 2024-09-14T04:00:55.794 INFO:tasks.rados.rados.0.smithi070.stdout:3718: expect (ObjNum 1044 snap 344 seq_num 1044) 2024-09-14T04:00:55.794 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 42 current snap is 381 2024-09-14T04:00:55.794 INFO:tasks.rados.rados.0.smithi070.stdout:3719: seq_num 1136 ranges {0=630784,630784=409600,1040384=753664,1794048=417792,2211840=24576} 2024-09-14T04:00:55.796 INFO:tasks.rados.rados.0.smithi070.stdout:3719: writing smithi07085154-42 from 0 to 630784 tid 1 2024-09-14T04:00:55.798 INFO:tasks.rados.rados.0.smithi070.stdout:3719: writing smithi07085154-42 from 630784 to 1040384 tid 2 2024-09-14T04:00:55.803 INFO:tasks.rados.rados.0.smithi070.stdout:3719: writing smithi07085154-42 from 1040384 to 1794048 tid 3 2024-09-14T04:00:55.806 INFO:tasks.rados.rados.0.smithi070.stdout:3719: writing smithi07085154-42 from 1794048 to 2211840 tid 4 2024-09-14T04:00:55.806 INFO:tasks.rados.rados.0.smithi070.stdout:3719: writing smithi07085154-42 from 2211840 to 2236416 tid 5 2024-09-14T04:00:55.806 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 27 current snap is 381 2024-09-14T04:00:55.807 INFO:tasks.rados.rados.0.smithi070.stdout:3717: finishing rollback tid 0 to smithi07085154-25 2024-09-14T04:00:55.807 INFO:tasks.rados.rados.0.smithi070.stdout:3717: finishing rollback tid 1 to smithi07085154-25 2024-09-14T04:00:55.807 INFO:tasks.rados.rados.0.smithi070.stdout:3720: seq_num 1137 ranges {3383296=647168,4030464=696320,4726784=557056,5283840=352256} 2024-09-14T04:00:55.827 INFO:tasks.rados.rados.0.smithi070.stdout:3720: writing smithi07085154-27 from 3383296 to 4030464 tid 1 2024-09-14T04:00:55.831 INFO:tasks.rados.rados.0.smithi070.stdout:3720: writing smithi07085154-27 from 4030464 to 4726784 tid 2 2024-09-14T04:00:55.834 INFO:tasks.rados.rados.0.smithi070.stdout:3720: writing smithi07085154-27 from 4726784 to 5283840 tid 3 2024-09-14T04:00:55.837 INFO:tasks.rados.rados.0.smithi070.stdout:3720: writing smithi07085154-27 from 5283840 to 5636096 tid 4 2024-09-14T04:00:55.837 INFO:tasks.rados.rados.0.smithi070.stdout:3721: rmattr oid 2 current snap is 381 2024-09-14T04:00:55.837 INFO:tasks.rados.rados.0.smithi070.stdout:3721: done (4 left) 2024-09-14T04:00:55.838 INFO:tasks.rados.rados.0.smithi070.stdout:3722: read oid 3 snap -1 2024-09-14T04:00:55.838 INFO:tasks.rados.rados.0.smithi070.stdout:3722: expect (ObjNum 1072 snap 347 seq_num 1072) 2024-09-14T04:00:55.838 INFO:tasks.rados.rados.0.smithi070.stdout:3723: rollback oid 36 current snap is 381 2024-09-14T04:00:55.838 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 36 to 376 2024-09-14T04:00:55.838 INFO:tasks.rados.rados.0.smithi070.stdout:3717: finishing rollback tid 2 to smithi07085154-25 2024-09-14T04:00:55.838 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 4248 (ObjNum 1019 snap 336 seq_num 1019) dirty exists 2024-09-14T04:00:55.838 INFO:tasks.rados.rados.0.smithi070.stdout:3718: expect (ObjNum 1044 snap 344 seq_num 1044) 2024-09-14T04:00:56.090 INFO:tasks.rados.rados.0.smithi070.stdout:3719: finishing write tid 1 to smithi07085154-42 2024-09-14T04:00:56.091 INFO:tasks.rados.rados.0.smithi070.stdout:3719: finishing write tid 2 to smithi07085154-42 2024-09-14T04:00:56.091 INFO:tasks.rados.rados.0.smithi070.stdout:3719: finishing write tid 3 to smithi07085154-42 2024-09-14T04:00:56.091 INFO:tasks.rados.rados.0.smithi070.stdout:3719: finishing write tid 4 to smithi07085154-42 2024-09-14T04:00:56.091 INFO:tasks.rados.rados.0.smithi070.stdout:3719: finishing write tid 5 to smithi07085154-42 2024-09-14T04:00:56.091 INFO:tasks.rados.rados.0.smithi070.stdout:3719: finishing write tid 6 to smithi07085154-42 2024-09-14T04:00:56.091 INFO:tasks.rados.rados.0.smithi070.stdout:3719: finishing write tid 7 to smithi07085154-42 2024-09-14T04:00:56.091 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 42 v 4435 (ObjNum 1136 snap 381 seq_num 1136) dirty exists 2024-09-14T04:00:56.091 INFO:tasks.rados.rados.0.smithi070.stdout:3719: left oid 42 (ObjNum 1136 snap 381 seq_num 1136) 2024-09-14T04:00:56.091 INFO:tasks.rados.rados.0.smithi070.stdout:3720: finishing write tid 1 to smithi07085154-27 2024-09-14T04:00:56.091 INFO:tasks.rados.rados.0.smithi070.stdout:3717: done (5 left) 2024-09-14T04:00:56.091 INFO:tasks.rados.rados.0.smithi070.stdout:3718: done (4 left) 2024-09-14T04:00:56.091 INFO:tasks.rados.rados.0.smithi070.stdout:3719: done (3 left) 2024-09-14T04:00:56.092 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 49 current snap is 381 2024-09-14T04:00:56.092 INFO:tasks.rados.rados.0.smithi070.stdout:3720: finishing write tid 2 to smithi07085154-27 2024-09-14T04:00:56.092 INFO:tasks.rados.rados.0.smithi070.stdout:3724: seq_num 1138 ranges {3481600=663552,4145152=581632,4726784=712704,5439488=311296} 2024-09-14T04:00:56.112 INFO:tasks.rados.rados.0.smithi070.stdout:3724: writing smithi07085154-49 from 3481600 to 4145152 tid 1 2024-09-14T04:00:56.115 INFO:tasks.rados.rados.0.smithi070.stdout:3724: writing smithi07085154-49 from 4145152 to 4726784 tid 2 2024-09-14T04:00:56.119 INFO:tasks.rados.rados.0.smithi070.stdout:3724: writing smithi07085154-49 from 4726784 to 5439488 tid 3 2024-09-14T04:00:56.122 INFO:tasks.rados.rados.0.smithi070.stdout:3724: writing smithi07085154-49 from 5439488 to 5750784 tid 4 2024-09-14T04:00:56.122 INFO:tasks.rados.rados.0.smithi070.stdout:3725: read oid 43 snap -1 2024-09-14T04:00:56.122 INFO:tasks.rados.rados.0.smithi070.stdout:3725: expect (ObjNum 1047 snap 344 seq_num 1047) 2024-09-14T04:00:56.122 INFO:tasks.rados.rados.0.smithi070.stdout:3726: rmattr oid 9 current snap is 381 2024-09-14T04:00:56.122 INFO:tasks.rados.rados.0.smithi070.stdout:3726: done (5 left) 2024-09-14T04:00:56.122 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 23 current snap is 381 2024-09-14T04:00:56.122 INFO:tasks.rados.rados.0.smithi070.stdout:3720: finishing write tid 3 to smithi07085154-27 2024-09-14T04:00:56.123 INFO:tasks.rados.rados.0.smithi070.stdout:3723: finishing rollback tid 0 to smithi07085154-36 2024-09-14T04:00:56.123 INFO:tasks.rados.rados.0.smithi070.stdout:3720: finishing write tid 4 to smithi07085154-27 2024-09-14T04:00:56.123 INFO:tasks.rados.rados.0.smithi070.stdout:3723: finishing rollback tid 1 to smithi07085154-36 2024-09-14T04:00:56.123 INFO:tasks.rados.rados.0.smithi070.stdout:3720: finishing write tid 5 to smithi07085154-27 2024-09-14T04:00:56.123 INFO:tasks.rados.rados.0.smithi070.stdout:3723: finishing rollback tid 2 to smithi07085154-36 2024-09-14T04:00:56.123 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 36 v 4243 (ObjNum 1122 snap 376 seq_num 1122) dirty exists 2024-09-14T04:00:56.123 INFO:tasks.rados.rados.0.smithi070.stdout:3722: expect (ObjNum 1072 snap 347 seq_num 1072) 2024-09-14T04:00:56.157 INFO:tasks.rados.rados.0.smithi070.stdout:3720: finishing write tid 6 to smithi07085154-27 2024-09-14T04:00:56.158 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 4441 (ObjNum 1137 snap 381 seq_num 1137) dirty exists 2024-09-14T04:00:56.158 INFO:tasks.rados.rados.0.smithi070.stdout:3720: left oid 27 (ObjNum 1137 snap 381 seq_num 1137) 2024-09-14T04:00:56.158 INFO:tasks.rados.rados.0.smithi070.stdout:3724: finishing write tid 1 to smithi07085154-49 2024-09-14T04:00:56.158 INFO:tasks.rados.rados.0.smithi070.stdout:3724: finishing write tid 2 to smithi07085154-49 2024-09-14T04:00:56.158 INFO:tasks.rados.rados.0.smithi070.stdout:3724: finishing write tid 3 to smithi07085154-49 2024-09-14T04:00:56.158 INFO:tasks.rados.rados.0.smithi070.stdout:3724: finishing write tid 4 to smithi07085154-49 2024-09-14T04:00:56.158 INFO:tasks.rados.rados.0.smithi070.stdout:3724: finishing write tid 5 to smithi07085154-49 2024-09-14T04:00:56.158 INFO:tasks.rados.rados.0.smithi070.stdout:3727: seq_num 1139 ranges {7331840=679936,8011776=458752,8470528=524288,8994816=540672,9535488=81920} 2024-09-14T04:00:56.200 INFO:tasks.rados.rados.0.smithi070.stdout:3727: writing smithi07085154-23 from 7331840 to 8011776 tid 1 2024-09-14T04:00:56.203 INFO:tasks.rados.rados.0.smithi070.stdout:3727: writing smithi07085154-23 from 8011776 to 8470528 tid 2 2024-09-14T04:00:56.206 INFO:tasks.rados.rados.0.smithi070.stdout:3727: writing smithi07085154-23 from 8470528 to 8994816 tid 3 2024-09-14T04:00:56.209 INFO:tasks.rados.rados.0.smithi070.stdout:3727: writing smithi07085154-23 from 8994816 to 9535488 tid 4 2024-09-14T04:00:56.210 INFO:tasks.rados.rados.0.smithi070.stdout:3727: writing smithi07085154-23 from 9535488 to 9617408 tid 5 2024-09-14T04:00:56.210 INFO:tasks.rados.rados.0.smithi070.stdout:3720: done (5 left) 2024-09-14T04:00:56.210 INFO:tasks.rados.rados.0.smithi070.stdout:3722: done (4 left) 2024-09-14T04:00:56.211 INFO:tasks.rados.rados.0.smithi070.stdout:3723: done (3 left) 2024-09-14T04:00:56.211 INFO:tasks.rados.rados.0.smithi070.stdout:3728: rmattr oid 39 current snap is 381 2024-09-14T04:00:56.211 INFO:tasks.rados.rados.0.smithi070.stdout:3728: done (3 left) 2024-09-14T04:00:56.211 INFO:tasks.rados.rados.0.smithi070.stdout:3729: read oid 9 snap 379 2024-09-14T04:00:56.211 INFO:tasks.rados.rados.0.smithi070.stdout:3729: expect (ObjNum 1120 snap 374 seq_num 1120) 2024-09-14T04:00:56.211 INFO:tasks.rados.rados.0.smithi070.stdout:3730: read oid 19 snap -1 2024-09-14T04:00:56.211 INFO:tasks.rados.rados.0.smithi070.stdout:3730: expect (ObjNum 659 snap 220 seq_num 659) 2024-09-14T04:00:56.211 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 26 current snap is 381 2024-09-14T04:00:56.211 INFO:tasks.rados.rados.0.smithi070.stdout:3724: finishing write tid 6 to smithi07085154-49 2024-09-14T04:00:56.211 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 49 v 3220 (ObjNum 1138 snap 381 seq_num 1138) dirty exists 2024-09-14T04:00:56.211 INFO:tasks.rados.rados.0.smithi070.stdout:3724: left oid 49 (ObjNum 1138 snap 381 seq_num 1138) 2024-09-14T04:00:56.212 INFO:tasks.rados.rados.0.smithi070.stdout:3725: expect (ObjNum 1047 snap 344 seq_num 1047) 2024-09-14T04:00:56.328 INFO:tasks.rados.rados.0.smithi070.stdout:3727: finishing write tid 1 to smithi07085154-23 2024-09-14T04:00:56.329 INFO:tasks.rados.rados.0.smithi070.stdout:3727: finishing write tid 2 to smithi07085154-23 2024-09-14T04:00:56.329 INFO:tasks.rados.rados.0.smithi070.stdout:3727: finishing write tid 3 to smithi07085154-23 2024-09-14T04:00:56.329 INFO:tasks.rados.rados.0.smithi070.stdout:3727: finishing write tid 4 to smithi07085154-23 2024-09-14T04:00:56.329 INFO:tasks.rados.rados.0.smithi070.stdout:3727: finishing write tid 5 to smithi07085154-23 2024-09-14T04:00:56.329 INFO:tasks.rados.rados.0.smithi070.stdout:3727: finishing write tid 6 to smithi07085154-23 2024-09-14T04:00:56.329 INFO:tasks.rados.rados.0.smithi070.stdout:3727: finishing write tid 7 to smithi07085154-23 2024-09-14T04:00:56.329 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 3264 (ObjNum 1139 snap 381 seq_num 1139) dirty exists 2024-09-14T04:00:56.329 INFO:tasks.rados.rados.0.smithi070.stdout:3727: left oid 23 (ObjNum 1139 snap 381 seq_num 1139) 2024-09-14T04:00:56.329 INFO:tasks.rados.rados.0.smithi070.stdout:3729: expect (ObjNum 1120 snap 374 seq_num 1120) 2024-09-14T04:00:56.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:56 smithi094 ceph-mon[86597]: osdmap e817: 8 total, 8 up, 8 in 2024-09-14T04:00:56.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:56 smithi094 ceph-mon[86597]: pgmap v826: 121 pgs: 121 active+clean; 856 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 511 B/s rd, 1.0 MiB/s wr, 3 op/s 2024-09-14T04:00:56.532 INFO:tasks.rados.rados.0.smithi070.stdout:3730: expect (ObjNum 659 snap 220 seq_num 659) 2024-09-14T04:00:56.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:56 smithi070 ceph-mon[96704]: osdmap e817: 8 total, 8 up, 8 in 2024-09-14T04:00:56.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:56 smithi070 ceph-mon[96704]: pgmap v826: 121 pgs: 121 active+clean; 856 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 511 B/s rd, 1.0 MiB/s wr, 3 op/s 2024-09-14T04:00:56.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:56 smithi070 ceph-mon[98640]: osdmap e817: 8 total, 8 up, 8 in 2024-09-14T04:00:56.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:56 smithi070 ceph-mon[98640]: pgmap v826: 121 pgs: 121 active+clean; 856 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 511 B/s rd, 1.0 MiB/s wr, 3 op/s 2024-09-14T04:00:57.170 INFO:tasks.rados.rados.0.smithi070.stdout:3731: seq_num 1140 ranges {0=696320,696320=745472,1441792=729088,2170880=131072} 2024-09-14T04:00:57.174 INFO:tasks.rados.rados.0.smithi070.stdout:3731: writing smithi07085154-26 from 0 to 696320 tid 1 2024-09-14T04:00:57.178 INFO:tasks.rados.rados.0.smithi070.stdout:3731: writing smithi07085154-26 from 696320 to 1441792 tid 2 2024-09-14T04:00:57.182 INFO:tasks.rados.rados.0.smithi070.stdout:3731: writing smithi07085154-26 from 1441792 to 2170880 tid 3 2024-09-14T04:00:57.184 INFO:tasks.rados.rados.0.smithi070.stdout:3731: writing smithi07085154-26 from 2170880 to 2301952 tid 4 2024-09-14T04:00:57.184 INFO:tasks.rados.rados.0.smithi070.stdout:3724: done (5 left) 2024-09-14T04:00:57.184 INFO:tasks.rados.rados.0.smithi070.stdout:3725: done (4 left) 2024-09-14T04:00:57.184 INFO:tasks.rados.rados.0.smithi070.stdout:3727: done (3 left) 2024-09-14T04:00:57.184 INFO:tasks.rados.rados.0.smithi070.stdout:3729: done (2 left) 2024-09-14T04:00:57.184 INFO:tasks.rados.rados.0.smithi070.stdout:3730: done (1 left) 2024-09-14T04:00:57.184 INFO:tasks.rados.rados.0.smithi070.stdout:3732: read oid 32 snap -1 2024-09-14T04:00:57.184 INFO:tasks.rados.rados.0.smithi070.stdout:3732: expect (ObjNum 1132 snap 381 seq_num 1132) 2024-09-14T04:00:57.184 INFO:tasks.rados.rados.0.smithi070.stdout:3733: snap_remove snap 332 2024-09-14T04:00:57.240 INFO:tasks.rados.rados.0.smithi070.stdout:3733: done (2 left) 2024-09-14T04:00:57.240 INFO:tasks.rados.rados.0.smithi070.stdout:3734: rollback oid 40 current snap is 381 2024-09-14T04:00:57.240 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 40 to 361 2024-09-14T04:00:57.240 INFO:tasks.rados.rados.0.smithi070.stdout:3731: finishing write tid 1 to smithi07085154-26 2024-09-14T04:00:57.240 INFO:tasks.rados.rados.0.smithi070.stdout:3731: finishing write tid 2 to smithi07085154-26 2024-09-14T04:00:57.240 INFO:tasks.rados.rados.0.smithi070.stdout:3732: expect (ObjNum 1132 snap 381 seq_num 1132) 2024-09-14T04:00:57.308 INFO:tasks.rados.rados.0.smithi070.stdout:3731: finishing write tid 3 to smithi07085154-26 2024-09-14T04:00:57.309 INFO:tasks.rados.rados.0.smithi070.stdout:3731: finishing write tid 4 to smithi07085154-26 2024-09-14T04:00:57.309 INFO:tasks.rados.rados.0.smithi070.stdout:3731: finishing write tid 5 to smithi07085154-26 2024-09-14T04:00:57.309 INFO:tasks.rados.rados.0.smithi070.stdout:3731: finishing write tid 6 to smithi07085154-26 2024-09-14T04:00:57.309 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 4446 (ObjNum 1140 snap 381 seq_num 1140) dirty exists 2024-09-14T04:00:57.309 INFO:tasks.rados.rados.0.smithi070.stdout:3731: left oid 26 (ObjNum 1140 snap 381 seq_num 1140) 2024-09-14T04:00:57.309 INFO:tasks.rados.rados.0.smithi070.stdout:3734: finishing rollback tid 0 to smithi07085154-40 2024-09-14T04:00:57.309 INFO:tasks.rados.rados.0.smithi070.stdout:3734: finishing rollback tid 1 to smithi07085154-40 2024-09-14T04:00:57.309 INFO:tasks.rados.rados.0.smithi070.stdout:3734: finishing rollback tid 2 to smithi07085154-40 2024-09-14T04:00:57.309 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 4246 (ObjNum 1079 snap 350 seq_num 1079) dirty exists 2024-09-14T04:00:57.309 INFO:tasks.rados.rados.0.smithi070.stdout:3731: done (2 left) 2024-09-14T04:00:57.309 INFO:tasks.rados.rados.0.smithi070.stdout:3732: done (1 left) 2024-09-14T04:00:57.310 INFO:tasks.rados.rados.0.smithi070.stdout:3734: done (0 left) 2024-09-14T04:00:57.310 INFO:tasks.rados.rados.0.smithi070.stdout:3735: rollback oid 5 current snap is 381 2024-09-14T04:00:57.310 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 5 to 346 2024-09-14T04:00:57.310 INFO:tasks.rados.rados.0.smithi070.stdout:3736: copy_from oid 17 from oid 48 current snap is 381 2024-09-14T04:00:57.310 INFO:tasks.rados.rados.0.smithi070.stdout:3737: setattr oid 32 current snap is 381 2024-09-14T04:00:57.310 INFO:tasks.rados.rados.0.smithi070.stdout:3738: rollback oid 27 current snap is 381 2024-09-14T04:00:57.310 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 27 to 368 2024-09-14T04:00:57.310 INFO:tasks.rados.rados.0.smithi070.stdout:3739: read oid 15 snap -1 2024-09-14T04:00:57.310 INFO:tasks.rados.rados.0.smithi070.stdout:3739: expect (ObjNum 1120 snap 374 seq_num 1120) 2024-09-14T04:00:57.310 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 49 current snap is 381 2024-09-14T04:00:57.310 INFO:tasks.rados.rados.0.smithi070.stdout:3740: seq_num 1141 ranges {5750784=712704,6463488=679936,7143424=540672,7684096=385024} 2024-09-14T04:00:57.343 INFO:tasks.rados.rados.0.smithi070.stdout:3740: writing smithi07085154-49 from 5750784 to 6463488 tid 1 2024-09-14T04:00:57.348 INFO:tasks.rados.rados.0.smithi070.stdout:3740: writing smithi07085154-49 from 6463488 to 7143424 tid 2 2024-09-14T04:00:57.351 INFO:tasks.rados.rados.0.smithi070.stdout:3740: writing smithi07085154-49 from 7143424 to 7684096 tid 3 2024-09-14T04:00:57.354 INFO:tasks.rados.rados.0.smithi070.stdout:3740: writing smithi07085154-49 from 7684096 to 8069120 tid 4 2024-09-14T04:00:57.355 INFO:tasks.rados.rados.0.smithi070.stdout:3741: read oid 47 snap -1 2024-09-14T04:00:57.355 INFO:tasks.rados.rados.0.smithi070.stdout:3741: expect (ObjNum 1014 snap 334 seq_num 1014) 2024-09-14T04:00:57.355 INFO:tasks.rados.rados.0.smithi070.stdout:3742: snap_remove snap 354 2024-09-14T04:00:58.248 INFO:tasks.rados.rados.0.smithi070.stdout:3742: done (7 left) 2024-09-14T04:00:58.248 INFO:tasks.rados.rados.0.smithi070.stdout:3743: copy_from oid 25 from oid 42 current snap is 381 2024-09-14T04:00:58.249 INFO:tasks.rados.rados.0.smithi070.stdout:3744: setattr oid 10 current snap is 381 2024-09-14T04:00:58.249 INFO:tasks.rados.rados.0.smithi070.stdout:3745: read oid 30 snap 361 2024-09-14T04:00:58.250 INFO:tasks.rados.rados.0.smithi070.stdout:3745: expect (ObjNum 1035 snap 341 seq_num 1035) 2024-09-14T04:00:58.250 INFO:tasks.rados.rados.0.smithi070.stdout:3746: snap_remove snap 376 2024-09-14T04:00:58.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:58 smithi094 ceph-mon[86597]: osdmap e818: 8 total, 8 up, 8 in 2024-09-14T04:00:58.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:58 smithi094 ceph-mon[86597]: pgmap v828: 121 pgs: 121 active+clean; 856 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 15 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2024-09-14T04:00:58.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:58 smithi070 ceph-mon[96704]: osdmap e818: 8 total, 8 up, 8 in 2024-09-14T04:00:58.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:58 smithi070 ceph-mon[96704]: pgmap v828: 121 pgs: 121 active+clean; 856 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 15 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2024-09-14T04:00:58.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:58 smithi070 ceph-mon[98640]: osdmap e818: 8 total, 8 up, 8 in 2024-09-14T04:00:58.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:58 smithi070 ceph-mon[98640]: pgmap v828: 121 pgs: 121 active+clean; 856 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 15 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2024-09-14T04:00:59.148 INFO:tasks.rados.rados.0.smithi070.stdout:3746: done (10 left) 2024-09-14T04:00:59.148 INFO:tasks.rados.rados.0.smithi070.stdout:3747: read oid 34 snap 370 2024-09-14T04:00:59.148 INFO:tasks.rados.rados.0.smithi070.stdout:3747: expect (ObjNum 1094 snap 358 seq_num 1094) 2024-09-14T04:00:59.148 INFO:tasks.rados.rados.0.smithi070.stdout:3748: rollback oid 21 current snap is 381 2024-09-14T04:00:59.148 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 21 to 370 2024-09-14T04:00:59.148 INFO:tasks.rados.rados.0.smithi070.stdout:3735: finishing rollback tid 1 to smithi07085154-5 2024-09-14T04:00:59.148 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 3259 (ObjNum 1132 snap 381 seq_num 1132) dirty exists 2024-09-14T04:00:59.148 INFO:tasks.rados.rados.0.smithi070.stdout:3735: finishing rollback tid 2 to smithi07085154-5 2024-09-14T04:00:59.148 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 3201 (ObjNum 1042 snap 342 seq_num 1042) dirty exists 2024-09-14T04:00:59.148 INFO:tasks.rados.rados.0.smithi070.stdout:3736: finishing copy_from racing read to smithi07085154-17 2024-09-14T04:00:59.148 INFO:tasks.rados.rados.0.smithi070.stdout:3735: done (11 left) 2024-09-14T04:00:59.149 INFO:tasks.rados.rados.0.smithi070.stdout:3737: done (10 left) 2024-09-14T04:00:59.149 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 45 current snap is 381 2024-09-14T04:00:59.149 INFO:tasks.rados.rados.0.smithi070.stdout:3749: seq_num 1142 ranges {6692864=729088,7421952=565248,7987200=696320,8683520=344064} 2024-09-14T04:00:59.189 INFO:tasks.rados.rados.0.smithi070.stdout:3749: writing smithi07085154-45 from 6692864 to 7421952 tid 1 2024-09-14T04:00:59.193 INFO:tasks.rados.rados.0.smithi070.stdout:3749: writing smithi07085154-45 from 7421952 to 7987200 tid 2 2024-09-14T04:00:59.197 INFO:tasks.rados.rados.0.smithi070.stdout:3749: writing smithi07085154-45 from 7987200 to 8683520 tid 3 2024-09-14T04:00:59.199 INFO:tasks.rados.rados.0.smithi070.stdout:3749: writing smithi07085154-45 from 8683520 to 9027584 tid 4 2024-09-14T04:00:59.200 INFO:tasks.rados.rados.0.smithi070.stdout:3750: snap_create 2024-09-14T04:00:59.200 INFO:tasks.rados.rados.0.smithi070.stdout:3738: finishing rollback tid 0 to smithi07085154-27 2024-09-14T04:00:59.200 INFO:tasks.rados.rados.0.smithi070.stdout:3738: finishing rollback tid 1 to smithi07085154-27 2024-09-14T04:00:59.200 INFO:tasks.rados.rados.0.smithi070.stdout:3738: finishing rollback tid 2 to smithi07085154-27 2024-09-14T04:00:59.200 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 4465 (ObjNum 829 snap 283 seq_num 829) dirty exists 2024-09-14T04:00:59.200 INFO:tasks.rados.rados.0.smithi070.stdout:3736: finishing copy_from to smithi07085154-17 2024-09-14T04:00:59.200 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 2277 (ObjNum 1001 snap 331 seq_num 1001) dirty exists 2024-09-14T04:00:59.200 INFO:tasks.rados.rados.0.smithi070.stdout:3739: expect (ObjNum 1120 snap 374 seq_num 1120) 2024-09-14T04:00:59.405 INFO:tasks.rados.rados.0.smithi070.stdout:3740: finishing write tid 1 to smithi07085154-49 2024-09-14T04:00:59.406 INFO:tasks.rados.rados.0.smithi070.stdout:3740: finishing write tid 2 to smithi07085154-49 2024-09-14T04:00:59.406 INFO:tasks.rados.rados.0.smithi070.stdout:3740: finishing write tid 3 to smithi07085154-49 2024-09-14T04:00:59.406 INFO:tasks.rados.rados.0.smithi070.stdout:3740: finishing write tid 4 to smithi07085154-49 2024-09-14T04:00:59.406 INFO:tasks.rados.rados.0.smithi070.stdout:3740: finishing write tid 5 to smithi07085154-49 2024-09-14T04:00:59.406 INFO:tasks.rados.rados.0.smithi070.stdout:3740: finishing write tid 6 to smithi07085154-49 2024-09-14T04:00:59.406 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 49 v 3225 (ObjNum 1141 snap 381 seq_num 1141) dirty exists 2024-09-14T04:00:59.406 INFO:tasks.rados.rados.0.smithi070.stdout:3740: left oid 49 (ObjNum 1141 snap 381 seq_num 1141) 2024-09-14T04:00:59.406 INFO:tasks.rados.rados.0.smithi070.stdout:3741: expect (ObjNum 1014 snap 334 seq_num 1014) 2024-09-14T04:00:59.507 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 4467 (ObjNum 1122 snap 376 seq_num 1122) dirty exists 2024-09-14T04:00:59.507 INFO:tasks.rados.rados.0.smithi070.stdout:3745: expect (ObjNum 1035 snap 341 seq_num 1035) 2024-09-14T04:00:59.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:59 smithi094 ceph-mon[86597]: osdmap e819: 8 total, 8 up, 8 in 2024-09-14T04:00:59.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:00:59 smithi094 ceph-mon[86597]: osdmap e820: 8 total, 8 up, 8 in 2024-09-14T04:00:59.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:59 smithi070 ceph-mon[96704]: osdmap e819: 8 total, 8 up, 8 in 2024-09-14T04:00:59.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:00:59 smithi070 ceph-mon[96704]: osdmap e820: 8 total, 8 up, 8 in 2024-09-14T04:00:59.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:59 smithi070 ceph-mon[98640]: osdmap e819: 8 total, 8 up, 8 in 2024-09-14T04:00:59.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:00:59 smithi070 ceph-mon[98640]: osdmap e820: 8 total, 8 up, 8 in 2024-09-14T04:00:59.731 INFO:tasks.rados.rados.0.smithi070.stdout:3743: finishing copy_from racing read to smithi07085154-25 2024-09-14T04:00:59.731 INFO:tasks.rados.rados.0.smithi070.stdout:3743: finishing copy_from to smithi07085154-25 2024-09-14T04:00:59.731 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 4435 (ObjNum 1136 snap 381 seq_num 1136) dirty exists 2024-09-14T04:00:59.731 INFO:tasks.rados.rados.0.smithi070.stdout:3748: finishing rollback tid 0 to smithi07085154-21 2024-09-14T04:00:59.732 INFO:tasks.rados.rados.0.smithi070.stdout:3748: finishing rollback tid 1 to smithi07085154-21 2024-09-14T04:00:59.732 INFO:tasks.rados.rados.0.smithi070.stdout:3748: finishing rollback tid 2 to smithi07085154-21 2024-09-14T04:00:59.732 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 1731 (ObjNum 1091 snap 355 seq_num 1091) dirty exists 2024-09-14T04:00:59.732 INFO:tasks.rados.rados.0.smithi070.stdout:3749: finishing write tid 1 to smithi07085154-45 2024-09-14T04:00:59.732 INFO:tasks.rados.rados.0.smithi070.stdout:3749: finishing write tid 2 to smithi07085154-45 2024-09-14T04:00:59.732 INFO:tasks.rados.rados.0.smithi070.stdout:3747: expect (ObjNum 1094 snap 358 seq_num 1094) 2024-09-14T04:01:00.328 INFO:tasks.rados.rados.0.smithi070.stdout:3749: finishing write tid 3 to smithi07085154-45 2024-09-14T04:01:00.328 INFO:tasks.rados.rados.0.smithi070.stdout:3749: finishing write tid 4 to smithi07085154-45 2024-09-14T04:01:00.328 INFO:tasks.rados.rados.0.smithi070.stdout:3749: finishing write tid 5 to smithi07085154-45 2024-09-14T04:01:00.328 INFO:tasks.rados.rados.0.smithi070.stdout:3749: finishing write tid 6 to smithi07085154-45 2024-09-14T04:01:00.328 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 45 v 3230 (ObjNum 1142 snap 381 seq_num 1142) dirty exists 2024-09-14T04:01:00.329 INFO:tasks.rados.rados.0.smithi070.stdout:3749: left oid 45 (ObjNum 1142 snap 381 seq_num 1142) 2024-09-14T04:01:00.329 INFO:tasks.rados.rados.0.smithi070.stdout:3736: done (11 left) 2024-09-14T04:01:00.329 INFO:tasks.rados.rados.0.smithi070.stdout:3738: done (10 left) 2024-09-14T04:01:00.329 INFO:tasks.rados.rados.0.smithi070.stdout:3739: done (9 left) 2024-09-14T04:01:00.329 INFO:tasks.rados.rados.0.smithi070.stdout:3740: done (8 left) 2024-09-14T04:01:00.329 INFO:tasks.rados.rados.0.smithi070.stdout:3741: done (7 left) 2024-09-14T04:01:00.329 INFO:tasks.rados.rados.0.smithi070.stdout:3743: done (6 left) 2024-09-14T04:01:00.329 INFO:tasks.rados.rados.0.smithi070.stdout:3744: done (5 left) 2024-09-14T04:01:00.329 INFO:tasks.rados.rados.0.smithi070.stdout:3745: done (4 left) 2024-09-14T04:01:00.329 INFO:tasks.rados.rados.0.smithi070.stdout:3747: done (3 left) 2024-09-14T04:01:00.329 INFO:tasks.rados.rados.0.smithi070.stdout:3748: done (2 left) 2024-09-14T04:01:00.329 INFO:tasks.rados.rados.0.smithi070.stdout:3749: done (1 left) 2024-09-14T04:01:00.329 INFO:tasks.rados.rados.0.smithi070.stdout:3750: done (0 left) 2024-09-14T04:01:00.330 INFO:tasks.rados.rados.0.smithi070.stdout:3751: copy_from oid 13 from oid 29 current snap is 382 2024-09-14T04:01:00.330 INFO:tasks.rados.rados.0.smithi070.stdout:3752: setattr oid 33 current snap is 382 2024-09-14T04:01:00.330 INFO:tasks.rados.rados.0.smithi070.stdout:3753: setattr oid 30 current snap is 382 2024-09-14T04:01:00.331 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 7 current snap is 382 2024-09-14T04:01:00.331 INFO:tasks.rados.rados.0.smithi070.stdout:3754: seq_num 1143 ranges {1662976=753664,2416640=450560,2867200=507904,3375104=589824,3964928=57344} 2024-09-14T04:01:00.345 INFO:tasks.rados.rados.0.smithi070.stdout:3754: writing smithi07085154-7 from 1662976 to 2416640 tid 1 2024-09-14T04:01:00.348 INFO:tasks.rados.rados.0.smithi070.stdout:3754: writing smithi07085154-7 from 2416640 to 2867200 tid 2 2024-09-14T04:01:00.351 INFO:tasks.rados.rados.0.smithi070.stdout:3754: writing smithi07085154-7 from 2867200 to 3375104 tid 3 2024-09-14T04:01:00.355 INFO:tasks.rados.rados.0.smithi070.stdout:3754: writing smithi07085154-7 from 3375104 to 3964928 tid 4 2024-09-14T04:01:00.355 INFO:tasks.rados.rados.0.smithi070.stdout:3754: writing smithi07085154-7 from 3964928 to 4022272 tid 5 2024-09-14T04:01:00.356 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 27 current snap is 382 2024-09-14T04:01:00.356 INFO:tasks.rados.rados.0.smithi070.stdout:3755: seq_num 1144 ranges {3383296=770048,4153344=737280,4890624=712704,5603328=155648} 2024-09-14T04:01:00.378 INFO:tasks.rados.rados.0.smithi070.stdout:3755: writing smithi07085154-27 from 3383296 to 4153344 tid 1 2024-09-14T04:01:00.382 INFO:tasks.rados.rados.0.smithi070.stdout:3755: writing smithi07085154-27 from 4153344 to 4890624 tid 2 2024-09-14T04:01:00.386 INFO:tasks.rados.rados.0.smithi070.stdout:3755: writing smithi07085154-27 from 4890624 to 5603328 tid 3 2024-09-14T04:01:00.387 INFO:tasks.rados.rados.0.smithi070.stdout:3755: writing smithi07085154-27 from 5603328 to 5758976 tid 4 2024-09-14T04:01:00.387 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 2 current snap is 382 2024-09-14T04:01:00.387 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 33 v 3231 (ObjNum 590 snap 195 seq_num 590) dirty exists 2024-09-14T04:01:00.387 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 30 v 3992 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-14T04:01:00.387 INFO:tasks.rados.rados.0.smithi070.stdout:3754: finishing write tid 1 to smithi07085154-7 2024-09-14T04:01:00.387 INFO:tasks.rados.rados.0.smithi070.stdout:3754: finishing write tid 2 to smithi07085154-7 2024-09-14T04:01:00.387 INFO:tasks.rados.rados.0.smithi070.stdout:3756: seq_num 1145 ranges {1753088=786432,2539520=622592,3162112=516096,3678208=417792,4096000=49152} 2024-09-14T04:01:00.400 INFO:tasks.rados.rados.0.smithi070.stdout:3756: writing smithi07085154-2 from 1753088 to 2539520 tid 1 2024-09-14T04:01:00.404 INFO:tasks.rados.rados.0.smithi070.stdout:3756: writing smithi07085154-2 from 2539520 to 3162112 tid 2 2024-09-14T04:01:00.407 INFO:tasks.rados.rados.0.smithi070.stdout:3756: writing smithi07085154-2 from 3162112 to 3678208 tid 3 2024-09-14T04:01:00.410 INFO:tasks.rados.rados.0.smithi070.stdout:3756: writing smithi07085154-2 from 3678208 to 4096000 tid 4 2024-09-14T04:01:00.410 INFO:tasks.rados.rados.0.smithi070.stdout:3756: writing smithi07085154-2 from 4096000 to 4145152 tid 5 2024-09-14T04:01:00.410 INFO:tasks.rados.rados.0.smithi070.stdout:3752: done (5 left) 2024-09-14T04:01:00.410 INFO:tasks.rados.rados.0.smithi070.stdout:3753: done (4 left) 2024-09-14T04:01:00.410 INFO:tasks.rados.rados.0.smithi070.stdout:3757: read oid 14 snap -1 2024-09-14T04:01:00.410 INFO:tasks.rados.rados.0.smithi070.stdout:3757: expect (ObjNum 1112 snap 370 seq_num 1112) 2024-09-14T04:01:00.410 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 45 current snap is 382 2024-09-14T04:01:00.410 INFO:tasks.rados.rados.0.smithi070.stdout:3754: finishing write tid 3 to smithi07085154-7 2024-09-14T04:01:00.410 INFO:tasks.rados.rados.0.smithi070.stdout:3754: finishing write tid 4 to smithi07085154-7 2024-09-14T04:01:00.411 INFO:tasks.rados.rados.0.smithi070.stdout:3758: seq_num 1146 ranges {9027584=802816,9830400=499712,10330112=679936,11010048=425984} 2024-09-14T04:01:00.462 INFO:tasks.rados.rados.0.smithi070.stdout:3758: writing smithi07085154-45 from 9027584 to 9830400 tid 1 2024-09-14T04:01:00.465 INFO:tasks.rados.rados.0.smithi070.stdout:3758: writing smithi07085154-45 from 9830400 to 10330112 tid 2 2024-09-14T04:01:00.469 INFO:tasks.rados.rados.0.smithi070.stdout:3758: writing smithi07085154-45 from 10330112 to 11010048 tid 3 2024-09-14T04:01:00.472 INFO:tasks.rados.rados.0.smithi070.stdout:3758: writing smithi07085154-45 from 11010048 to 11436032 tid 4 2024-09-14T04:01:00.472 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 9 current snap is 382 2024-09-14T04:01:00.472 INFO:tasks.rados.rados.0.smithi070.stdout:3751: finishing copy_from racing read to smithi07085154-13 2024-09-14T04:01:00.472 INFO:tasks.rados.rados.0.smithi070.stdout:3754: finishing write tid 5 to smithi07085154-7 2024-09-14T04:01:00.472 INFO:tasks.rados.rados.0.smithi070.stdout:3759: seq_num 1147 ranges {5193728=16384} 2024-09-14T04:01:00.500 INFO:tasks.rados.rados.0.smithi070.stdout:3759: writing smithi07085154-9 from 5193728 to 5210112 tid 1 2024-09-14T04:01:00.500 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 11 current snap is 382 2024-09-14T04:01:00.500 INFO:tasks.rados.rados.0.smithi070.stdout:3760: seq_num 1148 ranges {0=32768} 2024-09-14T04:01:00.500 INFO:tasks.rados.rados.0.smithi070.stdout:3760: writing smithi07085154-11 from 0 to 32768 tid 1 2024-09-14T04:01:00.500 INFO:tasks.rados.rados.0.smithi070.stdout:3761: rmattr oid 33 current snap is 382 2024-09-14T04:01:00.500 INFO:tasks.rados.rados.0.smithi070.stdout:3754: finishing write tid 6 to smithi07085154-7 2024-09-14T04:01:00.501 INFO:tasks.rados.rados.0.smithi070.stdout:3751: finishing copy_from to smithi07085154-13 2024-09-14T04:01:00.501 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 4238 (ObjNum 1078 snap 349 seq_num 1078) dirty exists 2024-09-14T04:01:00.501 INFO:tasks.rados.rados.0.smithi070.stdout:3754: finishing write tid 7 to smithi07085154-7 2024-09-14T04:01:00.501 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 4224 (ObjNum 1143 snap 382 seq_num 1143) dirty exists 2024-09-14T04:01:00.501 INFO:tasks.rados.rados.0.smithi070.stdout:3754: left oid 7 (ObjNum 1143 snap 382 seq_num 1143) 2024-09-14T04:01:00.501 INFO:tasks.rados.rados.0.smithi070.stdout:3755: finishing write tid 1 to smithi07085154-27 2024-09-14T04:01:00.501 INFO:tasks.rados.rados.0.smithi070.stdout:3751: done (8 left) 2024-09-14T04:01:00.501 INFO:tasks.rados.rados.0.smithi070.stdout:3754: done (7 left) 2024-09-14T04:01:00.501 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 29 current snap is 382 2024-09-14T04:01:00.501 INFO:tasks.rados.rados.0.smithi070.stdout:3762: seq_num 1149 ranges {5316608=49152} 2024-09-14T04:01:00.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:01:00 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:01:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:01:00.529 INFO:tasks.rados.rados.0.smithi070.stdout:3762: writing smithi07085154-29 from 5316608 to 5365760 tid 1 2024-09-14T04:01:00.530 INFO:tasks.rados.rados.0.smithi070.stdout:3763: snap_create 2024-09-14T04:01:00.530 INFO:tasks.rados.rados.0.smithi070.stdout:3755: finishing write tid 2 to smithi07085154-27 2024-09-14T04:01:00.530 INFO:tasks.rados.rados.0.smithi070.stdout:3755: finishing write tid 3 to smithi07085154-27 2024-09-14T04:01:00.530 INFO:tasks.rados.rados.0.smithi070.stdout:3755: finishing write tid 4 to smithi07085154-27 2024-09-14T04:01:00.530 INFO:tasks.rados.rados.0.smithi070.stdout:3755: finishing write tid 5 to smithi07085154-27 2024-09-14T04:01:00.530 INFO:tasks.rados.rados.0.smithi070.stdout:3755: finishing write tid 6 to smithi07085154-27 2024-09-14T04:01:00.530 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 4491 (ObjNum 1144 snap 382 seq_num 1144) dirty exists 2024-09-14T04:01:00.530 INFO:tasks.rados.rados.0.smithi070.stdout:3755: left oid 27 (ObjNum 1144 snap 382 seq_num 1144) 2024-09-14T04:01:00.530 INFO:tasks.rados.rados.0.smithi070.stdout:3756: finishing write tid 1 to smithi07085154-2 2024-09-14T04:01:00.530 INFO:tasks.rados.rados.0.smithi070.stdout:3756: finishing write tid 2 to smithi07085154-2 2024-09-14T04:01:00.530 INFO:tasks.rados.rados.0.smithi070.stdout:3756: finishing write tid 3 to smithi07085154-2 2024-09-14T04:01:00.530 INFO:tasks.rados.rados.0.smithi070.stdout:3756: finishing write tid 4 to smithi07085154-2 2024-09-14T04:01:00.530 INFO:tasks.rados.rados.0.smithi070.stdout:3756: finishing write tid 5 to smithi07085154-2 2024-09-14T04:01:00.531 INFO:tasks.rados.rados.0.smithi070.stdout:3756: finishing write tid 6 to smithi07085154-2 2024-09-14T04:01:00.531 INFO:tasks.rados.rados.0.smithi070.stdout:3757: expect (ObjNum 1112 snap 370 seq_num 1112) 2024-09-14T04:01:00.576 INFO:tasks.rados.rados.0.smithi070.stdout:3756: finishing write tid 7 to smithi07085154-2 2024-09-14T04:01:00.576 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 2 v 4498 (ObjNum 1145 snap 382 seq_num 1145) dirty exists 2024-09-14T04:01:00.576 INFO:tasks.rados.rados.0.smithi070.stdout:3756: left oid 2 (ObjNum 1145 snap 382 seq_num 1145) 2024-09-14T04:01:00.576 INFO:tasks.rados.rados.0.smithi070.stdout:3758: finishing write tid 1 to smithi07085154-45 2024-09-14T04:01:00.576 INFO:tasks.rados.rados.0.smithi070.stdout:3758: finishing write tid 2 to smithi07085154-45 2024-09-14T04:01:00.576 INFO:tasks.rados.rados.0.smithi070.stdout:3758: finishing write tid 3 to smithi07085154-45 2024-09-14T04:01:00.576 INFO:tasks.rados.rados.0.smithi070.stdout:3758: finishing write tid 4 to smithi07085154-45 2024-09-14T04:01:00.576 INFO:tasks.rados.rados.0.smithi070.stdout:3758: finishing write tid 5 to smithi07085154-45 2024-09-14T04:01:00.576 INFO:tasks.rados.rados.0.smithi070.stdout:3758: finishing write tid 6 to smithi07085154-45 2024-09-14T04:01:00.576 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 45 v 3236 (ObjNum 1146 snap 382 seq_num 1146) dirty exists 2024-09-14T04:01:00.577 INFO:tasks.rados.rados.0.smithi070.stdout:3758: left oid 45 (ObjNum 1146 snap 382 seq_num 1146) 2024-09-14T04:01:00.577 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 33 v 3237 (ObjNum 590 snap 195 seq_num 590) dirty exists 2024-09-14T04:01:00.577 INFO:tasks.rados.rados.0.smithi070.stdout:3760: finishing write tid 1 to smithi07085154-11 2024-09-14T04:01:00.577 INFO:tasks.rados.rados.0.smithi070.stdout:3760: finishing write tid 2 to smithi07085154-11 2024-09-14T04:01:00.577 INFO:tasks.rados.rados.0.smithi070.stdout:3760: finishing write tid 3 to smithi07085154-11 2024-09-14T04:01:00.577 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 11 v 4240 (ObjNum 1148 snap 382 seq_num 1148) dirty exists 2024-09-14T04:01:00.577 INFO:tasks.rados.rados.0.smithi070.stdout:3760: left oid 11 (ObjNum 1148 snap 382 seq_num 1148) 2024-09-14T04:01:00.577 INFO:tasks.rados.rados.0.smithi070.stdout:3759: finishing write tid 1 to smithi07085154-9 2024-09-14T04:01:00.577 INFO:tasks.rados.rados.0.smithi070.stdout:3759: finishing write tid 2 to smithi07085154-9 2024-09-14T04:01:00.577 INFO:tasks.rados.rados.0.smithi070.stdout:3759: finishing write tid 3 to smithi07085154-9 2024-09-14T04:01:00.577 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 4501 (ObjNum 1147 snap 382 seq_num 1147) dirty exists 2024-09-14T04:01:00.577 INFO:tasks.rados.rados.0.smithi070.stdout:3759: left oid 9 (ObjNum 1147 snap 382 seq_num 1147) 2024-09-14T04:01:00.578 INFO:tasks.rados.rados.0.smithi070.stdout:3762: finishing write tid 1 to smithi07085154-29 2024-09-14T04:01:00.578 INFO:tasks.rados.rados.0.smithi070.stdout:3762: finishing write tid 2 to smithi07085154-29 2024-09-14T04:01:00.578 INFO:tasks.rados.rados.0.smithi070.stdout:3762: finishing write tid 3 to smithi07085154-29 2024-09-14T04:01:00.578 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 4251 (ObjNum 1149 snap 382 seq_num 1149) dirty exists 2024-09-14T04:01:00.578 INFO:tasks.rados.rados.0.smithi070.stdout:3762: left oid 29 (ObjNum 1149 snap 382 seq_num 1149) 2024-09-14T04:01:01.151 INFO:tasks.rados.rados.0.smithi070.stdout:3755: done (8 left) 2024-09-14T04:01:01.151 INFO:tasks.rados.rados.0.smithi070.stdout:3756: done (7 left) 2024-09-14T04:01:01.151 INFO:tasks.rados.rados.0.smithi070.stdout:3757: done (6 left) 2024-09-14T04:01:01.151 INFO:tasks.rados.rados.0.smithi070.stdout:3758: done (5 left) 2024-09-14T04:01:01.151 INFO:tasks.rados.rados.0.smithi070.stdout:3759: done (4 left) 2024-09-14T04:01:01.151 INFO:tasks.rados.rados.0.smithi070.stdout:3760: done (3 left) 2024-09-14T04:01:01.151 INFO:tasks.rados.rados.0.smithi070.stdout:3761: done (2 left) 2024-09-14T04:01:01.151 INFO:tasks.rados.rados.0.smithi070.stdout:3762: done (1 left) 2024-09-14T04:01:01.151 INFO:tasks.rados.rados.0.smithi070.stdout:3763: done (0 left) 2024-09-14T04:01:01.151 INFO:tasks.rados.rados.0.smithi070.stdout:3764: snap_remove snap 360 2024-09-14T04:01:01.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:01 smithi094 ceph-mon[86597]: osdmap e821: 8 total, 8 up, 8 in 2024-09-14T04:01:01.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:01 smithi094 ceph-mon[86597]: pgmap v832: 121 pgs: 121 active+clean; 872 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 25 MiB/s rd, 6.6 MiB/s wr, 27 op/s 2024-09-14T04:01:01.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:01 smithi070 ceph-mon[96704]: osdmap e821: 8 total, 8 up, 8 in 2024-09-14T04:01:01.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:01 smithi070 ceph-mon[96704]: pgmap v832: 121 pgs: 121 active+clean; 872 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 25 MiB/s rd, 6.6 MiB/s wr, 27 op/s 2024-09-14T04:01:01.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:01 smithi070 ceph-mon[98640]: osdmap e821: 8 total, 8 up, 8 in 2024-09-14T04:01:01.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:01 smithi070 ceph-mon[98640]: pgmap v832: 121 pgs: 121 active+clean; 872 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 25 MiB/s rd, 6.6 MiB/s wr, 27 op/s 2024-09-14T04:01:02.154 INFO:tasks.rados.rados.0.smithi070.stdout:3764: done (0 left) 2024-09-14T04:01:02.154 INFO:tasks.rados.rados.0.smithi070.stdout:3765: copy_from oid 49 from oid 2 current snap is 383 2024-09-14T04:01:02.154 INFO:tasks.rados.rados.0.smithi070.stdout:3766: read oid 18 snap -1 2024-09-14T04:01:02.154 INFO:tasks.rados.rados.0.smithi070.stdout:3766: expect (ObjNum 596 snap 199 seq_num 596) 2024-09-14T04:01:02.154 INFO:tasks.rados.rados.0.smithi070.stdout:3767: read oid 4 snap 377 2024-09-14T04:01:02.154 INFO:tasks.rados.rados.0.smithi070.stdout:3767: expect deleted 2024-09-14T04:01:02.155 INFO:tasks.rados.rados.0.smithi070.stdout:3768: copy_from oid 6 from oid 29 current snap is 383 2024-09-14T04:01:02.155 INFO:tasks.rados.rados.0.smithi070.stdout:3769: delete oid 36 current snap is 383 2024-09-14T04:01:02.169 INFO:tasks.rados.rados.0.smithi070.stdout:3767: done (4 left) 2024-09-14T04:01:02.169 INFO:tasks.rados.rados.0.smithi070.stdout:3769: done (3 left) 2024-09-14T04:01:02.169 INFO:tasks.rados.rados.0.smithi070.stdout:3770: snap_remove snap 373 2024-09-14T04:01:02.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:02 smithi094 ceph-mon[86597]: osdmap e822: 8 total, 8 up, 8 in 2024-09-14T04:01:02.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:02 smithi070 ceph-mon[96704]: osdmap e822: 8 total, 8 up, 8 in 2024-09-14T04:01:02.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:02 smithi070 ceph-mon[98640]: osdmap e822: 8 total, 8 up, 8 in 2024-09-14T04:01:03.166 INFO:tasks.rados.rados.0.smithi070.stdout:3770: done (3 left) 2024-09-14T04:01:03.166 INFO:tasks.rados.rados.0.smithi070.stdout:3771: setattr oid 43 current snap is 383 2024-09-14T04:01:03.168 INFO:tasks.rados.rados.0.smithi070.stdout:3772: delete oid 24 current snap is 383 2024-09-14T04:01:03.168 INFO:tasks.rados.rados.0.smithi070.stdout:3766: expect (ObjNum 596 snap 199 seq_num 596) 2024-09-14T04:01:03.460 INFO:tasks.rados.rados.0.smithi070.stdout:3765: finishing copy_from racing read to smithi07085154-49 2024-09-14T04:01:03.461 INFO:tasks.rados.rados.0.smithi070.stdout:3765: finishing copy_from to smithi07085154-49 2024-09-14T04:01:03.461 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 49 v 4498 (ObjNum 1145 snap 382 seq_num 1145) dirty exists 2024-09-14T04:01:03.461 INFO:tasks.rados.rados.0.smithi070.stdout:3768: finishing copy_from racing read to smithi07085154-6 2024-09-14T04:01:03.461 INFO:tasks.rados.rados.0.smithi070.stdout:3765: done (4 left) 2024-09-14T04:01:03.461 INFO:tasks.rados.rados.0.smithi070.stdout:3766: done (3 left) 2024-09-14T04:01:03.461 INFO:tasks.rados.rados.0.smithi070.stdout:3772: done (2 left) 2024-09-14T04:01:03.461 INFO:tasks.rados.rados.0.smithi070.stdout:3773: read oid 28 snap -1 2024-09-14T04:01:03.461 INFO:tasks.rados.rados.0.smithi070.stdout:3773: expect (ObjNum 1044 snap 344 seq_num 1044) 2024-09-14T04:01:03.461 INFO:tasks.rados.rados.0.smithi070.stdout:3774: copy_from oid 7 from oid 32 current snap is 383 2024-09-14T04:01:03.461 INFO:tasks.rados.rados.0.smithi070.stdout:3768: finishing copy_from to smithi07085154-6 2024-09-14T04:01:03.461 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 4251 (ObjNum 1149 snap 382 seq_num 1149) dirty exists 2024-09-14T04:01:03.462 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 4154 (ObjNum 1047 snap 344 seq_num 1047) dirty exists 2024-09-14T04:01:03.462 INFO:tasks.rados.rados.0.smithi070.stdout:3768: done (3 left) 2024-09-14T04:01:03.462 INFO:tasks.rados.rados.0.smithi070.stdout:3771: done (2 left) 2024-09-14T04:01:03.462 INFO:tasks.rados.rados.0.smithi070.stdout:3775: read oid 13 snap -1 2024-09-14T04:01:03.462 INFO:tasks.rados.rados.0.smithi070.stdout:3775: expect (ObjNum 1078 snap 349 seq_num 1078) 2024-09-14T04:01:03.462 INFO:tasks.rados.rados.0.smithi070.stdout:3776: setattr oid 29 current snap is 383 2024-09-14T04:01:03.462 INFO:tasks.rados.rados.0.smithi070.stdout:3777: read oid 33 snap -1 2024-09-14T04:01:03.463 INFO:tasks.rados.rados.0.smithi070.stdout:3777: expect (ObjNum 590 snap 195 seq_num 590) 2024-09-14T04:01:03.463 INFO:tasks.rados.rados.0.smithi070.stdout:3778: snap_create 2024-09-14T04:01:03.469 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 4252 (ObjNum 1149 snap 382 seq_num 1149) dirty exists 2024-09-14T04:01:03.482 INFO:tasks.rados.rados.0.smithi070.stdout:3773: expect (ObjNum 1044 snap 344 seq_num 1044) 2024-09-14T04:01:03.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:03 smithi094 ceph-mon[86597]: osdmap e823: 8 total, 8 up, 8 in 2024-09-14T04:01:03.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:03 smithi094 ceph-mon[86597]: pgmap v835: 121 pgs: 121 active+clean; 888 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 27 MiB/s rd, 5.8 MiB/s wr, 40 op/s 2024-09-14T04:01:03.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:03 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:01:03.535 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:01:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:01:03] "GET /metrics HTTP/1.1" 200 36506 "" "Prometheus/2.51.0" 2024-09-14T04:01:03.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:03 smithi070 ceph-mon[96704]: osdmap e823: 8 total, 8 up, 8 in 2024-09-14T04:01:03.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:03 smithi070 ceph-mon[96704]: pgmap v835: 121 pgs: 121 active+clean; 888 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 27 MiB/s rd, 5.8 MiB/s wr, 40 op/s 2024-09-14T04:01:03.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:03 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:01:03.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:03 smithi070 ceph-mon[98640]: osdmap e823: 8 total, 8 up, 8 in 2024-09-14T04:01:03.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:03 smithi070 ceph-mon[98640]: pgmap v835: 121 pgs: 121 active+clean; 888 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 27 MiB/s rd, 5.8 MiB/s wr, 40 op/s 2024-09-14T04:01:03.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:03 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:01:03.744 INFO:tasks.rados.rados.0.smithi070.stdout:3775: expect (ObjNum 1078 snap 349 seq_num 1078) 2024-09-14T04:01:03.978 INFO:tasks.rados.rados.0.smithi070.stdout:3774: finishing copy_from racing read to smithi07085154-7 2024-09-14T04:01:03.978 INFO:tasks.rados.rados.0.smithi070.stdout:3774: finishing copy_from to smithi07085154-7 2024-09-14T04:01:03.978 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 3259 (ObjNum 1132 snap 381 seq_num 1132) dirty exists 2024-09-14T04:01:03.978 INFO:tasks.rados.rados.0.smithi070.stdout:3777: expect (ObjNum 590 snap 195 seq_num 590) 2024-09-14T04:01:04.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:04 smithi094 ceph-mon[86597]: osdmap e824: 8 total, 8 up, 8 in 2024-09-14T04:01:04.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:04 smithi094 ceph-mon[86597]: osdmap e825: 8 total, 8 up, 8 in 2024-09-14T04:01:04.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:04 smithi070 ceph-mon[98640]: osdmap e824: 8 total, 8 up, 8 in 2024-09-14T04:01:04.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:04 smithi070 ceph-mon[98640]: osdmap e825: 8 total, 8 up, 8 in 2024-09-14T04:01:04.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:04 smithi070 ceph-mon[96704]: osdmap e824: 8 total, 8 up, 8 in 2024-09-14T04:01:04.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:04 smithi070 ceph-mon[96704]: osdmap e825: 8 total, 8 up, 8 in 2024-09-14T04:01:05.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:05 smithi094 ceph-mon[86597]: pgmap v838: 121 pgs: 121 active+clean; 895 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 21 MiB/s rd, 5.9 MiB/s wr, 34 op/s 2024-09-14T04:01:05.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:05 smithi070 ceph-mon[96704]: pgmap v838: 121 pgs: 121 active+clean; 895 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 21 MiB/s rd, 5.9 MiB/s wr, 34 op/s 2024-09-14T04:01:05.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:05 smithi070 ceph-mon[98640]: pgmap v838: 121 pgs: 121 active+clean; 895 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 21 MiB/s rd, 5.9 MiB/s wr, 34 op/s 2024-09-14T04:01:05.892 INFO:tasks.rados.rados.0.smithi070.stdout:3773: done (5 left) 2024-09-14T04:01:05.892 INFO:tasks.rados.rados.0.smithi070.stdout:3774: done (4 left) 2024-09-14T04:01:05.892 INFO:tasks.rados.rados.0.smithi070.stdout:3775: done (3 left) 2024-09-14T04:01:05.892 INFO:tasks.rados.rados.0.smithi070.stdout:3776: done (2 left) 2024-09-14T04:01:05.892 INFO:tasks.rados.rados.0.smithi070.stdout:3777: done (1 left) 2024-09-14T04:01:05.892 INFO:tasks.rados.rados.0.smithi070.stdout:3778: done (0 left) 2024-09-14T04:01:05.892 INFO:tasks.rados.rados.0.smithi070.stdout:3779: copy_from oid 33 from oid 47 current snap is 384 2024-09-14T04:01:05.892 INFO:tasks.rados.rados.0.smithi070.stdout:3780: snap_create 2024-09-14T04:01:05.904 INFO:tasks.rados.rados.0.smithi070.stdout:3779: finishing copy_from racing read to smithi07085154-33 2024-09-14T04:01:05.906 INFO:tasks.rados.rados.0.smithi070.stdout:3779: finishing copy_from to smithi07085154-33 2024-09-14T04:01:05.906 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 33 v 3214 (ObjNum 1014 snap 334 seq_num 1014) dirty exists 2024-09-14T04:01:06.169 INFO:tasks.rados.rados.0.smithi070.stdout:3779: done (1 left) 2024-09-14T04:01:06.169 INFO:tasks.rados.rados.0.smithi070.stdout:3780: done (0 left) 2024-09-14T04:01:06.170 INFO:tasks.rados.rados.0.smithi070.stdout:3781: delete oid 8 current snap is 385 2024-09-14T04:01:06.174 INFO:tasks.rados.rados.0.smithi070.stdout:3781: done (0 left) 2024-09-14T04:01:06.174 INFO:tasks.rados.rados.0.smithi070.stdout:3782: snap_remove snap 362 2024-09-14T04:01:07.176 INFO:tasks.rados.rados.0.smithi070.stdout:3782: done (0 left) 2024-09-14T04:01:07.176 INFO:tasks.rados.rados.0.smithi070.stdout:3783: rmattr oid 5 current snap is 385 2024-09-14T04:01:07.176 INFO:tasks.rados.rados.0.smithi070.stdout:3784: snap_create 2024-09-14T04:01:07.189 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 3202 (ObjNum 1042 snap 342 seq_num 1042) dirty exists 2024-09-14T04:01:07.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:07 smithi094 ceph-mon[86597]: pgmap v839: 121 pgs: 121 active+clean; 903 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 29 MiB/s rd, 6.4 MiB/s wr, 40 op/s 2024-09-14T04:01:07.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:07 smithi094 ceph-mon[86597]: osdmap e826: 8 total, 8 up, 8 in 2024-09-14T04:01:07.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:07 smithi070 ceph-mon[96704]: pgmap v839: 121 pgs: 121 active+clean; 903 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 29 MiB/s rd, 6.4 MiB/s wr, 40 op/s 2024-09-14T04:01:07.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:07 smithi070 ceph-mon[96704]: osdmap e826: 8 total, 8 up, 8 in 2024-09-14T04:01:07.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:07 smithi070 ceph-mon[98640]: pgmap v839: 121 pgs: 121 active+clean; 903 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 29 MiB/s rd, 6.4 MiB/s wr, 40 op/s 2024-09-14T04:01:07.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:07 smithi070 ceph-mon[98640]: osdmap e826: 8 total, 8 up, 8 in 2024-09-14T04:01:08.181 INFO:tasks.rados.rados.0.smithi070.stdout:3783: done (1 left) 2024-09-14T04:01:08.181 INFO:tasks.rados.rados.0.smithi070.stdout:3784: done (0 left) 2024-09-14T04:01:08.181 INFO:tasks.rados.rados.0.smithi070.stdout:3785: snap_remove snap 330 2024-09-14T04:01:08.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:08 smithi094 ceph-mon[86597]: osdmap e827: 8 total, 8 up, 8 in 2024-09-14T04:01:08.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:08 smithi094 ceph-mon[86597]: pgmap v842: 121 pgs: 121 active+clean; 905 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 18 MiB/s rd, 3.6 MiB/s wr, 20 op/s 2024-09-14T04:01:08.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:08 smithi070 ceph-mon[96704]: osdmap e827: 8 total, 8 up, 8 in 2024-09-14T04:01:08.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:08 smithi070 ceph-mon[96704]: pgmap v842: 121 pgs: 121 active+clean; 905 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 18 MiB/s rd, 3.6 MiB/s wr, 20 op/s 2024-09-14T04:01:08.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:08 smithi070 ceph-mon[98640]: osdmap e827: 8 total, 8 up, 8 in 2024-09-14T04:01:08.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:08 smithi070 ceph-mon[98640]: pgmap v842: 121 pgs: 121 active+clean; 905 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 18 MiB/s rd, 3.6 MiB/s wr, 20 op/s 2024-09-14T04:01:09.148 INFO:tasks.rados.rados.0.smithi070.stdout:3785: done (0 left) 2024-09-14T04:01:09.148 INFO:tasks.rados.rados.0.smithi070.stdout:3786: snap_remove snap 352 2024-09-14T04:01:09.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:09 smithi094 ceph-mon[86597]: osdmap e828: 8 total, 8 up, 8 in 2024-09-14T04:01:09.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:09 smithi094 ceph-mon[86597]: osdmap e829: 8 total, 8 up, 8 in 2024-09-14T04:01:09.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:09 smithi070 ceph-mon[96704]: osdmap e828: 8 total, 8 up, 8 in 2024-09-14T04:01:09.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:09 smithi070 ceph-mon[96704]: osdmap e829: 8 total, 8 up, 8 in 2024-09-14T04:01:09.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:09 smithi070 ceph-mon[98640]: osdmap e828: 8 total, 8 up, 8 in 2024-09-14T04:01:09.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:09 smithi070 ceph-mon[98640]: osdmap e829: 8 total, 8 up, 8 in 2024-09-14T04:01:10.151 INFO:tasks.rados.rados.0.smithi070.stdout:3786: done (0 left) 2024-09-14T04:01:10.151 INFO:tasks.rados.rados.0.smithi070.stdout:3787: snap_create 2024-09-14T04:01:10.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:01:10 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:01:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:01:11.151 INFO:tasks.rados.rados.0.smithi070.stdout:3787: done (0 left) 2024-09-14T04:01:11.151 INFO:tasks.rados.rados.0.smithi070.stdout:3788: read oid 36 snap 371 2024-09-14T04:01:11.151 INFO:tasks.rados.rados.0.smithi070.stdout:3788: expect deleted 2024-09-14T04:01:11.151 INFO:tasks.rados.rados.0.smithi070.stdout:3789: setattr oid 19 current snap is 387 2024-09-14T04:01:11.153 INFO:tasks.rados.rados.0.smithi070.stdout:3788: done (1 left) 2024-09-14T04:01:11.153 INFO:tasks.rados.rados.0.smithi070.stdout:3790: copy_from oid 35 from oid 25 current snap is 387 2024-09-14T04:01:11.153 INFO:tasks.rados.rados.0.smithi070.stdout:3791: snap_remove snap 304 2024-09-14T04:01:11.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:11 smithi094 ceph-mon[86597]: osdmap e830: 8 total, 8 up, 8 in 2024-09-14T04:01:11.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:11 smithi094 ceph-mon[86597]: pgmap v846: 121 pgs: 121 active+clean; 908 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.3 MiB/s rd, 1.1 MiB/s wr, 8 op/s 2024-09-14T04:01:11.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:11 smithi070 ceph-mon[96704]: osdmap e830: 8 total, 8 up, 8 in 2024-09-14T04:01:11.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:11 smithi070 ceph-mon[96704]: pgmap v846: 121 pgs: 121 active+clean; 908 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.3 MiB/s rd, 1.1 MiB/s wr, 8 op/s 2024-09-14T04:01:11.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:11 smithi070 ceph-mon[98640]: osdmap e830: 8 total, 8 up, 8 in 2024-09-14T04:01:11.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:11 smithi070 ceph-mon[98640]: pgmap v846: 121 pgs: 121 active+clean; 908 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.3 MiB/s rd, 1.1 MiB/s wr, 8 op/s 2024-09-14T04:01:12.161 INFO:tasks.rados.rados.0.smithi070.stdout:3791: done (2 left) 2024-09-14T04:01:12.161 INFO:tasks.rados.rados.0.smithi070.stdout:3792: rmattr oid 12 current snap is 387 2024-09-14T04:01:12.161 INFO:tasks.rados.rados.0.smithi070.stdout:3792: done (2 left) 2024-09-14T04:01:12.161 INFO:tasks.rados.rados.0.smithi070.stdout:3793: rollback oid 1 current snap is 387 2024-09-14T04:01:12.161 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 1 to 291 2024-09-14T04:01:12.161 INFO:tasks.rados.rados.0.smithi070.stdout:3794: snap_remove snap 384 2024-09-14T04:01:12.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:12 smithi094 ceph-mon[86597]: osdmap e831: 8 total, 8 up, 8 in 2024-09-14T04:01:12.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:12 smithi070 ceph-mon[96704]: osdmap e831: 8 total, 8 up, 8 in 2024-09-14T04:01:12.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:12 smithi070 ceph-mon[98640]: osdmap e831: 8 total, 8 up, 8 in 2024-09-14T04:01:13.172 INFO:tasks.rados.rados.0.smithi070.stdout:3794: done (3 left) 2024-09-14T04:01:13.172 INFO:tasks.rados.rados.0.smithi070.stdout:3795: snap_remove snap 296 2024-09-14T04:01:13.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:13 smithi094 ceph-mon[86597]: pgmap v849: 121 pgs: 121 active+clean; 899 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 1.1 MiB/s wr, 7 op/s 2024-09-14T04:01:13.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:13 smithi094 ceph-mon[86597]: osdmap e832: 8 total, 8 up, 8 in 2024-09-14T04:01:13.536 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:01:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:01:13] "GET /metrics HTTP/1.1" 200 36506 "" "Prometheus/2.51.0" 2024-09-14T04:01:13.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:13 smithi070 ceph-mon[96704]: pgmap v849: 121 pgs: 121 active+clean; 899 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 1.1 MiB/s wr, 7 op/s 2024-09-14T04:01:13.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:13 smithi070 ceph-mon[96704]: osdmap e832: 8 total, 8 up, 8 in 2024-09-14T04:01:13.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:13 smithi070 ceph-mon[98640]: pgmap v849: 121 pgs: 121 active+clean; 899 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 1.1 MiB/s wr, 7 op/s 2024-09-14T04:01:13.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:13 smithi070 ceph-mon[98640]: osdmap e832: 8 total, 8 up, 8 in 2024-09-14T04:01:14.150 INFO:tasks.rados.rados.0.smithi070.stdout:3795: done (3 left) 2024-09-14T04:01:14.150 INFO:tasks.rados.rados.0.smithi070.stdout:3796: copy_from oid 40 from oid 29 current snap is 387 2024-09-14T04:01:14.151 INFO:tasks.rados.rados.0.smithi070.stdout:3797: copy_from oid 50 from oid 17 current snap is 387 2024-09-14T04:01:14.151 INFO:tasks.rados.rados.0.smithi070.stdout:3798: snap_remove snap 385 2024-09-14T04:01:14.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:14 smithi094 ceph-mon[86597]: osdmap e833: 8 total, 8 up, 8 in 2024-09-14T04:01:14.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:14 smithi094 ceph-mon[86597]: osdmap e834: 8 total, 8 up, 8 in 2024-09-14T04:01:14.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:14 smithi070 ceph-mon[96704]: osdmap e833: 8 total, 8 up, 8 in 2024-09-14T04:01:14.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:14 smithi070 ceph-mon[96704]: osdmap e834: 8 total, 8 up, 8 in 2024-09-14T04:01:14.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:14 smithi070 ceph-mon[98640]: osdmap e833: 8 total, 8 up, 8 in 2024-09-14T04:01:14.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:14 smithi070 ceph-mon[98640]: osdmap e834: 8 total, 8 up, 8 in 2024-09-14T04:01:15.154 INFO:tasks.rados.rados.0.smithi070.stdout:3798: done (5 left) 2024-09-14T04:01:15.154 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 11 current snap is 387 2024-09-14T04:01:15.155 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 19 v 4253 (ObjNum 659 snap 220 seq_num 659) dirty exists 2024-09-14T04:01:15.155 INFO:tasks.rados.rados.0.smithi070.stdout:3799: seq_num 1153 ranges {32768=114688} 2024-09-14T04:01:15.155 INFO:tasks.rados.rados.0.smithi070.stdout:3799: writing smithi07085154-11 from 32768 to 147456 tid 1 2024-09-14T04:01:15.155 INFO:tasks.rados.rados.0.smithi070.stdout:3789: done (5 left) 2024-09-14T04:01:15.155 INFO:tasks.rados.rados.0.smithi070.stdout:3800: rmattr oid 8 current snap is 387 2024-09-14T04:01:15.156 INFO:tasks.rados.rados.0.smithi070.stdout:3800: done (5 left) 2024-09-14T04:01:15.156 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 10 current snap is 387 2024-09-14T04:01:15.156 INFO:tasks.rados.rados.0.smithi070.stdout:3801: seq_num 1154 ranges {1998848=131072} 2024-09-14T04:01:15.168 INFO:tasks.rados.rados.0.smithi070.stdout:3801: writing smithi07085154-10 from 1998848 to 2129920 tid 1 2024-09-14T04:01:15.168 INFO:tasks.rados.rados.0.smithi070.stdout:3802: setattr oid 12 current snap is 387 2024-09-14T04:01:15.168 INFO:tasks.rados.rados.0.smithi070.stdout:3790: finishing copy_from racing read to smithi07085154-35 2024-09-14T04:01:15.168 INFO:tasks.rados.rados.0.smithi070.stdout:3790: finishing copy_from to smithi07085154-35 2024-09-14T04:01:15.168 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 35 v 4435 (ObjNum 1136 snap 381 seq_num 1136) dirty exists 2024-09-14T04:01:15.169 INFO:tasks.rados.rados.0.smithi070.stdout:3793: finishing rollback tid 0 to smithi07085154-1 2024-09-14T04:01:15.169 INFO:tasks.rados.rados.0.smithi070.stdout:3793: finishing rollback tid 1 to smithi07085154-1 2024-09-14T04:01:15.169 INFO:tasks.rados.rados.0.smithi070.stdout:3793: finishing rollback tid 2 to smithi07085154-1 2024-09-14T04:01:15.169 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 1 v 4250 (ObjNum 712 snap 243 seq_num 712) dirty exists 2024-09-14T04:01:15.169 INFO:tasks.rados.rados.0.smithi070.stdout:3797: finishing copy_from racing read to smithi07085154-50 2024-09-14T04:01:15.169 INFO:tasks.rados.rados.0.smithi070.stdout:3796: finishing copy_from racing read to smithi07085154-40 2024-09-14T04:01:15.169 INFO:tasks.rados.rados.0.smithi070.stdout:3797: finishing copy_from to smithi07085154-50 2024-09-14T04:01:15.169 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 2277 (ObjNum 1001 snap 331 seq_num 1001) dirty exists 2024-09-14T04:01:15.169 INFO:tasks.rados.rados.0.smithi070.stdout:3796: finishing copy_from to smithi07085154-40 2024-09-14T04:01:15.169 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 4252 (ObjNum 1149 snap 382 seq_num 1149) dirty exists 2024-09-14T04:01:15.169 INFO:tasks.rados.rados.0.smithi070.stdout:3799: finishing write tid 1 to smithi07085154-11 2024-09-14T04:01:15.169 INFO:tasks.rados.rados.0.smithi070.stdout:3799: finishing write tid 2 to smithi07085154-11 2024-09-14T04:01:15.170 INFO:tasks.rados.rados.0.smithi070.stdout:3799: finishing write tid 3 to smithi07085154-11 2024-09-14T04:01:15.170 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 11 v 4242 (ObjNum 1153 snap 387 seq_num 1153) dirty exists 2024-09-14T04:01:15.170 INFO:tasks.rados.rados.0.smithi070.stdout:3799: left oid 11 (ObjNum 1153 snap 387 seq_num 1153) 2024-09-14T04:01:15.170 INFO:tasks.rados.rados.0.smithi070.stdout:3790: done (6 left) 2024-09-14T04:01:15.170 INFO:tasks.rados.rados.0.smithi070.stdout:3793: done (5 left) 2024-09-14T04:01:15.171 INFO:tasks.rados.rados.0.smithi070.stdout:3796: done (4 left) 2024-09-14T04:01:15.171 INFO:tasks.rados.rados.0.smithi070.stdout:3797: done (3 left) 2024-09-14T04:01:15.171 INFO:tasks.rados.rados.0.smithi070.stdout:3799: done (2 left) 2024-09-14T04:01:15.171 INFO:tasks.rados.rados.0.smithi070.stdout:3803: read oid 3 snap -1 2024-09-14T04:01:15.171 INFO:tasks.rados.rados.0.smithi070.stdout:3803: expect (ObjNum 1072 snap 347 seq_num 1072) 2024-09-14T04:01:15.171 INFO:tasks.rados.rados.0.smithi070.stdout:3804: delete oid 19 current snap is 387 2024-09-14T04:01:15.173 INFO:tasks.rados.rados.0.smithi070.stdout:3804: done (3 left) 2024-09-14T04:01:15.173 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 1 current snap is 387 2024-09-14T04:01:15.173 INFO:tasks.rados.rados.0.smithi070.stdout:3805: seq_num 1156 ranges {2334720=163840} 2024-09-14T04:01:15.187 INFO:tasks.rados.rados.0.smithi070.stdout:3805: writing smithi07085154-1 from 2334720 to 2498560 tid 1 2024-09-14T04:01:15.187 INFO:tasks.rados.rados.0.smithi070.stdout:3806: snap_remove snap 386 2024-09-14T04:01:15.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:15 smithi094 ceph-mon[86597]: pgmap v852: 121 pgs: 121 active+clean; 899 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 546 KiB/s wr, 7 op/s 2024-09-14T04:01:15.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:15 smithi094 ceph-mon[86597]: osdmap e835: 8 total, 8 up, 8 in 2024-09-14T04:01:15.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:15 smithi070 ceph-mon[96704]: pgmap v852: 121 pgs: 121 active+clean; 899 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 546 KiB/s wr, 7 op/s 2024-09-14T04:01:15.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:15 smithi070 ceph-mon[96704]: osdmap e835: 8 total, 8 up, 8 in 2024-09-14T04:01:15.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:15 smithi070 ceph-mon[98640]: pgmap v852: 121 pgs: 121 active+clean; 899 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 546 KiB/s wr, 7 op/s 2024-09-14T04:01:15.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:15 smithi070 ceph-mon[98640]: osdmap e835: 8 total, 8 up, 8 in 2024-09-14T04:01:16.174 INFO:tasks.rados.rados.0.smithi070.stdout:3806: done (4 left) 2024-09-14T04:01:16.174 INFO:tasks.rados.rados.0.smithi070.stdout:3807: snap_create 2024-09-14T04:01:16.174 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 2278 (ObjNum 65 snap 0 seq_num 1818062187) dirty exists 2024-09-14T04:01:16.174 INFO:tasks.rados.rados.0.smithi070.stdout:3803: expect (ObjNum 1072 snap 347 seq_num 1072) 2024-09-14T04:01:16.207 INFO:tasks.rados.rados.0.smithi070.stdout:3801: finishing write tid 1 to smithi07085154-10 2024-09-14T04:01:16.207 INFO:tasks.rados.rados.0.smithi070.stdout:3801: finishing write tid 2 to smithi07085154-10 2024-09-14T04:01:16.207 INFO:tasks.rados.rados.0.smithi070.stdout:3801: finishing write tid 3 to smithi07085154-10 2024-09-14T04:01:16.207 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 4590 (ObjNum 1154 snap 387 seq_num 1154) dirty exists 2024-09-14T04:01:16.207 INFO:tasks.rados.rados.0.smithi070.stdout:3801: left oid 10 (ObjNum 1154 snap 387 seq_num 1154) 2024-09-14T04:01:16.207 INFO:tasks.rados.rados.0.smithi070.stdout:3805: finishing write tid 1 to smithi07085154-1 2024-09-14T04:01:16.208 INFO:tasks.rados.rados.0.smithi070.stdout:3805: finishing write tid 2 to smithi07085154-1 2024-09-14T04:01:16.208 INFO:tasks.rados.rados.0.smithi070.stdout:3805: finishing write tid 3 to smithi07085154-1 2024-09-14T04:01:16.208 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 1 v 4254 (ObjNum 1156 snap 387 seq_num 1156) dirty exists 2024-09-14T04:01:16.208 INFO:tasks.rados.rados.0.smithi070.stdout:3805: left oid 1 (ObjNum 1156 snap 387 seq_num 1156) 2024-09-14T04:01:17.176 INFO:tasks.rados.rados.0.smithi070.stdout:3801: done (4 left) 2024-09-14T04:01:17.176 INFO:tasks.rados.rados.0.smithi070.stdout:3802: done (3 left) 2024-09-14T04:01:17.176 INFO:tasks.rados.rados.0.smithi070.stdout:3803: done (2 left) 2024-09-14T04:01:17.176 INFO:tasks.rados.rados.0.smithi070.stdout:3805: done (1 left) 2024-09-14T04:01:17.176 INFO:tasks.rados.rados.0.smithi070.stdout:3807: done (0 left) 2024-09-14T04:01:17.176 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 28 current snap is 388 2024-09-14T04:01:17.176 INFO:tasks.rados.rados.0.smithi070.stdout:3808: seq_num 1157 ranges {4710400=180224} 2024-09-14T04:01:17.202 INFO:tasks.rados.rados.0.smithi070.stdout:3808: writing smithi07085154-28 from 4710400 to 4890624 tid 1 2024-09-14T04:01:17.202 INFO:tasks.rados.rados.0.smithi070.stdout:3809: snap_remove snap 372 2024-09-14T04:01:17.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:17 smithi094 ceph-mon[86597]: pgmap v854: 121 pgs: 121 active+clean; 899 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-14T04:01:17.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:17 smithi094 ceph-mon[86597]: osdmap e836: 8 total, 8 up, 8 in 2024-09-14T04:01:17.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:17 smithi070 ceph-mon[96704]: pgmap v854: 121 pgs: 121 active+clean; 899 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-14T04:01:17.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:17 smithi070 ceph-mon[96704]: osdmap e836: 8 total, 8 up, 8 in 2024-09-14T04:01:17.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:17 smithi070 ceph-mon[98640]: pgmap v854: 121 pgs: 121 active+clean; 899 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-14T04:01:17.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:17 smithi070 ceph-mon[98640]: osdmap e836: 8 total, 8 up, 8 in 2024-09-14T04:01:18.186 INFO:tasks.rados.rados.0.smithi070.stdout:3809: done (1 left) 2024-09-14T04:01:18.186 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 9 current snap is 388 2024-09-14T04:01:18.186 INFO:tasks.rados.rados.0.smithi070.stdout:3810: seq_num 1158 ranges {5210112=196608} 2024-09-14T04:01:18.217 INFO:tasks.rados.rados.0.smithi070.stdout:3810: writing smithi07085154-9 from 5210112 to 5406720 tid 1 2024-09-14T04:01:18.217 INFO:tasks.rados.rados.0.smithi070.stdout:3811: rollback oid 32 current snap is 388 2024-09-14T04:01:18.217 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 32 to 370 2024-09-14T04:01:18.217 INFO:tasks.rados.rados.0.smithi070.stdout:3808: finishing write tid 1 to smithi07085154-28 2024-09-14T04:01:18.218 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 3 current snap is 388 2024-09-14T04:01:18.218 INFO:tasks.rados.rados.0.smithi070.stdout:3808: finishing write tid 2 to smithi07085154-28 2024-09-14T04:01:18.218 INFO:tasks.rados.rados.0.smithi070.stdout:3812: seq_num 1159 ranges {1163264=212992} 2024-09-14T04:01:18.225 INFO:tasks.rados.rados.0.smithi070.stdout:3812: writing smithi07085154-3 from 1163264 to 1376256 tid 1 2024-09-14T04:01:18.225 INFO:tasks.rados.rados.0.smithi070.stdout:3813: copy_from oid 23 from oid 43 current snap is 388 2024-09-14T04:01:18.225 INFO:tasks.rados.rados.0.smithi070.stdout:3808: finishing write tid 3 to smithi07085154-28 2024-09-14T04:01:18.225 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 28 v 4226 (ObjNum 1157 snap 388 seq_num 1157) dirty exists 2024-09-14T04:01:18.225 INFO:tasks.rados.rados.0.smithi070.stdout:3808: left oid 28 (ObjNum 1157 snap 388 seq_num 1157) 2024-09-14T04:01:18.225 INFO:tasks.rados.rados.0.smithi070.stdout:3811: finishing rollback tid 0 to smithi07085154-32 2024-09-14T04:01:18.225 INFO:tasks.rados.rados.0.smithi070.stdout:3811: finishing rollback tid 1 to smithi07085154-32 2024-09-14T04:01:18.226 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 4244 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-14T04:01:18.226 INFO:tasks.rados.rados.0.smithi070.stdout:3810: finishing write tid 1 to smithi07085154-9 2024-09-14T04:01:18.226 INFO:tasks.rados.rados.0.smithi070.stdout:3810: finishing write tid 2 to smithi07085154-9 2024-09-14T04:01:18.226 INFO:tasks.rados.rados.0.smithi070.stdout:3808: done (4 left) 2024-09-14T04:01:18.226 INFO:tasks.rados.rados.0.smithi070.stdout:3811: done (3 left) 2024-09-14T04:01:18.226 INFO:tasks.rados.rados.0.smithi070.stdout:3814: read oid 20 snap -1 2024-09-14T04:01:18.226 INFO:tasks.rados.rados.0.smithi070.stdout:3814: expect (ObjNum 1083 snap 353 seq_num 1083) 2024-09-14T04:01:18.226 INFO:tasks.rados.rados.0.smithi070.stdout:3815: snap_remove snap 383 2024-09-14T04:01:18.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:18 smithi094 ceph-mon[86597]: osdmap e837: 8 total, 8 up, 8 in 2024-09-14T04:01:18.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:18 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:01:18.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:18 smithi094 ceph-mon[86597]: pgmap v857: 121 pgs: 121 active+clean; 896 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 864 KiB/s rd, 638 KiB/s wr, 3 op/s 2024-09-14T04:01:18.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:18 smithi070 ceph-mon[96704]: osdmap e837: 8 total, 8 up, 8 in 2024-09-14T04:01:18.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:18 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:01:18.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:18 smithi070 ceph-mon[96704]: pgmap v857: 121 pgs: 121 active+clean; 896 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 864 KiB/s rd, 638 KiB/s wr, 3 op/s 2024-09-14T04:01:18.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:18 smithi070 ceph-mon[98640]: osdmap e837: 8 total, 8 up, 8 in 2024-09-14T04:01:18.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:18 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:01:18.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:18 smithi070 ceph-mon[98640]: pgmap v857: 121 pgs: 121 active+clean; 896 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 864 KiB/s rd, 638 KiB/s wr, 3 op/s 2024-09-14T04:01:19.152 INFO:tasks.rados.rados.0.smithi070.stdout:3815: done (4 left) 2024-09-14T04:01:19.152 INFO:tasks.rados.rados.0.smithi070.stdout:3816: rollback oid 4 current snap is 388 2024-09-14T04:01:19.152 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 4 to 350 2024-09-14T04:01:19.153 INFO:tasks.rados.rados.0.smithi070.stdout:3810: finishing write tid 3 to smithi07085154-9 2024-09-14T04:01:19.153 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 4603 (ObjNum 1158 snap 388 seq_num 1158) dirty exists 2024-09-14T04:01:19.153 INFO:tasks.rados.rados.0.smithi070.stdout:3810: left oid 9 (ObjNum 1158 snap 388 seq_num 1158) 2024-09-14T04:01:19.153 INFO:tasks.rados.rados.0.smithi070.stdout:3812: finishing write tid 1 to smithi07085154-3 2024-09-14T04:01:19.153 INFO:tasks.rados.rados.0.smithi070.stdout:3812: finishing write tid 2 to smithi07085154-3 2024-09-14T04:01:19.153 INFO:tasks.rados.rados.0.smithi070.stdout:3812: finishing write tid 3 to smithi07085154-3 2024-09-14T04:01:19.153 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 3 v 3994 (ObjNum 1159 snap 388 seq_num 1159) dirty exists 2024-09-14T04:01:19.153 INFO:tasks.rados.rados.0.smithi070.stdout:3812: left oid 3 (ObjNum 1159 snap 388 seq_num 1159) 2024-09-14T04:01:19.153 INFO:tasks.rados.rados.0.smithi070.stdout:3813: finishing copy_from racing read to smithi07085154-23 2024-09-14T04:01:19.153 INFO:tasks.rados.rados.0.smithi070.stdout:3813: finishing copy_from to smithi07085154-23 2024-09-14T04:01:19.153 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 4154 (ObjNum 1047 snap 344 seq_num 1047) dirty exists 2024-09-14T04:01:19.153 INFO:tasks.rados.rados.0.smithi070.stdout:3814: expect (ObjNum 1083 snap 353 seq_num 1083) 2024-09-14T04:01:19.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:19 smithi094 ceph-mon[86597]: osdmap e838: 8 total, 8 up, 8 in 2024-09-14T04:01:19.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:19 smithi094 ceph-mon[86597]: osdmap e839: 8 total, 8 up, 8 in 2024-09-14T04:01:19.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:19 smithi070 ceph-mon[96704]: osdmap e838: 8 total, 8 up, 8 in 2024-09-14T04:01:19.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:19 smithi070 ceph-mon[96704]: osdmap e839: 8 total, 8 up, 8 in 2024-09-14T04:01:19.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:19 smithi070 ceph-mon[98640]: osdmap e838: 8 total, 8 up, 8 in 2024-09-14T04:01:19.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:19 smithi070 ceph-mon[98640]: osdmap e839: 8 total, 8 up, 8 in 2024-09-14T04:01:19.769 INFO:tasks.rados.rados.0.smithi070.stdout:3816: finishing rollback tid 0 to smithi07085154-4 2024-09-14T04:01:19.769 INFO:tasks.rados.rados.0.smithi070.stdout:3816: finishing rollback tid 1 to smithi07085154-4 2024-09-14T04:01:19.769 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 4 v 4228 (ObjNum 65 snap 0 seq_num 1433690209) dirty dne 2024-09-14T04:01:19.769 INFO:tasks.rados.rados.0.smithi070.stdout:3810: done (4 left) 2024-09-14T04:01:19.769 INFO:tasks.rados.rados.0.smithi070.stdout:3812: done (3 left) 2024-09-14T04:01:19.770 INFO:tasks.rados.rados.0.smithi070.stdout:3813: done (2 left) 2024-09-14T04:01:19.770 INFO:tasks.rados.rados.0.smithi070.stdout:3814: done (1 left) 2024-09-14T04:01:19.770 INFO:tasks.rados.rados.0.smithi070.stdout:3816: done (0 left) 2024-09-14T04:01:19.770 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 14 current snap is 388 2024-09-14T04:01:19.770 INFO:tasks.rados.rados.0.smithi070.stdout:3817: seq_num 1160 ranges {1835008=229376} 2024-09-14T04:01:19.779 INFO:tasks.rados.rados.0.smithi070.stdout:3817: writing smithi07085154-14 from 1835008 to 2064384 tid 1 2024-09-14T04:01:19.779 INFO:tasks.rados.rados.0.smithi070.stdout:3818: snap_remove snap 340 2024-09-14T04:01:20.192 INFO:tasks.rados.rados.0.smithi070.stdout:3817: finishing write tid 1 to smithi07085154-14 2024-09-14T04:01:20.192 INFO:tasks.rados.rados.0.smithi070.stdout:3817: finishing write tid 2 to smithi07085154-14 2024-09-14T04:01:20.192 INFO:tasks.rados.rados.0.smithi070.stdout:3817: finishing write tid 3 to smithi07085154-14 2024-09-14T04:01:20.192 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 4156 (ObjNum 1160 snap 388 seq_num 1160) dirty exists 2024-09-14T04:01:20.192 INFO:tasks.rados.rados.0.smithi070.stdout:3817: left oid 14 (ObjNum 1160 snap 388 seq_num 1160) 2024-09-14T04:01:20.192 INFO:tasks.rados.rados.0.smithi070.stdout:3817: done (1 left) 2024-09-14T04:01:20.193 INFO:tasks.rados.rados.0.smithi070.stdout:3818: done (0 left) 2024-09-14T04:01:20.193 INFO:tasks.rados.rados.0.smithi070.stdout:3819: read oid 44 snap -1 2024-09-14T04:01:20.193 INFO:tasks.rados.rados.0.smithi070.stdout:3819: expect (ObjNum 1130 snap 379 seq_num 1130) 2024-09-14T04:01:20.193 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 43 current snap is 388 2024-09-14T04:01:20.193 INFO:tasks.rados.rados.0.smithi070.stdout:3820: seq_num 1161 ranges {2482176=245760} 2024-09-14T04:01:20.207 INFO:tasks.rados.rados.0.smithi070.stdout:3820: writing smithi07085154-43 from 2482176 to 2727936 tid 1 2024-09-14T04:01:20.208 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 1 current snap is 388 2024-09-14T04:01:20.208 INFO:tasks.rados.rados.0.smithi070.stdout:3821: seq_num 1162 ranges {2498560=270336} 2024-09-14T04:01:20.224 INFO:tasks.rados.rados.0.smithi070.stdout:3821: writing smithi07085154-1 from 2498560 to 2768896 tid 1 2024-09-14T04:01:20.224 INFO:tasks.rados.rados.0.smithi070.stdout:3822: snap_remove snap 291 2024-09-14T04:01:20.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:20 smithi094 ceph-mon[86597]: pgmap v860: 121 pgs: 121 active+clean; 896 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 867 KiB/s rd, 1.9 MiB/s wr, 9 op/s 2024-09-14T04:01:20.527 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 04:01:20 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4[92016]: 2024-09-14T04:01:20.241+0000 7f6d3c1e3640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap 1a6 after trimming was started 2024-09-14T04:01:20.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:01:20 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:01:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:01:20.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:20 smithi070 ceph-mon[96704]: pgmap v860: 121 pgs: 121 active+clean; 896 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 867 KiB/s rd, 1.9 MiB/s wr, 9 op/s 2024-09-14T04:01:20.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:20 smithi070 ceph-mon[98640]: pgmap v860: 121 pgs: 121 active+clean; 896 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 867 KiB/s rd, 1.9 MiB/s wr, 9 op/s 2024-09-14T04:01:21.197 INFO:tasks.rados.rados.0.smithi070.stdout:3822: done (3 left) 2024-09-14T04:01:21.197 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 49 current snap is 388 2024-09-14T04:01:21.197 INFO:tasks.rados.rados.0.smithi070.stdout:3823: seq_num 1163 ranges {4145152=286720} 2024-09-14T04:01:21.223 INFO:tasks.rados.rados.0.smithi070.stdout:3823: writing smithi07085154-49 from 4145152 to 4431872 tid 1 2024-09-14T04:01:21.223 INFO:tasks.rados.rados.0.smithi070.stdout:3824: read oid 33 snap -1 2024-09-14T04:01:21.223 INFO:tasks.rados.rados.0.smithi070.stdout:3824: expect (ObjNum 1014 snap 334 seq_num 1014) 2024-09-14T04:01:21.223 INFO:tasks.rados.rados.0.smithi070.stdout:3825: snap_remove snap 387 2024-09-14T04:01:21.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:21 smithi094 ceph-mon[86597]: osdmap e840: 8 total, 8 up, 8 in 2024-09-14T04:01:21.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:21 smithi070 ceph-mon[96704]: osdmap e840: 8 total, 8 up, 8 in 2024-09-14T04:01:21.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:21 smithi070 ceph-mon[98640]: osdmap e840: 8 total, 8 up, 8 in 2024-09-14T04:01:22.204 INFO:tasks.rados.rados.0.smithi070.stdout:3825: done (5 left) 2024-09-14T04:01:22.204 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 25 current snap is 388 2024-09-14T04:01:22.204 INFO:tasks.rados.rados.0.smithi070.stdout:3826: seq_num 1164 ranges {2236416=303104} 2024-09-14T04:01:22.218 INFO:tasks.rados.rados.0.smithi070.stdout:3826: writing smithi07085154-25 from 2236416 to 2539520 tid 1 2024-09-14T04:01:22.219 INFO:tasks.rados.rados.0.smithi070.stdout:3827: setattr oid 26 current snap is 388 2024-09-14T04:01:22.219 INFO:tasks.rados.rados.0.smithi070.stdout:3820: finishing write tid 1 to smithi07085154-43 2024-09-14T04:01:22.219 INFO:tasks.rados.rados.0.smithi070.stdout:3820: finishing write tid 2 to smithi07085154-43 2024-09-14T04:01:22.219 INFO:tasks.rados.rados.0.smithi070.stdout:3820: finishing write tid 3 to smithi07085154-43 2024-09-14T04:01:22.219 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 4157 (ObjNum 1161 snap 388 seq_num 1161) dirty exists 2024-09-14T04:01:22.219 INFO:tasks.rados.rados.0.smithi070.stdout:3820: left oid 43 (ObjNum 1161 snap 388 seq_num 1161) 2024-09-14T04:01:22.219 INFO:tasks.rados.rados.0.smithi070.stdout:3819: expect (ObjNum 1130 snap 379 seq_num 1130) 2024-09-14T04:01:22.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:22 smithi094 ceph-mon[86597]: osdmap e841: 8 total, 8 up, 8 in 2024-09-14T04:01:22.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:22 smithi094 ceph-mon[86597]: pgmap v863: 121 pgs: 121 active+clean; 842 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.0 MiB/s rd, 2.1 MiB/s wr, 11 op/s 2024-09-14T04:01:22.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:22 smithi070 ceph-mon[96704]: osdmap e841: 8 total, 8 up, 8 in 2024-09-14T04:01:22.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:22 smithi070 ceph-mon[96704]: pgmap v863: 121 pgs: 121 active+clean; 842 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.0 MiB/s rd, 2.1 MiB/s wr, 11 op/s 2024-09-14T04:01:22.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:22 smithi070 ceph-mon[98640]: osdmap e841: 8 total, 8 up, 8 in 2024-09-14T04:01:22.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:22 smithi070 ceph-mon[98640]: pgmap v863: 121 pgs: 121 active+clean; 842 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.0 MiB/s rd, 2.1 MiB/s wr, 11 op/s 2024-09-14T04:01:22.854 INFO:tasks.rados.rados.0.smithi070.stdout:3821: finishing write tid 1 to smithi07085154-1 2024-09-14T04:01:22.854 INFO:tasks.rados.rados.0.smithi070.stdout:3821: finishing write tid 2 to smithi07085154-1 2024-09-14T04:01:22.854 INFO:tasks.rados.rados.0.smithi070.stdout:3821: finishing write tid 3 to smithi07085154-1 2024-09-14T04:01:22.855 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 1 v 4256 (ObjNum 1162 snap 388 seq_num 1162) dirty exists 2024-09-14T04:01:22.855 INFO:tasks.rados.rados.0.smithi070.stdout:3821: left oid 1 (ObjNum 1162 snap 388 seq_num 1162) 2024-09-14T04:01:22.855 INFO:tasks.rados.rados.0.smithi070.stdout:3823: finishing write tid 1 to smithi07085154-49 2024-09-14T04:01:22.855 INFO:tasks.rados.rados.0.smithi070.stdout:3823: finishing write tid 2 to smithi07085154-49 2024-09-14T04:01:22.855 INFO:tasks.rados.rados.0.smithi070.stdout:3823: finishing write tid 3 to smithi07085154-49 2024-09-14T04:01:22.855 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 49 v 4500 (ObjNum 1163 snap 388 seq_num 1163) dirty exists 2024-09-14T04:01:22.856 INFO:tasks.rados.rados.0.smithi070.stdout:3823: left oid 49 (ObjNum 1163 snap 388 seq_num 1163) 2024-09-14T04:01:22.856 INFO:tasks.rados.rados.0.smithi070.stdout:3824: expect (ObjNum 1014 snap 334 seq_num 1014) 2024-09-14T04:01:22.955 INFO:tasks.rados.rados.0.smithi070.stdout:3826: finishing write tid 1 to smithi07085154-25 2024-09-14T04:01:22.955 INFO:tasks.rados.rados.0.smithi070.stdout:3826: finishing write tid 2 to smithi07085154-25 2024-09-14T04:01:22.955 INFO:tasks.rados.rados.0.smithi070.stdout:3826: finishing write tid 3 to smithi07085154-25 2024-09-14T04:01:22.955 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 4437 (ObjNum 1164 snap 388 seq_num 1164) dirty exists 2024-09-14T04:01:22.956 INFO:tasks.rados.rados.0.smithi070.stdout:3826: left oid 25 (ObjNum 1164 snap 388 seq_num 1164) 2024-09-14T04:01:22.956 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 4645 (ObjNum 1140 snap 381 seq_num 1140) dirty exists 2024-09-14T04:01:22.956 INFO:tasks.rados.rados.0.smithi070.stdout:3819: done (6 left) 2024-09-14T04:01:22.956 INFO:tasks.rados.rados.0.smithi070.stdout:3820: done (5 left) 2024-09-14T04:01:22.956 INFO:tasks.rados.rados.0.smithi070.stdout:3821: done (4 left) 2024-09-14T04:01:22.956 INFO:tasks.rados.rados.0.smithi070.stdout:3823: done (3 left) 2024-09-14T04:01:22.956 INFO:tasks.rados.rados.0.smithi070.stdout:3824: done (2 left) 2024-09-14T04:01:22.956 INFO:tasks.rados.rados.0.smithi070.stdout:3826: done (1 left) 2024-09-14T04:01:22.956 INFO:tasks.rados.rados.0.smithi070.stdout:3827: done (0 left) 2024-09-14T04:01:22.956 INFO:tasks.rados.rados.0.smithi070.stdout:3828: snap_create 2024-09-14T04:01:23.202 INFO:tasks.rados.rados.0.smithi070.stdout:3828: done (0 left) 2024-09-14T04:01:23.202 INFO:tasks.rados.rados.0.smithi070.stdout:3829: delete oid 11 current snap is 389 2024-09-14T04:01:23.207 INFO:tasks.rados.rados.0.smithi070.stdout:3829: done (0 left) 2024-09-14T04:01:23.207 INFO:tasks.rados.rados.0.smithi070.stdout:3830: snap_remove snap 348 2024-09-14T04:01:23.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:23 smithi094 ceph-mon[86597]: osdmap e842: 8 total, 8 up, 8 in 2024-09-14T04:01:23.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:23 smithi070 ceph-mon[98640]: osdmap e842: 8 total, 8 up, 8 in 2024-09-14T04:01:23.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:23 smithi070 ceph-mon[96704]: osdmap e842: 8 total, 8 up, 8 in 2024-09-14T04:01:23.536 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:01:23 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:01:23] "GET /metrics HTTP/1.1" 200 36502 "" "Prometheus/2.51.0" 2024-09-14T04:01:24.153 INFO:tasks.rados.rados.0.smithi070.stdout:3830: done (0 left) 2024-09-14T04:01:24.153 INFO:tasks.rados.rados.0.smithi070.stdout:3831: read oid 26 snap -1 2024-09-14T04:01:24.154 INFO:tasks.rados.rados.0.smithi070.stdout:3831: expect (ObjNum 1140 snap 381 seq_num 1140) 2024-09-14T04:01:24.154 INFO:tasks.rados.rados.0.smithi070.stdout:3832: read oid 21 snap -1 2024-09-14T04:01:24.154 INFO:tasks.rados.rados.0.smithi070.stdout:3832: expect (ObjNum 1091 snap 355 seq_num 1091) 2024-09-14T04:01:24.154 INFO:tasks.rados.rados.0.smithi070.stdout:3833: copy_from oid 17 from oid 24 current snap is 389 2024-09-14T04:01:24.154 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 16 current snap is 389 2024-09-14T04:01:24.154 INFO:tasks.rados.rados.0.smithi070.stdout:3834: seq_num 1166 ranges {4259840=335872} 2024-09-14T04:01:24.180 INFO:tasks.rados.rados.0.smithi070.stdout:3834: writing smithi07085154-16 from 4259840 to 4595712 tid 1 2024-09-14T04:01:24.181 INFO:tasks.rados.rados.0.smithi070.stdout:3835: delete oid 31 current snap is 389 2024-09-14T04:01:24.181 INFO:tasks.rados.rados.0.smithi070.stdout:3833: finishing copy_from to smithi07085154-17 2024-09-14T04:01:24.181 INFO:tasks.rados.rados.0.smithi070.stdout:3833: got expected ENOENT (src dne) 2024-09-14T04:01:24.181 INFO:tasks.rados.rados.0.smithi070.stdout:3833: finishing copy_from racing read to smithi07085154-17 2024-09-14T04:01:24.183 INFO:tasks.rados.rados.0.smithi070.stdout:3832: expect (ObjNum 1091 snap 355 seq_num 1091) 2024-09-14T04:01:24.316 INFO:tasks.rados.rados.0.smithi070.stdout:3831: expect (ObjNum 1140 snap 381 seq_num 1140) 2024-09-14T04:01:24.376 INFO:tasks.rados.rados.0.smithi070.stdout:3834: finishing write tid 1 to smithi07085154-16 2024-09-14T04:01:24.376 INFO:tasks.rados.rados.0.smithi070.stdout:3834: finishing write tid 2 to smithi07085154-16 2024-09-14T04:01:24.377 INFO:tasks.rados.rados.0.smithi070.stdout:3834: finishing write tid 3 to smithi07085154-16 2024-09-14T04:01:24.377 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 4648 (ObjNum 1166 snap 389 seq_num 1166) dirty exists 2024-09-14T04:01:24.377 INFO:tasks.rados.rados.0.smithi070.stdout:3834: left oid 16 (ObjNum 1166 snap 389 seq_num 1166) 2024-09-14T04:01:24.377 INFO:tasks.rados.rados.0.smithi070.stdout:3831: done (4 left) 2024-09-14T04:01:24.377 INFO:tasks.rados.rados.0.smithi070.stdout:3832: done (3 left) 2024-09-14T04:01:24.377 INFO:tasks.rados.rados.0.smithi070.stdout:3833: done (2 left) 2024-09-14T04:01:24.377 INFO:tasks.rados.rados.0.smithi070.stdout:3834: done (1 left) 2024-09-14T04:01:24.377 INFO:tasks.rados.rados.0.smithi070.stdout:3835: done (0 left) 2024-09-14T04:01:24.377 INFO:tasks.rados.rados.0.smithi070.stdout:3836: copy_from oid 6 from oid 26 current snap is 389 2024-09-14T04:01:24.377 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 38 current snap is 389 2024-09-14T04:01:24.377 INFO:tasks.rados.rados.0.smithi070.stdout:3837: seq_num 1168 ranges {1130496=368640} 2024-09-14T04:01:24.384 INFO:tasks.rados.rados.0.smithi070.stdout:3837: writing smithi07085154-38 from 1130496 to 1499136 tid 1 2024-09-14T04:01:24.384 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 13 current snap is 389 2024-09-14T04:01:24.384 INFO:tasks.rados.rados.0.smithi070.stdout:3838: seq_num 1169 ranges {5316608=385024} 2024-09-14T04:01:24.415 INFO:tasks.rados.rados.0.smithi070.stdout:3838: writing smithi07085154-13 from 5316608 to 5701632 tid 1 2024-09-14T04:01:24.416 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 14 current snap is 389 2024-09-14T04:01:24.416 INFO:tasks.rados.rados.0.smithi070.stdout:3836: finishing copy_from racing read to smithi07085154-6 2024-09-14T04:01:24.416 INFO:tasks.rados.rados.0.smithi070.stdout:3836: finishing copy_from to smithi07085154-6 2024-09-14T04:01:24.416 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 6 v 4645 (ObjNum 1140 snap 381 seq_num 1140) dirty exists 2024-09-14T04:01:24.416 INFO:tasks.rados.rados.0.smithi070.stdout:3837: finishing write tid 1 to smithi07085154-38 2024-09-14T04:01:24.416 INFO:tasks.rados.rados.0.smithi070.stdout:3837: finishing write tid 2 to smithi07085154-38 2024-09-14T04:01:24.416 INFO:tasks.rados.rados.0.smithi070.stdout:3837: finishing write tid 3 to smithi07085154-38 2024-09-14T04:01:24.416 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 38 v 3996 (ObjNum 1168 snap 389 seq_num 1168) dirty exists 2024-09-14T04:01:24.416 INFO:tasks.rados.rados.0.smithi070.stdout:3837: left oid 38 (ObjNum 1168 snap 389 seq_num 1168) 2024-09-14T04:01:24.416 INFO:tasks.rados.rados.0.smithi070.stdout:3839: seq_num 1170 ranges {2064384=401408} 2024-09-14T04:01:24.429 INFO:tasks.rados.rados.0.smithi070.stdout:3839: writing smithi07085154-14 from 2064384 to 2465792 tid 1 2024-09-14T04:01:24.429 INFO:tasks.rados.rados.0.smithi070.stdout:3836: done (3 left) 2024-09-14T04:01:24.429 INFO:tasks.rados.rados.0.smithi070.stdout:3837: done (2 left) 2024-09-14T04:01:24.429 INFO:tasks.rados.rados.0.smithi070.stdout:3840: snap_remove snap 388 2024-09-14T04:01:24.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:24 smithi094 ceph-mon[86597]: osdmap e843: 8 total, 8 up, 8 in 2024-09-14T04:01:24.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:24 smithi094 ceph-mon[86597]: osdmap e844: 8 total, 8 up, 8 in 2024-09-14T04:01:24.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:24 smithi094 ceph-mon[86597]: pgmap v867: 121 pgs: 121 active+clean; 812 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 14 MiB/s rd, 952 KiB/s wr, 8 op/s 2024-09-14T04:01:24.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:24 smithi070 ceph-mon[96704]: osdmap e843: 8 total, 8 up, 8 in 2024-09-14T04:01:24.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:24 smithi070 ceph-mon[96704]: osdmap e844: 8 total, 8 up, 8 in 2024-09-14T04:01:24.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:24 smithi070 ceph-mon[96704]: pgmap v867: 121 pgs: 121 active+clean; 812 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 14 MiB/s rd, 952 KiB/s wr, 8 op/s 2024-09-14T04:01:24.536 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 04:01:24 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2[113018]: 2024-09-14T04:01:24.177+0000 7f5c8bca3640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap 1ae after trimming was started 2024-09-14T04:01:24.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:24 smithi070 ceph-mon[98640]: osdmap e843: 8 total, 8 up, 8 in 2024-09-14T04:01:24.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:24 smithi070 ceph-mon[98640]: osdmap e844: 8 total, 8 up, 8 in 2024-09-14T04:01:24.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:24 smithi070 ceph-mon[98640]: pgmap v867: 121 pgs: 121 active+clean; 812 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 14 MiB/s rd, 952 KiB/s wr, 8 op/s 2024-09-14T04:01:25.211 INFO:tasks.rados.rados.0.smithi070.stdout:3840: done (2 left) 2024-09-14T04:01:25.211 INFO:tasks.rados.rados.0.smithi070.stdout:3841: rollback oid 44 current snap is 389 2024-09-14T04:01:25.212 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 44 to 315 2024-09-14T04:01:25.212 INFO:tasks.rados.rados.0.smithi070.stdout:3838: finishing write tid 1 to smithi07085154-13 2024-09-14T04:01:25.212 INFO:tasks.rados.rados.0.smithi070.stdout:3838: finishing write tid 2 to smithi07085154-13 2024-09-14T04:01:25.212 INFO:tasks.rados.rados.0.smithi070.stdout:3838: finishing write tid 3 to smithi07085154-13 2024-09-14T04:01:25.212 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 13 v 4247 (ObjNum 1169 snap 389 seq_num 1169) dirty exists 2024-09-14T04:01:25.212 INFO:tasks.rados.rados.0.smithi070.stdout:3838: left oid 13 (ObjNum 1169 snap 389 seq_num 1169) 2024-09-14T04:01:25.212 INFO:tasks.rados.rados.0.smithi070.stdout:3839: finishing write tid 1 to smithi07085154-14 2024-09-14T04:01:25.212 INFO:tasks.rados.rados.0.smithi070.stdout:3839: finishing write tid 2 to smithi07085154-14 2024-09-14T04:01:25.212 INFO:tasks.rados.rados.0.smithi070.stdout:3838: done (2 left) 2024-09-14T04:01:25.212 INFO:tasks.rados.rados.0.smithi070.stdout:3842: rollback oid 15 current snap is 389 2024-09-14T04:01:25.212 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 15 to 375 2024-09-14T04:01:25.212 INFO:tasks.rados.rados.0.smithi070.stdout:3839: finishing write tid 3 to smithi07085154-14 2024-09-14T04:01:25.213 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 4158 (ObjNum 1170 snap 389 seq_num 1170) dirty exists 2024-09-14T04:01:25.213 INFO:tasks.rados.rados.0.smithi070.stdout:3839: left oid 14 (ObjNum 1170 snap 389 seq_num 1170) 2024-09-14T04:01:25.213 INFO:tasks.rados.rados.0.smithi070.stdout:3839: done (2 left) 2024-09-14T04:01:25.213 INFO:tasks.rados.rados.0.smithi070.stdout:3843: delete oid 11 current snap is 389 2024-09-14T04:01:25.217 INFO:tasks.rados.rados.0.smithi070.stdout:3841: finishing rollback tid 0 to smithi07085154-44 2024-09-14T04:01:25.218 INFO:tasks.rados.rados.0.smithi070.stdout:3841: finishing rollback tid 1 to smithi07085154-44 2024-09-14T04:01:25.218 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 4258 (ObjNum 225 snap 0 seq_num 2931756448) dirty dne 2024-09-14T04:01:25.218 INFO:tasks.rados.rados.0.smithi070.stdout:3841: done (2 left) 2024-09-14T04:01:25.218 INFO:tasks.rados.rados.0.smithi070.stdout:3843: done (1 left) 2024-09-14T04:01:25.219 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 16 current snap is 389 2024-09-14T04:01:25.219 INFO:tasks.rados.rados.0.smithi070.stdout:3844: seq_num 1172 ranges {4595712=434176} 2024-09-14T04:01:25.246 INFO:tasks.rados.rados.0.smithi070.stdout:3844: writing smithi07085154-16 from 4595712 to 5029888 tid 1 2024-09-14T04:01:25.246 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 39 current snap is 389 2024-09-14T04:01:25.246 INFO:tasks.rados.rados.0.smithi070.stdout:3842: finishing rollback tid 0 to smithi07085154-15 2024-09-14T04:01:25.246 INFO:tasks.rados.rados.0.smithi070.stdout:3842: finishing rollback tid 1 to smithi07085154-15 2024-09-14T04:01:25.246 INFO:tasks.rados.rados.0.smithi070.stdout:3842: finishing rollback tid 2 to smithi07085154-15 2024-09-14T04:01:25.246 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 15 v 4160 (ObjNum 1120 snap 374 seq_num 1120) dirty exists 2024-09-14T04:01:25.246 INFO:tasks.rados.rados.0.smithi070.stdout:3845: seq_num 1173 ranges {1736704=450560} 2024-09-14T04:01:25.257 INFO:tasks.rados.rados.0.smithi070.stdout:3845: writing smithi07085154-39 from 1736704 to 2187264 tid 1 2024-09-14T04:01:25.258 INFO:tasks.rados.rados.0.smithi070.stdout:3842: done (2 left) 2024-09-14T04:01:25.258 INFO:tasks.rados.rados.0.smithi070.stdout:3846: snap_remove snap 381 2024-09-14T04:01:26.216 INFO:tasks.rados.rados.0.smithi070.stdout:3846: done (2 left) 2024-09-14T04:01:26.216 INFO:tasks.rados.rados.0.smithi070.stdout:3847: delete oid 7 current snap is 389 2024-09-14T04:01:26.216 INFO:tasks.rados.rados.0.smithi070.stdout:3844: finishing write tid 1 to smithi07085154-16 2024-09-14T04:01:26.216 INFO:tasks.rados.rados.0.smithi070.stdout:3844: finishing write tid 2 to smithi07085154-16 2024-09-14T04:01:26.216 INFO:tasks.rados.rados.0.smithi070.stdout:3844: finishing write tid 3 to smithi07085154-16 2024-09-14T04:01:26.217 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 4662 (ObjNum 1172 snap 389 seq_num 1172) dirty exists 2024-09-14T04:01:26.217 INFO:tasks.rados.rados.0.smithi070.stdout:3844: left oid 16 (ObjNum 1172 snap 389 seq_num 1172) 2024-09-14T04:01:26.217 INFO:tasks.rados.rados.0.smithi070.stdout:3845: finishing write tid 1 to smithi07085154-39 2024-09-14T04:01:26.217 INFO:tasks.rados.rados.0.smithi070.stdout:3845: finishing write tid 2 to smithi07085154-39 2024-09-14T04:01:26.217 INFO:tasks.rados.rados.0.smithi070.stdout:3845: finishing write tid 3 to smithi07085154-39 2024-09-14T04:01:26.217 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 39 v 2644 (ObjNum 1173 snap 389 seq_num 1173) dirty exists 2024-09-14T04:01:26.217 INFO:tasks.rados.rados.0.smithi070.stdout:3845: left oid 39 (ObjNum 1173 snap 389 seq_num 1173) 2024-09-14T04:01:26.231 INFO:tasks.rados.rados.0.smithi070.stdout:3844: done (2 left) 2024-09-14T04:01:26.231 INFO:tasks.rados.rados.0.smithi070.stdout:3845: done (1 left) 2024-09-14T04:01:26.231 INFO:tasks.rados.rados.0.smithi070.stdout:3847: done (0 left) 2024-09-14T04:01:26.232 INFO:tasks.rados.rados.0.smithi070.stdout:3848: read oid 22 snap -1 2024-09-14T04:01:26.232 INFO:tasks.rados.rados.0.smithi070.stdout:3848: expect deleted 2024-09-14T04:01:26.232 INFO:tasks.rados.rados.0.smithi070.stdout:3849: rollback oid 32 current snap is 389 2024-09-14T04:01:26.232 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 32 to 361 2024-09-14T04:01:26.232 INFO:tasks.rados.rados.0.smithi070.stdout:3850: rmattr oid 10 current snap is 389 2024-09-14T04:01:26.232 INFO:tasks.rados.rados.0.smithi070.stdout:3851: rollback oid 25 current snap is 389 2024-09-14T04:01:26.232 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 25 to 346 2024-09-14T04:01:26.232 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 36 current snap is 389 2024-09-14T04:01:26.232 INFO:tasks.rados.rados.0.smithi070.stdout:3852: seq_num 1175 ranges {0=483328} 2024-09-14T04:01:26.234 INFO:tasks.rados.rados.0.smithi070.stdout:3852: writing smithi07085154-36 from 0 to 483328 tid 1 2024-09-14T04:01:26.235 INFO:tasks.rados.rados.0.smithi070.stdout:3848: done (4 left) 2024-09-14T04:01:26.235 INFO:tasks.rados.rados.0.smithi070.stdout:3853: read oid 23 snap -1 2024-09-14T04:01:26.235 INFO:tasks.rados.rados.0.smithi070.stdout:3853: expect (ObjNum 1047 snap 344 seq_num 1047) 2024-09-14T04:01:26.235 INFO:tasks.rados.rados.0.smithi070.stdout:3854: snap_remove snap 363 2024-09-14T04:01:26.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:26 smithi094 ceph-mon[86597]: osdmap e845: 8 total, 8 up, 8 in 2024-09-14T04:01:26.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:26 smithi094 ceph-mon[86597]: pgmap v869: 121 pgs: 121 active+clean; 808 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.8 MiB/s rd, 368 KiB/s wr, 5 op/s 2024-09-14T04:01:26.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:26 smithi070 ceph-mon[96704]: osdmap e845: 8 total, 8 up, 8 in 2024-09-14T04:01:26.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:26 smithi070 ceph-mon[96704]: pgmap v869: 121 pgs: 121 active+clean; 808 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.8 MiB/s rd, 368 KiB/s wr, 5 op/s 2024-09-14T04:01:26.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:26 smithi070 ceph-mon[98640]: osdmap e845: 8 total, 8 up, 8 in 2024-09-14T04:01:26.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:26 smithi070 ceph-mon[98640]: pgmap v869: 121 pgs: 121 active+clean; 808 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.8 MiB/s rd, 368 KiB/s wr, 5 op/s 2024-09-14T04:01:27.220 INFO:tasks.rados.rados.0.smithi070.stdout:3854: done (5 left) 2024-09-14T04:01:27.220 INFO:tasks.rados.rados.0.smithi070.stdout:3855: delete oid 49 current snap is 389 2024-09-14T04:01:27.220 INFO:tasks.rados.rados.0.smithi070.stdout:3849: finishing rollback tid 1 to smithi07085154-32 2024-09-14T04:01:27.221 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 32 v 4244 (ObjNum 65 snap 0 seq_num 1667195240) dirty dne 2024-09-14T04:01:27.221 INFO:tasks.rados.rados.0.smithi070.stdout:3851: finishing rollback tid 0 to smithi07085154-25 2024-09-14T04:01:27.221 INFO:tasks.rados.rados.0.smithi070.stdout:3851: finishing rollback tid 1 to smithi07085154-25 2024-09-14T04:01:27.221 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 10 v 4667 (ObjNum 1154 snap 387 seq_num 1154) dirty exists 2024-09-14T04:01:27.221 INFO:tasks.rados.rados.0.smithi070.stdout:3851: finishing rollback tid 2 to smithi07085154-25 2024-09-14T04:01:27.221 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 4648 (ObjNum 1055 snap 344 seq_num 1055) dirty exists 2024-09-14T04:01:27.221 INFO:tasks.rados.rados.0.smithi070.stdout:3852: finishing write tid 1 to smithi07085154-36 2024-09-14T04:01:27.221 INFO:tasks.rados.rados.0.smithi070.stdout:3852: finishing write tid 2 to smithi07085154-36 2024-09-14T04:01:27.221 INFO:tasks.rados.rados.0.smithi070.stdout:3853: expect (ObjNum 1047 snap 344 seq_num 1047) 2024-09-14T04:01:27.348 INFO:tasks.rados.rados.0.smithi070.stdout:3852: finishing write tid 3 to smithi07085154-36 2024-09-14T04:01:27.349 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 36 v 4260 (ObjNum 1175 snap 389 seq_num 1175) dirty exists 2024-09-14T04:01:27.349 INFO:tasks.rados.rados.0.smithi070.stdout:3852: left oid 36 (ObjNum 1175 snap 389 seq_num 1175) 2024-09-14T04:01:27.349 INFO:tasks.rados.rados.0.smithi070.stdout:3849: done (5 left) 2024-09-14T04:01:27.349 INFO:tasks.rados.rados.0.smithi070.stdout:3850: done (4 left) 2024-09-14T04:01:27.349 INFO:tasks.rados.rados.0.smithi070.stdout:3851: done (3 left) 2024-09-14T04:01:27.349 INFO:tasks.rados.rados.0.smithi070.stdout:3852: done (2 left) 2024-09-14T04:01:27.349 INFO:tasks.rados.rados.0.smithi070.stdout:3853: done (1 left) 2024-09-14T04:01:27.349 INFO:tasks.rados.rados.0.smithi070.stdout:3855: done (0 left) 2024-09-14T04:01:27.349 INFO:tasks.rados.rados.0.smithi070.stdout:3856: read oid 19 snap -1 2024-09-14T04:01:27.349 INFO:tasks.rados.rados.0.smithi070.stdout:3856: expect deleted 2024-09-14T04:01:27.349 INFO:tasks.rados.rados.0.smithi070.stdout:3857: snap_create 2024-09-14T04:01:27.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:27 smithi094 ceph-mon[86597]: osdmap e846: 8 total, 8 up, 8 in 2024-09-14T04:01:27.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:27 smithi070 ceph-mon[96704]: osdmap e846: 8 total, 8 up, 8 in 2024-09-14T04:01:27.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:27 smithi070 ceph-mon[98640]: osdmap e846: 8 total, 8 up, 8 in 2024-09-14T04:01:28.225 INFO:tasks.rados.rados.0.smithi070.stdout:3856: done (1 left) 2024-09-14T04:01:28.225 INFO:tasks.rados.rados.0.smithi070.stdout:3857: done (0 left) 2024-09-14T04:01:28.225 INFO:tasks.rados.rados.0.smithi070.stdout:3858: snap_create 2024-09-14T04:01:28.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:28 smithi094 ceph-mon[86597]: osdmap e847: 8 total, 8 up, 8 in 2024-09-14T04:01:28.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:28 smithi094 ceph-mon[86597]: pgmap v872: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 795 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 5.8 MiB/s rd, 598 KiB/s wr, 15 op/s 2024-09-14T04:01:28.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:28 smithi070 ceph-mon[96704]: osdmap e847: 8 total, 8 up, 8 in 2024-09-14T04:01:28.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:28 smithi070 ceph-mon[96704]: pgmap v872: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 795 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 5.8 MiB/s rd, 598 KiB/s wr, 15 op/s 2024-09-14T04:01:28.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:28 smithi070 ceph-mon[98640]: osdmap e847: 8 total, 8 up, 8 in 2024-09-14T04:01:28.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:28 smithi070 ceph-mon[98640]: pgmap v872: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 795 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 5.8 MiB/s rd, 598 KiB/s wr, 15 op/s 2024-09-14T04:01:29.155 INFO:tasks.rados.rados.0.smithi070.stdout:3858: done (0 left) 2024-09-14T04:01:29.155 INFO:tasks.rados.rados.0.smithi070.stdout:3859: read oid 20 snap -1 2024-09-14T04:01:29.155 INFO:tasks.rados.rados.0.smithi070.stdout:3859: expect (ObjNum 1083 snap 353 seq_num 1083) 2024-09-14T04:01:29.155 INFO:tasks.rados.rados.0.smithi070.stdout:3860: delete oid 47 current snap is 391 2024-09-14T04:01:29.161 INFO:tasks.rados.rados.0.smithi070.stdout:3860: done (1 left) 2024-09-14T04:01:29.162 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 9 current snap is 391 2024-09-14T04:01:29.162 INFO:tasks.rados.rados.0.smithi070.stdout:3861: seq_num 1178 ranges {5406720=532480} 2024-09-14T04:01:29.195 INFO:tasks.rados.rados.0.smithi070.stdout:3861: writing smithi07085154-9 from 5406720 to 5939200 tid 1 2024-09-14T04:01:29.195 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 43 current snap is 391 2024-09-14T04:01:29.195 INFO:tasks.rados.rados.0.smithi070.stdout:3862: seq_num 1179 ranges {2727936=548864} 2024-09-14T04:01:29.213 INFO:tasks.rados.rados.0.smithi070.stdout:3862: writing smithi07085154-43 from 2727936 to 3276800 tid 1 2024-09-14T04:01:29.213 INFO:tasks.rados.rados.0.smithi070.stdout:3863: rollback oid 48 current snap is 391 2024-09-14T04:01:29.213 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 48 to 389 2024-09-14T04:01:29.213 INFO:tasks.rados.rados.0.smithi070.stdout:3859: expect (ObjNum 1083 snap 353 seq_num 1083) 2024-09-14T04:01:29.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:29 smithi094 ceph-mon[86597]: osdmap e848: 8 total, 8 up, 8 in 2024-09-14T04:01:29.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:29 smithi094 ceph-mon[86597]: osdmap e849: 8 total, 8 up, 8 in 2024-09-14T04:01:29.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:29 smithi070 ceph-mon[98640]: osdmap e848: 8 total, 8 up, 8 in 2024-09-14T04:01:29.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:29 smithi070 ceph-mon[98640]: osdmap e849: 8 total, 8 up, 8 in 2024-09-14T04:01:29.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:29 smithi070 ceph-mon[96704]: osdmap e848: 8 total, 8 up, 8 in 2024-09-14T04:01:29.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:29 smithi070 ceph-mon[96704]: osdmap e849: 8 total, 8 up, 8 in 2024-09-14T04:01:29.824 INFO:tasks.rados.rados.0.smithi070.stdout:3861: finishing write tid 1 to smithi07085154-9 2024-09-14T04:01:29.824 INFO:tasks.rados.rados.0.smithi070.stdout:3861: finishing write tid 2 to smithi07085154-9 2024-09-14T04:01:29.824 INFO:tasks.rados.rados.0.smithi070.stdout:3861: finishing write tid 3 to smithi07085154-9 2024-09-14T04:01:29.824 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 4690 (ObjNum 1178 snap 391 seq_num 1178) dirty exists 2024-09-14T04:01:29.824 INFO:tasks.rados.rados.0.smithi070.stdout:3861: left oid 9 (ObjNum 1178 snap 391 seq_num 1178) 2024-09-14T04:01:29.824 INFO:tasks.rados.rados.0.smithi070.stdout:3863: finishing rollback tid 0 to smithi07085154-48 2024-09-14T04:01:29.825 INFO:tasks.rados.rados.0.smithi070.stdout:3863: finishing rollback tid 1 to smithi07085154-48 2024-09-14T04:01:29.825 INFO:tasks.rados.rados.0.smithi070.stdout:3859: done (3 left) 2024-09-14T04:01:29.825 INFO:tasks.rados.rados.0.smithi070.stdout:3861: done (2 left) 2024-09-14T04:01:29.825 INFO:tasks.rados.rados.0.smithi070.stdout:3864: read oid 29 snap -1 2024-09-14T04:01:29.825 INFO:tasks.rados.rados.0.smithi070.stdout:3864: expect (ObjNum 1149 snap 382 seq_num 1149) 2024-09-14T04:01:29.825 INFO:tasks.rados.rados.0.smithi070.stdout:3865: snap_create 2024-09-14T04:01:29.825 INFO:tasks.rados.rados.0.smithi070.stdout:3863: finishing rollback tid 2 to smithi07085154-48 2024-09-14T04:01:29.825 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 2281 (ObjNum 1001 snap 331 seq_num 1001) dirty exists 2024-09-14T04:01:29.825 INFO:tasks.rados.rados.0.smithi070.stdout:3862: finishing write tid 1 to smithi07085154-43 2024-09-14T04:01:29.825 INFO:tasks.rados.rados.0.smithi070.stdout:3862: finishing write tid 2 to smithi07085154-43 2024-09-14T04:01:29.825 INFO:tasks.rados.rados.0.smithi070.stdout:3862: finishing write tid 3 to smithi07085154-43 2024-09-14T04:01:29.825 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 4162 (ObjNum 1179 snap 391 seq_num 1179) dirty exists 2024-09-14T04:01:29.825 INFO:tasks.rados.rados.0.smithi070.stdout:3862: left oid 43 (ObjNum 1179 snap 391 seq_num 1179) 2024-09-14T04:01:29.848 INFO:tasks.rados.rados.0.smithi070.stdout:3864: expect (ObjNum 1149 snap 382 seq_num 1149) 2024-09-14T04:01:30.229 INFO:tasks.rados.rados.0.smithi070.stdout:3862: done (3 left) 2024-09-14T04:01:30.229 INFO:tasks.rados.rados.0.smithi070.stdout:3863: done (2 left) 2024-09-14T04:01:30.229 INFO:tasks.rados.rados.0.smithi070.stdout:3864: done (1 left) 2024-09-14T04:01:30.230 INFO:tasks.rados.rados.0.smithi070.stdout:3865: done (0 left) 2024-09-14T04:01:30.230 INFO:tasks.rados.rados.0.smithi070.stdout:3866: snap_remove snap 370 2024-09-14T04:01:30.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:30 smithi094 ceph-mon[86597]: pgmap v875: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 790 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2024-09-14T04:01:30.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:01:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:01:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:01:30.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:30 smithi070 ceph-mon[96704]: pgmap v875: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 790 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2024-09-14T04:01:30.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:30 smithi070 ceph-mon[98640]: pgmap v875: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 790 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2024-09-14T04:01:31.234 INFO:tasks.rados.rados.0.smithi070.stdout:3866: done (0 left) 2024-09-14T04:01:31.234 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 30 current snap is 392 2024-09-14T04:01:31.234 INFO:tasks.rados.rados.0.smithi070.stdout:3867: seq_num 1180 ranges {0=565248} 2024-09-14T04:01:31.238 INFO:tasks.rados.rados.0.smithi070.stdout:3867: writing smithi07085154-30 from 0 to 565248 tid 1 2024-09-14T04:01:31.238 INFO:tasks.rados.rados.0.smithi070.stdout:3868: setattr oid 36 current snap is 392 2024-09-14T04:01:31.241 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 26 current snap is 392 2024-09-14T04:01:31.241 INFO:tasks.rados.rados.0.smithi070.stdout:3869: seq_num 1181 ranges {2301952=581632} 2024-09-14T04:01:31.259 INFO:tasks.rados.rados.0.smithi070.stdout:3869: writing smithi07085154-26 from 2301952 to 2883584 tid 1 2024-09-14T04:01:31.259 INFO:tasks.rados.rados.0.smithi070.stdout:3870: snap_create 2024-09-14T04:01:31.259 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 36 v 4261 (ObjNum 1175 snap 389 seq_num 1175) dirty exists 2024-09-14T04:01:31.259 INFO:tasks.rados.rados.0.smithi070.stdout:3867: finishing write tid 1 to smithi07085154-30 2024-09-14T04:01:31.259 INFO:tasks.rados.rados.0.smithi070.stdout:3867: finishing write tid 2 to smithi07085154-30 2024-09-14T04:01:31.260 INFO:tasks.rados.rados.0.smithi070.stdout:3867: finishing write tid 3 to smithi07085154-30 2024-09-14T04:01:31.260 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 30 v 3999 (ObjNum 1180 snap 392 seq_num 1180) dirty exists 2024-09-14T04:01:31.260 INFO:tasks.rados.rados.0.smithi070.stdout:3867: left oid 30 (ObjNum 1180 snap 392 seq_num 1180) 2024-09-14T04:01:31.274 INFO:tasks.rados.rados.0.smithi070.stdout:3869: finishing write tid 1 to smithi07085154-26 2024-09-14T04:01:31.275 INFO:tasks.rados.rados.0.smithi070.stdout:3869: finishing write tid 2 to smithi07085154-26 2024-09-14T04:01:31.276 INFO:tasks.rados.rados.0.smithi070.stdout:3869: finishing write tid 3 to smithi07085154-26 2024-09-14T04:01:31.277 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 4701 (ObjNum 1181 snap 392 seq_num 1181) dirty exists 2024-09-14T04:01:31.277 INFO:tasks.rados.rados.0.smithi070.stdout:3869: left oid 26 (ObjNum 1181 snap 392 seq_num 1181) 2024-09-14T04:01:31.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:31 smithi094 ceph-mon[86597]: osdmap e850: 8 total, 8 up, 8 in 2024-09-14T04:01:31.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:31 smithi070 ceph-mon[98640]: osdmap e850: 8 total, 8 up, 8 in 2024-09-14T04:01:31.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:31 smithi070 ceph-mon[96704]: osdmap e850: 8 total, 8 up, 8 in 2024-09-14T04:01:32.247 INFO:tasks.rados.rados.0.smithi070.stdout:3867: done (3 left) 2024-09-14T04:01:32.247 INFO:tasks.rados.rados.0.smithi070.stdout:3868: done (2 left) 2024-09-14T04:01:32.247 INFO:tasks.rados.rados.0.smithi070.stdout:3869: done (1 left) 2024-09-14T04:01:32.247 INFO:tasks.rados.rados.0.smithi070.stdout:3870: done (0 left) 2024-09-14T04:01:32.247 INFO:tasks.rados.rados.0.smithi070.stdout:3871: rollback oid 46 current snap is 393 2024-09-14T04:01:32.247 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 46 to 315 2024-09-14T04:01:32.247 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 48 current snap is 393 2024-09-14T04:01:32.247 INFO:tasks.rados.rados.0.smithi070.stdout:3872: seq_num 1182 ranges {2375680=606208} 2024-09-14T04:01:32.263 INFO:tasks.rados.rados.0.smithi070.stdout:3872: writing smithi07085154-48 from 2375680 to 2981888 tid 1 2024-09-14T04:01:32.263 INFO:tasks.rados.rados.0.smithi070.stdout:3873: read oid 33 snap -1 2024-09-14T04:01:32.263 INFO:tasks.rados.rados.0.smithi070.stdout:3873: expect (ObjNum 1014 snap 334 seq_num 1014) 2024-09-14T04:01:32.263 INFO:tasks.rados.rados.0.smithi070.stdout:3874: setattr oid 14 current snap is 393 2024-09-14T04:01:32.263 INFO:tasks.rados.rados.0.smithi070.stdout:3871: finishing rollback tid 1 to smithi07085154-46 2024-09-14T04:01:32.263 INFO:tasks.rados.rados.0.smithi070.stdout:3871: finishing rollback tid 2 to smithi07085154-46 2024-09-14T04:01:32.264 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 46 v 4709 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-14T04:01:32.265 INFO:tasks.rados.rados.0.smithi070.stdout:3871: done (3 left) 2024-09-14T04:01:32.265 INFO:tasks.rados.rados.0.smithi070.stdout:3875: read oid 3 snap -1 2024-09-14T04:01:32.265 INFO:tasks.rados.rados.0.smithi070.stdout:3875: expect (ObjNum 1159 snap 388 seq_num 1159) 2024-09-14T04:01:32.265 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 12 current snap is 393 2024-09-14T04:01:32.265 INFO:tasks.rados.rados.0.smithi070.stdout:3876: seq_num 1183 ranges {0=622592} 2024-09-14T04:01:32.270 INFO:tasks.rados.rados.0.smithi070.stdout:3876: writing smithi07085154-12 from 0 to 622592 tid 1 2024-09-14T04:01:32.270 INFO:tasks.rados.rados.0.smithi070.stdout:3877: delete oid 30 current snap is 393 2024-09-14T04:01:32.271 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 14 v 4159 (ObjNum 1170 snap 389 seq_num 1170) dirty exists 2024-09-14T04:01:32.271 INFO:tasks.rados.rados.0.smithi070.stdout:3872: finishing write tid 1 to smithi07085154-48 2024-09-14T04:01:32.274 INFO:tasks.rados.rados.0.smithi070.stdout:3872: finishing write tid 2 to smithi07085154-48 2024-09-14T04:01:32.276 INFO:tasks.rados.rados.0.smithi070.stdout:3875: expect (ObjNum 1159 snap 388 seq_num 1159) 2024-09-14T04:01:32.332 INFO:tasks.rados.rados.0.smithi070.stdout:3872: finishing write tid 3 to smithi07085154-48 2024-09-14T04:01:32.332 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 2283 (ObjNum 1182 snap 393 seq_num 1182) dirty exists 2024-09-14T04:01:32.332 INFO:tasks.rados.rados.0.smithi070.stdout:3872: left oid 48 (ObjNum 1182 snap 393 seq_num 1182) 2024-09-14T04:01:32.332 INFO:tasks.rados.rados.0.smithi070.stdout:3873: expect (ObjNum 1014 snap 334 seq_num 1014) 2024-09-14T04:01:32.433 INFO:tasks.rados.rados.0.smithi070.stdout:3876: finishing write tid 1 to smithi07085154-12 2024-09-14T04:01:32.434 INFO:tasks.rados.rados.0.smithi070.stdout:3876: finishing write tid 2 to smithi07085154-12 2024-09-14T04:01:32.434 INFO:tasks.rados.rados.0.smithi070.stdout:3876: finishing write tid 3 to smithi07085154-12 2024-09-14T04:01:32.434 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 2285 (ObjNum 1183 snap 393 seq_num 1183) dirty exists 2024-09-14T04:01:32.434 INFO:tasks.rados.rados.0.smithi070.stdout:3876: left oid 12 (ObjNum 1183 snap 393 seq_num 1183) 2024-09-14T04:01:32.434 INFO:tasks.rados.rados.0.smithi070.stdout:3872: done (5 left) 2024-09-14T04:01:32.434 INFO:tasks.rados.rados.0.smithi070.stdout:3873: done (4 left) 2024-09-14T04:01:32.434 INFO:tasks.rados.rados.0.smithi070.stdout:3874: done (3 left) 2024-09-14T04:01:32.434 INFO:tasks.rados.rados.0.smithi070.stdout:3875: done (2 left) 2024-09-14T04:01:32.434 INFO:tasks.rados.rados.0.smithi070.stdout:3876: done (1 left) 2024-09-14T04:01:32.434 INFO:tasks.rados.rados.0.smithi070.stdout:3877: done (0 left) 2024-09-14T04:01:32.434 INFO:tasks.rados.rados.0.smithi070.stdout:3878: copy_from oid 30 from oid 40 current snap is 393 2024-09-14T04:01:32.434 INFO:tasks.rados.rados.0.smithi070.stdout:3879: copy_from oid 31 from oid 36 current snap is 393 2024-09-14T04:01:32.434 INFO:tasks.rados.rados.0.smithi070.stdout:3880: read oid 4 snap -1 2024-09-14T04:01:32.435 INFO:tasks.rados.rados.0.smithi070.stdout:3880: expect deleted 2024-09-14T04:01:32.435 INFO:tasks.rados.rados.0.smithi070.stdout:3881: snap_remove snap 213 2024-09-14T04:01:32.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:32 smithi094 ceph-mon[86597]: osdmap e851: 8 total, 8 up, 8 in 2024-09-14T04:01:32.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:32 smithi094 ceph-mon[86597]: pgmap v878: 121 pgs: 121 active+clean; 791 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 13 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-14T04:01:32.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:32 smithi070 ceph-mon[96704]: osdmap e851: 8 total, 8 up, 8 in 2024-09-14T04:01:32.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:32 smithi070 ceph-mon[96704]: pgmap v878: 121 pgs: 121 active+clean; 791 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 13 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-14T04:01:32.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:32 smithi070 ceph-mon[98640]: osdmap e851: 8 total, 8 up, 8 in 2024-09-14T04:01:32.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:32 smithi070 ceph-mon[98640]: pgmap v878: 121 pgs: 121 active+clean; 791 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 13 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-14T04:01:33.248 INFO:tasks.rados.rados.0.smithi070.stdout:3881: done (3 left) 2024-09-14T04:01:33.248 INFO:tasks.rados.rados.0.smithi070.stdout:3882: setattr oid 15 current snap is 393 2024-09-14T04:01:33.248 INFO:tasks.rados.rados.0.smithi070.stdout:3879: finishing copy_from racing read to smithi07085154-31 2024-09-14T04:01:33.248 INFO:tasks.rados.rados.0.smithi070.stdout:3879: finishing copy_from to smithi07085154-31 2024-09-14T04:01:33.248 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 4261 (ObjNum 1175 snap 389 seq_num 1175) dirty exists 2024-09-14T04:01:33.249 INFO:tasks.rados.rados.0.smithi070.stdout:3878: finishing copy_from racing read to smithi07085154-30 2024-09-14T04:01:33.249 INFO:tasks.rados.rados.0.smithi070.stdout:3878: finishing copy_from to smithi07085154-30 2024-09-14T04:01:33.249 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 30 v 4252 (ObjNum 1149 snap 382 seq_num 1149) dirty exists 2024-09-14T04:01:33.251 INFO:tasks.rados.rados.0.smithi070.stdout:3878: done (3 left) 2024-09-14T04:01:33.251 INFO:tasks.rados.rados.0.smithi070.stdout:3879: done (2 left) 2024-09-14T04:01:33.252 INFO:tasks.rados.rados.0.smithi070.stdout:3880: done (1 left) 2024-09-14T04:01:33.252 INFO:tasks.rados.rados.0.smithi070.stdout:3883: copy_from oid 9 from oid 16 current snap is 393 2024-09-14T04:01:33.252 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 3 current snap is 393 2024-09-14T04:01:33.252 INFO:tasks.rados.rados.0.smithi070.stdout:3884: seq_num 1185 ranges {1376256=655360} 2024-09-14T04:01:33.263 INFO:tasks.rados.rados.0.smithi070.stdout:3884: writing smithi07085154-3 from 1376256 to 2031616 tid 1 2024-09-14T04:01:33.264 INFO:tasks.rados.rados.0.smithi070.stdout:3885: copy_from oid 31 from oid 49 current snap is 393 2024-09-14T04:01:33.265 INFO:tasks.rados.rados.0.smithi070.stdout:3886: rollback oid 27 current snap is 393 2024-09-14T04:01:33.265 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 27 to 361 2024-09-14T04:01:33.265 INFO:tasks.rados.rados.0.smithi070.stdout:3887: rollback oid 40 current snap is 393 2024-09-14T04:01:33.265 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 40 to 379 2024-09-14T04:01:33.265 INFO:tasks.rados.rados.0.smithi070.stdout:3888: snap_remove snap 351 2024-09-14T04:01:33.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:33 smithi094 ceph-mon[86597]: osdmap e852: 8 total, 8 up, 8 in 2024-09-14T04:01:33.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:33 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:01:33.535 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:01:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:01:33] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.51.0" 2024-09-14T04:01:33.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:33 smithi070 ceph-mon[96704]: osdmap e852: 8 total, 8 up, 8 in 2024-09-14T04:01:33.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:33 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:01:33.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:33 smithi070 ceph-mon[98640]: osdmap e852: 8 total, 8 up, 8 in 2024-09-14T04:01:33.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:33 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:01:34.157 INFO:tasks.rados.rados.0.smithi070.stdout:3888: done (6 left) 2024-09-14T04:01:34.157 INFO:tasks.rados.rados.0.smithi070.stdout:3889: rollback oid 22 current snap is 393 2024-09-14T04:01:34.157 INFO:tasks.rados.rados.0.smithi070.stdout:3885: finishing copy_from to smithi07085154-31 2024-09-14T04:01:34.157 INFO:tasks.rados.rados.0.smithi070.stdout:3885: got expected ENOENT (src dne) 2024-09-14T04:01:34.157 INFO:tasks.rados.rados.0.smithi070.stdout:3885: finishing copy_from racing read to smithi07085154-31 2024-09-14T04:01:34.157 INFO:tasks.rados.rados.0.smithi070.stdout:3884: finishing write tid 1 to smithi07085154-3 2024-09-14T04:01:34.157 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 15 v 4163 (ObjNum 1120 snap 374 seq_num 1120) dirty exists 2024-09-14T04:01:34.157 INFO:tasks.rados.rados.0.smithi070.stdout:3884: finishing write tid 2 to smithi07085154-3 2024-09-14T04:01:34.157 INFO:tasks.rados.rados.0.smithi070.stdout:3884: finishing write tid 3 to smithi07085154-3 2024-09-14T04:01:34.158 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 3 v 4254 (ObjNum 1185 snap 393 seq_num 1185) dirty exists 2024-09-14T04:01:34.158 INFO:tasks.rados.rados.0.smithi070.stdout:3884: left oid 3 (ObjNum 1185 snap 393 seq_num 1185) 2024-09-14T04:01:34.158 INFO:tasks.rados.rados.0.smithi070.stdout:3887: finishing rollback tid 0 to smithi07085154-40 2024-09-14T04:01:34.158 INFO:tasks.rados.rados.0.smithi070.stdout:3887: finishing rollback tid 1 to smithi07085154-40 2024-09-14T04:01:34.158 INFO:tasks.rados.rados.0.smithi070.stdout:3887: finishing rollback tid 2 to smithi07085154-40 2024-09-14T04:01:34.158 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 4264 (ObjNum 1079 snap 350 seq_num 1079) dirty exists 2024-09-14T04:01:34.158 INFO:tasks.rados.rados.0.smithi070.stdout:3883: finishing copy_from racing read to smithi07085154-9 2024-09-14T04:01:34.158 INFO:tasks.rados.rados.0.smithi070.stdout:3886: finishing rollback tid 0 to smithi07085154-27 2024-09-14T04:01:34.158 INFO:tasks.rados.rados.0.smithi070.stdout:3886: finishing rollback tid 1 to smithi07085154-27 2024-09-14T04:01:34.158 INFO:tasks.rados.rados.0.smithi070.stdout:3886: finishing rollback tid 2 to smithi07085154-27 2024-09-14T04:01:34.158 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 27 v 4713 (ObjNum 829 snap 283 seq_num 829) dirty exists 2024-09-14T04:01:34.158 INFO:tasks.rados.rados.0.smithi070.stdout:3883: finishing copy_from to smithi07085154-9 2024-09-14T04:01:34.158 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 9 v 4662 (ObjNum 1172 snap 389 seq_num 1172) dirty exists 2024-09-14T04:01:34.159 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 22 to 350 2024-09-14T04:01:34.159 INFO:tasks.rados.rados.0.smithi070.stdout:3882: done (6 left) 2024-09-14T04:01:34.159 INFO:tasks.rados.rados.0.smithi070.stdout:3883: done (5 left) 2024-09-14T04:01:34.159 INFO:tasks.rados.rados.0.smithi070.stdout:3884: done (4 left) 2024-09-14T04:01:34.159 INFO:tasks.rados.rados.0.smithi070.stdout:3885: done (3 left) 2024-09-14T04:01:34.159 INFO:tasks.rados.rados.0.smithi070.stdout:3886: done (2 left) 2024-09-14T04:01:34.159 INFO:tasks.rados.rados.0.smithi070.stdout:3887: done (1 left) 2024-09-14T04:01:34.159 INFO:tasks.rados.rados.0.smithi070.stdout:3890: snap_remove snap 346 2024-09-14T04:01:34.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:34 smithi094 ceph-mon[86597]: osdmap e853: 8 total, 8 up, 8 in 2024-09-14T04:01:34.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:34 smithi094 ceph-mon[86597]: osdmap e854: 8 total, 8 up, 8 in 2024-09-14T04:01:34.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:34 smithi094 ceph-mon[86597]: pgmap v882: 121 pgs: 121 active+clean; 792 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 13 MiB/s rd, 701 KiB/s wr, 12 op/s 2024-09-14T04:01:34.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:34 smithi070 ceph-mon[98640]: osdmap e853: 8 total, 8 up, 8 in 2024-09-14T04:01:34.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:34 smithi070 ceph-mon[98640]: osdmap e854: 8 total, 8 up, 8 in 2024-09-14T04:01:34.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:34 smithi070 ceph-mon[98640]: pgmap v882: 121 pgs: 121 active+clean; 792 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 13 MiB/s rd, 701 KiB/s wr, 12 op/s 2024-09-14T04:01:34.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:34 smithi070 ceph-mon[96704]: osdmap e853: 8 total, 8 up, 8 in 2024-09-14T04:01:34.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:34 smithi070 ceph-mon[96704]: osdmap e854: 8 total, 8 up, 8 in 2024-09-14T04:01:34.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:34 smithi070 ceph-mon[96704]: pgmap v882: 121 pgs: 121 active+clean; 792 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 13 MiB/s rd, 701 KiB/s wr, 12 op/s 2024-09-14T04:01:35.163 INFO:tasks.rados.rados.0.smithi070.stdout:3889: finishing rollback tid 1 to smithi07085154-22 2024-09-14T04:01:35.163 INFO:tasks.rados.rados.0.smithi070.stdout:3889: finishing rollback tid 2 to smithi07085154-22 2024-09-14T04:01:35.164 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 4256 (ObjNum 1071 snap 347 seq_num 1071) dirty exists 2024-09-14T04:01:35.164 INFO:tasks.rados.rados.0.smithi070.stdout:3889: done (1 left) 2024-09-14T04:01:35.164 INFO:tasks.rados.rados.0.smithi070.stdout:3890: done (0 left) 2024-09-14T04:01:35.164 INFO:tasks.rados.rados.0.smithi070.stdout:3891: setattr oid 47 current snap is 393 2024-09-14T04:01:35.166 INFO:tasks.rados.rados.0.smithi070.stdout:3892: rmattr oid 34 current snap is 393 2024-09-14T04:01:35.166 INFO:tasks.rados.rados.0.smithi070.stdout:3892: done (1 left) 2024-09-14T04:01:35.166 INFO:tasks.rados.rados.0.smithi070.stdout:3893: snap_remove snap 382 2024-09-14T04:01:36.166 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 47 v 4255 (ObjNum 65 snap 0 seq_num 1347904366) dirty exists 2024-09-14T04:01:36.167 INFO:tasks.rados.rados.0.smithi070.stdout:3891: done (1 left) 2024-09-14T04:01:36.167 INFO:tasks.rados.rados.0.smithi070.stdout:3893: done (0 left) 2024-09-14T04:01:36.167 INFO:tasks.rados.rados.0.smithi070.stdout:3894: snap_remove snap 337 2024-09-14T04:01:36.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:36 smithi094 ceph-mon[86597]: osdmap e855: 8 total, 8 up, 8 in 2024-09-14T04:01:36.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:36 smithi070 ceph-mon[98640]: osdmap e855: 8 total, 8 up, 8 in 2024-09-14T04:01:36.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:36 smithi070 ceph-mon[96704]: osdmap e855: 8 total, 8 up, 8 in 2024-09-14T04:01:37.173 INFO:tasks.rados.rados.0.smithi070.stdout:3894: done (0 left) 2024-09-14T04:01:37.173 INFO:tasks.rados.rados.0.smithi070.stdout:3895: copy_from oid 25 from oid 30 current snap is 393 2024-09-14T04:01:37.173 INFO:tasks.rados.rados.0.smithi070.stdout:3896: copy_from oid 17 from oid 5 current snap is 393 2024-09-14T04:01:37.173 INFO:tasks.rados.rados.0.smithi070.stdout:3897: snap_create 2024-09-14T04:01:37.220 INFO:tasks.rados.rados.0.smithi070.stdout:3895: finishing copy_from racing read to smithi07085154-25 2024-09-14T04:01:37.220 INFO:tasks.rados.rados.0.smithi070.stdout:3896: finishing copy_from racing read to smithi07085154-17 2024-09-14T04:01:37.220 INFO:tasks.rados.rados.0.smithi070.stdout:3896: finishing copy_from to smithi07085154-17 2024-09-14T04:01:37.220 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 17 v 3202 (ObjNum 1042 snap 342 seq_num 1042) dirty exists 2024-09-14T04:01:37.223 INFO:tasks.rados.rados.0.smithi070.stdout:3895: finishing copy_from to smithi07085154-25 2024-09-14T04:01:37.223 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 25 v 4252 (ObjNum 1149 snap 382 seq_num 1149) dirty exists 2024-09-14T04:01:37.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:37 smithi094 ceph-mon[86597]: osdmap e856: 8 total, 8 up, 8 in 2024-09-14T04:01:37.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:37 smithi094 ceph-mon[86597]: pgmap v885: 121 pgs: 121 active+clean; 715 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 1.8 MiB/s rd, 426 KiB/s wr, 18 op/s 2024-09-14T04:01:37.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:37 smithi070 ceph-mon[96704]: osdmap e856: 8 total, 8 up, 8 in 2024-09-14T04:01:37.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:37 smithi070 ceph-mon[96704]: pgmap v885: 121 pgs: 121 active+clean; 715 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 1.8 MiB/s rd, 426 KiB/s wr, 18 op/s 2024-09-14T04:01:37.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:37 smithi070 ceph-mon[98640]: osdmap e856: 8 total, 8 up, 8 in 2024-09-14T04:01:37.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:37 smithi070 ceph-mon[98640]: pgmap v885: 121 pgs: 121 active+clean; 715 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 1.8 MiB/s rd, 426 KiB/s wr, 18 op/s 2024-09-14T04:01:38.178 INFO:tasks.rados.rados.0.smithi070.stdout:3895: done (2 left) 2024-09-14T04:01:38.178 INFO:tasks.rados.rados.0.smithi070.stdout:3896: done (1 left) 2024-09-14T04:01:38.178 INFO:tasks.rados.rados.0.smithi070.stdout:3897: done (0 left) 2024-09-14T04:01:38.178 INFO:tasks.rados.rados.0.smithi070.stdout:3898: rollback oid 12 current snap is 394 2024-09-14T04:01:38.178 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 12 to 378 2024-09-14T04:01:38.178 INFO:tasks.rados.rados.0.smithi070.stdout:3899: read oid 10 snap -1 2024-09-14T04:01:38.178 INFO:tasks.rados.rados.0.smithi070.stdout:3899: expect (ObjNum 1154 snap 387 seq_num 1154) 2024-09-14T04:01:38.178 INFO:tasks.rados.rados.0.smithi070.stdout:3900: rollback oid 38 current snap is 394 2024-09-14T04:01:38.178 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 38 to 379 2024-09-14T04:01:38.178 INFO:tasks.rados.rados.0.smithi070.stdout:3901: setattr oid 47 current snap is 394 2024-09-14T04:01:38.179 INFO:tasks.rados.rados.0.smithi070.stdout:3902: copy_from oid 44 from oid 48 current snap is 394 2024-09-14T04:01:38.179 INFO:tasks.rados.rados.0.smithi070.stdout:3903: snap_create 2024-09-14T04:01:38.186 INFO:tasks.rados.rados.0.smithi070.stdout:3898: finishing rollback tid 0 to smithi07085154-12 2024-09-14T04:01:38.186 INFO:tasks.rados.rados.0.smithi070.stdout:3900: finishing rollback tid 0 to smithi07085154-38 2024-09-14T04:01:38.186 INFO:tasks.rados.rados.0.smithi070.stdout:3900: finishing rollback tid 1 to smithi07085154-38 2024-09-14T04:01:38.187 INFO:tasks.rados.rados.0.smithi070.stdout:3900: finishing rollback tid 2 to smithi07085154-38 2024-09-14T04:01:38.187 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 38 v 4258 (ObjNum 1115 snap 371 seq_num 1115) dirty exists 2024-09-14T04:01:38.189 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 47 v 4259 (ObjNum 1921 snap 0 seq_num 4194279856) dirty exists 2024-09-14T04:01:38.190 INFO:tasks.rados.rados.0.smithi070.stdout:3899: expect (ObjNum 1154 snap 387 seq_num 1154) 2024-09-14T04:01:38.277 INFO:tasks.rados.rados.0.smithi070.stdout:3898: finishing rollback tid 1 to smithi07085154-12 2024-09-14T04:01:38.277 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 2287 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-14T04:01:38.277 INFO:tasks.rados.rados.0.smithi070.stdout:3902: finishing copy_from racing read to smithi07085154-44 2024-09-14T04:01:38.277 INFO:tasks.rados.rados.0.smithi070.stdout:3902: finishing copy_from to smithi07085154-44 2024-09-14T04:01:38.277 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 44 v 2283 (ObjNum 1182 snap 393 seq_num 1182) dirty exists 2024-09-14T04:01:38.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:38 smithi094 ceph-mon[86597]: osdmap e857: 8 total, 8 up, 8 in 2024-09-14T04:01:38.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:38 smithi070 ceph-mon[96704]: osdmap e857: 8 total, 8 up, 8 in 2024-09-14T04:01:38.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:38 smithi070 ceph-mon[98640]: osdmap e857: 8 total, 8 up, 8 in 2024-09-14T04:01:39.158 INFO:tasks.rados.rados.0.smithi070.stdout:3898: done (5 left) 2024-09-14T04:01:39.158 INFO:tasks.rados.rados.0.smithi070.stdout:3899: done (4 left) 2024-09-14T04:01:39.158 INFO:tasks.rados.rados.0.smithi070.stdout:3900: done (3 left) 2024-09-14T04:01:39.158 INFO:tasks.rados.rados.0.smithi070.stdout:3901: done (2 left) 2024-09-14T04:01:39.158 INFO:tasks.rados.rados.0.smithi070.stdout:3902: done (1 left) 2024-09-14T04:01:39.158 INFO:tasks.rados.rados.0.smithi070.stdout:3903: done (0 left) 2024-09-14T04:01:39.158 INFO:tasks.rados.rados.0.smithi070.stdout:3904: read oid 6 snap -1 2024-09-14T04:01:39.158 INFO:tasks.rados.rados.0.smithi070.stdout:3904: expect (ObjNum 1140 snap 381 seq_num 1140) 2024-09-14T04:01:39.158 INFO:tasks.rados.rados.0.smithi070.stdout:3905: snap_create 2024-09-14T04:01:39.169 INFO:tasks.rados.rados.0.smithi070.stdout:3904: expect (ObjNum 1140 snap 381 seq_num 1140) 2024-09-14T04:01:39.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:39 smithi094 ceph-mon[86597]: pgmap v887: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 584 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.8 MiB/s wr, 20 op/s 2024-09-14T04:01:39.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:39 smithi094 ceph-mon[86597]: osdmap e858: 8 total, 8 up, 8 in 2024-09-14T04:01:39.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:39 smithi094 ceph-mon[86597]: osdmap e859: 8 total, 8 up, 8 in 2024-09-14T04:01:39.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:39 smithi070 ceph-mon[96704]: pgmap v887: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 584 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.8 MiB/s wr, 20 op/s 2024-09-14T04:01:39.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:39 smithi070 ceph-mon[96704]: osdmap e858: 8 total, 8 up, 8 in 2024-09-14T04:01:39.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:39 smithi070 ceph-mon[96704]: osdmap e859: 8 total, 8 up, 8 in 2024-09-14T04:01:39.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:39 smithi070 ceph-mon[98640]: pgmap v887: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 584 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.8 MiB/s wr, 20 op/s 2024-09-14T04:01:39.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:39 smithi070 ceph-mon[98640]: osdmap e858: 8 total, 8 up, 8 in 2024-09-14T04:01:39.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:39 smithi070 ceph-mon[98640]: osdmap e859: 8 total, 8 up, 8 in 2024-09-14T04:01:40.159 INFO:tasks.rados.rados.0.smithi070.stdout:3904: done (1 left) 2024-09-14T04:01:40.159 INFO:tasks.rados.rados.0.smithi070.stdout:3905: done (0 left) 2024-09-14T04:01:40.159 INFO:tasks.rados.rados.0.smithi070.stdout:3906: rollback oid 35 current snap is 396 2024-09-14T04:01:40.159 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 35 to 375 2024-09-14T04:01:40.160 INFO:tasks.rados.rados.0.smithi070.stdout:3907: snap_remove snap 368 2024-09-14T04:01:40.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:01:40 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:01:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:01:41.165 INFO:tasks.rados.rados.0.smithi070.stdout:3907: done (1 left) 2024-09-14T04:01:41.165 INFO:tasks.rados.rados.0.smithi070.stdout:3908: read oid 21 snap 375 2024-09-14T04:01:41.166 INFO:tasks.rados.rados.0.smithi070.stdout:3908: expect (ObjNum 1091 snap 355 seq_num 1091) 2024-09-14T04:01:41.166 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 19 current snap is 396 2024-09-14T04:01:41.166 INFO:tasks.rados.rados.0.smithi070.stdout:3909: seq_num 1186 ranges {0=671744} 2024-09-14T04:01:41.170 INFO:tasks.rados.rados.0.smithi070.stdout:3909: writing smithi07085154-19 from 0 to 671744 tid 1 2024-09-14T04:01:41.171 INFO:tasks.rados.rados.0.smithi070.stdout:3906: finishing rollback tid 0 to smithi07085154-35 2024-09-14T04:01:41.171 INFO:tasks.rados.rados.0.smithi070.stdout:3906: finishing rollback tid 1 to smithi07085154-35 2024-09-14T04:01:41.171 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 35 v 4437 (ObjNum 65 snap 0 seq_num 1836211039) dirty dne 2024-09-14T04:01:41.172 INFO:tasks.rados.rados.0.smithi070.stdout:3906: done (2 left) 2024-09-14T04:01:41.172 INFO:tasks.rados.rados.0.smithi070.stdout:3910: rmattr oid 7 current snap is 396 2024-09-14T04:01:41.172 INFO:tasks.rados.rados.0.smithi070.stdout:3910: done (2 left) 2024-09-14T04:01:41.172 INFO:tasks.rados.rados.0.smithi070.stdout:3911: delete oid 11 current snap is 396 2024-09-14T04:01:41.182 INFO:tasks.rados.rados.0.smithi070.stdout:3911: done (2 left) 2024-09-14T04:01:41.182 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 20 current snap is 396 2024-09-14T04:01:41.182 INFO:tasks.rados.rados.0.smithi070.stdout:3912: seq_num 1188 ranges {9789440=704512} 2024-09-14T04:01:41.239 INFO:tasks.rados.rados.0.smithi070.stdout:3912: writing smithi07085154-20 from 9789440 to 10493952 tid 1 2024-09-14T04:01:41.239 INFO:tasks.rados.rados.0.smithi070.stdout:3913: read oid 23 snap 361 2024-09-14T04:01:41.239 INFO:tasks.rados.rados.0.smithi070.stdout:3913: expect (ObjNum 1078 snap 349 seq_num 1078) 2024-09-14T04:01:41.240 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 16 current snap is 396 2024-09-14T04:01:41.240 INFO:tasks.rados.rados.0.smithi070.stdout:3914: seq_num 1189 ranges {5029888=720896} 2024-09-14T04:01:41.271 INFO:tasks.rados.rados.0.smithi070.stdout:3914: writing smithi07085154-16 from 5029888 to 5750784 tid 1 2024-09-14T04:01:41.271 INFO:tasks.rados.rados.0.smithi070.stdout:3915: read oid 15 snap -1 2024-09-14T04:01:41.271 INFO:tasks.rados.rados.0.smithi070.stdout:3915: expect (ObjNum 1120 snap 374 seq_num 1120) 2024-09-14T04:01:41.271 INFO:tasks.rados.rados.0.smithi070.stdout:3916: copy_from oid 12 from oid 33 current snap is 396 2024-09-14T04:01:41.271 INFO:tasks.rados.rados.0.smithi070.stdout:3909: finishing write tid 1 to smithi07085154-19 2024-09-14T04:01:41.272 INFO:tasks.rados.rados.0.smithi070.stdout:3909: finishing write tid 2 to smithi07085154-19 2024-09-14T04:01:41.272 INFO:tasks.rados.rados.0.smithi070.stdout:3909: finishing write tid 3 to smithi07085154-19 2024-09-14T04:01:41.272 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 19 v 4258 (ObjNum 1186 snap 396 seq_num 1186) dirty exists 2024-09-14T04:01:41.272 INFO:tasks.rados.rados.0.smithi070.stdout:3909: left oid 19 (ObjNum 1186 snap 396 seq_num 1186) 2024-09-14T04:01:41.272 INFO:tasks.rados.rados.0.smithi070.stdout:3908: expect (ObjNum 1091 snap 355 seq_num 1091) 2024-09-14T04:01:41.404 INFO:tasks.rados.rados.0.smithi070.stdout:3912: finishing write tid 1 to smithi07085154-20 2024-09-14T04:01:41.404 INFO:tasks.rados.rados.0.smithi070.stdout:3912: finishing write tid 2 to smithi07085154-20 2024-09-14T04:01:41.404 INFO:tasks.rados.rados.0.smithi070.stdout:3908: done (6 left) 2024-09-14T04:01:41.404 INFO:tasks.rados.rados.0.smithi070.stdout:3909: done (5 left) 2024-09-14T04:01:41.404 INFO:tasks.rados.rados.0.smithi070.stdout:3917: setattr oid 29 current snap is 396 2024-09-14T04:01:41.404 INFO:tasks.rados.rados.0.smithi070.stdout:3912: finishing write tid 3 to smithi07085154-20 2024-09-14T04:01:41.404 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 20 v 4231 (ObjNum 1188 snap 396 seq_num 1188) dirty exists 2024-09-14T04:01:41.404 INFO:tasks.rados.rados.0.smithi070.stdout:3912: left oid 20 (ObjNum 1188 snap 396 seq_num 1188) 2024-09-14T04:01:41.404 INFO:tasks.rados.rados.0.smithi070.stdout:3913: expect (ObjNum 1078 snap 349 seq_num 1078) 2024-09-14T04:01:41.444 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:41 smithi094 ceph-mon[86597]: osdmap e860: 8 total, 8 up, 8 in 2024-09-14T04:01:41.444 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:41 smithi094 ceph-mon[86597]: pgmap v891: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 494 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 1023 KiB/s rd, 5.4 MiB/s wr, 12 op/s 2024-09-14T04:01:41.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:41 smithi070 ceph-mon[96704]: osdmap e860: 8 total, 8 up, 8 in 2024-09-14T04:01:41.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:41 smithi070 ceph-mon[96704]: pgmap v891: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 494 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 1023 KiB/s rd, 5.4 MiB/s wr, 12 op/s 2024-09-14T04:01:41.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:41 smithi070 ceph-mon[98640]: osdmap e860: 8 total, 8 up, 8 in 2024-09-14T04:01:41.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:41 smithi070 ceph-mon[98640]: pgmap v891: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 494 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 1023 KiB/s rd, 5.4 MiB/s wr, 12 op/s 2024-09-14T04:01:41.640 INFO:tasks.rados.rados.0.smithi070.stdout:3914: finishing write tid 1 to smithi07085154-16 2024-09-14T04:01:41.640 INFO:tasks.rados.rados.0.smithi070.stdout:3914: finishing write tid 2 to smithi07085154-16 2024-09-14T04:01:41.640 INFO:tasks.rados.rados.0.smithi070.stdout:3914: finishing write tid 3 to smithi07085154-16 2024-09-14T04:01:41.640 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 16 v 4790 (ObjNum 1189 snap 396 seq_num 1189) dirty exists 2024-09-14T04:01:41.640 INFO:tasks.rados.rados.0.smithi070.stdout:3914: left oid 16 (ObjNum 1189 snap 396 seq_num 1189) 2024-09-14T04:01:41.641 INFO:tasks.rados.rados.0.smithi070.stdout:3916: finishing copy_from racing read to smithi07085154-12 2024-09-14T04:01:41.641 INFO:tasks.rados.rados.0.smithi070.stdout:3916: finishing copy_from to smithi07085154-12 2024-09-14T04:01:41.641 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 12 v 3214 (ObjNum 1014 snap 334 seq_num 1014) dirty exists 2024-09-14T04:01:41.641 INFO:tasks.rados.rados.0.smithi070.stdout:3915: expect (ObjNum 1120 snap 374 seq_num 1120) 2024-09-14T04:01:41.846 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 29 v 4259 (ObjNum 1149 snap 382 seq_num 1149) dirty exists 2024-09-14T04:01:41.846 INFO:tasks.rados.rados.0.smithi070.stdout:3912: done (5 left) 2024-09-14T04:01:41.846 INFO:tasks.rados.rados.0.smithi070.stdout:3913: done (4 left) 2024-09-14T04:01:41.847 INFO:tasks.rados.rados.0.smithi070.stdout:3914: done (3 left) 2024-09-14T04:01:41.847 INFO:tasks.rados.rados.0.smithi070.stdout:3915: done (2 left) 2024-09-14T04:01:41.847 INFO:tasks.rados.rados.0.smithi070.stdout:3916: done (1 left) 2024-09-14T04:01:41.847 INFO:tasks.rados.rados.0.smithi070.stdout:3917: done (0 left) 2024-09-14T04:01:41.847 INFO:tasks.rados.rados.0.smithi070.stdout:3918: read oid 29 snap -1 2024-09-14T04:01:41.847 INFO:tasks.rados.rados.0.smithi070.stdout:3918: expect (ObjNum 1149 snap 382 seq_num 1149) 2024-09-14T04:01:41.847 INFO:tasks.rados.rados.0.smithi070.stdout:3919: rmattr oid 11 current snap is 396 2024-09-14T04:01:41.847 INFO:tasks.rados.rados.0.smithi070.stdout:3919: done (1 left) 2024-09-14T04:01:41.847 INFO:tasks.rados.rados.0.smithi070.stdout:3920: snap_create 2024-09-14T04:01:41.867 INFO:tasks.rados.rados.0.smithi070.stdout:3918: expect (ObjNum 1149 snap 382 seq_num 1149) 2024-09-14T04:01:42.191 INFO:tasks.rados.rados.0.smithi070.stdout:3918: done (1 left) 2024-09-14T04:01:42.191 INFO:tasks.rados.rados.0.smithi070.stdout:3920: done (0 left) 2024-09-14T04:01:42.191 INFO:tasks.rados.rados.0.smithi070.stdout:3921: snap_remove snap 366 2024-09-14T04:01:42.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:42 smithi094 ceph-mon[86597]: osdmap e861: 8 total, 8 up, 8 in 2024-09-14T04:01:42.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:42 smithi070 ceph-mon[96704]: osdmap e861: 8 total, 8 up, 8 in 2024-09-14T04:01:42.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:42 smithi070 ceph-mon[98640]: osdmap e861: 8 total, 8 up, 8 in 2024-09-14T04:01:43.176 INFO:tasks.rados.rados.0.smithi070.stdout:3921: done (0 left) 2024-09-14T04:01:43.176 INFO:tasks.rados.rados.0.smithi070.stdout:3922: rollback oid 37 current snap is 397 2024-09-14T04:01:43.176 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 37 to 380 2024-09-14T04:01:43.178 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 36 current snap is 397 2024-09-14T04:01:43.178 INFO:tasks.rados.rados.0.smithi070.stdout:3923: seq_num 1190 ranges {483328=737280} 2024-09-14T04:01:43.184 INFO:tasks.rados.rados.0.smithi070.stdout:3923: writing smithi07085154-36 from 483328 to 1220608 tid 1 2024-09-14T04:01:43.184 INFO:tasks.rados.rados.0.smithi070.stdout:3924: read oid 21 snap -1 2024-09-14T04:01:43.184 INFO:tasks.rados.rados.0.smithi070.stdout:3924: expect (ObjNum 1091 snap 355 seq_num 1091) 2024-09-14T04:01:43.185 INFO:tasks.rados.rados.0.smithi070.stdout:3925: read oid 18 snap -1 2024-09-14T04:01:43.185 INFO:tasks.rados.rados.0.smithi070.stdout:3925: expect (ObjNum 596 snap 199 seq_num 596) 2024-09-14T04:01:43.185 INFO:tasks.rados.rados.0.smithi070.stdout:3926: delete oid 34 current snap is 397 2024-09-14T04:01:43.197 INFO:tasks.rados.rados.0.smithi070.stdout:3923: finishing write tid 1 to smithi07085154-36 2024-09-14T04:01:43.197 INFO:tasks.rados.rados.0.smithi070.stdout:3923: finishing write tid 2 to smithi07085154-36 2024-09-14T04:01:43.198 INFO:tasks.rados.rados.0.smithi070.stdout:3923: finishing write tid 3 to smithi07085154-36 2024-09-14T04:01:43.198 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 36 v 4266 (ObjNum 1190 snap 397 seq_num 1190) dirty exists 2024-09-14T04:01:43.198 INFO:tasks.rados.rados.0.smithi070.stdout:3923: left oid 36 (ObjNum 1190 snap 397 seq_num 1190) 2024-09-14T04:01:43.202 INFO:tasks.rados.rados.0.smithi070.stdout:3923: done (4 left) 2024-09-14T04:01:43.202 INFO:tasks.rados.rados.0.smithi070.stdout:3926: done (3 left) 2024-09-14T04:01:43.202 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 2 current snap is 397 2024-09-14T04:01:43.202 INFO:tasks.rados.rados.0.smithi070.stdout:3927: seq_num 1192 ranges {4145152=770048} 2024-09-14T04:01:43.230 INFO:tasks.rados.rados.0.smithi070.stdout:3927: writing smithi07085154-2 from 4145152 to 4915200 tid 1 2024-09-14T04:01:43.230 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 33 current snap is 397 2024-09-14T04:01:43.230 INFO:tasks.rados.rados.0.smithi070.stdout:3922: finishing rollback tid 1 to smithi07085154-37 2024-09-14T04:01:43.230 INFO:tasks.rados.rados.0.smithi070.stdout:3928: seq_num 1193 ranges {2514944=786432} 2024-09-14T04:01:43.248 INFO:tasks.rados.rados.0.smithi070.stdout:3928: writing smithi07085154-33 from 2514944 to 3301376 tid 1 2024-09-14T04:01:43.249 INFO:tasks.rados.rados.0.smithi070.stdout:3929: read oid 28 snap -1 2024-09-14T04:01:43.249 INFO:tasks.rados.rados.0.smithi070.stdout:3929: expect (ObjNum 1157 snap 388 seq_num 1157) 2024-09-14T04:01:43.249 INFO:tasks.rados.rados.0.smithi070.stdout:3930: rollback oid 40 current snap is 397 2024-09-14T04:01:43.249 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 40 to 395 2024-09-14T04:01:43.249 INFO:tasks.rados.rados.0.smithi070.stdout:3924: expect (ObjNum 1091 snap 355 seq_num 1091) 2024-09-14T04:01:43.378 INFO:tasks.rados.rados.0.smithi070.stdout:3922: finishing rollback tid 2 to smithi07085154-37 2024-09-14T04:01:43.378 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 37 v 4165 (ObjNum 1121 snap 375 seq_num 1121) dirty exists 2024-09-14T04:01:43.378 INFO:tasks.rados.rados.0.smithi070.stdout:3925: expect (ObjNum 596 snap 199 seq_num 596) 2024-09-14T04:01:43.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:43 smithi094 ceph-mon[86597]: osdmap e862: 8 total, 8 up, 8 in 2024-09-14T04:01:43.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:43 smithi094 ceph-mon[86597]: pgmap v894: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 491 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.9 MiB/s rd, 3.1 MiB/s wr, 13 op/s 2024-09-14T04:01:43.535 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:01:43 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:01:43] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.51.0" 2024-09-14T04:01:43.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:43 smithi070 ceph-mon[96704]: osdmap e862: 8 total, 8 up, 8 in 2024-09-14T04:01:43.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:43 smithi070 ceph-mon[96704]: pgmap v894: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 491 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.9 MiB/s rd, 3.1 MiB/s wr, 13 op/s 2024-09-14T04:01:43.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:43 smithi070 ceph-mon[98640]: osdmap e862: 8 total, 8 up, 8 in 2024-09-14T04:01:43.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:43 smithi070 ceph-mon[98640]: pgmap v894: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 491 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.9 MiB/s rd, 3.1 MiB/s wr, 13 op/s 2024-09-14T04:01:43.660 INFO:tasks.rados.rados.0.smithi070.stdout:3927: finishing write tid 1 to smithi07085154-2 2024-09-14T04:01:43.660 INFO:tasks.rados.rados.0.smithi070.stdout:3927: finishing write tid 2 to smithi07085154-2 2024-09-14T04:01:43.660 INFO:tasks.rados.rados.0.smithi070.stdout:3927: finishing write tid 3 to smithi07085154-2 2024-09-14T04:01:43.660 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 2 v 4805 (ObjNum 1192 snap 397 seq_num 1192) dirty exists 2024-09-14T04:01:43.660 INFO:tasks.rados.rados.0.smithi070.stdout:3927: left oid 2 (ObjNum 1192 snap 397 seq_num 1192) 2024-09-14T04:01:43.660 INFO:tasks.rados.rados.0.smithi070.stdout:3928: finishing write tid 1 to smithi07085154-33 2024-09-14T04:01:43.660 INFO:tasks.rados.rados.0.smithi070.stdout:3930: finishing rollback tid 0 to smithi07085154-40 2024-09-14T04:01:43.660 INFO:tasks.rados.rados.0.smithi070.stdout:3930: finishing rollback tid 1 to smithi07085154-40 2024-09-14T04:01:43.660 INFO:tasks.rados.rados.0.smithi070.stdout:3930: finishing rollback tid 2 to smithi07085154-40 2024-09-14T04:01:43.660 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 40 v 4269 (ObjNum 1079 snap 350 seq_num 1079) dirty exists 2024-09-14T04:01:43.660 INFO:tasks.rados.rados.0.smithi070.stdout:3928: finishing write tid 2 to smithi07085154-33 2024-09-14T04:01:43.660 INFO:tasks.rados.rados.0.smithi070.stdout:3922: done (6 left) 2024-09-14T04:01:43.660 INFO:tasks.rados.rados.0.smithi070.stdout:3924: done (5 left) 2024-09-14T04:01:43.661 INFO:tasks.rados.rados.0.smithi070.stdout:3925: done (4 left) 2024-09-14T04:01:43.661 INFO:tasks.rados.rados.0.smithi070.stdout:3927: done (3 left) 2024-09-14T04:01:43.661 INFO:tasks.rados.rados.0.smithi070.stdout:3930: done (2 left) 2024-09-14T04:01:43.661 INFO:tasks.rados.rados.0.smithi070.stdout:3931: delete oid 46 current snap is 397 2024-09-14T04:01:43.661 INFO:tasks.rados.rados.0.smithi070.stdout:3928: finishing write tid 3 to smithi07085154-33 2024-09-14T04:01:43.661 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 33 v 4503 (ObjNum 1193 snap 397 seq_num 1193) dirty exists 2024-09-14T04:01:43.661 INFO:tasks.rados.rados.0.smithi070.stdout:3928: left oid 33 (ObjNum 1193 snap 397 seq_num 1193) 2024-09-14T04:01:43.661 INFO:tasks.rados.rados.0.smithi070.stdout:3929: expect (ObjNum 1157 snap 388 seq_num 1157) 2024-09-14T04:01:43.988 INFO:tasks.rados.rados.0.smithi070.stdout:3928: done (2 left) 2024-09-14T04:01:43.988 INFO:tasks.rados.rados.0.smithi070.stdout:3929: done (1 left) 2024-09-14T04:01:43.988 INFO:tasks.rados.rados.0.smithi070.stdout:3931: done (0 left) 2024-09-14T04:01:43.988 INFO:tasks.rados.rados.0.smithi070.stdout:3932: snap_remove snap 350 2024-09-14T04:01:44.158 INFO:tasks.rados.rados.0.smithi070.stdout:3932: done (0 left) 2024-09-14T04:01:44.158 INFO:tasks.rados.rados.0.smithi070.stdout:3933: rmattr oid 2 current snap is 397 2024-09-14T04:01:44.158 INFO:tasks.rados.rados.0.smithi070.stdout:3933: done (0 left) 2024-09-14T04:01:44.158 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 3 current snap is 397 2024-09-14T04:01:44.158 INFO:tasks.rados.rados.0.smithi070.stdout:3934: seq_num 1195 ranges {2031616=417792,2449408=401408} 2024-09-14T04:01:44.172 INFO:tasks.rados.rados.0.smithi070.stdout:3934: writing smithi07085154-3 from 2031616 to 2449408 tid 1 2024-09-14T04:01:44.175 INFO:tasks.rados.rados.0.smithi070.stdout:3934: writing smithi07085154-3 from 2449408 to 2850816 tid 2 2024-09-14T04:01:44.175 INFO:tasks.rados.rados.0.smithi070.stdout:3935: rollback oid 43 current snap is 397 2024-09-14T04:01:44.175 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 43 to 301 2024-09-14T04:01:44.176 INFO:tasks.rados.rados.0.smithi070.stdout:3936: rollback oid 18 current snap is 397 2024-09-14T04:01:44.176 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 18 to 392 2024-09-14T04:01:44.176 INFO:tasks.rados.rados.0.smithi070.stdout:3937: snap_create 2024-09-14T04:01:44.185 INFO:tasks.rados.rados.0.smithi070.stdout:3934: finishing write tid 1 to smithi07085154-3 2024-09-14T04:01:44.185 INFO:tasks.rados.rados.0.smithi070.stdout:3934: finishing write tid 2 to smithi07085154-3 2024-09-14T04:01:44.186 INFO:tasks.rados.rados.0.smithi070.stdout:3934: finishing write tid 3 to smithi07085154-3 2024-09-14T04:01:44.189 INFO:tasks.rados.rados.0.smithi070.stdout:3934: finishing write tid 4 to smithi07085154-3 2024-09-14T04:01:44.189 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 3 v 4262 (ObjNum 1195 snap 397 seq_num 1195) dirty exists 2024-09-14T04:01:44.189 INFO:tasks.rados.rados.0.smithi070.stdout:3934: left oid 3 (ObjNum 1195 snap 397 seq_num 1195) 2024-09-14T04:01:44.189 INFO:tasks.rados.rados.0.smithi070.stdout:3935: finishing rollback tid 0 to smithi07085154-43 2024-09-14T04:01:44.193 INFO:tasks.rados.rados.0.smithi070.stdout:3935: finishing rollback tid 1 to smithi07085154-43 2024-09-14T04:01:44.194 INFO:tasks.rados.rados.0.smithi070.stdout:3935: finishing rollback tid 2 to smithi07085154-43 2024-09-14T04:01:44.194 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 4168 (ObjNum 681 snap 230 seq_num 681) dirty exists 2024-09-14T04:01:44.194 INFO:tasks.rados.rados.0.smithi070.stdout:3936: finishing rollback tid 0 to smithi07085154-18 2024-09-14T04:01:44.194 INFO:tasks.rados.rados.0.smithi070.stdout:3936: finishing rollback tid 1 to smithi07085154-18 2024-09-14T04:01:44.194 INFO:tasks.rados.rados.0.smithi070.stdout:3936: finishing rollback tid 2 to smithi07085154-18 2024-09-14T04:01:44.194 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 18 v 4171 (ObjNum 596 snap 199 seq_num 596) dirty exists 2024-09-14T04:01:44.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:44 smithi094 ceph-mon[86597]: osdmap e863: 8 total, 8 up, 8 in 2024-09-14T04:01:44.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:44 smithi094 ceph-mon[86597]: osdmap e864: 8 total, 8 up, 8 in 2024-09-14T04:01:44.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:44 smithi070 ceph-mon[96704]: osdmap e863: 8 total, 8 up, 8 in 2024-09-14T04:01:44.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:44 smithi070 ceph-mon[96704]: osdmap e864: 8 total, 8 up, 8 in 2024-09-14T04:01:44.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:44 smithi070 ceph-mon[98640]: osdmap e863: 8 total, 8 up, 8 in 2024-09-14T04:01:44.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:44 smithi070 ceph-mon[98640]: osdmap e864: 8 total, 8 up, 8 in 2024-09-14T04:01:45.180 INFO:tasks.rados.rados.0.smithi070.stdout:3934: done (3 left) 2024-09-14T04:01:45.181 INFO:tasks.rados.rados.0.smithi070.stdout:3935: done (2 left) 2024-09-14T04:01:45.181 INFO:tasks.rados.rados.0.smithi070.stdout:3936: done (1 left) 2024-09-14T04:01:45.181 INFO:tasks.rados.rados.0.smithi070.stdout:3937: done (0 left) 2024-09-14T04:01:45.181 INFO:tasks.rados.rados.0.smithi070.stdout:3938: copy_from oid 1 from oid 29 current snap is 398 2024-09-14T04:01:45.181 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 49 current snap is 398 2024-09-14T04:01:45.181 INFO:tasks.rados.rados.0.smithi070.stdout:3939: seq_num 1196 ranges {0=434176,434176=401408} 2024-09-14T04:01:45.183 INFO:tasks.rados.rados.0.smithi070.stdout:3939: writing smithi07085154-49 from 0 to 434176 tid 1 2024-09-14T04:01:45.185 INFO:tasks.rados.rados.0.smithi070.stdout:3939: writing smithi07085154-49 from 434176 to 835584 tid 2 2024-09-14T04:01:45.185 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 23 current snap is 398 2024-09-14T04:01:45.185 INFO:tasks.rados.rados.0.smithi070.stdout:3940: seq_num 1197 ranges {2482176=450560,2932736=401408} 2024-09-14T04:01:45.201 INFO:tasks.rados.rados.0.smithi070.stdout:3940: writing smithi07085154-23 from 2482176 to 2932736 tid 1 2024-09-14T04:01:45.204 INFO:tasks.rados.rados.0.smithi070.stdout:3940: writing smithi07085154-23 from 2932736 to 3334144 tid 2 2024-09-14T04:01:45.204 INFO:tasks.rados.rados.0.smithi070.stdout:3941: rmattr oid 37 current snap is 398 2024-09-14T04:01:45.204 INFO:tasks.rados.rados.0.smithi070.stdout:3941: done (3 left) 2024-09-14T04:01:45.204 INFO:tasks.rados.rados.0.smithi070.stdout:3942: snap_create 2024-09-14T04:01:45.205 INFO:tasks.rados.rados.0.smithi070.stdout:3939: finishing write tid 1 to smithi07085154-49 2024-09-14T04:01:45.205 INFO:tasks.rados.rados.0.smithi070.stdout:3939: finishing write tid 2 to smithi07085154-49 2024-09-14T04:01:45.205 INFO:tasks.rados.rados.0.smithi070.stdout:3939: finishing write tid 3 to smithi07085154-49 2024-09-14T04:01:45.205 INFO:tasks.rados.rados.0.smithi070.stdout:3939: finishing write tid 4 to smithi07085154-49 2024-09-14T04:01:45.205 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 49 v 4506 (ObjNum 1196 snap 398 seq_num 1196) dirty exists 2024-09-14T04:01:45.205 INFO:tasks.rados.rados.0.smithi070.stdout:3939: left oid 49 (ObjNum 1196 snap 398 seq_num 1196) 2024-09-14T04:01:45.206 INFO:tasks.rados.rados.0.smithi070.stdout:3938: finishing copy_from racing read to smithi07085154-1 2024-09-14T04:01:45.208 INFO:tasks.rados.rados.0.smithi070.stdout:3938: finishing copy_from to smithi07085154-1 2024-09-14T04:01:45.209 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 1 v 4259 (ObjNum 1149 snap 382 seq_num 1149) dirty exists 2024-09-14T04:01:45.209 INFO:tasks.rados.rados.0.smithi070.stdout:3940: finishing write tid 1 to smithi07085154-23 2024-09-14T04:01:45.210 INFO:tasks.rados.rados.0.smithi070.stdout:3940: finishing write tid 2 to smithi07085154-23 2024-09-14T04:01:45.210 INFO:tasks.rados.rados.0.smithi070.stdout:3940: finishing write tid 3 to smithi07085154-23 2024-09-14T04:01:45.211 INFO:tasks.rados.rados.0.smithi070.stdout:3940: finishing write tid 4 to smithi07085154-23 2024-09-14T04:01:45.211 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 4162 (ObjNum 1197 snap 398 seq_num 1197) dirty exists 2024-09-14T04:01:45.211 INFO:tasks.rados.rados.0.smithi070.stdout:3940: left oid 23 (ObjNum 1197 snap 398 seq_num 1197) 2024-09-14T04:01:45.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:45 smithi094 ceph-mon[86597]: pgmap v897: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 492 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.6 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2024-09-14T04:01:45.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:45 smithi070 ceph-mon[96704]: pgmap v897: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 492 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.6 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2024-09-14T04:01:45.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:45 smithi070 ceph-mon[98640]: pgmap v897: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 492 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.6 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2024-09-14T04:01:46.187 INFO:tasks.rados.rados.0.smithi070.stdout:3938: done (3 left) 2024-09-14T04:01:46.187 INFO:tasks.rados.rados.0.smithi070.stdout:3939: done (2 left) 2024-09-14T04:01:46.187 INFO:tasks.rados.rados.0.smithi070.stdout:3940: done (1 left) 2024-09-14T04:01:46.187 INFO:tasks.rados.rados.0.smithi070.stdout:3942: done (0 left) 2024-09-14T04:01:46.187 INFO:tasks.rados.rados.0.smithi070.stdout:3943: snap_create 2024-09-14T04:01:46.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:46 smithi094 ceph-mon[86597]: osdmap e865: 8 total, 8 up, 8 in 2024-09-14T04:01:46.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:46 smithi070 ceph-mon[96704]: osdmap e865: 8 total, 8 up, 8 in 2024-09-14T04:01:46.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:46 smithi070 ceph-mon[98640]: osdmap e865: 8 total, 8 up, 8 in 2024-09-14T04:01:47.188 INFO:tasks.rados.rados.0.smithi070.stdout:3943: done (0 left) 2024-09-14T04:01:47.188 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 48 current snap is 400 2024-09-14T04:01:47.188 INFO:tasks.rados.rados.0.smithi070.stdout:3944: seq_num 1198 ranges {2981888=466944,3448832=401408} 2024-09-14T04:01:47.206 INFO:tasks.rados.rados.0.smithi070.stdout:3944: writing smithi07085154-48 from 2981888 to 3448832 tid 1 2024-09-14T04:01:47.209 INFO:tasks.rados.rados.0.smithi070.stdout:3944: writing smithi07085154-48 from 3448832 to 3850240 tid 2 2024-09-14T04:01:47.209 INFO:tasks.rados.rados.0.smithi070.stdout:3945: snap_create 2024-09-14T04:01:47.214 INFO:tasks.rados.rados.0.smithi070.stdout:3944: finishing write tid 1 to smithi07085154-48 2024-09-14T04:01:47.215 INFO:tasks.rados.rados.0.smithi070.stdout:3944: finishing write tid 2 to smithi07085154-48 2024-09-14T04:01:47.215 INFO:tasks.rados.rados.0.smithi070.stdout:3944: finishing write tid 3 to smithi07085154-48 2024-09-14T04:01:47.216 INFO:tasks.rados.rados.0.smithi070.stdout:3944: finishing write tid 4 to smithi07085154-48 2024-09-14T04:01:47.216 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 3217 (ObjNum 1198 snap 400 seq_num 1198) dirty exists 2024-09-14T04:01:47.216 INFO:tasks.rados.rados.0.smithi070.stdout:3944: left oid 48 (ObjNum 1198 snap 400 seq_num 1198) 2024-09-14T04:01:47.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:47 smithi094 ceph-mon[86597]: pgmap v899: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 495 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 13 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-14T04:01:47.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:47 smithi094 ceph-mon[86597]: osdmap e866: 8 total, 8 up, 8 in 2024-09-14T04:01:47.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:47 smithi070 ceph-mon[96704]: pgmap v899: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 495 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 13 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-14T04:01:47.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:47 smithi070 ceph-mon[96704]: osdmap e866: 8 total, 8 up, 8 in 2024-09-14T04:01:47.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:47 smithi070 ceph-mon[98640]: pgmap v899: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 495 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 13 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-14T04:01:47.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:47 smithi070 ceph-mon[98640]: osdmap e866: 8 total, 8 up, 8 in 2024-09-14T04:01:48.198 INFO:tasks.rados.rados.0.smithi070.stdout:3944: done (1 left) 2024-09-14T04:01:48.198 INFO:tasks.rados.rados.0.smithi070.stdout:3945: done (0 left) 2024-09-14T04:01:48.198 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 31 current snap is 401 2024-09-14T04:01:48.198 INFO:tasks.rados.rados.0.smithi070.stdout:3946: seq_num 1199 ranges {483328=483328,966656=401408} 2024-09-14T04:01:48.203 INFO:tasks.rados.rados.0.smithi070.stdout:3946: writing smithi07085154-31 from 483328 to 966656 tid 1 2024-09-14T04:01:48.205 INFO:tasks.rados.rados.0.smithi070.stdout:3946: writing smithi07085154-31 from 966656 to 1368064 tid 2 2024-09-14T04:01:48.205 INFO:tasks.rados.rados.0.smithi070.stdout:3947: copy_from oid 50 from oid 15 current snap is 401 2024-09-14T04:01:48.205 INFO:tasks.rados.rados.0.smithi070.stdout:3948: snap_create 2024-09-14T04:01:48.211 INFO:tasks.rados.rados.0.smithi070.stdout:3946: finishing write tid 1 to smithi07085154-31 2024-09-14T04:01:48.211 INFO:tasks.rados.rados.0.smithi070.stdout:3946: finishing write tid 2 to smithi07085154-31 2024-09-14T04:01:48.211 INFO:tasks.rados.rados.0.smithi070.stdout:3946: finishing write tid 3 to smithi07085154-31 2024-09-14T04:01:48.213 INFO:tasks.rados.rados.0.smithi070.stdout:3946: finishing write tid 4 to smithi07085154-31 2024-09-14T04:01:48.213 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 4651 (ObjNum 1199 snap 401 seq_num 1199) dirty exists 2024-09-14T04:01:48.213 INFO:tasks.rados.rados.0.smithi070.stdout:3946: left oid 31 (ObjNum 1199 snap 401 seq_num 1199) 2024-09-14T04:01:48.235 INFO:tasks.rados.rados.0.smithi070.stdout:3947: finishing copy_from racing read to smithi07085154-50 2024-09-14T04:01:48.238 INFO:tasks.rados.rados.0.smithi070.stdout:3947: finishing copy_from to smithi07085154-50 2024-09-14T04:01:48.238 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 4163 (ObjNum 1120 snap 374 seq_num 1120) dirty exists 2024-09-14T04:01:48.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:48 smithi094 ceph-mon[86597]: osdmap e867: 8 total, 8 up, 8 in 2024-09-14T04:01:48.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:48 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:01:48.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:48 smithi094 ceph-mon[86597]: pgmap v902: 121 pgs: 121 active+clean; 491 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 21 MiB/s rd, 785 KiB/s wr, 16 op/s 2024-09-14T04:01:48.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:48 smithi070 ceph-mon[96704]: osdmap e867: 8 total, 8 up, 8 in 2024-09-14T04:01:48.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:48 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:01:48.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:48 smithi070 ceph-mon[96704]: pgmap v902: 121 pgs: 121 active+clean; 491 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 21 MiB/s rd, 785 KiB/s wr, 16 op/s 2024-09-14T04:01:48.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:48 smithi070 ceph-mon[98640]: osdmap e867: 8 total, 8 up, 8 in 2024-09-14T04:01:48.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:48 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:01:48.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:48 smithi070 ceph-mon[98640]: pgmap v902: 121 pgs: 121 active+clean; 491 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 21 MiB/s rd, 785 KiB/s wr, 16 op/s 2024-09-14T04:01:49.160 INFO:tasks.rados.rados.0.smithi070.stdout:3946: done (2 left) 2024-09-14T04:01:49.160 INFO:tasks.rados.rados.0.smithi070.stdout:3947: done (1 left) 2024-09-14T04:01:49.161 INFO:tasks.rados.rados.0.smithi070.stdout:3948: done (0 left) 2024-09-14T04:01:49.161 INFO:tasks.rados.rados.0.smithi070.stdout:3949: rollback oid 31 current snap is 402 2024-09-14T04:01:49.161 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 31 to 399 2024-09-14T04:01:49.161 INFO:tasks.rados.rados.0.smithi070.stdout:3950: read oid 29 snap -1 2024-09-14T04:01:49.161 INFO:tasks.rados.rados.0.smithi070.stdout:3950: expect (ObjNum 1149 snap 382 seq_num 1149) 2024-09-14T04:01:49.161 INFO:tasks.rados.rados.0.smithi070.stdout:3951: copy_from oid 11 from oid 18 current snap is 402 2024-09-14T04:01:49.161 INFO:tasks.rados.rados.0.smithi070.stdout:3952: snap_remove snap 390 2024-09-14T04:01:49.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:49 smithi094 ceph-mon[86597]: osdmap e868: 8 total, 8 up, 8 in 2024-09-14T04:01:49.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:49 smithi094 ceph-mon[86597]: osdmap e869: 8 total, 8 up, 8 in 2024-09-14T04:01:49.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:49 smithi070 ceph-mon[96704]: osdmap e868: 8 total, 8 up, 8 in 2024-09-14T04:01:49.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:49 smithi070 ceph-mon[96704]: osdmap e869: 8 total, 8 up, 8 in 2024-09-14T04:01:49.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:49 smithi070 ceph-mon[98640]: osdmap e868: 8 total, 8 up, 8 in 2024-09-14T04:01:49.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:49 smithi070 ceph-mon[98640]: osdmap e869: 8 total, 8 up, 8 in 2024-09-14T04:01:50.164 INFO:tasks.rados.rados.0.smithi070.stdout:3952: done (3 left) 2024-09-14T04:01:50.164 INFO:tasks.rados.rados.0.smithi070.stdout:3953: read oid 8 snap -1 2024-09-14T04:01:50.164 INFO:tasks.rados.rados.0.smithi070.stdout:3953: expect deleted 2024-09-14T04:01:50.164 INFO:tasks.rados.rados.0.smithi070.stdout:3954: read oid 49 snap 361 2024-09-14T04:01:50.165 INFO:tasks.rados.rados.0.smithi070.stdout:3954: expect (ObjNum 1080 snap 350 seq_num 1080) 2024-09-14T04:01:50.165 INFO:tasks.rados.rados.0.smithi070.stdout:3955: read oid 33 snap -1 2024-09-14T04:01:50.165 INFO:tasks.rados.rados.0.smithi070.stdout:3955: expect (ObjNum 1193 snap 397 seq_num 1193) 2024-09-14T04:01:50.165 INFO:tasks.rados.rados.0.smithi070.stdout:3956: read oid 24 snap -1 2024-09-14T04:01:50.165 INFO:tasks.rados.rados.0.smithi070.stdout:3956: expect deleted 2024-09-14T04:01:50.165 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 23 current snap is 402 2024-09-14T04:01:50.165 INFO:tasks.rados.rados.0.smithi070.stdout:3957: seq_num 1200 ranges {3334144=499712,3833856=401408} 2024-09-14T04:01:50.188 INFO:tasks.rados.rados.0.smithi070.stdout:3957: writing smithi07085154-23 from 3334144 to 3833856 tid 1 2024-09-14T04:01:50.191 INFO:tasks.rados.rados.0.smithi070.stdout:3957: writing smithi07085154-23 from 3833856 to 4235264 tid 2 2024-09-14T04:01:50.192 INFO:tasks.rados.rados.0.smithi070.stdout:3958: read oid 17 snap -1 2024-09-14T04:01:50.192 INFO:tasks.rados.rados.0.smithi070.stdout:3958: expect (ObjNum 1042 snap 342 seq_num 1042) 2024-09-14T04:01:50.192 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 28 current snap is 402 2024-09-14T04:01:50.192 INFO:tasks.rados.rados.0.smithi070.stdout:3959: seq_num 1201 ranges {4890624=524288,5414912=401408} 2024-09-14T04:01:50.221 INFO:tasks.rados.rados.0.smithi070.stdout:3959: writing smithi07085154-28 from 4890624 to 5414912 tid 1 2024-09-14T04:01:50.224 INFO:tasks.rados.rados.0.smithi070.stdout:3959: writing smithi07085154-28 from 5414912 to 5816320 tid 2 2024-09-14T04:01:50.224 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 48 current snap is 402 2024-09-14T04:01:50.224 INFO:tasks.rados.rados.0.smithi070.stdout:3949: finishing rollback tid 0 to smithi07085154-31 2024-09-14T04:01:50.224 INFO:tasks.rados.rados.0.smithi070.stdout:3960: seq_num 1202 ranges {3850240=540672,4390912=401408} 2024-09-14T04:01:50.247 INFO:tasks.rados.rados.0.smithi070.stdout:3960: writing smithi07085154-48 from 3850240 to 4390912 tid 1 2024-09-14T04:01:50.250 INFO:tasks.rados.rados.0.smithi070.stdout:3960: writing smithi07085154-48 from 4390912 to 4792320 tid 2 2024-09-14T04:01:50.250 INFO:tasks.rados.rados.0.smithi070.stdout:3961: copy_from oid 50 from oid 1 current snap is 402 2024-09-14T04:01:50.250 INFO:tasks.rados.rados.0.smithi070.stdout:3949: finishing rollback tid 1 to smithi07085154-31 2024-09-14T04:01:50.250 INFO:tasks.rados.rados.0.smithi070.stdout:3949: finishing rollback tid 2 to smithi07085154-31 2024-09-14T04:01:50.250 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 31 v 4654 (ObjNum 1175 snap 389 seq_num 1175) dirty exists 2024-09-14T04:01:50.250 INFO:tasks.rados.rados.0.smithi070.stdout:3950: expect (ObjNum 1149 snap 382 seq_num 1149) 2024-09-14T04:01:50.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:01:50 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:01:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:01:50.535 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 04:01:50 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1[108413]: 2024-09-14T04:01:50.188+0000 7fde30371640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap 1d8 after trimming was started 2024-09-14T04:01:50.578 INFO:tasks.rados.rados.0.smithi070.stdout:3951: finishing copy_from racing read to smithi07085154-11 2024-09-14T04:01:50.578 INFO:tasks.rados.rados.0.smithi070.stdout:3951: finishing copy_from to smithi07085154-11 2024-09-14T04:01:50.578 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 11 v 4171 (ObjNum 596 snap 199 seq_num 596) dirty exists 2024-09-14T04:01:50.578 INFO:tasks.rados.rados.0.smithi070.stdout:3954: expect (ObjNum 1080 snap 350 seq_num 1080) 2024-09-14T04:01:50.610 INFO:tasks.rados.rados.0.smithi070.stdout:3955: expect (ObjNum 1193 snap 397 seq_num 1193) 2024-09-14T04:01:50.767 INFO:tasks.rados.rados.0.smithi070.stdout:3957: finishing write tid 1 to smithi07085154-23 2024-09-14T04:01:50.767 INFO:tasks.rados.rados.0.smithi070.stdout:3957: finishing write tid 2 to smithi07085154-23 2024-09-14T04:01:50.767 INFO:tasks.rados.rados.0.smithi070.stdout:3949: done (11 left) 2024-09-14T04:01:50.767 INFO:tasks.rados.rados.0.smithi070.stdout:3950: done (10 left) 2024-09-14T04:01:50.767 INFO:tasks.rados.rados.0.smithi070.stdout:3951: done (9 left) 2024-09-14T04:01:50.768 INFO:tasks.rados.rados.0.smithi070.stdout:3953: done (8 left) 2024-09-14T04:01:50.768 INFO:tasks.rados.rados.0.smithi070.stdout:3954: done (7 left) 2024-09-14T04:01:50.768 INFO:tasks.rados.rados.0.smithi070.stdout:3955: done (6 left) 2024-09-14T04:01:50.768 INFO:tasks.rados.rados.0.smithi070.stdout:3956: done (5 left) 2024-09-14T04:01:50.768 INFO:tasks.rados.rados.0.smithi070.stdout:3962: read oid 44 snap -1 2024-09-14T04:01:50.768 INFO:tasks.rados.rados.0.smithi070.stdout:3962: expect (ObjNum 1182 snap 393 seq_num 1182) 2024-09-14T04:01:50.768 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 3 current snap is 402 2024-09-14T04:01:50.768 INFO:tasks.rados.rados.0.smithi070.stdout:3957: finishing write tid 3 to smithi07085154-23 2024-09-14T04:01:50.768 INFO:tasks.rados.rados.0.smithi070.stdout:3963: seq_num 1203 ranges {2850816=557056,3407872=401408} 2024-09-14T04:01:50.785 INFO:tasks.rados.rados.0.smithi070.stdout:3963: writing smithi07085154-3 from 2850816 to 3407872 tid 1 2024-09-14T04:01:50.787 INFO:tasks.rados.rados.0.smithi070.stdout:3963: writing smithi07085154-3 from 3407872 to 3809280 tid 2 2024-09-14T04:01:50.788 INFO:tasks.rados.rados.0.smithi070.stdout:3964: snap_create 2024-09-14T04:01:50.788 INFO:tasks.rados.rados.0.smithi070.stdout:3957: finishing write tid 4 to smithi07085154-23 2024-09-14T04:01:50.788 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 23 v 4165 (ObjNum 1200 snap 402 seq_num 1200) dirty exists 2024-09-14T04:01:50.788 INFO:tasks.rados.rados.0.smithi070.stdout:3957: left oid 23 (ObjNum 1200 snap 402 seq_num 1200) 2024-09-14T04:01:50.788 INFO:tasks.rados.rados.0.smithi070.stdout:3958: expect (ObjNum 1042 snap 342 seq_num 1042) 2024-09-14T04:01:50.882 INFO:tasks.rados.rados.0.smithi070.stdout:3959: finishing write tid 1 to smithi07085154-28 2024-09-14T04:01:50.883 INFO:tasks.rados.rados.0.smithi070.stdout:3959: finishing write tid 2 to smithi07085154-28 2024-09-14T04:01:50.883 INFO:tasks.rados.rados.0.smithi070.stdout:3959: finishing write tid 3 to smithi07085154-28 2024-09-14T04:01:50.883 INFO:tasks.rados.rados.0.smithi070.stdout:3959: finishing write tid 4 to smithi07085154-28 2024-09-14T04:01:50.883 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 28 v 4234 (ObjNum 1201 snap 402 seq_num 1201) dirty exists 2024-09-14T04:01:50.883 INFO:tasks.rados.rados.0.smithi070.stdout:3959: left oid 28 (ObjNum 1201 snap 402 seq_num 1201) 2024-09-14T04:01:50.883 INFO:tasks.rados.rados.0.smithi070.stdout:3960: finishing write tid 1 to smithi07085154-48 2024-09-14T04:01:50.883 INFO:tasks.rados.rados.0.smithi070.stdout:3960: finishing write tid 2 to smithi07085154-48 2024-09-14T04:01:50.883 INFO:tasks.rados.rados.0.smithi070.stdout:3960: finishing write tid 3 to smithi07085154-48 2024-09-14T04:01:50.883 INFO:tasks.rados.rados.0.smithi070.stdout:3960: finishing write tid 4 to smithi07085154-48 2024-09-14T04:01:50.883 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 48 v 3220 (ObjNum 1202 snap 402 seq_num 1202) dirty exists 2024-09-14T04:01:50.883 INFO:tasks.rados.rados.0.smithi070.stdout:3960: left oid 48 (ObjNum 1202 snap 402 seq_num 1202) 2024-09-14T04:01:50.883 INFO:tasks.rados.rados.0.smithi070.stdout:3961: finishing copy_from racing read to smithi07085154-50 2024-09-14T04:01:50.883 INFO:tasks.rados.rados.0.smithi070.stdout:3961: finishing copy_from to smithi07085154-50 2024-09-14T04:01:50.884 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 4259 (ObjNum 1149 snap 382 seq_num 1149) dirty exists 2024-09-14T04:01:50.884 INFO:tasks.rados.rados.0.smithi070.stdout:3962: expect (ObjNum 1182 snap 393 seq_num 1182) 2024-09-14T04:01:50.993 INFO:tasks.rados.rados.0.smithi070.stdout:3963: finishing write tid 1 to smithi07085154-3 2024-09-14T04:01:50.993 INFO:tasks.rados.rados.0.smithi070.stdout:3963: finishing write tid 2 to smithi07085154-3 2024-09-14T04:01:50.993 INFO:tasks.rados.rados.0.smithi070.stdout:3963: finishing write tid 3 to smithi07085154-3 2024-09-14T04:01:50.993 INFO:tasks.rados.rados.0.smithi070.stdout:3963: finishing write tid 4 to smithi07085154-3 2024-09-14T04:01:50.993 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 3 v 4265 (ObjNum 1203 snap 402 seq_num 1203) dirty exists 2024-09-14T04:01:50.993 INFO:tasks.rados.rados.0.smithi070.stdout:3963: left oid 3 (ObjNum 1203 snap 402 seq_num 1203) 2024-09-14T04:01:51.166 INFO:tasks.rados.rados.0.smithi070.stdout:3957: done (7 left) 2024-09-14T04:01:51.167 INFO:tasks.rados.rados.0.smithi070.stdout:3958: done (6 left) 2024-09-14T04:01:51.167 INFO:tasks.rados.rados.0.smithi070.stdout:3959: done (5 left) 2024-09-14T04:01:51.167 INFO:tasks.rados.rados.0.smithi070.stdout:3960: done (4 left) 2024-09-14T04:01:51.167 INFO:tasks.rados.rados.0.smithi070.stdout:3961: done (3 left) 2024-09-14T04:01:51.167 INFO:tasks.rados.rados.0.smithi070.stdout:3962: done (2 left) 2024-09-14T04:01:51.167 INFO:tasks.rados.rados.0.smithi070.stdout:3963: done (1 left) 2024-09-14T04:01:51.167 INFO:tasks.rados.rados.0.smithi070.stdout:3964: done (0 left) 2024-09-14T04:01:51.167 INFO:tasks.rados.rados.0.smithi070.stdout:3965: read oid 13 snap -1 2024-09-14T04:01:51.167 INFO:tasks.rados.rados.0.smithi070.stdout:3965: expect (ObjNum 1169 snap 389 seq_num 1169) 2024-09-14T04:01:51.167 INFO:tasks.rados.rados.0.smithi070.stdout:3966: delete oid 29 current snap is 403 2024-09-14T04:01:51.174 INFO:tasks.rados.rados.0.smithi070.stdout:3966: done (1 left) 2024-09-14T04:01:51.174 INFO:tasks.rados.rados.0.smithi070.stdout:3967: snap_remove snap 371 2024-09-14T04:01:51.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:51 smithi094 ceph-mon[86597]: osdmap e870: 8 total, 8 up, 8 in 2024-09-14T04:01:51.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:51 smithi094 ceph-mon[86597]: pgmap v906: 121 pgs: 121 active+clean; 497 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.9 MiB/s rd, 2.3 MiB/s wr, 14 op/s 2024-09-14T04:01:51.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:51 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:01:51.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:51 smithi070 ceph-mon[98640]: osdmap e870: 8 total, 8 up, 8 in 2024-09-14T04:01:51.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:51 smithi070 ceph-mon[98640]: pgmap v906: 121 pgs: 121 active+clean; 497 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.9 MiB/s rd, 2.3 MiB/s wr, 14 op/s 2024-09-14T04:01:51.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:51 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:01:51.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:51 smithi070 ceph-mon[96704]: osdmap e870: 8 total, 8 up, 8 in 2024-09-14T04:01:51.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:51 smithi070 ceph-mon[96704]: pgmap v906: 121 pgs: 121 active+clean; 497 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.9 MiB/s rd, 2.3 MiB/s wr, 14 op/s 2024-09-14T04:01:51.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:51 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:01:52.170 INFO:tasks.rados.rados.0.smithi070.stdout:3967: done (1 left) 2024-09-14T04:01:52.170 INFO:tasks.rados.rados.0.smithi070.stdout:3968: snap_create 2024-09-14T04:01:52.170 INFO:tasks.rados.rados.0.smithi070.stdout:3965: expect (ObjNum 1169 snap 389 seq_num 1169) 2024-09-14T04:01:52.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:52 smithi094 ceph-mon[86597]: osdmap e871: 8 total, 8 up, 8 in 2024-09-14T04:01:52.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:52 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:52.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:52 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:01:52.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:52 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:01:52.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:52 smithi070 ceph-mon[96704]: osdmap e871: 8 total, 8 up, 8 in 2024-09-14T04:01:52.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:52 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:52.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:52 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:01:52.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:52 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:01:52.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:52 smithi070 ceph-mon[98640]: osdmap e871: 8 total, 8 up, 8 in 2024-09-14T04:01:52.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:52 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:52.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:52 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:01:52.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:52 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:01:53.173 INFO:tasks.rados.rados.0.smithi070.stdout:3965: done (1 left) 2024-09-14T04:01:53.173 INFO:tasks.rados.rados.0.smithi070.stdout:3968: done (0 left) 2024-09-14T04:01:53.173 INFO:tasks.rados.rados.0.smithi070.stdout:3969: copy_from oid 22 from oid 12 current snap is 404 2024-09-14T04:01:53.173 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 45 current snap is 404 2024-09-14T04:01:53.173 INFO:tasks.rados.rados.0.smithi070.stdout:3970: seq_num 1205 ranges {11436032=589824,12025856=401408} 2024-09-14T04:01:53.237 INFO:tasks.rados.rados.0.smithi070.stdout:3970: writing smithi07085154-45 from 11436032 to 12025856 tid 1 2024-09-14T04:01:53.240 INFO:tasks.rados.rados.0.smithi070.stdout:3970: writing smithi07085154-45 from 12025856 to 12427264 tid 2 2024-09-14T04:01:53.240 INFO:tasks.rados.rados.0.smithi070.stdout:3971: rollback oid 21 current snap is 404 2024-09-14T04:01:53.240 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 21 to 377 2024-09-14T04:01:53.240 INFO:tasks.rados.rados.0.smithi070.stdout:3969: finishing copy_from racing read to smithi07085154-22 2024-09-14T04:01:53.241 INFO:tasks.rados.rados.0.smithi070.stdout:3969: finishing copy_from to smithi07085154-22 2024-09-14T04:01:53.241 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 22 v 3214 (ObjNum 1014 snap 334 seq_num 1014) dirty exists 2024-09-14T04:01:53.241 INFO:tasks.rados.rados.0.smithi070.stdout:3969: done (2 left) 2024-09-14T04:01:53.241 INFO:tasks.rados.rados.0.smithi070.stdout:3972: read oid 26 snap -1 2024-09-14T04:01:53.241 INFO:tasks.rados.rados.0.smithi070.stdout:3972: expect (ObjNum 1181 snap 392 seq_num 1181) 2024-09-14T04:01:53.241 INFO:tasks.rados.rados.0.smithi070.stdout:3973: read oid 3 snap -1 2024-09-14T04:01:53.241 INFO:tasks.rados.rados.0.smithi070.stdout:3973: expect (ObjNum 1203 snap 402 seq_num 1203) 2024-09-14T04:01:53.241 INFO:tasks.rados.rados.0.smithi070.stdout:3974: snap_remove snap 358 2024-09-14T04:01:53.437 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:01:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:01:53] "GET /metrics HTTP/1.1" 200 36498 "" "Prometheus/2.51.0" 2024-09-14T04:01:53.437 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:53 smithi070 ceph-mon[98640]: osdmap e872: 8 total, 8 up, 8 in 2024-09-14T04:01:53.437 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:53 smithi070 ceph-mon[98640]: pgmap v909: 121 pgs: 121 active+clean; 514 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.1 MiB/s rd, 6.3 MiB/s wr, 10 op/s 2024-09-14T04:01:53.438 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:53 smithi070 ceph-mon[96704]: osdmap e872: 8 total, 8 up, 8 in 2024-09-14T04:01:53.438 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:53 smithi070 ceph-mon[96704]: pgmap v909: 121 pgs: 121 active+clean; 514 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.1 MiB/s rd, 6.3 MiB/s wr, 10 op/s 2024-09-14T04:01:53.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:53 smithi094 ceph-mon[86597]: osdmap e872: 8 total, 8 up, 8 in 2024-09-14T04:01:53.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:53 smithi094 ceph-mon[86597]: pgmap v909: 121 pgs: 121 active+clean; 514 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.1 MiB/s rd, 6.3 MiB/s wr, 10 op/s 2024-09-14T04:01:54.162 INFO:tasks.rados.rados.0.smithi070.stdout:3970: finishing write tid 1 to smithi07085154-45 2024-09-14T04:01:54.162 INFO:tasks.rados.rados.0.smithi070.stdout:3970: finishing write tid 2 to smithi07085154-45 2024-09-14T04:01:54.162 INFO:tasks.rados.rados.0.smithi070.stdout:3970: finishing write tid 3 to smithi07085154-45 2024-09-14T04:01:54.162 INFO:tasks.rados.rados.0.smithi070.stdout:3971: finishing rollback tid 0 to smithi07085154-21 2024-09-14T04:01:54.162 INFO:tasks.rados.rados.0.smithi070.stdout:3970: finishing write tid 4 to smithi07085154-45 2024-09-14T04:01:54.162 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 45 v 4509 (ObjNum 1205 snap 404 seq_num 1205) dirty exists 2024-09-14T04:01:54.162 INFO:tasks.rados.rados.0.smithi070.stdout:3970: left oid 45 (ObjNum 1205 snap 404 seq_num 1205) 2024-09-14T04:01:54.163 INFO:tasks.rados.rados.0.smithi070.stdout:3971: finishing rollback tid 1 to smithi07085154-21 2024-09-14T04:01:54.163 INFO:tasks.rados.rados.0.smithi070.stdout:3971: finishing rollback tid 2 to smithi07085154-21 2024-09-14T04:01:54.163 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 21 v 1734 (ObjNum 1091 snap 355 seq_num 1091) dirty exists 2024-09-14T04:01:54.163 INFO:tasks.rados.rados.0.smithi070.stdout:3972: expect (ObjNum 1181 snap 392 seq_num 1181) 2024-09-14T04:01:54.288 INFO:tasks.rados.rados.0.smithi070.stdout:3973: expect (ObjNum 1203 snap 402 seq_num 1203) 2024-09-14T04:01:54.501 INFO:tasks.rados.rados.0.smithi070.stdout:3970: done (4 left) 2024-09-14T04:01:54.501 INFO:tasks.rados.rados.0.smithi070.stdout:3971: done (3 left) 2024-09-14T04:01:54.501 INFO:tasks.rados.rados.0.smithi070.stdout:3972: done (2 left) 2024-09-14T04:01:54.501 INFO:tasks.rados.rados.0.smithi070.stdout:3973: done (1 left) 2024-09-14T04:01:54.501 INFO:tasks.rados.rados.0.smithi070.stdout:3974: done (0 left) 2024-09-14T04:01:54.501 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 5 current snap is 404 2024-09-14T04:01:54.501 INFO:tasks.rados.rados.0.smithi070.stdout:3975: seq_num 1206 ranges {2555904=606208,3162112=401408} 2024-09-14T04:01:54.517 INFO:tasks.rados.rados.0.smithi070.stdout:3975: writing smithi07085154-5 from 2555904 to 3162112 tid 1 2024-09-14T04:01:54.520 INFO:tasks.rados.rados.0.smithi070.stdout:3975: writing smithi07085154-5 from 3162112 to 3563520 tid 2 2024-09-14T04:01:54.520 INFO:tasks.rados.rados.0.smithi070.stdout:3976: rmattr oid 43 current snap is 404 2024-09-14T04:01:54.520 INFO:tasks.rados.rados.0.smithi070.stdout:3977: copy_from oid 31 from oid 34 current snap is 404 2024-09-14T04:01:54.520 INFO:tasks.rados.rados.0.smithi070.stdout:3978: snap_create 2024-09-14T04:01:54.521 INFO:tasks.rados.rados.0.smithi070.stdout:3977: finishing copy_from to smithi07085154-31 2024-09-14T04:01:54.521 INFO:tasks.rados.rados.0.smithi070.stdout:3977: got expected ENOENT (src dne) 2024-09-14T04:01:54.521 INFO:tasks.rados.rados.0.smithi070.stdout:3977: finishing copy_from racing read to smithi07085154-31 2024-09-14T04:01:54.522 INFO:tasks.rados.rados.0.smithi070.stdout:3975: finishing write tid 1 to smithi07085154-5 2024-09-14T04:01:54.522 INFO:tasks.rados.rados.0.smithi070.stdout:3975: finishing write tid 2 to smithi07085154-5 2024-09-14T04:01:54.523 INFO:tasks.rados.rados.0.smithi070.stdout:3975: finishing write tid 3 to smithi07085154-5 2024-09-14T04:01:54.523 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 4172 (ObjNum 681 snap 230 seq_num 681) dirty exists 2024-09-14T04:01:54.523 INFO:tasks.rados.rados.0.smithi070.stdout:3975: finishing write tid 4 to smithi07085154-5 2024-09-14T04:01:54.524 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 5 v 3205 (ObjNum 1206 snap 404 seq_num 1206) dirty exists 2024-09-14T04:01:54.524 INFO:tasks.rados.rados.0.smithi070.stdout:3975: left oid 5 (ObjNum 1206 snap 404 seq_num 1206) 2024-09-14T04:01:54.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:54 smithi094 ceph-mon[86597]: osdmap e873: 8 total, 8 up, 8 in 2024-09-14T04:01:54.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:54 smithi094 ceph-mon[86597]: osdmap e874: 8 total, 8 up, 8 in 2024-09-14T04:01:54.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:54 smithi070 ceph-mon[96704]: osdmap e873: 8 total, 8 up, 8 in 2024-09-14T04:01:54.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:54 smithi070 ceph-mon[96704]: osdmap e874: 8 total, 8 up, 8 in 2024-09-14T04:01:54.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:54 smithi070 ceph-mon[98640]: osdmap e873: 8 total, 8 up, 8 in 2024-09-14T04:01:54.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:54 smithi070 ceph-mon[98640]: osdmap e874: 8 total, 8 up, 8 in 2024-09-14T04:01:55.180 INFO:tasks.rados.rados.0.smithi070.stdout:3975: done (3 left) 2024-09-14T04:01:55.181 INFO:tasks.rados.rados.0.smithi070.stdout:3976: done (2 left) 2024-09-14T04:01:55.181 INFO:tasks.rados.rados.0.smithi070.stdout:3977: done (1 left) 2024-09-14T04:01:55.181 INFO:tasks.rados.rados.0.smithi070.stdout:3978: done (0 left) 2024-09-14T04:01:55.181 INFO:tasks.rados.rados.0.smithi070.stdout:3979: rollback oid 50 current snap is 405 2024-09-14T04:01:55.181 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 50 to 377 2024-09-14T04:01:55.181 INFO:tasks.rados.rados.0.smithi070.stdout:3980: rollback oid 8 current snap is 405 2024-09-14T04:01:55.181 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 8 to 396 2024-09-14T04:01:55.181 INFO:tasks.rados.rados.0.smithi070.stdout:3981: snap_remove snap 391 2024-09-14T04:01:55.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:55 smithi094 ceph-mon[86597]: pgmap v912: 121 pgs: 121 active+clean; 515 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 12 MiB/s rd, 4.6 MiB/s wr, 10 op/s 2024-09-14T04:01:55.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:55 smithi070 ceph-mon[96704]: pgmap v912: 121 pgs: 121 active+clean; 515 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 12 MiB/s rd, 4.6 MiB/s wr, 10 op/s 2024-09-14T04:01:55.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:55 smithi070 ceph-mon[98640]: pgmap v912: 121 pgs: 121 active+clean; 515 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 12 MiB/s rd, 4.6 MiB/s wr, 10 op/s 2024-09-14T04:01:56.193 INFO:tasks.rados.rados.0.smithi070.stdout:3980: finishing rollback tid 1 to smithi07085154-8 2024-09-14T04:01:56.193 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 8 v 4155 (ObjNum 33 snap 0 seq_num 2930190768) dirty dne 2024-09-14T04:01:56.193 INFO:tasks.rados.rados.0.smithi070.stdout:3979: finishing rollback tid 0 to smithi07085154-50 2024-09-14T04:01:56.193 INFO:tasks.rados.rados.0.smithi070.stdout:3979: finishing rollback tid 1 to smithi07085154-50 2024-09-14T04:01:56.193 INFO:tasks.rados.rados.0.smithi070.stdout:3979: finishing rollback tid 2 to smithi07085154-50 2024-09-14T04:01:56.193 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 4264 (ObjNum 1116 snap 371 seq_num 1116) dirty exists 2024-09-14T04:01:56.193 INFO:tasks.rados.rados.0.smithi070.stdout:3979: done (2 left) 2024-09-14T04:01:56.193 INFO:tasks.rados.rados.0.smithi070.stdout:3980: done (1 left) 2024-09-14T04:01:56.194 INFO:tasks.rados.rados.0.smithi070.stdout:3981: done (0 left) 2024-09-14T04:01:56.194 INFO:tasks.rados.rados.0.smithi070.stdout:3982: snap_remove snap 399 2024-09-14T04:01:56.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:56 smithi094 ceph-mon[86597]: osdmap e875: 8 total, 8 up, 8 in 2024-09-14T04:01:56.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:56 smithi070 ceph-mon[98640]: osdmap e875: 8 total, 8 up, 8 in 2024-09-14T04:01:56.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:56 smithi070 ceph-mon[96704]: osdmap e875: 8 total, 8 up, 8 in 2024-09-14T04:01:57.194 INFO:tasks.rados.rados.0.smithi070.stdout:3982: done (0 left) 2024-09-14T04:01:57.194 INFO:tasks.rados.rados.0.smithi070.stdout:3983: snap_create 2024-09-14T04:01:57.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:57 smithi094 ceph-mon[86597]: pgmap v914: 121 pgs: 121 active+clean; 516 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.3 MiB/s rd, 471 KiB/s wr, 7 op/s 2024-09-14T04:01:57.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:57 smithi094 ceph-mon[86597]: osdmap e876: 8 total, 8 up, 8 in 2024-09-14T04:01:57.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:57 smithi070 ceph-mon[98640]: pgmap v914: 121 pgs: 121 active+clean; 516 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.3 MiB/s rd, 471 KiB/s wr, 7 op/s 2024-09-14T04:01:57.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:57 smithi070 ceph-mon[98640]: osdmap e876: 8 total, 8 up, 8 in 2024-09-14T04:01:57.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:57 smithi070 ceph-mon[96704]: pgmap v914: 121 pgs: 121 active+clean; 516 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.3 MiB/s rd, 471 KiB/s wr, 7 op/s 2024-09-14T04:01:57.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:57 smithi070 ceph-mon[96704]: osdmap e876: 8 total, 8 up, 8 in 2024-09-14T04:01:58.203 INFO:tasks.rados.rados.0.smithi070.stdout:3983: done (0 left) 2024-09-14T04:01:58.204 INFO:tasks.rados.rados.0.smithi070.stdout:3984: rmattr oid 26 current snap is 406 2024-09-14T04:01:58.204 INFO:tasks.rados.rados.0.smithi070.stdout:3985: snap_remove snap 400 2024-09-14T04:01:58.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:58 smithi094 ceph-mon[86597]: osdmap e877: 8 total, 8 up, 8 in 2024-09-14T04:01:58.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:58 smithi094 ceph-mon[86597]: pgmap v917: 121 pgs: 121 active+clean; 521 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.1 MiB/s rd, 1.1 MiB/s wr, 8 op/s 2024-09-14T04:01:58.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:58 smithi070 ceph-mon[98640]: osdmap e877: 8 total, 8 up, 8 in 2024-09-14T04:01:58.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:58 smithi070 ceph-mon[98640]: pgmap v917: 121 pgs: 121 active+clean; 521 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.1 MiB/s rd, 1.1 MiB/s wr, 8 op/s 2024-09-14T04:01:58.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:58 smithi070 ceph-mon[96704]: osdmap e877: 8 total, 8 up, 8 in 2024-09-14T04:01:58.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:58 smithi070 ceph-mon[96704]: pgmap v917: 121 pgs: 121 active+clean; 521 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.1 MiB/s rd, 1.1 MiB/s wr, 8 op/s 2024-09-14T04:01:59.164 INFO:tasks.rados.rados.0.smithi070.stdout:3985: done (1 left) 2024-09-14T04:01:59.164 INFO:tasks.rados.rados.0.smithi070.stdout:3986: snap_create 2024-09-14T04:01:59.164 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 26 v 4867 (ObjNum 1181 snap 392 seq_num 1181) dirty exists 2024-09-14T04:01:59.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:59 smithi094 ceph-mon[86597]: osdmap e878: 8 total, 8 up, 8 in 2024-09-14T04:01:59.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:01:59 smithi094 ceph-mon[86597]: osdmap e879: 8 total, 8 up, 8 in 2024-09-14T04:01:59.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:59 smithi070 ceph-mon[98640]: osdmap e878: 8 total, 8 up, 8 in 2024-09-14T04:01:59.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:01:59 smithi070 ceph-mon[98640]: osdmap e879: 8 total, 8 up, 8 in 2024-09-14T04:01:59.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:59 smithi070 ceph-mon[96704]: osdmap e878: 8 total, 8 up, 8 in 2024-09-14T04:01:59.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:01:59 smithi070 ceph-mon[96704]: osdmap e879: 8 total, 8 up, 8 in 2024-09-14T04:02:00.164 INFO:tasks.rados.rados.0.smithi070.stdout:3984: done (1 left) 2024-09-14T04:02:00.164 INFO:tasks.rados.rados.0.smithi070.stdout:3986: done (0 left) 2024-09-14T04:02:00.164 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 50 current snap is 407 2024-09-14T04:02:00.165 INFO:tasks.rados.rados.0.smithi070.stdout:3987: seq_num 1207 ranges {5439488=622592,6062080=401408} 2024-09-14T04:02:00.196 INFO:tasks.rados.rados.0.smithi070.stdout:3987: writing smithi07085154-50 from 5439488 to 6062080 tid 1 2024-09-14T04:02:00.198 INFO:tasks.rados.rados.0.smithi070.stdout:3987: writing smithi07085154-50 from 6062080 to 6463488 tid 2 2024-09-14T04:02:00.198 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 7 current snap is 407 2024-09-14T04:02:00.199 INFO:tasks.rados.rados.0.smithi070.stdout:3988: seq_num 1208 ranges {0=638976,638976=401408} 2024-09-14T04:02:00.202 INFO:tasks.rados.rados.0.smithi070.stdout:3988: writing smithi07085154-7 from 0 to 638976 tid 1 2024-09-14T04:02:00.205 INFO:tasks.rados.rados.0.smithi070.stdout:3988: writing smithi07085154-7 from 638976 to 1040384 tid 2 2024-09-14T04:02:00.205 INFO:tasks.rados.rados.0.smithi070.stdout:3989: rollback oid 43 current snap is 407 2024-09-14T04:02:00.205 INFO:tasks.rados.rados.0.smithi070.stdout:rollback oid 43 to 375 2024-09-14T04:02:00.205 INFO:tasks.rados.rados.0.smithi070.stdout:3990: delete oid 23 current snap is 407 2024-09-14T04:02:00.209 INFO:tasks.rados.rados.0.smithi070.stdout:3987: finishing write tid 1 to smithi07085154-50 2024-09-14T04:02:00.209 INFO:tasks.rados.rados.0.smithi070.stdout:3987: finishing write tid 2 to smithi07085154-50 2024-09-14T04:02:00.209 INFO:tasks.rados.rados.0.smithi070.stdout:3987: finishing write tid 3 to smithi07085154-50 2024-09-14T04:02:00.211 INFO:tasks.rados.rados.0.smithi070.stdout:3988: finishing write tid 1 to smithi07085154-7 2024-09-14T04:02:00.211 INFO:tasks.rados.rados.0.smithi070.stdout:3990: done (3 left) 2024-09-14T04:02:00.211 INFO:tasks.rados.rados.0.smithi070.stdout:3991: read oid 1 snap 343 2024-09-14T04:02:00.212 INFO:tasks.rados.rados.0.smithi070.stdout:3991: expect (ObjNum 1002 snap 331 seq_num 1002) 2024-09-14T04:02:00.212 INFO:tasks.rados.rados.0.smithi070.stdout:append oid 49 current snap is 407 2024-09-14T04:02:00.212 INFO:tasks.rados.rados.0.smithi070.stdout:3992: seq_num 1210 ranges {835584=671744,1507328=401408} 2024-09-14T04:02:00.220 INFO:tasks.rados.rados.0.smithi070.stdout:3992: writing smithi07085154-49 from 835584 to 1507328 tid 1 2024-09-14T04:02:00.223 INFO:tasks.rados.rados.0.smithi070.stdout:3992: writing smithi07085154-49 from 1507328 to 1908736 tid 2 2024-09-14T04:02:00.223 INFO:tasks.rados.rados.0.smithi070.stdout:3993: delete oid 23 current snap is 407 2024-09-14T04:02:00.223 INFO:tasks.rados.rados.0.smithi070.stdout:3988: finishing write tid 2 to smithi07085154-7 2024-09-14T04:02:00.223 INFO:tasks.rados.rados.0.smithi070.stdout:3987: finishing write tid 4 to smithi07085154-50 2024-09-14T04:02:00.223 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 50 v 4267 (ObjNum 1207 snap 407 seq_num 1207) dirty exists 2024-09-14T04:02:00.223 INFO:tasks.rados.rados.0.smithi070.stdout:3987: left oid 50 (ObjNum 1207 snap 407 seq_num 1207) 2024-09-14T04:02:00.223 INFO:tasks.rados.rados.0.smithi070.stdout:3988: finishing write tid 3 to smithi07085154-7 2024-09-14T04:02:00.223 INFO:tasks.rados.rados.0.smithi070.stdout:3988: finishing write tid 4 to smithi07085154-7 2024-09-14T04:02:00.223 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 7 v 4237 (ObjNum 1208 snap 407 seq_num 1208) dirty exists 2024-09-14T04:02:00.224 INFO:tasks.rados.rados.0.smithi070.stdout:3988: left oid 7 (ObjNum 1208 snap 407 seq_num 1208) 2024-09-14T04:02:00.224 INFO:tasks.rados.rados.0.smithi070.stdout:3989: finishing rollback tid 0 to smithi07085154-43 2024-09-14T04:02:00.224 INFO:tasks.rados.rados.0.smithi070.stdout:3989: finishing rollback tid 1 to smithi07085154-43 2024-09-14T04:02:00.225 INFO:tasks.rados.rados.0.smithi070.stdout:3989: finishing rollback tid 2 to smithi07085154-43 2024-09-14T04:02:00.225 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 43 v 4175 (ObjNum 1047 snap 344 seq_num 1047) dirty exists 2024-09-14T04:02:00.225 INFO:tasks.rados.rados.0.smithi070.stdout:3987: done (5 left) 2024-09-14T04:02:00.225 INFO:tasks.rados.rados.0.smithi070.stdout:3988: done (4 left) 2024-09-14T04:02:00.225 INFO:tasks.rados.rados.0.smithi070.stdout:3989: done (3 left) 2024-09-14T04:02:00.226 INFO:tasks.rados.rados.0.smithi070.stdout:3993: done (2 left) 2024-09-14T04:02:00.226 INFO:tasks.rados.rados.0.smithi070.stdout:3994: delete oid 48 current snap is 407 2024-09-14T04:02:00.228 INFO:tasks.rados.rados.0.smithi070.stdout:3994: done (2 left) 2024-09-14T04:02:00.228 INFO:tasks.rados.rados.0.smithi070.stdout:3995: copy_from oid 20 from oid 47 current snap is 407 2024-09-14T04:02:00.228 INFO:tasks.rados.rados.0.smithi070.stdout:3996: read oid 24 snap -1 2024-09-14T04:02:00.228 INFO:tasks.rados.rados.0.smithi070.stdout:3996: expect deleted 2024-09-14T04:02:00.228 INFO:tasks.rados.rados.0.smithi070.stdout:3997: snap_create 2024-09-14T04:02:00.229 INFO:tasks.rados.rados.0.smithi070.stdout:3992: finishing write tid 1 to smithi07085154-49 2024-09-14T04:02:00.229 INFO:tasks.rados.rados.0.smithi070.stdout:3992: finishing write tid 2 to smithi07085154-49 2024-09-14T04:02:00.230 INFO:tasks.rados.rados.0.smithi070.stdout:3992: finishing write tid 3 to smithi07085154-49 2024-09-14T04:02:00.232 INFO:tasks.rados.rados.0.smithi070.stdout:3992: finishing write tid 4 to smithi07085154-49 2024-09-14T04:02:00.232 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 49 v 4512 (ObjNum 1210 snap 407 seq_num 1210) dirty exists 2024-09-14T04:02:00.232 INFO:tasks.rados.rados.0.smithi070.stdout:3992: left oid 49 (ObjNum 1210 snap 407 seq_num 1210) 2024-09-14T04:02:00.234 INFO:tasks.rados.rados.0.smithi070.stdout:3995: finishing copy_from racing read to smithi07085154-20 2024-09-14T04:02:00.235 INFO:tasks.rados.rados.0.smithi070.stdout:3995: finishing copy_from to smithi07085154-20 2024-09-14T04:02:00.235 INFO:tasks.rados.rados.0.smithi070.stdout:update_object_version oid 20 v 4259 (ObjNum 65 snap 0 seq_num 1869505133) dirty exists 2024-09-14T04:02:00.245 INFO:tasks.rados.rados.0.smithi070.stdout:3991: expect (ObjNum 1002 snap 331 seq_num 1002) 2024-09-14T04:02:00.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:02:00 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:02:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:02:01.169 INFO:tasks.rados.rados.0.smithi070.stdout:3991: done (4 left) 2024-09-14T04:02:01.169 INFO:tasks.rados.rados.0.smithi070.stdout:3992: done (3 left) 2024-09-14T04:02:01.169 INFO:tasks.rados.rados.0.smithi070.stdout:3995: done (2 left) 2024-09-14T04:02:01.169 INFO:tasks.rados.rados.0.smithi070.stdout:3996: done (1 left) 2024-09-14T04:02:01.169 INFO:tasks.rados.rados.0.smithi070.stdout:3997: done (0 left) 2024-09-14T04:02:01.169 INFO:tasks.rados.rados.0.smithi070.stdout:3998: snap_create 2024-09-14T04:02:01.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:01 smithi094 ceph-mon[86597]: osdmap e880: 8 total, 8 up, 8 in 2024-09-14T04:02:01.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:01 smithi094 ceph-mon[86597]: pgmap v921: 121 pgs: 121 active+clean; 521 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 4.8 MiB/s rd, 863 KiB/s wr, 4 op/s 2024-09-14T04:02:01.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:01 smithi070 ceph-mon[98640]: osdmap e880: 8 total, 8 up, 8 in 2024-09-14T04:02:01.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:01 smithi070 ceph-mon[98640]: pgmap v921: 121 pgs: 121 active+clean; 521 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 4.8 MiB/s rd, 863 KiB/s wr, 4 op/s 2024-09-14T04:02:01.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:01 smithi070 ceph-mon[96704]: osdmap e880: 8 total, 8 up, 8 in 2024-09-14T04:02:01.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:01 smithi070 ceph-mon[96704]: pgmap v921: 121 pgs: 121 active+clean; 521 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 4.8 MiB/s rd, 863 KiB/s wr, 4 op/s 2024-09-14T04:02:02.170 INFO:tasks.rados.rados.0.smithi070.stdout:3998: done (0 left) 2024-09-14T04:02:02.171 INFO:tasks.rados.rados.0.smithi070.stdout:3999: snap_create 2024-09-14T04:02:02.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:02 smithi094 ceph-mon[86597]: osdmap e881: 8 total, 8 up, 8 in 2024-09-14T04:02:02.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:02 smithi070 ceph-mon[98640]: osdmap e881: 8 total, 8 up, 8 in 2024-09-14T04:02:02.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:02 smithi070 ceph-mon[96704]: osdmap e881: 8 total, 8 up, 8 in 2024-09-14T04:02:03.177 INFO:tasks.rados.rados.0.smithi070.stdout:3999: done (0 left) 2024-09-14T04:02:03.180 INFO:tasks.rados.rados.0.smithi070.stderr:0 errors. 2024-09-14T04:02:03.180 INFO:tasks.rados.rados.0.smithi070.stderr: 2024-09-14T04:02:03.186 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T04:02:03.437 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:02:03.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:03 smithi094 ceph-mon[86597]: osdmap e882: 8 total, 8 up, 8 in 2024-09-14T04:02:03.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:03 smithi094 ceph-mon[86597]: pgmap v924: 121 pgs: 121 active+clean; 523 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 515 B/s rd, 507 KiB/s wr, 4 op/s 2024-09-14T04:02:03.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:03 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:03.535 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:02:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:02:03] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.51.0" 2024-09-14T04:02:03.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:03 smithi070 ceph-mon[96704]: osdmap e882: 8 total, 8 up, 8 in 2024-09-14T04:02:03.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:03 smithi070 ceph-mon[96704]: pgmap v924: 121 pgs: 121 active+clean; 523 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 515 B/s rd, 507 KiB/s wr, 4 op/s 2024-09-14T04:02:03.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:03 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:03.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:03 smithi070 ceph-mon[98640]: osdmap e882: 8 total, 8 up, 8 in 2024-09-14T04:02:03.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:03 smithi070 ceph-mon[98640]: pgmap v924: 121 pgs: 121 active+clean; 523 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 515 B/s rd, 507 KiB/s wr, 4 op/s 2024-09-14T04:02:03.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:03 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:04.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:04 smithi094 ceph-mon[86597]: osdmap e883: 8 total, 8 up, 8 in 2024-09-14T04:02:04.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:04 smithi094 ceph-mon[86597]: osdmap e884: 8 total, 8 up, 8 in 2024-09-14T04:02:04.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:04 smithi070 ceph-mon[96704]: osdmap e883: 8 total, 8 up, 8 in 2024-09-14T04:02:04.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:04 smithi070 ceph-mon[96704]: osdmap e884: 8 total, 8 up, 8 in 2024-09-14T04:02:04.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:04 smithi070 ceph-mon[98640]: osdmap e883: 8 total, 8 up, 8 in 2024-09-14T04:02:04.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:04 smithi070 ceph-mon[98640]: osdmap e884: 8 total, 8 up, 8 in 2024-09-14T04:02:05.282 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:05 smithi070 ceph-mon[96704]: pgmap v927: 121 pgs: 121 active+clean; 523 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 4.6 MiB/s rd, 504 KiB/s wr, 5 op/s 2024-09-14T04:02:05.283 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:05 smithi070 ceph-mon[98640]: pgmap v927: 121 pgs: 121 active+clean; 523 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 4.6 MiB/s rd, 504 KiB/s wr, 5 op/s 2024-09-14T04:02:05.458 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T04:02:05.459 INFO:teuthology.orchestra.run.smithi070.stdout:{"epoch":884,"fsid":"b6380d42-724a-11ef-bceb-c7b262605968","created":"2024-09-14T03:38:29.406762+0000","modified":"2024-09-14T04:02:04.158060+0000","last_up_change":"2024-09-14T03:56:42.433445+0000","last_in_change":"2024-09-14T03:43:17.652311+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:41:38.745294+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-14T03:43:57.402183+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-14T03:44:00.803078+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"rbd","create_time":"2024-09-14T03:45:08.059178+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":67,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":3,"score_stable":3,"optimal_score":1,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":5,"pool_name":"unique_pool_0","create_time":"2024-09-14T03:48:03.203059+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":3,"size":3,"min_size":2,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"883","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":491,"snap_epoch":883,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"teuthologyprofile","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"a7fabda1-9e60-42f2-af60-eec626181688","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":333,"up_from":346,"up_thru":571,"down_at":334,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6802","nonce":1152327285},{"type":"v1","addr":"172.21.15.70:6803","nonce":1152327285}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6804","nonce":1152327285},{"type":"v1","addr":"172.21.15.70:6805","nonce":1152327285}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6808","nonce":1152327285},{"type":"v1","addr":"172.21.15.70:6809","nonce":1152327285}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6806","nonce":1152327285},{"type":"v1","addr":"172.21.15.70:6807","nonce":1152327285}]},"public_addr":"172.21.15.70:6803/1152327285","cluster_addr":"172.21.15.70:6805/1152327285","heartbeat_back_addr":"172.21.15.70:6809/1152327285","heartbeat_front_addr":"172.21.15.70:6807/1152327285","state":["exists","up"]},{"osd":1,"uuid":"f5b9ef91-59af-4ef0-a101-1c00ba6007ad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":366,"up_from":379,"up_thru":571,"down_at":367,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6810","nonce":478220551},{"type":"v1","addr":"172.21.15.70:6811","nonce":478220551}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6812","nonce":478220551},{"type":"v1","addr":"172.21.15.70:6813","nonce":478220551}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6816","nonce":478220551},{"type":"v1","addr":"172.21.15.70:6817","nonce":478220551}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6814","nonce":478220551},{"type":"v1","addr":"172.21.15.70:6815","nonce":478220551}]},"public_addr":"172.21.15.70:6811/478220551","cluster_addr":"172.21.15.70:6813/478220551","heartbeat_back_addr":"172.21.15.70:6817/478220551","heartbeat_front_addr":"172.21.15.70:6815/478220551","state":["exists","up"]},{"osd":2,"uuid":"0dc07495-c6a5-4762-8f63-575eb0fe1671","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":399,"up_from":411,"up_thru":571,"down_at":400,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6818","nonce":2093874787},{"type":"v1","addr":"172.21.15.70:6819","nonce":2093874787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6820","nonce":2093874787},{"type":"v1","addr":"172.21.15.70:6821","nonce":2093874787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6824","nonce":2093874787},{"type":"v1","addr":"172.21.15.70:6825","nonce":2093874787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6822","nonce":2093874787},{"type":"v1","addr":"172.21.15.70:6823","nonce":2093874787}]},"public_addr":"172.21.15.70:6819/2093874787","cluster_addr":"172.21.15.70:6821/2093874787","heartbeat_back_addr":"172.21.15.70:6825/2093874787","heartbeat_front_addr":"172.21.15.70:6823/2093874787","state":["exists","up"]},{"osd":3,"uuid":"583e5165-ea10-4d03-b731-36cfb1108362","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":431,"up_from":444,"up_thru":571,"down_at":432,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6826","nonce":3491567009},{"type":"v1","addr":"172.21.15.70:6827","nonce":3491567009}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6828","nonce":3491567009},{"type":"v1","addr":"172.21.15.70:6829","nonce":3491567009}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6832","nonce":3491567009},{"type":"v1","addr":"172.21.15.70:6833","nonce":3491567009}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6830","nonce":3491567009},{"type":"v1","addr":"172.21.15.70:6831","nonce":3491567009}]},"public_addr":"172.21.15.70:6827/3491567009","cluster_addr":"172.21.15.70:6829/3491567009","heartbeat_back_addr":"172.21.15.70:6833/3491567009","heartbeat_front_addr":"172.21.15.70:6831/3491567009","state":["exists","up"]},{"osd":4,"uuid":"33177119-9632-431c-9180-9d27e7bc0c42","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":31,"last_clean_end":461,"up_from":474,"up_thru":571,"down_at":462,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6800","nonce":930498880},{"type":"v1","addr":"172.21.15.94:6801","nonce":930498880}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6802","nonce":930498880},{"type":"v1","addr":"172.21.15.94:6803","nonce":930498880}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6806","nonce":930498880},{"type":"v1","addr":"172.21.15.94:6807","nonce":930498880}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6804","nonce":930498880},{"type":"v1","addr":"172.21.15.94:6805","nonce":930498880}]},"public_addr":"172.21.15.94:6801/930498880","cluster_addr":"172.21.15.94:6803/930498880","heartbeat_back_addr":"172.21.15.94:6807/930498880","heartbeat_front_addr":"172.21.15.94:6805/930498880","state":["exists","up"]},{"osd":5,"uuid":"eb037130-5a05-42e0-947e-09bfba8fac22","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":37,"last_clean_end":492,"up_from":505,"up_thru":571,"down_at":493,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6808","nonce":1267079794},{"type":"v1","addr":"172.21.15.94:6809","nonce":1267079794}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6810","nonce":1267079794},{"type":"v1","addr":"172.21.15.94:6811","nonce":1267079794}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6814","nonce":1267079794},{"type":"v1","addr":"172.21.15.94:6815","nonce":1267079794}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6812","nonce":1267079794},{"type":"v1","addr":"172.21.15.94:6813","nonce":1267079794}]},"public_addr":"172.21.15.94:6809/1267079794","cluster_addr":"172.21.15.94:6811/1267079794","heartbeat_back_addr":"172.21.15.94:6815/1267079794","heartbeat_front_addr":"172.21.15.94:6813/1267079794","state":["exists","up"]},{"osd":6,"uuid":"f5dca4b5-8906-4eac-84e4-fcb7d1a29674","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":43,"last_clean_end":525,"up_from":536,"up_thru":571,"down_at":526,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6816","nonce":1180082390},{"type":"v1","addr":"172.21.15.94:6817","nonce":1180082390}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6818","nonce":1180082390},{"type":"v1","addr":"172.21.15.94:6819","nonce":1180082390}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6822","nonce":1180082390},{"type":"v1","addr":"172.21.15.94:6823","nonce":1180082390}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6820","nonce":1180082390},{"type":"v1","addr":"172.21.15.94:6821","nonce":1180082390}]},"public_addr":"172.21.15.94:6817/1180082390","cluster_addr":"172.21.15.94:6819/1180082390","heartbeat_back_addr":"172.21.15.94:6823/1180082390","heartbeat_front_addr":"172.21.15.94:6821/1180082390","state":["exists","up"]},{"osd":7,"uuid":"86e3ef7d-6ab8-47c3-a4bb-db923af2b49b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":49,"last_clean_end":557,"up_from":571,"up_thru":571,"down_at":558,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6824","nonce":1152477702},{"type":"v1","addr":"172.21.15.94:6825","nonce":1152477702}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6826","nonce":1152477702},{"type":"v1","addr":"172.21.15.94:6827","nonce":1152477702}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6830","nonce":1152477702},{"type":"v1","addr":"172.21.15.94:6831","nonce":1152477702}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6828","nonce":1152477702},{"type":"v1","addr":"172.21.15.94:6829","nonce":1152477702}]},"public_addr":"172.21.15.94:6825/1152477702","cluster_addr":"172.21.15.94:6827/1152477702","heartbeat_back_addr":"172.21.15.94:6831/1152477702","heartbeat_front_addr":"172.21.15.94:6829/1152477702","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:40:47.681447+0000","dead_epoch":333},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:41:11.801118+0000","dead_epoch":366},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:41:35.764497+0000","dead_epoch":399},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:42:00.424802+0000","dead_epoch":431},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:42:21.618849+0000","dead_epoch":461},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:42:45.384437+0000","dead_epoch":492},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:43:09.350263+0000","dead_epoch":525},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:43:32.472815+0000","dead_epoch":557}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.18","mappings":[{"from":7,"to":2}]},{"pgid":"3.3","mappings":[{"from":0,"to":1}]},{"pgid":"3.a","mappings":[{"from":6,"to":2}]},{"pgid":"3.b","mappings":[{"from":0,"to":2},{"from":4,"to":1}]},{"pgid":"3.26","mappings":[{"from":4,"to":3}]},{"pgid":"3.36","mappings":[{"from":5,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.94:6833/3620660683":"2024-09-15T03:57:07.271721+0000","172.21.15.94:6832/3620660683":"2024-09-15T03:57:07.271721+0000","172.21.15.70:6835/421917097":"2024-09-15T03:56:47.027591+0000","172.21.15.70:6834/421917097":"2024-09-15T03:56:47.027591+0000","172.21.15.70:0/22543101":"2024-09-15T03:39:01.589983+0000","172.21.15.70:0/1338085373":"2024-09-15T03:51:31.997365+0000","172.21.15.70:0/2239676154":"2024-09-15T03:51:31.997365+0000","172.21.15.94:0/3953959655":"2024-09-15T03:50:51.201434+0000","172.21.15.70:6800/2555772132":"2024-09-15T03:51:31.997365+0000","172.21.15.70:6800/2541724886":"2024-09-15T03:49:34.505287+0000","172.21.15.70:0/2339853520":"2024-09-15T03:45:06.535242+0000","172.21.15.70:0/1654192586":"2024-09-15T03:49:34.505287+0000","172.21.15.70:6801/673916264":"2024-09-15T03:39:24.448616+0000","172.21.15.70:0/3005166882":"2024-09-15T03:39:24.448616+0000","172.21.15.94:0/2129663473":"2024-09-15T03:50:51.201434+0000","172.21.15.70:0/4130351200":"2024-09-15T03:51:31.997365+0000","172.21.15.70:0/3134643770":"2024-09-15T03:39:24.448616+0000","172.21.15.70:6801/2254107694":"2024-09-15T03:39:01.589983+0000","172.21.15.70:6800/2254107694":"2024-09-15T03:39:01.589983+0000","172.21.15.70:0/4203647763":"2024-09-15T03:39:01.589983+0000","172.21.15.70:0/3515912540":"2024-09-15T03:51:31.997365+0000","172.21.15.70:0/2892154750":"2024-09-15T03:49:34.505287+0000","172.21.15.70:0/869883831":"2024-09-15T03:49:34.505287+0000","172.21.15.94:6834/4056574908":"2024-09-15T03:50:51.201434+0000","172.21.15.70:0/2978576061":"2024-09-15T03:39:24.448616+0000","172.21.15.70:0/3407824580":"2024-09-15T03:51:31.997365+0000","172.21.15.70:0/3199751552":"2024-09-15T03:45:06.535242+0000","172.21.15.94:0/3098594889":"2024-09-15T03:50:51.201434+0000","172.21.15.70:0/2324380307":"2024-09-15T03:49:34.505287+0000","172.21.15.70:0/2447056811":"2024-09-15T03:39:01.589983+0000","172.21.15.70:0/2818410174":"2024-09-15T03:51:31.997365+0000","172.21.15.70:0/3204520938":"2024-09-15T03:51:31.997365+0000","172.21.15.70:6800/3164926893":"2024-09-15T03:45:06.535242+0000","172.21.15.94:0/2404500339":"2024-09-15T03:50:51.201434+0000","172.21.15.70:0/3067035520":"2024-09-15T03:45:06.535242+0000","172.21.15.70:6801/3164926893":"2024-09-15T03:45:06.535242+0000","172.21.15.70:6801/2555772132":"2024-09-15T03:51:31.997365+0000","172.21.15.70:6801/2541724886":"2024-09-15T03:49:34.505287+0000","172.21.15.94:0/1940159411":"2024-09-15T03:50:51.201434+0000","172.21.15.94:0/3503149243":"2024-09-15T03:50:51.201434+0000","172.21.15.70:6800/673916264":"2024-09-15T03:39:24.448616+0000","172.21.15.94:0/3283822429":"2024-09-15T03:50:51.201434+0000","172.21.15.94:6835/4056574908":"2024-09-15T03:50:51.201434+0000","172.21.15.70:0/4275249763":"2024-09-15T03:45:06.535242+0000","172.21.15.70:0/2993355975":"2024-09-15T03:49:34.505287+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"},"teuthologyprofile":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van","w":"8"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":482,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[{"pool":5,"snaps":[{"begin":440,"length":1},{"begin":473,"length":1},{"begin":481,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T04:02:05.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:05 smithi094 ceph-mon[86597]: pgmap v927: 121 pgs: 121 active+clean; 523 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 4.6 MiB/s rd, 504 KiB/s wr, 5 op/s 2024-09-14T04:02:06.211 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:02:06.456 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:02:06.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:06 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/1348504947' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T04:02:06.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:06 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/1348504947' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T04:02:06.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:06 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/1348504947' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T04:02:07.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:07 smithi094 ceph-mon[86597]: pgmap v928: 121 pgs: 121 active+clean; 526 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 3.7 MiB/s rd, 612 KiB/s wr, 5 op/s 2024-09-14T04:02:07.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:07 smithi070 ceph-mon[96704]: pgmap v928: 121 pgs: 121 active+clean; 526 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 3.7 MiB/s rd, 612 KiB/s wr, 5 op/s 2024-09-14T04:02:07.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:07 smithi070 ceph-mon[98640]: pgmap v928: 121 pgs: 121 active+clean; 526 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 3.7 MiB/s rd, 612 KiB/s wr, 5 op/s 2024-09-14T04:02:08.294 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T04:02:08.296 INFO:teuthology.orchestra.run.smithi070.stderr:dumped all 2024-09-14T04:02:08.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:08 smithi094 ceph-mon[86597]: pgmap v929: 121 pgs: 121 active+clean; 526 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 3.0 MiB/s rd, 174 KiB/s wr, 1 op/s 2024-09-14T04:02:08.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:08 smithi070 ceph-mon[96704]: pgmap v929: 121 pgs: 121 active+clean; 526 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 3.0 MiB/s rd, 174 KiB/s wr, 1 op/s 2024-09-14T04:02:08.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:08 smithi070 ceph-mon[98640]: pgmap v929: 121 pgs: 121 active+clean; 526 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 3.0 MiB/s rd, 174 KiB/s wr, 1 op/s 2024-09-14T04:02:08.949 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":929,"stamp":"2024-09-14T04:02:08.173792+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":552000509,"num_objects":318,"num_object_clones":242,"num_object_copies":954,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":318,"num_whiteouts":10,"num_read":4752,"num_read_kb":6253571,"num_write":8962,"num_write_kb":1972497,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":133,"num_bytes_recovered":526959136,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":28084,"ondisk_log_size":28084,"up":363,"acting":363,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":363,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":4346680,"kb_used_data":807040,"kb_used_omap":480,"kb_used_meta":3539039,"kb_avail":745614536,"statfs":{"total":767960285184,"available":763509284864,"internally_reserved":0,"allocated":826408960,"data_stored":1449968867,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":491617,"internal_metadata":3623976863},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":3555328,"num_objects":3,"num_object_clones":3,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":1,"num_read":5,"num_read_kb":18746,"num_write":6,"num_write_kb":1048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.007234"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":1142,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.361290+0000","last_change":"2024-09-14T03:56:43.463859+0000","last_active":"2024-09-14T04:02:04.361290+0000","last_peered":"2024-09-14T04:02:04.361290+0000","last_clean":"2024-09-14T04:02:04.361290+0000","last_became_active":"2024-09-14T03:56:43.462704+0000","last_became_peered":"2024-09-14T03:56:43.462704+0000","last_unstale":"2024-09-14T04:02:04.361290+0000","last_undegraded":"2024-09-14T04:02:04.361290+0000","last_fullsized":"2024-09-14T04:02:04.361290+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:56:47.012613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":1334,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.161741+0000","last_change":"2024-09-14T03:56:08.138394+0000","last_active":"2024-09-14T04:02:04.161741+0000","last_peered":"2024-09-14T04:02:04.161741+0000","last_clean":"2024-09-14T04:02:04.161741+0000","last_became_active":"2024-09-14T03:56:08.138094+0000","last_became_peered":"2024-09-14T03:56:08.138094+0000","last_unstale":"2024-09-14T04:02:04.161741+0000","last_undegraded":"2024-09-14T04:02:04.161741+0000","last_fullsized":"2024-09-14T04:02:04.161741+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:47:18.205804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":1356,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.196392+0000","last_change":"2024-09-14T03:53:54.050269+0000","last_active":"2024-09-14T04:02:04.196392+0000","last_peered":"2024-09-14T04:02:04.196392+0000","last_clean":"2024-09-14T04:02:04.196392+0000","last_became_active":"2024-09-14T03:53:54.049998+0000","last_became_peered":"2024-09-14T03:53:54.049998+0000","last_unstale":"2024-09-14T04:02:04.196392+0000","last_undegraded":"2024-09-14T04:02:04.196392+0000","last_fullsized":"2024-09-14T04:02:04.196392+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:04:54.299765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":1211,"reported_epoch":881,"state":"active+clean","last_fresh":"2024-09-14T04:02:01.420717+0000","last_change":"2024-09-14T03:56:43.451836+0000","last_active":"2024-09-14T04:02:01.420717+0000","last_peered":"2024-09-14T04:02:01.420717+0000","last_clean":"2024-09-14T04:02:01.420717+0000","last_became_active":"2024-09-14T03:56:43.451520+0000","last_became_peered":"2024-09-14T03:56:43.451520+0000","last_unstale":"2024-09-14T04:02:01.420717+0000","last_undegraded":"2024-09-14T04:02:01.420717+0000","last_fullsized":"2024-09-14T04:02:01.420717+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:24:03.898106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":1162,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.362641+0000","last_change":"2024-09-14T03:56:43.456134+0000","last_active":"2024-09-14T04:02:04.362641+0000","last_peered":"2024-09-14T04:02:04.362641+0000","last_clean":"2024-09-14T04:02:04.362641+0000","last_became_active":"2024-09-14T03:56:43.454457+0000","last_became_peered":"2024-09-14T03:56:43.454457+0000","last_unstale":"2024-09-14T04:02:04.362641+0000","last_undegraded":"2024-09-14T04:02:04.362641+0000","last_fullsized":"2024-09-14T04:02:04.362641+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:08:53.515296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":1192,"reported_epoch":881,"state":"active+clean","last_fresh":"2024-09-14T04:02:01.420290+0000","last_change":"2024-09-14T03:55:35.120879+0000","last_active":"2024-09-14T04:02:01.420290+0000","last_peered":"2024-09-14T04:02:01.420290+0000","last_clean":"2024-09-14T04:02:01.420290+0000","last_became_active":"2024-09-14T03:55:35.120207+0000","last_became_peered":"2024-09-14T03:55:35.120207+0000","last_unstale":"2024-09-14T04:02:01.420290+0000","last_undegraded":"2024-09-14T04:02:01.420290+0000","last_fullsized":"2024-09-14T04:02:01.420290+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:35:40.381365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":1162,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.364340+0000","last_change":"2024-09-14T03:56:43.457154+0000","last_active":"2024-09-14T04:02:04.364340+0000","last_peered":"2024-09-14T04:02:04.364340+0000","last_clean":"2024-09-14T04:02:04.364340+0000","last_became_active":"2024-09-14T03:56:43.456442+0000","last_became_peered":"2024-09-14T03:56:43.456442+0000","last_unstale":"2024-09-14T04:02:04.364340+0000","last_undegraded":"2024-09-14T04:02:04.364340+0000","last_fullsized":"2024-09-14T04:02:04.364340+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:54:36.395589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":1276,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.248751+0000","last_change":"2024-09-14T03:56:08.123044+0000","last_active":"2024-09-14T04:02:04.248751+0000","last_peered":"2024-09-14T04:02:04.248751+0000","last_clean":"2024-09-14T04:02:04.248751+0000","last_became_active":"2024-09-14T03:56:08.121835+0000","last_became_peered":"2024-09-14T03:56:08.121835+0000","last_unstale":"2024-09-14T04:02:04.248751+0000","last_undegraded":"2024-09-14T04:02:04.248751+0000","last_fullsized":"2024-09-14T04:02:04.248751+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:31:11.012507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":1153,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.361402+0000","last_change":"2024-09-14T03:56:43.458399+0000","last_active":"2024-09-14T04:02:04.361402+0000","last_peered":"2024-09-14T04:02:04.361402+0000","last_clean":"2024-09-14T04:02:04.361402+0000","last_became_active":"2024-09-14T03:56:43.458066+0000","last_became_peered":"2024-09-14T03:56:43.458066+0000","last_unstale":"2024-09-14T04:02:04.361402+0000","last_undegraded":"2024-09-14T04:02:04.361402+0000","last_fullsized":"2024-09-14T04:02:04.361402+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:50:42.506723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":1176,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.363638+0000","last_change":"2024-09-14T03:56:08.144313+0000","last_active":"2024-09-14T04:02:04.363638+0000","last_peered":"2024-09-14T04:02:04.363638+0000","last_clean":"2024-09-14T04:02:04.363638+0000","last_became_active":"2024-09-14T03:56:08.143269+0000","last_became_peered":"2024-09-14T03:56:08.143269+0000","last_unstale":"2024-09-14T04:02:04.363638+0000","last_undegraded":"2024-09-14T04:02:04.363638+0000","last_fullsized":"2024-09-14T04:02:04.363638+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:55:37.717465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":1177,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.363046+0000","last_change":"2024-09-14T03:56:08.136643+0000","last_active":"2024-09-14T04:02:04.363046+0000","last_peered":"2024-09-14T04:02:04.363046+0000","last_clean":"2024-09-14T04:02:04.363046+0000","last_became_active":"2024-09-14T03:56:08.136108+0000","last_became_peered":"2024-09-14T03:56:08.136108+0000","last_unstale":"2024-09-14T04:02:04.363046+0000","last_undegraded":"2024-09-14T04:02:04.363046+0000","last_fullsized":"2024-09-14T04:02:04.363046+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:40:22.948578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":1233,"reported_epoch":882,"state":"active+clean","last_fresh":"2024-09-14T04:02:02.168683+0000","last_change":"2024-09-14T03:56:08.132298+0000","last_active":"2024-09-14T04:02:02.168683+0000","last_peered":"2024-09-14T04:02:02.168683+0000","last_clean":"2024-09-14T04:02:02.168683+0000","last_became_active":"2024-09-14T03:56:08.131948+0000","last_became_peered":"2024-09-14T03:56:08.131948+0000","last_unstale":"2024-09-14T04:02:02.168683+0000","last_undegraded":"2024-09-14T04:02:02.168683+0000","last_fullsized":"2024-09-14T04:02:02.168683+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:06:11.094627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":1278,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.249120+0000","last_change":"2024-09-14T03:56:08.131742+0000","last_active":"2024-09-14T04:02:04.249120+0000","last_peered":"2024-09-14T04:02:04.249120+0000","last_clean":"2024-09-14T04:02:04.249120+0000","last_became_active":"2024-09-14T03:56:08.131044+0000","last_became_peered":"2024-09-14T03:56:08.131044+0000","last_unstale":"2024-09-14T04:02:04.249120+0000","last_undegraded":"2024-09-14T04:02:04.249120+0000","last_fullsized":"2024-09-14T04:02:04.249120+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:20:32.161702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":1356,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.195907+0000","last_change":"2024-09-14T03:55:35.111689+0000","last_active":"2024-09-14T04:02:04.195907+0000","last_peered":"2024-09-14T04:02:04.195907+0000","last_clean":"2024-09-14T04:02:04.195907+0000","last_became_active":"2024-09-14T03:55:35.111184+0000","last_became_peered":"2024-09-14T03:55:35.111184+0000","last_unstale":"2024-09-14T04:02:04.195907+0000","last_undegraded":"2024-09-14T04:02:04.195907+0000","last_fullsized":"2024-09-14T04:02:04.195907+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:08:23.817174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":1324,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.161124+0000","last_change":"2024-09-14T03:55:35.117430+0000","last_active":"2024-09-14T04:02:04.161124+0000","last_peered":"2024-09-14T04:02:04.161124+0000","last_clean":"2024-09-14T04:02:04.161124+0000","last_became_active":"2024-09-14T03:55:35.117126+0000","last_became_peered":"2024-09-14T03:55:35.117126+0000","last_unstale":"2024-09-14T04:02:04.161124+0000","last_undegraded":"2024-09-14T04:02:04.161124+0000","last_fullsized":"2024-09-14T04:02:04.161124+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:18:45.271178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":1194,"reported_epoch":881,"state":"active+clean","last_fresh":"2024-09-14T04:02:01.419121+0000","last_change":"2024-09-14T03:56:43.455178+0000","last_active":"2024-09-14T04:02:01.419121+0000","last_peered":"2024-09-14T04:02:01.419121+0000","last_clean":"2024-09-14T04:02:01.419121+0000","last_became_active":"2024-09-14T03:56:43.454865+0000","last_became_peered":"2024-09-14T03:56:43.454865+0000","last_unstale":"2024-09-14T04:02:01.419121+0000","last_undegraded":"2024-09-14T04:02:01.419121+0000","last_fullsized":"2024-09-14T04:02:01.419121+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:44:14.290752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":1198,"reported_epoch":881,"state":"active+clean","last_fresh":"2024-09-14T04:02:01.420687+0000","last_change":"2024-09-14T03:56:43.452708+0000","last_active":"2024-09-14T04:02:01.420687+0000","last_peered":"2024-09-14T04:02:01.420687+0000","last_clean":"2024-09-14T04:02:01.420687+0000","last_became_active":"2024-09-14T03:56:43.451617+0000","last_became_peered":"2024-09-14T03:56:43.451617+0000","last_unstale":"2024-09-14T04:02:01.420687+0000","last_undegraded":"2024-09-14T04:02:01.420687+0000","last_fullsized":"2024-09-14T04:02:01.420687+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:06:48.112005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":1291,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.340838+0000","last_change":"2024-09-14T03:56:43.451870+0000","last_active":"2024-09-14T04:02:04.340838+0000","last_peered":"2024-09-14T04:02:04.340838+0000","last_clean":"2024-09-14T04:02:04.340838+0000","last_became_active":"2024-09-14T03:56:43.451496+0000","last_became_peered":"2024-09-14T03:56:43.451496+0000","last_unstale":"2024-09-14T04:02:04.340838+0000","last_undegraded":"2024-09-14T04:02:04.340838+0000","last_fullsized":"2024-09-14T04:02:04.340838+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:38:18.872102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":1142,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.362770+0000","last_change":"2024-09-14T03:56:43.457531+0000","last_active":"2024-09-14T04:02:04.362770+0000","last_peered":"2024-09-14T04:02:04.362770+0000","last_clean":"2024-09-14T04:02:04.362770+0000","last_became_active":"2024-09-14T03:56:43.456964+0000","last_became_peered":"2024-09-14T03:56:43.456964+0000","last_unstale":"2024-09-14T04:02:04.362770+0000","last_undegraded":"2024-09-14T04:02:04.362770+0000","last_fullsized":"2024-09-14T04:02:04.362770+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:06:48.804407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":1188,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.363064+0000","last_change":"2024-09-14T03:56:08.133725+0000","last_active":"2024-09-14T04:02:04.363064+0000","last_peered":"2024-09-14T04:02:04.363064+0000","last_clean":"2024-09-14T04:02:04.363064+0000","last_became_active":"2024-09-14T03:56:08.133446+0000","last_became_peered":"2024-09-14T03:56:08.133446+0000","last_unstale":"2024-09-14T04:02:04.363064+0000","last_undegraded":"2024-09-14T04:02:04.363064+0000","last_fullsized":"2024-09-14T04:02:04.363064+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:57:03.461550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":1356,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.195411+0000","last_change":"2024-09-14T03:56:43.470164+0000","last_active":"2024-09-14T04:02:04.195411+0000","last_peered":"2024-09-14T04:02:04.195411+0000","last_clean":"2024-09-14T04:02:04.195411+0000","last_became_active":"2024-09-14T03:56:43.469270+0000","last_became_peered":"2024-09-14T03:56:43.469270+0000","last_unstale":"2024-09-14T04:02:04.195411+0000","last_undegraded":"2024-09-14T04:02:04.195411+0000","last_fullsized":"2024-09-14T04:02:04.195411+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:24:51.787644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":1184,"reported_epoch":881,"state":"active+clean","last_fresh":"2024-09-14T04:02:01.419937+0000","last_change":"2024-09-14T03:56:43.452247+0000","last_active":"2024-09-14T04:02:01.419937+0000","last_peered":"2024-09-14T04:02:01.419937+0000","last_clean":"2024-09-14T04:02:01.419937+0000","last_became_active":"2024-09-14T03:56:43.451961+0000","last_became_peered":"2024-09-14T03:56:43.451961+0000","last_unstale":"2024-09-14T04:02:01.419937+0000","last_undegraded":"2024-09-14T04:02:01.419937+0000","last_fullsized":"2024-09-14T04:02:01.419937+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:34:10.837199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":1278,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.248944+0000","last_change":"2024-09-14T03:55:00.456425+0000","last_active":"2024-09-14T04:02:04.248944+0000","last_peered":"2024-09-14T04:02:04.248944+0000","last_clean":"2024-09-14T04:02:04.248944+0000","last_became_active":"2024-09-14T03:55:00.456121+0000","last_became_peered":"2024-09-14T03:55:00.456121+0000","last_unstale":"2024-09-14T04:02:04.248944+0000","last_undegraded":"2024-09-14T04:02:04.248944+0000","last_fullsized":"2024-09-14T04:02:04.248944+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:19:22.985732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":1217,"reported_epoch":881,"state":"active+clean","last_fresh":"2024-09-14T04:02:01.419415+0000","last_change":"2024-09-14T03:55:35.123767+0000","last_active":"2024-09-14T04:02:01.419415+0000","last_peered":"2024-09-14T04:02:01.419415+0000","last_clean":"2024-09-14T04:02:01.419415+0000","last_became_active":"2024-09-14T03:55:35.123302+0000","last_became_peered":"2024-09-14T03:55:35.123302+0000","last_unstale":"2024-09-14T04:02:01.419415+0000","last_undegraded":"2024-09-14T04:02:01.419415+0000","last_fullsized":"2024-09-14T04:02:01.419415+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:28:38.051538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":1269,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.249481+0000","last_change":"2024-09-14T03:55:00.452942+0000","last_active":"2024-09-14T04:02:04.249481+0000","last_peered":"2024-09-14T04:02:04.249481+0000","last_clean":"2024-09-14T04:02:04.249481+0000","last_became_active":"2024-09-14T03:55:00.452352+0000","last_became_peered":"2024-09-14T03:55:00.452352+0000","last_unstale":"2024-09-14T04:02:04.249481+0000","last_undegraded":"2024-09-14T04:02:04.249481+0000","last_fullsized":"2024-09-14T04:02:04.249481+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:15:46.717132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":1162,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.363868+0000","last_change":"2024-09-14T03:56:43.464374+0000","last_active":"2024-09-14T04:02:04.363868+0000","last_peered":"2024-09-14T04:02:04.363868+0000","last_clean":"2024-09-14T04:02:04.363868+0000","last_became_active":"2024-09-14T03:56:43.464050+0000","last_became_peered":"2024-09-14T03:56:43.464050+0000","last_unstale":"2024-09-14T04:02:04.363868+0000","last_undegraded":"2024-09-14T04:02:04.363868+0000","last_fullsized":"2024-09-14T04:02:04.363868+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:30:54.682253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":1325,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.161433+0000","last_change":"2024-09-14T03:54:27.087347+0000","last_active":"2024-09-14T04:02:04.161433+0000","last_peered":"2024-09-14T04:02:04.161433+0000","last_clean":"2024-09-14T04:02:04.161433+0000","last_became_active":"2024-09-14T03:54:27.087094+0000","last_became_peered":"2024-09-14T03:54:27.087094+0000","last_unstale":"2024-09-14T04:02:04.161433+0000","last_undegraded":"2024-09-14T04:02:04.161433+0000","last_fullsized":"2024-09-14T04:02:04.161433+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:34:37.330440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":1286,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.249186+0000","last_change":"2024-09-14T03:55:00.453183+0000","last_active":"2024-09-14T04:02:04.249186+0000","last_peered":"2024-09-14T04:02:04.249186+0000","last_clean":"2024-09-14T04:02:04.249186+0000","last_became_active":"2024-09-14T03:55:00.452348+0000","last_became_peered":"2024-09-14T03:55:00.452348+0000","last_unstale":"2024-09-14T04:02:04.249186+0000","last_undegraded":"2024-09-14T04:02:04.249186+0000","last_fullsized":"2024-09-14T04:02:04.249186+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:34:06.653688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":1356,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.196032+0000","last_change":"2024-09-14T03:55:00.447583+0000","last_active":"2024-09-14T04:02:04.196032+0000","last_peered":"2024-09-14T04:02:04.196032+0000","last_clean":"2024-09-14T04:02:04.196032+0000","last_became_active":"2024-09-14T03:55:00.446159+0000","last_became_peered":"2024-09-14T03:55:00.446159+0000","last_unstale":"2024-09-14T04:02:04.196032+0000","last_undegraded":"2024-09-14T04:02:04.196032+0000","last_fullsized":"2024-09-14T04:02:04.196032+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:55:37.240550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":1217,"reported_epoch":881,"state":"active+clean","last_fresh":"2024-09-14T04:02:01.420034+0000","last_change":"2024-09-14T03:55:35.126988+0000","last_active":"2024-09-14T04:02:01.420034+0000","last_peered":"2024-09-14T04:02:01.420034+0000","last_clean":"2024-09-14T04:02:01.420034+0000","last_became_active":"2024-09-14T03:55:35.122897+0000","last_became_peered":"2024-09-14T03:55:35.122897+0000","last_unstale":"2024-09-14T04:02:01.420034+0000","last_undegraded":"2024-09-14T04:02:01.420034+0000","last_fullsized":"2024-09-14T04:02:01.420034+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:43:38.979780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":1291,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.340383+0000","last_change":"2024-09-14T03:56:08.143607+0000","last_active":"2024-09-14T04:02:04.340383+0000","last_peered":"2024-09-14T04:02:04.340383+0000","last_clean":"2024-09-14T04:02:04.340383+0000","last_became_active":"2024-09-14T03:56:08.143312+0000","last_became_peered":"2024-09-14T03:56:08.143312+0000","last_unstale":"2024-09-14T04:02:04.340383+0000","last_undegraded":"2024-09-14T04:02:04.340383+0000","last_fullsized":"2024-09-14T04:02:04.340383+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:21:13.021495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":1177,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.362462+0000","last_change":"2024-09-14T03:56:08.133262+0000","last_active":"2024-09-14T04:02:04.362462+0000","last_peered":"2024-09-14T04:02:04.362462+0000","last_clean":"2024-09-14T04:02:04.362462+0000","last_became_active":"2024-09-14T03:56:08.132977+0000","last_became_peered":"2024-09-14T03:56:08.132977+0000","last_unstale":"2024-09-14T04:02:04.362462+0000","last_undegraded":"2024-09-14T04:02:04.362462+0000","last_fullsized":"2024-09-14T04:02:04.362462+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:24:32.878597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":1263,"reported_epoch":882,"state":"active+clean","last_fresh":"2024-09-14T04:02:02.168146+0000","last_change":"2024-09-14T03:55:00.455096+0000","last_active":"2024-09-14T04:02:02.168146+0000","last_peered":"2024-09-14T04:02:02.168146+0000","last_clean":"2024-09-14T04:02:02.168146+0000","last_became_active":"2024-09-14T03:55:00.454026+0000","last_became_peered":"2024-09-14T03:55:00.454026+0000","last_unstale":"2024-09-14T04:02:02.168146+0000","last_undegraded":"2024-09-14T04:02:02.168146+0000","last_fullsized":"2024-09-14T04:02:02.168146+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:11:21.594058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"596'3","reported_seq":1283,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.249166+0000","last_change":"2024-09-14T03:56:43.468671+0000","last_active":"2024-09-14T04:02:04.249166+0000","last_peered":"2024-09-14T04:02:04.249166+0000","last_clean":"2024-09-14T04:02:04.249166+0000","last_became_active":"2024-09-14T03:56:43.468296+0000","last_became_peered":"2024-09-14T03:56:43.468296+0000","last_unstale":"2024-09-14T04:02:04.249166+0000","last_undegraded":"2024-09-14T04:02:04.249166+0000","last_fullsized":"2024-09-14T04:02:04.249166+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:18:56.900011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":1355,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.195282+0000","last_change":"2024-09-14T03:56:43.454701+0000","last_active":"2024-09-14T04:02:04.195282+0000","last_peered":"2024-09-14T04:02:04.195282+0000","last_clean":"2024-09-14T04:02:04.195282+0000","last_became_active":"2024-09-14T03:56:43.454379+0000","last_became_peered":"2024-09-14T03:56:43.454379+0000","last_unstale":"2024-09-14T04:02:04.195282+0000","last_undegraded":"2024-09-14T04:02:04.195282+0000","last_fullsized":"2024-09-14T04:02:04.195282+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:37:36.027028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"596'3","reported_seq":1188,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.361379+0000","last_change":"2024-09-14T03:56:43.461667+0000","last_active":"2024-09-14T04:02:04.361379+0000","last_peered":"2024-09-14T04:02:04.361379+0000","last_clean":"2024-09-14T04:02:04.361379+0000","last_became_active":"2024-09-14T03:56:43.461302+0000","last_became_peered":"2024-09-14T03:56:43.461302+0000","last_unstale":"2024-09-14T04:02:04.361379+0000","last_undegraded":"2024-09-14T04:02:04.361379+0000","last_fullsized":"2024-09-14T04:02:04.361379+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:48:33.190979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":1288,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.248792+0000","last_change":"2024-09-14T03:56:08.143229+0000","last_active":"2024-09-14T04:02:04.248792+0000","last_peered":"2024-09-14T04:02:04.248792+0000","last_clean":"2024-09-14T04:02:04.248792+0000","last_became_active":"2024-09-14T03:56:08.142913+0000","last_became_peered":"2024-09-14T03:56:08.142913+0000","last_unstale":"2024-09-14T04:02:04.248792+0000","last_undegraded":"2024-09-14T04:02:04.248792+0000","last_fullsized":"2024-09-14T04:02:04.248792+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:19:30.459934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1284,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.248799+0000","last_change":"2024-09-14T03:54:27.092464+0000","last_active":"2024-09-14T04:02:04.248799+0000","last_peered":"2024-09-14T04:02:04.248799+0000","last_clean":"2024-09-14T04:02:04.248799+0000","last_became_active":"2024-09-14T03:54:27.090632+0000","last_became_peered":"2024-09-14T03:54:27.090632+0000","last_unstale":"2024-09-14T04:02:04.248799+0000","last_undegraded":"2024-09-14T04:02:04.248799+0000","last_fullsized":"2024-09-14T04:02:04.248799+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:07:30.553983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":1217,"reported_epoch":881,"state":"active+clean","last_fresh":"2024-09-14T04:02:01.419240+0000","last_change":"2024-09-14T03:55:35.114733+0000","last_active":"2024-09-14T04:02:01.419240+0000","last_peered":"2024-09-14T04:02:01.419240+0000","last_clean":"2024-09-14T04:02:01.419240+0000","last_became_active":"2024-09-14T03:55:35.114464+0000","last_became_peered":"2024-09-14T03:55:35.114464+0000","last_unstale":"2024-09-14T04:02:01.419240+0000","last_undegraded":"2024-09-14T04:02:01.419240+0000","last_fullsized":"2024-09-14T04:02:01.419240+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:11:30.403861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1324,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.161099+0000","last_change":"2024-09-14T03:55:00.449243+0000","last_active":"2024-09-14T04:02:04.161099+0000","last_peered":"2024-09-14T04:02:04.161099+0000","last_clean":"2024-09-14T04:02:04.161099+0000","last_became_active":"2024-09-14T03:55:00.448955+0000","last_became_peered":"2024-09-14T03:55:00.448955+0000","last_unstale":"2024-09-14T04:02:04.161099+0000","last_undegraded":"2024-09-14T04:02:04.161099+0000","last_fullsized":"2024-09-14T04:02:04.161099+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:36:02.192152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":1180,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.363654+0000","last_change":"2024-09-14T03:56:08.143602+0000","last_active":"2024-09-14T04:02:04.363654+0000","last_peered":"2024-09-14T04:02:04.363654+0000","last_clean":"2024-09-14T04:02:04.363654+0000","last_became_active":"2024-09-14T03:56:08.143313+0000","last_became_peered":"2024-09-14T03:56:08.143313+0000","last_unstale":"2024-09-14T04:02:04.363654+0000","last_undegraded":"2024-09-14T04:02:04.363654+0000","last_fullsized":"2024-09-14T04:02:04.363654+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:31:55.004521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1186,"reported_epoch":881,"state":"active+clean","last_fresh":"2024-09-14T04:02:01.419926+0000","last_change":"2024-09-14T03:56:43.462405+0000","last_active":"2024-09-14T04:02:01.419926+0000","last_peered":"2024-09-14T04:02:01.419926+0000","last_clean":"2024-09-14T04:02:01.419926+0000","last_became_active":"2024-09-14T03:56:43.462037+0000","last_became_peered":"2024-09-14T03:56:43.462037+0000","last_unstale":"2024-09-14T04:02:01.419926+0000","last_undegraded":"2024-09-14T04:02:01.419926+0000","last_fullsized":"2024-09-14T04:02:01.419926+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:53:14.543798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":1180,"reported_epoch":881,"state":"active+clean","last_fresh":"2024-09-14T04:02:01.420425+0000","last_change":"2024-09-14T03:56:08.134371+0000","last_active":"2024-09-14T04:02:01.420425+0000","last_peered":"2024-09-14T04:02:01.420425+0000","last_clean":"2024-09-14T04:02:01.420425+0000","last_became_active":"2024-09-14T03:56:08.133454+0000","last_became_peered":"2024-09-14T03:56:08.133454+0000","last_unstale":"2024-09-14T04:02:01.420425+0000","last_undegraded":"2024-09-14T04:02:01.420425+0000","last_fullsized":"2024-09-14T04:02:01.420425+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:57:55.499563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1355,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.196397+0000","last_change":"2024-09-14T03:55:35.112828+0000","last_active":"2024-09-14T04:02:04.196397+0000","last_peered":"2024-09-14T04:02:04.196397+0000","last_clean":"2024-09-14T04:02:04.196397+0000","last_became_active":"2024-09-14T03:55:35.112537+0000","last_became_peered":"2024-09-14T03:55:35.112537+0000","last_unstale":"2024-09-14T04:02:04.196397+0000","last_undegraded":"2024-09-14T04:02:04.196397+0000","last_fullsized":"2024-09-14T04:02:04.196397+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:48:54.915621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"58'3","reported_seq":1343,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.161588+0000","last_change":"2024-09-14T03:55:35.114925+0000","last_active":"2024-09-14T04:02:04.161588+0000","last_peered":"2024-09-14T04:02:04.161588+0000","last_clean":"2024-09-14T04:02:04.161588+0000","last_became_active":"2024-09-14T03:55:35.114569+0000","last_became_peered":"2024-09-14T03:55:35.114569+0000","last_unstale":"2024-09-14T04:02:04.161588+0000","last_undegraded":"2024-09-14T04:02:04.161588+0000","last_fullsized":"2024-09-14T04:02:04.161588+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:13:56.121098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":11,"num_read_kb":11,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":1224,"reported_epoch":882,"state":"active+clean","last_fresh":"2024-09-14T04:02:02.168792+0000","last_change":"2024-09-14T03:56:43.462896+0000","last_active":"2024-09-14T04:02:02.168792+0000","last_peered":"2024-09-14T04:02:02.168792+0000","last_clean":"2024-09-14T04:02:02.168792+0000","last_became_active":"2024-09-14T03:56:43.462599+0000","last_became_peered":"2024-09-14T03:56:43.462599+0000","last_unstale":"2024-09-14T04:02:02.168792+0000","last_undegraded":"2024-09-14T04:02:02.168792+0000","last_fullsized":"2024-09-14T04:02:02.168792+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:09:26.835817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"596'30","reported_seq":1221,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.363218+0000","last_change":"2024-09-14T03:56:08.134528+0000","last_active":"2024-09-14T04:02:04.363218+0000","last_peered":"2024-09-14T04:02:04.363218+0000","last_clean":"2024-09-14T04:02:04.363218+0000","last_became_active":"2024-09-14T03:56:08.134127+0000","last_became_peered":"2024-09-14T03:56:08.134127+0000","last_unstale":"2024-09-14T04:02:04.363218+0000","last_undegraded":"2024-09-14T04:02:04.363218+0000","last_fullsized":"2024-09-14T04:02:04.363218+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":30,"log_dups_size":0,"ondisk_log_size":30,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:22:24.530649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5816,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":17,"num_read_kb":25,"num_write":28,"num_write_kb":25,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":1162,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.362570+0000","last_change":"2024-09-14T03:56:43.470524+0000","last_active":"2024-09-14T04:02:04.362570+0000","last_peered":"2024-09-14T04:02:04.362570+0000","last_clean":"2024-09-14T04:02:04.362570+0000","last_became_active":"2024-09-14T03:56:43.470252+0000","last_became_peered":"2024-09-14T03:56:43.470252+0000","last_unstale":"2024-09-14T04:02:04.362570+0000","last_undegraded":"2024-09-14T04:02:04.362570+0000","last_fullsized":"2024-09-14T04:02:04.362570+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:22:16.472366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"58'2","reported_seq":1367,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.195936+0000","last_change":"2024-09-14T03:55:00.446283+0000","last_active":"2024-09-14T04:02:04.195936+0000","last_peered":"2024-09-14T04:02:04.195936+0000","last_clean":"2024-09-14T04:02:04.195936+0000","last_became_active":"2024-09-14T03:55:00.445727+0000","last_became_peered":"2024-09-14T03:55:00.445727+0000","last_unstale":"2024-09-14T04:02:04.195936+0000","last_undegraded":"2024-09-14T04:02:04.195936+0000","last_fullsized":"2024-09-14T04:02:04.195936+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:19:53.864090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1356,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.195122+0000","last_change":"2024-09-14T03:56:43.465741+0000","last_active":"2024-09-14T04:02:04.195122+0000","last_peered":"2024-09-14T04:02:04.195122+0000","last_clean":"2024-09-14T04:02:04.195122+0000","last_became_active":"2024-09-14T03:56:43.465416+0000","last_became_peered":"2024-09-14T03:56:43.465416+0000","last_unstale":"2024-09-14T04:02:04.195122+0000","last_undegraded":"2024-09-14T04:02:04.195122+0000","last_fullsized":"2024-09-14T04:02:04.195122+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:01:16.881422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":1213,"reported_epoch":881,"state":"active+clean","last_fresh":"2024-09-14T04:02:01.420394+0000","last_change":"2024-09-14T03:56:43.460735+0000","last_active":"2024-09-14T04:02:01.420394+0000","last_peered":"2024-09-14T04:02:01.420394+0000","last_clean":"2024-09-14T04:02:01.420394+0000","last_became_active":"2024-09-14T03:56:43.460454+0000","last_became_peered":"2024-09-14T03:56:43.460454+0000","last_unstale":"2024-09-14T04:02:01.420394+0000","last_undegraded":"2024-09-14T04:02:01.420394+0000","last_fullsized":"2024-09-14T04:02:01.420394+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:19:47.130646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":1162,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.361954+0000","last_change":"2024-09-14T03:56:43.457085+0000","last_active":"2024-09-14T04:02:04.361954+0000","last_peered":"2024-09-14T04:02:04.361954+0000","last_clean":"2024-09-14T04:02:04.361954+0000","last_became_active":"2024-09-14T03:56:43.456794+0000","last_became_peered":"2024-09-14T03:56:43.456794+0000","last_unstale":"2024-09-14T04:02:04.361954+0000","last_undegraded":"2024-09-14T04:02:04.361954+0000","last_fullsized":"2024-09-14T04:02:04.361954+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:18:08.813716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":1191,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.361497+0000","last_change":"2024-09-14T03:56:08.141464+0000","last_active":"2024-09-14T04:02:04.361497+0000","last_peered":"2024-09-14T04:02:04.361497+0000","last_clean":"2024-09-14T04:02:04.361497+0000","last_became_active":"2024-09-14T03:56:08.141196+0000","last_became_peered":"2024-09-14T03:56:08.141196+0000","last_unstale":"2024-09-14T04:02:04.361497+0000","last_undegraded":"2024-09-14T04:02:04.361497+0000","last_fullsized":"2024-09-14T04:02:04.361497+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:20:57.256057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":1177,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.362590+0000","last_change":"2024-09-14T03:56:08.137805+0000","last_active":"2024-09-14T04:02:04.362590+0000","last_peered":"2024-09-14T04:02:04.362590+0000","last_clean":"2024-09-14T04:02:04.362590+0000","last_became_active":"2024-09-14T03:56:08.133631+0000","last_became_peered":"2024-09-14T03:56:08.133631+0000","last_unstale":"2024-09-14T04:02:04.362590+0000","last_undegraded":"2024-09-14T04:02:04.362590+0000","last_fullsized":"2024-09-14T04:02:04.362590+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:08:39.608118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"58'8","reported_seq":1321,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.340507+0000","last_change":"2024-09-14T03:53:54.056504+0000","last_active":"2024-09-14T04:02:04.340507+0000","last_peered":"2024-09-14T04:02:04.340507+0000","last_clean":"2024-09-14T04:02:04.340507+0000","last_became_active":"2024-09-14T03:53:54.055616+0000","last_became_peered":"2024-09-14T03:53:54.055616+0000","last_unstale":"2024-09-14T04:02:04.340507+0000","last_undegraded":"2024-09-14T04:02:04.340507+0000","last_fullsized":"2024-09-14T04:02:04.340507+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:24:07.424336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":36,"num_read_kb":36,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1142,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.364847+0000","last_change":"2024-09-14T03:56:43.457032+0000","last_active":"2024-09-14T04:02:04.364847+0000","last_peered":"2024-09-14T04:02:04.364847+0000","last_clean":"2024-09-14T04:02:04.364847+0000","last_became_active":"2024-09-14T03:56:43.455863+0000","last_became_peered":"2024-09-14T03:56:43.455863+0000","last_unstale":"2024-09-14T04:02:04.364847+0000","last_undegraded":"2024-09-14T04:02:04.364847+0000","last_fullsized":"2024-09-14T04:02:04.364847+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:46:01.983382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.8","version":"880'2136","reported_seq":5300,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.340531+0000","last_change":"2024-09-14T04:01:59.171954+0000","last_active":"2024-09-14T04:02:04.340531+0000","last_peered":"2024-09-14T04:02:04.340531+0000","last_clean":"2024-09-14T04:02:04.340531+0000","last_became_active":"2024-09-14T03:53:54.057271+0000","last_became_peered":"2024-09-14T03:53:54.057271+0000","last_unstale":"2024-09-14T04:02:04.340531+0000","last_undegraded":"2024-09-14T04:02:04.340531+0000","last_fullsized":"2024-09-14T04:02:04.340531+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:48:03.942146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:48:03.942146+0000","last_clean_scrub_stamp":"2024-09-14T03:48:03.942146+0000","objects_scrubbed":0,"log_size":2136,"log_dups_size":0,"ondisk_log_size":2136,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:55:57.734195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.01111256,"stat_sum":{"num_bytes":35184640,"num_objects":21,"num_object_clones":17,"num_object_copies":63,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":1,"num_read":274,"num_read_kb":427522,"num_write":716,"num_write_kb":137920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":44261376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"1e2","length":"1"}]},{"pgid":"2.f","version":"58'1","reported_seq":1239,"reported_epoch":882,"state":"active+clean","last_fresh":"2024-09-14T04:02:02.169018+0000","last_change":"2024-09-14T03:56:43.461645+0000","last_active":"2024-09-14T04:02:02.169018+0000","last_peered":"2024-09-14T04:02:02.169018+0000","last_clean":"2024-09-14T04:02:02.169018+0000","last_became_active":"2024-09-14T03:56:43.461318+0000","last_became_peered":"2024-09-14T03:56:43.461318+0000","last_unstale":"2024-09-14T04:02:02.169018+0000","last_undegraded":"2024-09-14T04:02:02.169018+0000","last_fullsized":"2024-09-14T04:02:02.169018+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:12:11.886020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":1162,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.362238+0000","last_change":"2024-09-14T03:56:43.474778+0000","last_active":"2024-09-14T04:02:04.362238+0000","last_peered":"2024-09-14T04:02:04.362238+0000","last_clean":"2024-09-14T04:02:04.362238+0000","last_became_active":"2024-09-14T03:56:43.473291+0000","last_became_peered":"2024-09-14T03:56:43.473291+0000","last_unstale":"2024-09-14T04:02:04.362238+0000","last_undegraded":"2024-09-14T04:02:04.362238+0000","last_fullsized":"2024-09-14T04:02:04.362238+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:15:19.998552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.9","version":"880'2982","reported_seq":5766,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.364570+0000","last_change":"2024-09-14T04:01:59.188094+0000","last_active":"2024-09-14T04:02:04.364570+0000","last_peered":"2024-09-14T04:02:04.364570+0000","last_clean":"2024-09-14T04:02:04.364570+0000","last_became_active":"2024-09-14T03:56:43.458767+0000","last_became_peered":"2024-09-14T03:56:43.458767+0000","last_unstale":"2024-09-14T04:02:04.364570+0000","last_undegraded":"2024-09-14T04:02:04.364570+0000","last_fullsized":"2024-09-14T04:02:04.364570+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:48:03.942146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:48:03.942146+0000","last_clean_scrub_stamp":"2024-09-14T03:48:03.942146+0000","objects_scrubbed":0,"log_size":2982,"log_dups_size":0,"ondisk_log_size":2982,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:43:01.760128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.013597052,"stat_sum":{"num_bytes":75341824,"num_objects":38,"num_object_clones":33,"num_object_copies":114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":38,"num_whiteouts":2,"num_read":367,"num_read_kb":619988,"num_write":991,"num_write_kb":211408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":54960128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"1e2","length":"1"}]},{"pgid":"2.e","version":"58'2","reported_seq":1302,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.340946+0000","last_change":"2024-09-14T03:56:43.449467+0000","last_active":"2024-09-14T04:02:04.340946+0000","last_peered":"2024-09-14T04:02:04.340946+0000","last_clean":"2024-09-14T04:02:04.340946+0000","last_became_active":"2024-09-14T03:56:43.449107+0000","last_became_peered":"2024-09-14T03:56:43.449107+0000","last_unstale":"2024-09-14T04:02:04.340946+0000","last_undegraded":"2024-09-14T04:02:04.340946+0000","last_fullsized":"2024-09-14T04:02:04.340946+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:02:13.551939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":9,"num_read_kb":9,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":1162,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.361167+0000","last_change":"2024-09-14T03:56:43.471841+0000","last_active":"2024-09-14T04:02:04.361167+0000","last_peered":"2024-09-14T04:02:04.361167+0000","last_clean":"2024-09-14T04:02:04.361167+0000","last_became_active":"2024-09-14T03:56:43.471507+0000","last_became_peered":"2024-09-14T03:56:43.471507+0000","last_unstale":"2024-09-14T04:02:04.361167+0000","last_undegraded":"2024-09-14T04:02:04.361167+0000","last_fullsized":"2024-09-14T04:02:04.361167+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:33:31.254237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.a","version":"879'531","reported_seq":3890,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.340521+0000","last_change":"2024-09-14T04:01:59.167937+0000","last_active":"2024-09-14T04:02:04.340521+0000","last_peered":"2024-09-14T04:02:04.340521+0000","last_clean":"2024-09-14T04:02:04.340521+0000","last_became_active":"2024-09-14T03:55:00.456350+0000","last_became_peered":"2024-09-14T03:55:00.456350+0000","last_unstale":"2024-09-14T04:02:04.340521+0000","last_undegraded":"2024-09-14T04:02:04.340521+0000","last_fullsized":"2024-09-14T04:02:04.340521+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:48:03.942146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:48:03.942146+0000","last_clean_scrub_stamp":"2024-09-14T03:48:03.942146+0000","objects_scrubbed":0,"log_size":531,"log_dups_size":0,"ondisk_log_size":531,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:58:18.727615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0070870200000000003,"stat_sum":{"num_bytes":6307840,"num_objects":7,"num_object_clones":6,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":0,"num_read":83,"num_read_kb":137543,"num_write":148,"num_write_kb":32080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":2408448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"1e2","length":"1"}]},{"pgid":"2.d","version":"58'1","reported_seq":1331,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.161685+0000","last_change":"2024-09-14T03:55:00.444739+0000","last_active":"2024-09-14T04:02:04.161685+0000","last_peered":"2024-09-14T04:02:04.161685+0000","last_clean":"2024-09-14T04:02:04.161685+0000","last_became_active":"2024-09-14T03:55:00.444290+0000","last_became_peered":"2024-09-14T03:55:00.444290+0000","last_unstale":"2024-09-14T04:02:04.161685+0000","last_undegraded":"2024-09-14T04:02:04.161685+0000","last_fullsized":"2024-09-14T04:02:04.161685+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:13:04.726693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1210,"reported_epoch":881,"state":"active+clean","last_fresh":"2024-09-14T04:02:01.420861+0000","last_change":"2024-09-14T03:56:08.135157+0000","last_active":"2024-09-14T04:02:01.420861+0000","last_peered":"2024-09-14T04:02:01.420861+0000","last_clean":"2024-09-14T04:02:01.420861+0000","last_became_active":"2024-09-14T03:56:08.134872+0000","last_became_peered":"2024-09-14T03:56:08.134872+0000","last_unstale":"2024-09-14T04:02:01.420861+0000","last_undegraded":"2024-09-14T04:02:01.420861+0000","last_fullsized":"2024-09-14T04:02:01.420861+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:27:50.147707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.b","version":"879'540","reported_seq":3905,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.340767+0000","last_change":"2024-09-14T04:01:59.174307+0000","last_active":"2024-09-14T04:02:04.340767+0000","last_peered":"2024-09-14T04:02:04.340767+0000","last_clean":"2024-09-14T04:02:04.340767+0000","last_became_active":"2024-09-14T03:55:35.121298+0000","last_became_peered":"2024-09-14T03:55:35.121298+0000","last_unstale":"2024-09-14T04:02:04.340767+0000","last_undegraded":"2024-09-14T04:02:04.340767+0000","last_fullsized":"2024-09-14T04:02:04.340767+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:48:03.942146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:48:03.942146+0000","last_clean_scrub_stamp":"2024-09-14T03:48:03.942146+0000","objects_scrubbed":0,"log_size":540,"log_dups_size":0,"ondisk_log_size":540,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:49:38.580218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.012536253000000001,"stat_sum":{"num_bytes":8396800,"num_objects":7,"num_object_clones":6,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":0,"num_read":80,"num_read_kb":105198,"num_write":178,"num_write_kb":24904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4341760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"1e2","length":"1"}]},{"pgid":"2.c","version":"0'0","reported_seq":1304,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.340563+0000","last_change":"2024-09-14T03:55:35.119021+0000","last_active":"2024-09-14T04:02:04.340563+0000","last_peered":"2024-09-14T04:02:04.340563+0000","last_clean":"2024-09-14T04:02:04.340563+0000","last_became_active":"2024-09-14T03:55:35.118738+0000","last_became_peered":"2024-09-14T03:55:35.118738+0000","last_unstale":"2024-09-14T04:02:04.340563+0000","last_undegraded":"2024-09-14T04:02:04.340563+0000","last_fullsized":"2024-09-14T04:02:04.340563+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:53:30.824999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":1142,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.362084+0000","last_change":"2024-09-14T03:56:43.455244+0000","last_active":"2024-09-14T04:02:04.362084+0000","last_peered":"2024-09-14T04:02:04.362084+0000","last_clean":"2024-09-14T04:02:04.362084+0000","last_became_active":"2024-09-14T03:56:43.454947+0000","last_became_peered":"2024-09-14T03:56:43.454947+0000","last_unstale":"2024-09-14T04:02:04.362084+0000","last_undegraded":"2024-09-14T04:02:04.362084+0000","last_fullsized":"2024-09-14T04:02:04.362084+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:18:48.730827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.4","version":"879'1945","reported_seq":4894,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.362868+0000","last_change":"2024-09-14T04:01:59.183344+0000","last_active":"2024-09-14T04:02:04.362868+0000","last_peered":"2024-09-14T04:02:04.362868+0000","last_clean":"2024-09-14T04:02:04.362868+0000","last_became_active":"2024-09-14T03:56:43.454406+0000","last_became_peered":"2024-09-14T03:56:43.454406+0000","last_unstale":"2024-09-14T04:02:04.362868+0000","last_undegraded":"2024-09-14T04:02:04.362868+0000","last_fullsized":"2024-09-14T04:02:04.362868+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:48:03.942146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:48:03.942146+0000","last_clean_scrub_stamp":"2024-09-14T03:48:03.942146+0000","objects_scrubbed":0,"log_size":1945,"log_dups_size":0,"ondisk_log_size":1945,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:24:25.788106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0070274090000000001,"stat_sum":{"num_bytes":33538048,"num_objects":24,"num_object_clones":20,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":313,"num_read_kb":499473,"num_write":664,"num_write_kb":96128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":12288000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"1e2","length":"1"}]},{"pgid":"4.5","version":"0'0","reported_seq":1181,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.363670+0000","last_change":"2024-09-14T03:56:43.466813+0000","last_active":"2024-09-14T04:02:04.363670+0000","last_peered":"2024-09-14T04:02:04.363670+0000","last_clean":"2024-09-14T04:02:04.363670+0000","last_became_active":"2024-09-14T03:56:43.466470+0000","last_became_peered":"2024-09-14T03:56:43.466470+0000","last_unstale":"2024-09-14T04:02:04.363670+0000","last_undegraded":"2024-09-14T04:02:04.363670+0000","last_fullsized":"2024-09-14T04:02:04.363670+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_clean_scrub_stamp":"2024-09-14T03:45:08.693776+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:58:33.924123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039339699999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"286'160","reported_seq":1472,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.361102+0000","last_change":"2024-09-14T03:56:43.459877+0000","last_active":"2024-09-14T04:02:04.361102+0000","last_peered":"2024-09-14T04:02:04.361102+0000","last_clean":"2024-09-14T04:02:04.361102+0000","last_became_active":"2024-09-14T03:56:43.458042+0000","last_became_peered":"2024-09-14T03:56:43.458042+0000","last_unstale":"2024-09-14T04:02:04.361102+0000","last_undegraded":"2024-09-14T04:02:04.361102+0000","last_fullsized":"2024-09-14T04:02:04.361102+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:41:39.458815+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:41:39.458815+0000","last_clean_scrub_stamp":"2024-09-14T03:41:39.458815+0000","objects_scrubbed":0,"log_size":160,"log_dups_size":0,"ondisk_log_size":160,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:42:29.486061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":840,"num_read_kb":1709,"num_write":269,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1181,"reported_epoch":881,"state":"active+clean","last_fresh":"2024-09-14T04:02:01.419912+0000","last_change":"2024-09-14T03:56:43.451003+0000","last_active":"2024-09-14T04:02:01.419912+0000","last_peered":"2024-09-14T04:02:01.419912+0000","last_clean":"2024-09-14T04:02:01.419912+0000","last_became_active":"2024-09-14T03:56:43.450740+0000","last_became_peered":"2024-09-14T03:56:43.450740+0000","last_unstale":"2024-09-14T04:02:01.419912+0000","last_undegraded":"2024-09-14T04:02:01.419912+0000","last_fullsized":"2024-09-14T04:02:01.419912+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:48:24.329919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1276,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.248954+0000","last_change":"2024-09-14T03:56:08.130641+0000","last_active":"2024-09-14T04:02:04.248954+0000","last_peered":"2024-09-14T04:02:04.248954+0000","last_clean":"2024-09-14T04:02:04.248954+0000","last_became_active":"2024-09-14T03:56:08.130333+0000","last_became_peered":"2024-09-14T03:56:08.130333+0000","last_unstale":"2024-09-14T04:02:04.248954+0000","last_undegraded":"2024-09-14T04:02:04.248954+0000","last_fullsized":"2024-09-14T04:02:04.248954+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:05:52.237586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.7","version":"880'1285","reported_seq":4314,"reported_epoch":881,"state":"active+clean","last_fresh":"2024-09-14T04:02:01.421287+0000","last_change":"2024-09-14T04:01:59.173487+0000","last_active":"2024-09-14T04:02:01.421287+0000","last_peered":"2024-09-14T04:02:01.421287+0000","last_clean":"2024-09-14T04:02:01.421287+0000","last_became_active":"2024-09-14T03:55:35.119369+0000","last_became_peered":"2024-09-14T03:55:35.119369+0000","last_unstale":"2024-09-14T04:02:01.421287+0000","last_undegraded":"2024-09-14T04:02:01.421287+0000","last_fullsized":"2024-09-14T04:02:01.421287+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:48:03.942146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:48:03.942146+0000","last_clean_scrub_stamp":"2024-09-14T03:48:03.942146+0000","objects_scrubbed":0,"log_size":1285,"log_dups_size":0,"ondisk_log_size":1285,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:29:13.652784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0078329620000000006,"stat_sum":{"num_bytes":18997248,"num_objects":16,"num_object_clones":14,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16,"num_whiteouts":1,"num_read":139,"num_read_kb":142256,"num_write":443,"num_write_kb":81544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":17260544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"1b8","length":"1"},{"start":"1d9","length":"1"},{"start":"1e1","length":"2"}]},{"pgid":"4.6","version":"0'0","reported_seq":1367,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.195995+0000","last_change":"2024-09-14T03:53:54.049373+0000","last_active":"2024-09-14T04:02:04.195995+0000","last_peered":"2024-09-14T04:02:04.195995+0000","last_clean":"2024-09-14T04:02:04.195995+0000","last_became_active":"2024-09-14T03:53:54.049063+0000","last_became_peered":"2024-09-14T03:53:54.049063+0000","last_unstale":"2024-09-14T04:02:04.195995+0000","last_undegraded":"2024-09-14T04:02:04.195995+0000","last_fullsized":"2024-09-14T04:02:04.195995+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_clean_scrub_stamp":"2024-09-14T03:45:08.693776+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:49:02.127796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1153,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.361177+0000","last_change":"2024-09-14T03:56:43.459145+0000","last_active":"2024-09-14T04:02:04.361177+0000","last_peered":"2024-09-14T04:02:04.361177+0000","last_clean":"2024-09-14T04:02:04.361177+0000","last_became_active":"2024-09-14T03:56:43.458542+0000","last_became_peered":"2024-09-14T03:56:43.458542+0000","last_unstale":"2024-09-14T04:02:04.361177+0000","last_undegraded":"2024-09-14T04:02:04.361177+0000","last_fullsized":"2024-09-14T04:02:04.361177+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:24:39.657556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1356,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.195681+0000","last_change":"2024-09-14T03:55:00.447007+0000","last_active":"2024-09-14T04:02:04.195681+0000","last_peered":"2024-09-14T04:02:04.195681+0000","last_clean":"2024-09-14T04:02:04.195681+0000","last_became_active":"2024-09-14T03:55:00.446678+0000","last_became_peered":"2024-09-14T03:55:00.446678+0000","last_unstale":"2024-09-14T04:02:04.195681+0000","last_undegraded":"2024-09-14T04:02:04.195681+0000","last_fullsized":"2024-09-14T04:02:04.195681+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:49:34.332471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.6","version":"876'183","reported_seq":3511,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.341206+0000","last_change":"2024-09-14T04:01:59.168652+0000","last_active":"2024-09-14T04:02:04.341206+0000","last_peered":"2024-09-14T04:02:04.341206+0000","last_clean":"2024-09-14T04:02:04.341206+0000","last_became_active":"2024-09-14T03:55:35.120393+0000","last_became_peered":"2024-09-14T03:55:35.120393+0000","last_unstale":"2024-09-14T04:02:04.341206+0000","last_undegraded":"2024-09-14T04:02:04.341206+0000","last_fullsized":"2024-09-14T04:02:04.341206+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:48:03.942146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:48:03.942146+0000","last_clean_scrub_stamp":"2024-09-14T03:48:03.942146+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:17:10.474833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.007447464,"stat_sum":{"num_bytes":4284416,"num_objects":4,"num_object_clones":3,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":1,"num_read":23,"num_read_kb":6080,"num_write":72,"num_write_kb":10752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"1e2","length":"1"}]},{"pgid":"4.7","version":"0'0","reported_seq":1332,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.162048+0000","last_change":"2024-09-14T03:55:35.116531+0000","last_active":"2024-09-14T04:02:04.162048+0000","last_peered":"2024-09-14T04:02:04.162048+0000","last_clean":"2024-09-14T04:02:04.162048+0000","last_became_active":"2024-09-14T03:55:35.116241+0000","last_became_peered":"2024-09-14T03:55:35.116241+0000","last_unstale":"2024-09-14T04:02:04.162048+0000","last_undegraded":"2024-09-14T04:02:04.162048+0000","last_fullsized":"2024-09-14T04:02:04.162048+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_clean_scrub_stamp":"2024-09-14T03:45:08.693776+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:50:11.700786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057164100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1304,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.340453+0000","last_change":"2024-09-14T03:54:27.093302+0000","last_active":"2024-09-14T04:02:04.340453+0000","last_peered":"2024-09-14T04:02:04.340453+0000","last_clean":"2024-09-14T04:02:04.340453+0000","last_became_active":"2024-09-14T03:54:27.092985+0000","last_became_peered":"2024-09-14T03:54:27.092985+0000","last_unstale":"2024-09-14T04:02:04.340453+0000","last_undegraded":"2024-09-14T04:02:04.340453+0000","last_fullsized":"2024-09-14T04:02:04.340453+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:04:39.378654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1325,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.161171+0000","last_change":"2024-09-14T03:56:08.139504+0000","last_active":"2024-09-14T04:02:04.161171+0000","last_peered":"2024-09-14T04:02:04.161171+0000","last_clean":"2024-09-14T04:02:04.161171+0000","last_became_active":"2024-09-14T03:56:08.139131+0000","last_became_peered":"2024-09-14T03:56:08.139131+0000","last_unstale":"2024-09-14T04:02:04.161171+0000","last_undegraded":"2024-09-14T04:02:04.161171+0000","last_fullsized":"2024-09-14T04:02:04.161171+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:47:00.832980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.5","version":"872'1723","reported_seq":5066,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.196541+0000","last_change":"2024-09-14T04:01:59.167088+0000","last_active":"2024-09-14T04:02:04.196541+0000","last_peered":"2024-09-14T04:02:04.196541+0000","last_clean":"2024-09-14T04:02:04.196541+0000","last_became_active":"2024-09-14T03:55:00.444908+0000","last_became_peered":"2024-09-14T03:55:00.444908+0000","last_unstale":"2024-09-14T04:02:04.196541+0000","last_undegraded":"2024-09-14T04:02:04.196541+0000","last_fullsized":"2024-09-14T04:02:04.196541+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:48:03.942146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:48:03.942146+0000","last_clean_scrub_stamp":"2024-09-14T03:48:03.942146+0000","objects_scrubbed":0,"log_size":1723,"log_dups_size":0,"ondisk_log_size":1723,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:45:54.781724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001425777,"stat_sum":{"num_bytes":23748608,"num_objects":11,"num_object_clones":8,"num_object_copies":33,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11,"num_whiteouts":1,"num_read":223,"num_read_kb":415961,"num_write":462,"num_write_kb":146160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":57917440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"1e2","length":"1"}]},{"pgid":"4.4","version":"0'0","reported_seq":1335,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.161628+0000","last_change":"2024-09-14T03:54:27.084142+0000","last_active":"2024-09-14T04:02:04.161628+0000","last_peered":"2024-09-14T04:02:04.161628+0000","last_clean":"2024-09-14T04:02:04.161628+0000","last_became_active":"2024-09-14T03:54:27.083857+0000","last_became_peered":"2024-09-14T03:54:27.083857+0000","last_unstale":"2024-09-14T04:02:04.161628+0000","last_undegraded":"2024-09-14T04:02:04.161628+0000","last_fullsized":"2024-09-14T04:02:04.161628+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_clean_scrub_stamp":"2024-09-14T03:45:08.693776+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:54:14.576221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1190,"reported_epoch":881,"state":"active+clean","last_fresh":"2024-09-14T04:02:01.420191+0000","last_change":"2024-09-14T03:56:08.134476+0000","last_active":"2024-09-14T04:02:01.420191+0000","last_peered":"2024-09-14T04:02:01.420191+0000","last_clean":"2024-09-14T04:02:01.420191+0000","last_became_active":"2024-09-14T03:56:08.133788+0000","last_became_peered":"2024-09-14T03:56:08.133788+0000","last_unstale":"2024-09-14T04:02:01.420191+0000","last_undegraded":"2024-09-14T04:02:01.420191+0000","last_fullsized":"2024-09-14T04:02:01.420191+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:28:40.592346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1236,"reported_epoch":882,"state":"active+clean","last_fresh":"2024-09-14T04:02:02.168578+0000","last_change":"2024-09-14T03:56:08.130534+0000","last_active":"2024-09-14T04:02:02.168578+0000","last_peered":"2024-09-14T04:02:02.168578+0000","last_clean":"2024-09-14T04:02:02.168578+0000","last_became_active":"2024-09-14T03:56:08.130270+0000","last_became_peered":"2024-09-14T03:56:08.130270+0000","last_unstale":"2024-09-14T04:02:02.168578+0000","last_undegraded":"2024-09-14T04:02:02.168578+0000","last_fullsized":"2024-09-14T04:02:02.168578+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:29:00.439422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.3","version":"0'0","reported_seq":3455,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.196297+0000","last_change":"2024-09-14T04:01:59.166594+0000","last_active":"2024-09-14T04:02:04.196297+0000","last_peered":"2024-09-14T04:02:04.196297+0000","last_clean":"2024-09-14T04:02:04.196297+0000","last_became_active":"2024-09-14T03:56:08.131927+0000","last_became_peered":"2024-09-14T03:56:08.131927+0000","last_unstale":"2024-09-14T04:02:04.196297+0000","last_undegraded":"2024-09-14T04:02:04.196297+0000","last_fullsized":"2024-09-14T04:02:04.196297+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:48:03.942146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:48:03.942146+0000","last_clean_scrub_stamp":"2024-09-14T03:48:03.942146+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:23:11.051086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0012824830000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"1e2","length":"1"}]},{"pgid":"4.2","version":"67'2","reported_seq":1337,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.161575+0000","last_change":"2024-09-14T03:55:35.115630+0000","last_active":"2024-09-14T04:02:04.161575+0000","last_peered":"2024-09-14T04:02:04.161575+0000","last_clean":"2024-09-14T04:02:04.161575+0000","last_became_active":"2024-09-14T03:55:35.115276+0000","last_became_peered":"2024-09-14T03:55:35.115276+0000","last_unstale":"2024-09-14T04:02:04.161575+0000","last_undegraded":"2024-09-14T04:02:04.161575+0000","last_fullsized":"2024-09-14T04:02:04.161575+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_clean_scrub_stamp":"2024-09-14T03:45:08.693776+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:44:18.021116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000682786,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1336,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.161500+0000","last_change":"2024-09-14T03:56:43.463406+0000","last_active":"2024-09-14T04:02:04.161500+0000","last_peered":"2024-09-14T04:02:04.161500+0000","last_clean":"2024-09-14T04:02:04.161500+0000","last_became_active":"2024-09-14T03:56:43.463093+0000","last_became_peered":"2024-09-14T03:56:43.463093+0000","last_unstale":"2024-09-14T04:02:04.161500+0000","last_undegraded":"2024-09-14T04:02:04.161500+0000","last_fullsized":"2024-09-14T04:02:04.161500+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:12:56.569312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1217,"reported_epoch":881,"state":"active+clean","last_fresh":"2024-09-14T04:02:01.420325+0000","last_change":"2024-09-14T03:55:35.122985+0000","last_active":"2024-09-14T04:02:01.420325+0000","last_peered":"2024-09-14T04:02:01.420325+0000","last_clean":"2024-09-14T04:02:01.420325+0000","last_became_active":"2024-09-14T03:55:35.122695+0000","last_became_peered":"2024-09-14T03:55:35.122695+0000","last_unstale":"2024-09-14T04:02:01.420325+0000","last_undegraded":"2024-09-14T04:02:01.420325+0000","last_fullsized":"2024-09-14T04:02:01.420325+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:35:13.026164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.2","version":"880'1636","reported_seq":4633,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.361750+0000","last_change":"2024-09-14T04:01:59.184685+0000","last_active":"2024-09-14T04:02:04.361750+0000","last_peered":"2024-09-14T04:02:04.361750+0000","last_clean":"2024-09-14T04:02:04.361750+0000","last_became_active":"2024-09-14T03:56:08.136604+0000","last_became_peered":"2024-09-14T03:56:08.136604+0000","last_unstale":"2024-09-14T04:02:04.361750+0000","last_undegraded":"2024-09-14T04:02:04.361750+0000","last_fullsized":"2024-09-14T04:02:04.361750+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:48:03.942146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:48:03.942146+0000","last_clean_scrub_stamp":"2024-09-14T03:48:03.942146+0000","objects_scrubbed":0,"log_size":1636,"log_dups_size":0,"ondisk_log_size":1636,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:51:33.300207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0056362010000000004,"stat_sum":{"num_bytes":19570688,"num_objects":16,"num_object_clones":13,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16,"num_whiteouts":2,"num_read":229,"num_read_kb":294527,"num_write":526,"num_write_kb":155120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":4218880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[{"start":"1e2","length":"1"}]},{"pgid":"4.3","version":"65'1","reported_seq":1368,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.195112+0000","last_change":"2024-09-14T03:56:43.470019+0000","last_active":"2024-09-14T04:02:04.195112+0000","last_peered":"2024-09-14T04:02:04.195112+0000","last_clean":"2024-09-14T04:02:04.195112+0000","last_became_active":"2024-09-14T03:56:43.468862+0000","last_became_peered":"2024-09-14T03:56:43.468862+0000","last_unstale":"2024-09-14T04:02:04.195112+0000","last_undegraded":"2024-09-14T04:02:04.195112+0000","last_fullsized":"2024-09-14T04:02:04.195112+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_clean_scrub_stamp":"2024-09-14T03:45:08.693776+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:01:40.401351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1165,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.362244+0000","last_change":"2024-09-14T03:56:43.469826+0000","last_active":"2024-09-14T04:02:04.362244+0000","last_peered":"2024-09-14T04:02:04.362244+0000","last_clean":"2024-09-14T04:02:04.362244+0000","last_became_active":"2024-09-14T03:56:43.469307+0000","last_became_peered":"2024-09-14T03:56:43.469307+0000","last_unstale":"2024-09-14T04:02:04.362244+0000","last_undegraded":"2024-09-14T04:02:04.362244+0000","last_fullsized":"2024-09-14T04:02:04.362244+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:54:44.254256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1325,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.161411+0000","last_change":"2024-09-14T03:55:35.116461+0000","last_active":"2024-09-14T04:02:04.161411+0000","last_peered":"2024-09-14T04:02:04.161411+0000","last_clean":"2024-09-14T04:02:04.161411+0000","last_became_active":"2024-09-14T03:55:35.115800+0000","last_became_peered":"2024-09-14T03:55:35.115800+0000","last_unstale":"2024-09-14T04:02:04.161411+0000","last_undegraded":"2024-09-14T04:02:04.161411+0000","last_fullsized":"2024-09-14T04:02:04.161411+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:05:46.628658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.1","version":"876'2250","reported_seq":5243,"reported_epoch":882,"state":"active+clean","last_fresh":"2024-09-14T04:02:02.168231+0000","last_change":"2024-09-14T04:01:59.162057+0000","last_active":"2024-09-14T04:02:02.168231+0000","last_peered":"2024-09-14T04:02:02.168231+0000","last_clean":"2024-09-14T04:02:02.168231+0000","last_became_active":"2024-09-14T03:56:43.460248+0000","last_became_peered":"2024-09-14T03:56:43.460248+0000","last_unstale":"2024-09-14T04:02:02.168231+0000","last_undegraded":"2024-09-14T04:02:02.168231+0000","last_fullsized":"2024-09-14T04:02:02.168231+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:48:03.942146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:48:03.942146+0000","last_clean_scrub_stamp":"2024-09-14T03:48:03.942146+0000","objects_scrubbed":0,"log_size":2250,"log_dups_size":0,"ondisk_log_size":2250,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:57:21.091991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00077465399999999995,"stat_sum":{"num_bytes":51404800,"num_objects":28,"num_object_clones":24,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":219,"num_read_kb":308353,"num_write":672,"num_write_kb":162976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":102203392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[{"start":"1b8","length":"1"},{"start":"1d9","length":"1"},{"start":"1e1","length":"2"}]},{"pgid":"4.0","version":"0'0","reported_seq":1284,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.248666+0000","last_change":"2024-09-14T03:56:43.465066+0000","last_active":"2024-09-14T04:02:04.248666+0000","last_peered":"2024-09-14T04:02:04.248666+0000","last_clean":"2024-09-14T04:02:04.248666+0000","last_became_active":"2024-09-14T03:56:43.464394+0000","last_became_peered":"2024-09-14T03:56:43.464394+0000","last_unstale":"2024-09-14T04:02:04.248666+0000","last_undegraded":"2024-09-14T04:02:04.248666+0000","last_fullsized":"2024-09-14T04:02:04.248666+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_clean_scrub_stamp":"2024-09-14T03:45:08.693776+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:42:40.493433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031572099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"599'3","reported_seq":1334,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.161878+0000","last_change":"2024-09-14T03:56:08.139485+0000","last_active":"2024-09-14T04:02:04.161878+0000","last_peered":"2024-09-14T04:02:04.161878+0000","last_clean":"2024-09-14T04:02:04.161878+0000","last_became_active":"2024-09-14T03:56:08.139136+0000","last_became_peered":"2024-09-14T03:56:08.139136+0000","last_unstale":"2024-09-14T04:02:04.161878+0000","last_undegraded":"2024-09-14T04:02:04.161878+0000","last_fullsized":"2024-09-14T04:02:04.161878+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:54.094774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":1286,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.248669+0000","last_change":"2024-09-14T03:56:43.468699+0000","last_active":"2024-09-14T04:02:04.248669+0000","last_peered":"2024-09-14T04:02:04.248669+0000","last_clean":"2024-09-14T04:02:04.248669+0000","last_became_active":"2024-09-14T03:56:43.468320+0000","last_became_peered":"2024-09-14T03:56:43.468320+0000","last_unstale":"2024-09-14T04:02:04.248669+0000","last_undegraded":"2024-09-14T04:02:04.248669+0000","last_fullsized":"2024-09-14T04:02:04.248669+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:42:10.554271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.0","version":"880'3203","reported_seq":6217,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.249001+0000","last_change":"2024-09-14T04:01:59.177134+0000","last_active":"2024-09-14T04:02:04.249001+0000","last_peered":"2024-09-14T04:02:04.249001+0000","last_clean":"2024-09-14T04:02:04.249001+0000","last_became_active":"2024-09-14T03:55:00.452650+0000","last_became_peered":"2024-09-14T03:55:00.452650+0000","last_unstale":"2024-09-14T04:02:04.249001+0000","last_undegraded":"2024-09-14T04:02:04.249001+0000","last_fullsized":"2024-09-14T04:02:04.249001+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:48:03.942146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:48:03.942146+0000","last_clean_scrub_stamp":"2024-09-14T03:48:03.942146+0000","objects_scrubbed":0,"log_size":3203,"log_dups_size":0,"ondisk_log_size":3203,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:16:17.741186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0073477289999999999,"stat_sum":{"num_bytes":56262656,"num_objects":33,"num_object_clones":27,"num_object_copies":99,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":33,"num_whiteouts":1,"num_read":456,"num_read_kb":644313,"num_write":946,"num_write_kb":213616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":63430656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"1e2","length":"1"}]},{"pgid":"4.1","version":"0'0","reported_seq":1235,"reported_epoch":882,"state":"active+clean","last_fresh":"2024-09-14T04:02:02.168335+0000","last_change":"2024-09-14T03:56:08.141545+0000","last_active":"2024-09-14T04:02:02.168335+0000","last_peered":"2024-09-14T04:02:02.168335+0000","last_clean":"2024-09-14T04:02:02.168335+0000","last_became_active":"2024-09-14T03:56:08.141272+0000","last_became_peered":"2024-09-14T03:56:08.141272+0000","last_unstale":"2024-09-14T04:02:02.168335+0000","last_undegraded":"2024-09-14T04:02:02.168335+0000","last_fullsized":"2024-09-14T04:02:02.168335+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":62,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:45:08.693776+0000","last_clean_scrub_stamp":"2024-09-14T03:45:08.693776+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:25:58.023749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"58'2","reported_seq":1168,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.363536+0000","last_change":"2024-09-14T03:56:43.467159+0000","last_active":"2024-09-14T04:02:04.363536+0000","last_peered":"2024-09-14T04:02:04.363536+0000","last_clean":"2024-09-14T04:02:04.363536+0000","last_became_active":"2024-09-14T03:56:43.466829+0000","last_became_peered":"2024-09-14T03:56:43.466829+0000","last_unstale":"2024-09-14T04:02:04.363536+0000","last_undegraded":"2024-09-14T04:02:04.363536+0000","last_fullsized":"2024-09-14T04:02:04.363536+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:20:57.915505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":9,"num_read_kb":9,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":1356,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.196050+0000","last_change":"2024-09-14T03:55:00.450788+0000","last_active":"2024-09-14T04:02:04.196050+0000","last_peered":"2024-09-14T04:02:04.196050+0000","last_clean":"2024-09-14T04:02:04.196050+0000","last_became_active":"2024-09-14T03:55:00.450480+0000","last_became_peered":"2024-09-14T03:55:00.450480+0000","last_unstale":"2024-09-14T04:02:04.196050+0000","last_undegraded":"2024-09-14T04:02:04.196050+0000","last_fullsized":"2024-09-14T04:02:04.196050+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:04:21.413974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.f","version":"874'491","reported_seq":3717,"reported_epoch":881,"state":"active+clean","last_fresh":"2024-09-14T04:02:01.421014+0000","last_change":"2024-09-14T04:01:59.165896+0000","last_active":"2024-09-14T04:02:01.421014+0000","last_peered":"2024-09-14T04:02:01.421014+0000","last_clean":"2024-09-14T04:02:01.421014+0000","last_became_active":"2024-09-14T03:56:08.134611+0000","last_became_peered":"2024-09-14T03:56:08.134611+0000","last_unstale":"2024-09-14T04:02:01.421014+0000","last_undegraded":"2024-09-14T04:02:01.421014+0000","last_fullsized":"2024-09-14T04:02:01.421014+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:48:03.942146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:48:03.942146+0000","last_clean_scrub_stamp":"2024-09-14T03:48:03.942146+0000","objects_scrubbed":0,"log_size":491,"log_dups_size":0,"ondisk_log_size":491,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:32:17.702579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00070287799999999999,"stat_sum":{"num_bytes":2187264,"num_objects":2,"num_object_clones":1,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":222235,"num_write":179,"num_write_kb":25640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":909312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"1b8","length":"1"},{"start":"1d9","length":"1"},{"start":"1e1","length":"2"}]},{"pgid":"2.8","version":"0'0","reported_seq":1153,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.361032+0000","last_change":"2024-09-14T03:56:43.459450+0000","last_active":"2024-09-14T04:02:04.361032+0000","last_peered":"2024-09-14T04:02:04.361032+0000","last_clean":"2024-09-14T04:02:04.361032+0000","last_became_active":"2024-09-14T03:56:43.458897+0000","last_became_peered":"2024-09-14T03:56:43.458897+0000","last_unstale":"2024-09-14T04:02:04.361032+0000","last_undegraded":"2024-09-14T04:02:04.361032+0000","last_fullsized":"2024-09-14T04:02:04.361032+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:58:16.837503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1258,"reported_epoch":882,"state":"active+clean","last_fresh":"2024-09-14T04:02:02.168441+0000","last_change":"2024-09-14T03:56:43.459488+0000","last_active":"2024-09-14T04:02:02.168441+0000","last_peered":"2024-09-14T04:02:02.168441+0000","last_clean":"2024-09-14T04:02:02.168441+0000","last_became_active":"2024-09-14T03:56:43.459188+0000","last_became_peered":"2024-09-14T03:56:43.459188+0000","last_unstale":"2024-09-14T04:02:02.168441+0000","last_undegraded":"2024-09-14T04:02:02.168441+0000","last_fullsized":"2024-09-14T04:02:02.168441+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:42:06.498660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"879'2465","reported_seq":5497,"reported_epoch":882,"state":"active+clean","last_fresh":"2024-09-14T04:02:02.169086+0000","last_change":"2024-09-14T04:01:59.172186+0000","last_active":"2024-09-14T04:02:02.169086+0000","last_peered":"2024-09-14T04:02:02.169086+0000","last_clean":"2024-09-14T04:02:02.169086+0000","last_became_active":"2024-09-14T03:55:35.114101+0000","last_became_peered":"2024-09-14T03:55:35.114101+0000","last_unstale":"2024-09-14T04:02:02.169086+0000","last_undegraded":"2024-09-14T04:02:02.169086+0000","last_fullsized":"2024-09-14T04:02:02.169086+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:48:03.942146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:48:03.942146+0000","last_clean_scrub_stamp":"2024-09-14T03:48:03.942146+0000","objects_scrubbed":0,"log_size":2465,"log_dups_size":0,"ondisk_log_size":2465,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:36:18.352048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.01055765,"stat_sum":{"num_bytes":31293440,"num_objects":23,"num_object_clones":19,"num_object_copies":69,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":23,"num_whiteouts":0,"num_read":337,"num_read_kb":631662,"num_write":854,"num_write_kb":184744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":80699392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[{"start":"1b8","length":"1"},{"start":"1d9","length":"1"},{"start":"1e1","length":"2"}]},{"pgid":"2.9","version":"0'0","reported_seq":1324,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.161204+0000","last_change":"2024-09-14T03:56:43.477216+0000","last_active":"2024-09-14T04:02:04.161204+0000","last_peered":"2024-09-14T04:02:04.161204+0000","last_clean":"2024-09-14T04:02:04.161204+0000","last_became_active":"2024-09-14T03:56:43.476911+0000","last_became_peered":"2024-09-14T03:56:43.476911+0000","last_unstale":"2024-09-14T04:02:04.161204+0000","last_undegraded":"2024-09-14T04:02:04.161204+0000","last_fullsized":"2024-09-14T04:02:04.161204+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:58:56.569147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1278,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.248869+0000","last_change":"2024-09-14T03:56:43.463957+0000","last_active":"2024-09-14T04:02:04.248869+0000","last_peered":"2024-09-14T04:02:04.248869+0000","last_clean":"2024-09-14T04:02:04.248869+0000","last_became_active":"2024-09-14T03:56:43.463575+0000","last_became_peered":"2024-09-14T03:56:43.463575+0000","last_unstale":"2024-09-14T04:02:04.248869+0000","last_undegraded":"2024-09-14T04:02:04.248869+0000","last_fullsized":"2024-09-14T04:02:04.248869+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:41:48.802957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.d","version":"879'4869","reported_seq":7687,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.340329+0000","last_change":"2024-09-14T04:01:59.179587+0000","last_active":"2024-09-14T04:02:04.340329+0000","last_peered":"2024-09-14T04:02:04.340329+0000","last_clean":"2024-09-14T04:02:04.340329+0000","last_became_active":"2024-09-14T03:56:43.459908+0000","last_became_peered":"2024-09-14T03:56:43.459908+0000","last_unstale":"2024-09-14T04:02:04.340329+0000","last_undegraded":"2024-09-14T04:02:04.340329+0000","last_fullsized":"2024-09-14T04:02:04.340329+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:48:03.942146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:48:03.942146+0000","last_clean_scrub_stamp":"2024-09-14T03:48:03.942146+0000","objects_scrubbed":0,"log_size":4869,"log_dups_size":0,"ondisk_log_size":4869,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:30:39.744881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.015072644,"stat_sum":{"num_bytes":126844928,"num_objects":46,"num_object_clones":38,"num_object_copies":138,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":46,"num_whiteouts":1,"num_read":644,"num_read_kb":1274691,"num_write":1352,"num_write_kb":341960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":56729600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"1e2","length":"1"}]},{"pgid":"2.a","version":"0'0","reported_seq":1324,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.161367+0000","last_change":"2024-09-14T03:56:43.454609+0000","last_active":"2024-09-14T04:02:04.161367+0000","last_peered":"2024-09-14T04:02:04.161367+0000","last_clean":"2024-09-14T04:02:04.161367+0000","last_became_active":"2024-09-14T03:56:43.454321+0000","last_became_peered":"2024-09-14T03:56:43.454321+0000","last_unstale":"2024-09-14T04:02:04.161367+0000","last_undegraded":"2024-09-14T04:02:04.161367+0000","last_fullsized":"2024-09-14T04:02:04.161367+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:29:35.122922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":1285,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.249214+0000","last_change":"2024-09-14T03:54:27.095618+0000","last_active":"2024-09-14T04:02:04.249214+0000","last_peered":"2024-09-14T04:02:04.249214+0000","last_clean":"2024-09-14T04:02:04.249214+0000","last_became_active":"2024-09-14T03:54:27.095332+0000","last_became_peered":"2024-09-14T03:54:27.095332+0000","last_unstale":"2024-09-14T04:02:04.249214+0000","last_undegraded":"2024-09-14T04:02:04.249214+0000","last_fullsized":"2024-09-14T04:02:04.249214+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:13:16.480032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.c","version":"880'1607","reported_seq":4962,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.161533+0000","last_change":"2024-09-14T04:01:59.175215+0000","last_active":"2024-09-14T04:02:04.161533+0000","last_peered":"2024-09-14T04:02:04.161533+0000","last_clean":"2024-09-14T04:02:04.161533+0000","last_became_active":"2024-09-14T03:55:00.454239+0000","last_became_peered":"2024-09-14T03:55:00.454239+0000","last_unstale":"2024-09-14T04:02:04.161533+0000","last_undegraded":"2024-09-14T04:02:04.161533+0000","last_fullsized":"2024-09-14T04:02:04.161533+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:48:03.942146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:48:03.942146+0000","last_clean_scrub_stamp":"2024-09-14T03:48:03.942146+0000","objects_scrubbed":0,"log_size":1607,"log_dups_size":0,"ondisk_log_size":1607,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:47:26.799798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0061235359999999997,"stat_sum":{"num_bytes":58040320,"num_objects":16,"num_object_clones":13,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16,"num_whiteouts":0,"num_read":291,"num_read_kb":521918,"num_write":419,"num_write_kb":145880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":24739840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"1e2","length":"1"}]},{"pgid":"2.b","version":"596'4","reported_seq":1162,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.361789+0000","last_change":"2024-09-14T03:56:43.469732+0000","last_active":"2024-09-14T04:02:04.361789+0000","last_peered":"2024-09-14T04:02:04.361789+0000","last_clean":"2024-09-14T04:02:04.361789+0000","last_became_active":"2024-09-14T03:56:43.469352+0000","last_became_peered":"2024-09-14T03:56:43.469352+0000","last_unstale":"2024-09-14T04:02:04.361789+0000","last_undegraded":"2024-09-14T04:02:04.361789+0000","last_fullsized":"2024-09-14T04:02:04.361789+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:22:06.039574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1290,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.340383+0000","last_change":"2024-09-14T03:55:00.458346+0000","last_active":"2024-09-14T04:02:04.340383+0000","last_peered":"2024-09-14T04:02:04.340383+0000","last_clean":"2024-09-14T04:02:04.340383+0000","last_became_active":"2024-09-14T03:55:00.457918+0000","last_became_peered":"2024-09-14T03:55:00.457918+0000","last_unstale":"2024-09-14T04:02:04.340383+0000","last_undegraded":"2024-09-14T04:02:04.340383+0000","last_fullsized":"2024-09-14T04:02:04.340383+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:14:51.593558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":1198,"reported_epoch":881,"state":"active+clean","last_fresh":"2024-09-14T04:02:01.420126+0000","last_change":"2024-09-14T03:56:08.136207+0000","last_active":"2024-09-14T04:02:01.420126+0000","last_peered":"2024-09-14T04:02:01.420126+0000","last_clean":"2024-09-14T04:02:01.420126+0000","last_became_active":"2024-09-14T03:56:08.135357+0000","last_became_peered":"2024-09-14T03:56:08.135357+0000","last_unstale":"2024-09-14T04:02:01.420126+0000","last_undegraded":"2024-09-14T04:02:01.420126+0000","last_fullsized":"2024-09-14T04:02:01.420126+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:52.834101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"58'2","reported_seq":1161,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.362245+0000","last_change":"2024-09-14T03:56:43.454119+0000","last_active":"2024-09-14T04:02:04.362245+0000","last_peered":"2024-09-14T04:02:04.362245+0000","last_clean":"2024-09-14T04:02:04.362245+0000","last_became_active":"2024-09-14T03:56:43.453335+0000","last_became_peered":"2024-09-14T03:56:43.453335+0000","last_unstale":"2024-09-14T04:02:04.362245+0000","last_undegraded":"2024-09-14T04:02:04.362245+0000","last_fullsized":"2024-09-14T04:02:04.362245+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:41:51.656608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":9,"num_read_kb":9,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":1204,"reported_epoch":881,"state":"active+clean","last_fresh":"2024-09-14T04:02:01.420943+0000","last_change":"2024-09-14T03:55:35.119385+0000","last_active":"2024-09-14T04:02:01.420943+0000","last_peered":"2024-09-14T04:02:01.420943+0000","last_clean":"2024-09-14T04:02:01.420943+0000","last_became_active":"2024-09-14T03:55:35.119077+0000","last_became_peered":"2024-09-14T03:55:35.119077+0000","last_unstale":"2024-09-14T04:02:01.420943+0000","last_undegraded":"2024-09-14T04:02:01.420943+0000","last_fullsized":"2024-09-14T04:02:01.420943+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:15:24.048483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1d","version":"58'2","reported_seq":1150,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.362351+0000","last_change":"2024-09-14T03:56:43.465628+0000","last_active":"2024-09-14T04:02:04.362351+0000","last_peered":"2024-09-14T04:02:04.362351+0000","last_clean":"2024-09-14T04:02:04.362351+0000","last_became_active":"2024-09-14T03:56:43.465072+0000","last_became_peered":"2024-09-14T03:56:43.465072+0000","last_unstale":"2024-09-14T04:02:04.362351+0000","last_undegraded":"2024-09-14T04:02:04.362351+0000","last_fullsized":"2024-09-14T04:02:04.362351+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:25:21.135964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":9,"num_read_kb":9,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":1356,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.196055+0000","last_change":"2024-09-14T03:55:35.112641+0000","last_active":"2024-09-14T04:02:04.196055+0000","last_peered":"2024-09-14T04:02:04.196055+0000","last_clean":"2024-09-14T04:02:04.196055+0000","last_became_active":"2024-09-14T03:55:35.112354+0000","last_became_peered":"2024-09-14T03:55:35.112354+0000","last_unstale":"2024-09-14T04:02:04.196055+0000","last_undegraded":"2024-09-14T04:02:04.196055+0000","last_fullsized":"2024-09-14T04:02:04.196055+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:39:50.388190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"596'3","reported_seq":1291,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.249290+0000","last_change":"2024-09-14T03:55:35.112780+0000","last_active":"2024-09-14T04:02:04.249290+0000","last_peered":"2024-09-14T04:02:04.249290+0000","last_clean":"2024-09-14T04:02:04.249290+0000","last_became_active":"2024-09-14T03:55:35.112464+0000","last_became_peered":"2024-09-14T03:55:35.112464+0000","last_unstale":"2024-09-14T04:02:04.249290+0000","last_undegraded":"2024-09-14T04:02:04.249290+0000","last_fullsized":"2024-09-14T04:02:04.249290+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:48:07.511981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":1269,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.249371+0000","last_change":"2024-09-14T03:56:08.130776+0000","last_active":"2024-09-14T04:02:04.249371+0000","last_peered":"2024-09-14T04:02:04.249371+0000","last_clean":"2024-09-14T04:02:04.249371+0000","last_became_active":"2024-09-14T03:56:08.130496+0000","last_became_peered":"2024-09-14T03:56:08.130496+0000","last_unstale":"2024-09-14T04:02:04.249371+0000","last_undegraded":"2024-09-14T04:02:04.249371+0000","last_fullsized":"2024-09-14T04:02:04.249371+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:44:00.853212+0000","last_clean_scrub_stamp":"2024-09-14T03:44:00.853212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:48:27.451311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"596'6","reported_seq":1371,"reported_epoch":884,"state":"active+clean","last_fresh":"2024-09-14T04:02:04.195355+0000","last_change":"2024-09-14T03:56:43.469092+0000","last_active":"2024-09-14T04:02:04.195355+0000","last_peered":"2024-09-14T04:02:04.195355+0000","last_clean":"2024-09-14T04:02:04.195355+0000","last_became_active":"2024-09-14T03:56:43.468410+0000","last_became_peered":"2024-09-14T03:56:43.468410+0000","last_unstale":"2024-09-14T04:02:04.195355+0000","last_undegraded":"2024-09-14T04:02:04.195355+0000","last_fullsized":"2024-09-14T04:02:04.195355+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:43:57.713731+0000","last_clean_scrub_stamp":"2024-09-14T03:43:57.713731+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:29:56.763114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":8,"num_read_kb":8,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":551403520,"num_objects":292,"num_object_clones":242,"num_object_copies":876,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":292,"num_whiteouts":10,"num_read":3784,"num_read_kb":6251720,"num_write":8622,"num_write_kb":1970832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":131,"num_bytes_recovered":526368768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":717680640,"data_stored":1385299968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":27846,"ondisk_log_size":27846,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":6602,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":128,"num_read_kb":142,"num_write":69,"num_write_kb":35,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":19806,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":46271,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":840,"num_read_kb":1709,"num_write":269,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":160,"ondisk_log_size":160,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":571,"seq":2452426326082,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":591996,"kb_used_data":136040,"kb_used_omap":78,"kb_used_meta":455857,"kb_avail":93153156,"statfs":{"total":95995035648,"available":95388831744,"internally_reserved":0,"allocated":139304960,"data_stored":253778039,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":80385,"internal_metadata":466798079},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":536,"seq":2302102470729,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299936,"kb_used_data":52816,"kb_used_omap":34,"kb_used_meta":247069,"kb_avail":93445216,"statfs":{"total":95995035648,"available":95687901184,"internally_reserved":0,"allocated":54083584,"data_stored":110373796,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35465,"internal_metadata":252999031},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":505,"seq":2168958484559,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":592488,"kb_used_data":123228,"kb_used_omap":82,"kb_used_meta":469165,"kb_avail":93152664,"statfs":{"total":95995035648,"available":95388327936,"internally_reserved":0,"allocated":126185472,"data_stored":204037050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":84051,"internal_metadata":480425901},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":474,"seq":2035814498390,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":575908,"kb_used_data":108824,"kb_used_omap":74,"kb_used_meta":466997,"kb_avail":93169244,"statfs":{"total":95995035648,"available":95405305856,"internally_reserved":0,"allocated":111435776,"data_stored":182500799,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76790,"internal_metadata":478204938},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":444,"seq":1906965479517,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":273516,"kb_used_data":61152,"kb_used_omap":36,"kb_used_meta":212315,"kb_avail":93471636,"statfs":{"total":95995035648,"available":95714955264,"internally_reserved":0,"allocated":62619648,"data_stored":103900645,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37504,"internal_metadata":217410944},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":411,"seq":1765231558756,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":569624,"kb_used_data":105160,"kb_used_omap":52,"kb_used_meta":464395,"kb_avail":93175528,"statfs":{"total":95995035648,"available":95411740672,"internally_reserved":0,"allocated":107683840,"data_stored":182555068,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":53819,"internal_metadata":475540933},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":379,"seq":1627792605290,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":833628,"kb_used_data":123408,"kb_used_omap":66,"kb_used_meta":710141,"kb_avail":92911524,"statfs":{"total":95995035648,"available":95141400576,"internally_reserved":0,"allocated":126369792,"data_stored":246822878,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67592,"internal_metadata":727185400},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":346,"seq":1486058684529,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":609584,"kb_used_data":96412,"kb_used_omap":54,"kb_used_meta":513097,"kb_avail":93135568,"statfs":{"total":95995035648,"available":95370821632,"internally_reserved":0,"allocated":98725888,"data_stored":166000592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56011,"internal_metadata":525411637},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6147,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4694,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5518,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6102,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6400,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5906,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5989,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":5816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5948,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5473,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":84746240,"data_stored":157540352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":113029120,"data_stored":238972928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":94314496,"data_stored":174686208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":49242112,"data_stored":96034816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":98074624,"data_stored":174649344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":112840704,"data_stored":196190208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":40103936,"data_stored":101908480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":125329408,"data_stored":245317632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:02:08.955 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2024-09-14T04:02:08.955 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2024-09-14T04:02:09.197 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:02:10.357 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:10 smithi070 ceph-mon[96704]: from='client.34605 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:02:10.357 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:10 smithi070 ceph-mon[96704]: osdmap e885: 8 total, 8 up, 8 in 2024-09-14T04:02:10.357 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:10 smithi070 ceph-mon[98640]: from='client.34605 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:02:10.357 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:10 smithi070 ceph-mon[98640]: osdmap e885: 8 total, 8 up, 8 in 2024-09-14T04:02:10.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:10 smithi094 ceph-mon[86597]: from='client.34605 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:02:10.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:10 smithi094 ceph-mon[86597]: osdmap e885: 8 total, 8 up, 8 in 2024-09-14T04:02:10.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:02:10 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:02:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:02:11.167 INFO:teuthology.orchestra.run.smithi070.stderr:pool 'unique_pool_0' removed 2024-09-14T04:02:11.496 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:11 smithi070 ceph-mon[98640]: pgmap v931: 121 pgs: 121 active+clean; 526 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.6 MiB/s rd, 150 KiB/s wr, 1 op/s 2024-09-14T04:02:11.496 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:11 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/333160146' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:02:11.497 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:11 smithi070 ceph-mon[96704]: pgmap v931: 121 pgs: 121 active+clean; 526 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.6 MiB/s rd, 150 KiB/s wr, 1 op/s 2024-09-14T04:02:11.497 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:11 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/333160146' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:02:11.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:11 smithi094 ceph-mon[86597]: pgmap v931: 121 pgs: 121 active+clean; 526 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.6 MiB/s rd, 150 KiB/s wr, 1 op/s 2024-09-14T04:02:11.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:11 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/333160146' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:02:11.821 INFO:teuthology.task.full_sequential:In full_sequential, running task print... 2024-09-14T04:02:11.821 INFO:teuthology.task.print:**** done end ec-rados-default.yaml 2024-09-14T04:02:11.821 INFO:teuthology.task.full_sequential:In full_sequential, running task print... 2024-09-14T04:02:11.821 INFO:teuthology.task.print:**** done start rados_api.yaml 2024-09-14T04:02:11.821 INFO:teuthology.task.full_sequential:In full_sequential, running task workunit... 2024-09-14T04:02:11.822 INFO:tasks.workunit:Pulling workunits from ref quincy 2024-09-14T04:02:11.823 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-14T04:02:11.823 DEBUG:teuthology.orchestra.run.smithi070:> stat -- /home/ubuntu/cephtest/mnt.0 2024-09-14T04:02:11.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:02:11.840 INFO:teuthology.orchestra.run.smithi070.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-09-14T04:02:11.840 DEBUG:teuthology.orchestra.run.smithi070:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-09-14T04:02:11.896 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-09-14T04:02:11.896 DEBUG:teuthology.orchestra.run.smithi070:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-09-14T04:02:11.951 INFO:tasks.workunit:timeout=3h 2024-09-14T04:02:11.951 INFO:tasks.workunit:cleanup=True 2024-09-14T04:02:11.952 DEBUG:teuthology.orchestra.run.smithi070:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-09-14T04:02:12.007 INFO:tasks.workunit.client.0.smithi070.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-09-14T04:02:12.529 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:12 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/333160146' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:02:12.529 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:12 smithi094 ceph-mon[86597]: osdmap e886: 8 total, 8 up, 8 in 2024-09-14T04:02:12.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:12 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/333160146' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:02:12.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:12 smithi070 ceph-mon[96704]: osdmap e886: 8 total, 8 up, 8 in 2024-09-14T04:02:12.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:12 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/333160146' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:02:12.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:12 smithi070 ceph-mon[98640]: osdmap e886: 8 total, 8 up, 8 in 2024-09-14T04:02:13.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:13 smithi094 ceph-mon[86597]: pgmap v933: 105 pgs: 105 active+clean; 583 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:02:13.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:13 smithi070 ceph-mon[98640]: pgmap v933: 105 pgs: 105 active+clean; 583 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:02:13.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:13 smithi070 ceph-mon[96704]: pgmap v933: 105 pgs: 105 active+clean; 583 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:02:13.536 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:02:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:02:13] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.51.0" 2024-09-14T04:02:15.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:15 smithi094 ceph-mon[86597]: pgmap v934: 105 pgs: 105 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:02:15.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:15 smithi070 ceph-mon[96704]: pgmap v934: 105 pgs: 105 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:02:15.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:15 smithi070 ceph-mon[98640]: pgmap v934: 105 pgs: 105 active+clean; 583 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:02:17.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:17 smithi094 ceph-mon[86597]: pgmap v935: 105 pgs: 105 active+clean; 583 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:02:17.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:17 smithi070 ceph-mon[96704]: pgmap v935: 105 pgs: 105 active+clean; 583 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:02:17.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:17 smithi070 ceph-mon[98640]: pgmap v935: 105 pgs: 105 active+clean; 583 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:02:18.192 DEBUG:teuthology.orchestra.run.smithi070:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-09-14T04:02:18.248 INFO:tasks.workunit.client.0.smithi070.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-14T04:02:18.250 INFO:tasks.workunit.client.0.smithi070.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-14T04:02:18.250 INFO:tasks.workunit.client.0.smithi070.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-14T04:02:18.298 INFO:tasks.workunit.client.0.smithi070.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-14T04:02:18.347 INFO:tasks.workunit.client.0.smithi070.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-14T04:02:18.386 INFO:tasks.workunit.client.0.smithi070.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-14T04:02:18.388 INFO:tasks.workunit.client.0.smithi070.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-14T04:02:18.388 INFO:tasks.workunit.client.0.smithi070.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-14T04:02:18.427 INFO:tasks.workunit.client.0.smithi070.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-14T04:02:18.432 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2024-09-14T04:02:18.432 DEBUG:teuthology.orchestra.run.smithi070:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-09-14T04:02:18.488 INFO:tasks.workunit:Running workunits matching cls on client.0... 2024-09-14T04:02:18.489 INFO:tasks.workunit:Running workunit cls/test_cls_2pc_queue.sh... 2024-09-14T04:02:18.489 DEBUG:teuthology.orchestra.run.smithi070:workunit test cls/test_cls_2pc_queue.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_2pc_queue.sh 2024-09-14T04:02:18.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:18 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:18.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:18 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:18.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:18 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:18.554 INFO:tasks.workunit.client.0.smithi070.stdout:Running main() from gmock_main.cc 2024-09-14T04:02:18.554 INFO:tasks.workunit.client.0.smithi070.stdout:[==========] Running 22 tests from 1 test suite. 2024-09-14T04:02:18.554 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] Global test environment set-up. 2024-09-14T04:02:18.554 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] 22 tests from TestCls2PCQueue 2024-09-14T04:02:18.554 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestCls2PCQueue.GetCapacity 2024-09-14T04:02:19.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:19 smithi094 ceph-mon[86597]: pgmap v936: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:19.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:19 smithi070 ceph-mon[96704]: pgmap v936: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:19.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:19 smithi070 ceph-mon[98640]: pgmap v936: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:20.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:02:20 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:02:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:02:20.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:20 smithi094 ceph-mon[86597]: osdmap e887: 8 total, 8 up, 8 in 2024-09-14T04:02:20.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:20 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/2975343261' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:20.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:20 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:20.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:20 smithi094 ceph-mon[86597]: pgmap v938: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:20.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:20 smithi070 ceph-mon[96704]: osdmap e887: 8 total, 8 up, 8 in 2024-09-14T04:02:20.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:20 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/2975343261' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:20.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:20 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:20.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:20 smithi070 ceph-mon[96704]: pgmap v938: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:20.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:20 smithi070 ceph-mon[98640]: osdmap e887: 8 total, 8 up, 8 in 2024-09-14T04:02:20.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:20 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/2975343261' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:20.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:20 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:20.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:20 smithi070 ceph-mon[98640]: pgmap v938: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:21.209 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestCls2PCQueue.GetCapacity (2654 ms) 2024-09-14T04:02:21.209 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestCls2PCQueue.AsyncGetCapacity 2024-09-14T04:02:21.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:21 smithi094 ceph-mon[86597]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:02:21.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:21 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:21.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:21 smithi094 ceph-mon[86597]: osdmap e888: 8 total, 8 up, 8 in 2024-09-14T04:02:21.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:21 smithi070 ceph-mon[96704]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:02:21.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:21 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:21.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:21 smithi070 ceph-mon[96704]: osdmap e888: 8 total, 8 up, 8 in 2024-09-14T04:02:21.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:21 smithi070 ceph-mon[98640]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:02:21.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:21 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:21.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:21 smithi070 ceph-mon[98640]: osdmap e888: 8 total, 8 up, 8 in 2024-09-14T04:02:22.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:22 smithi094 ceph-mon[86597]: osdmap e889: 8 total, 8 up, 8 in 2024-09-14T04:02:22.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:22 smithi094 ceph-mon[86597]: pgmap v941: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:22.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:22 smithi070 ceph-mon[96704]: osdmap e889: 8 total, 8 up, 8 in 2024-09-14T04:02:22.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:22 smithi070 ceph-mon[96704]: pgmap v941: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:22.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:22 smithi070 ceph-mon[98640]: osdmap e889: 8 total, 8 up, 8 in 2024-09-14T04:02:22.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:22 smithi070 ceph-mon[98640]: pgmap v941: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:23.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:23 smithi094 ceph-mon[86597]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:02:23.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:23 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T04:02:23.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:23 smithi094 ceph-mon[86597]: osdmap e890: 8 total, 8 up, 8 in 2024-09-14T04:02:23.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:23 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/346972880' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:23.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:23 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:23.535 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:02:23 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:02:23] "GET /metrics HTTP/1.1" 200 34149 "" "Prometheus/2.51.0" 2024-09-14T04:02:23.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:23 smithi070 ceph-mon[96704]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:02:23.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:23 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T04:02:23.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:23 smithi070 ceph-mon[96704]: osdmap e890: 8 total, 8 up, 8 in 2024-09-14T04:02:23.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:23 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/346972880' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:23.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:23 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:23.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:23 smithi070 ceph-mon[98640]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:02:23.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:23 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T04:02:23.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:23 smithi070 ceph-mon[98640]: osdmap e890: 8 total, 8 up, 8 in 2024-09-14T04:02:23.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:23 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/346972880' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:23.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:23 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:24.226 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestCls2PCQueue.AsyncGetCapacity (3018 ms) 2024-09-14T04:02:24.226 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestCls2PCQueue.Reserve 2024-09-14T04:02:24.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:24 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:24.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:24 smithi094 ceph-mon[86597]: osdmap e891: 8 total, 8 up, 8 in 2024-09-14T04:02:24.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:24 smithi094 ceph-mon[86597]: pgmap v944: 137 pgs: 3 creating+activating, 2 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:24.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:24 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:24.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:24 smithi070 ceph-mon[96704]: osdmap e891: 8 total, 8 up, 8 in 2024-09-14T04:02:24.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:24 smithi070 ceph-mon[96704]: pgmap v944: 137 pgs: 3 creating+activating, 2 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:24.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:24 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:24.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:24 smithi070 ceph-mon[98640]: osdmap e891: 8 total, 8 up, 8 in 2024-09-14T04:02:24.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:24 smithi070 ceph-mon[98640]: pgmap v944: 137 pgs: 3 creating+activating, 2 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:25.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:25 smithi094 ceph-mon[86597]: osdmap e892: 8 total, 8 up, 8 in 2024-09-14T04:02:25.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:25 smithi070 ceph-mon[96704]: osdmap e892: 8 total, 8 up, 8 in 2024-09-14T04:02:25.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:25 smithi070 ceph-mon[98640]: osdmap e892: 8 total, 8 up, 8 in 2024-09-14T04:02:26.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:26 smithi094 ceph-mon[86597]: osdmap e893: 8 total, 8 up, 8 in 2024-09-14T04:02:26.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:26 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/362836732' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:26.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:26 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:26.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:26 smithi094 ceph-mon[86597]: pgmap v947: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:26.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:26 smithi070 ceph-mon[96704]: osdmap e893: 8 total, 8 up, 8 in 2024-09-14T04:02:26.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:26 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/362836732' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:26.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:26 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:26.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:26 smithi070 ceph-mon[96704]: pgmap v947: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:26.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:26 smithi070 ceph-mon[98640]: osdmap e893: 8 total, 8 up, 8 in 2024-09-14T04:02:26.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:26 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/362836732' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:26.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:26 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:26.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:26 smithi070 ceph-mon[98640]: pgmap v947: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:27.240 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestCls2PCQueue.Reserve (3014 ms) 2024-09-14T04:02:27.240 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestCls2PCQueue.AsyncReserve 2024-09-14T04:02:27.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:27 smithi094 ceph-mon[86597]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:02:27.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:27 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:27.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:27 smithi094 ceph-mon[86597]: osdmap e894: 8 total, 8 up, 8 in 2024-09-14T04:02:27.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:27 smithi070 ceph-mon[96704]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:02:27.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:27 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:27.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:27 smithi070 ceph-mon[96704]: osdmap e894: 8 total, 8 up, 8 in 2024-09-14T04:02:27.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:27 smithi070 ceph-mon[98640]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:02:27.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:27 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:27.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:27 smithi070 ceph-mon[98640]: osdmap e894: 8 total, 8 up, 8 in 2024-09-14T04:02:28.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:28 smithi094 ceph-mon[86597]: osdmap e895: 8 total, 8 up, 8 in 2024-09-14T04:02:28.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:28 smithi094 ceph-mon[86597]: pgmap v950: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:28.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:28 smithi070 ceph-mon[96704]: osdmap e895: 8 total, 8 up, 8 in 2024-09-14T04:02:28.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:28 smithi070 ceph-mon[96704]: pgmap v950: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:28.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:28 smithi070 ceph-mon[98640]: osdmap e895: 8 total, 8 up, 8 in 2024-09-14T04:02:28.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:28 smithi070 ceph-mon[98640]: pgmap v950: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:29.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:29 smithi094 ceph-mon[86597]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:02:29.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:29 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T04:02:29.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:29 smithi094 ceph-mon[86597]: osdmap e896: 8 total, 8 up, 8 in 2024-09-14T04:02:29.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:29 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/944585730' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:29.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:29 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:29.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:29 smithi070 ceph-mon[96704]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:02:29.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:29 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T04:02:29.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:29 smithi070 ceph-mon[96704]: osdmap e896: 8 total, 8 up, 8 in 2024-09-14T04:02:29.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:29 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/944585730' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:29.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:29 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:29.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:29 smithi070 ceph-mon[98640]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:02:29.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:29 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T04:02:29.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:29 smithi070 ceph-mon[98640]: osdmap e896: 8 total, 8 up, 8 in 2024-09-14T04:02:29.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:29 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/944585730' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:29.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:29 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:30.260 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestCls2PCQueue.AsyncReserve (3020 ms) 2024-09-14T04:02:30.260 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestCls2PCQueue.Commit 2024-09-14T04:02:30.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:30 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:30.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:30 smithi094 ceph-mon[86597]: osdmap e897: 8 total, 8 up, 8 in 2024-09-14T04:02:30.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:30 smithi094 ceph-mon[86597]: pgmap v953: 137 pgs: 4 creating+peering, 21 unknown, 112 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:30.529 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:02:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:02:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:02:30.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:30 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:30.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:30 smithi070 ceph-mon[96704]: osdmap e897: 8 total, 8 up, 8 in 2024-09-14T04:02:30.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:30 smithi070 ceph-mon[96704]: pgmap v953: 137 pgs: 4 creating+peering, 21 unknown, 112 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:30.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:30 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:30.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:30 smithi070 ceph-mon[98640]: osdmap e897: 8 total, 8 up, 8 in 2024-09-14T04:02:30.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:30 smithi070 ceph-mon[98640]: pgmap v953: 137 pgs: 4 creating+peering, 21 unknown, 112 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:31.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:31 smithi094 ceph-mon[86597]: osdmap e898: 8 total, 8 up, 8 in 2024-09-14T04:02:31.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:31 smithi070 ceph-mon[96704]: osdmap e898: 8 total, 8 up, 8 in 2024-09-14T04:02:31.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:31 smithi070 ceph-mon[98640]: osdmap e898: 8 total, 8 up, 8 in 2024-09-14T04:02:32.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:32 smithi094 ceph-mon[86597]: osdmap e899: 8 total, 8 up, 8 in 2024-09-14T04:02:32.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:32 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/3849357213' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:32.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:32 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:32.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:32 smithi094 ceph-mon[86597]: pgmap v956: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:32.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:32 smithi070 ceph-mon[96704]: osdmap e899: 8 total, 8 up, 8 in 2024-09-14T04:02:32.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:32 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/3849357213' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:32.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:32 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:32.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:32 smithi070 ceph-mon[96704]: pgmap v956: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:32.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:32 smithi070 ceph-mon[98640]: osdmap e899: 8 total, 8 up, 8 in 2024-09-14T04:02:32.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:32 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/3849357213' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:32.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:32 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:32.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:32 smithi070 ceph-mon[98640]: pgmap v956: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:33.275 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestCls2PCQueue.Commit (3014 ms) 2024-09-14T04:02:33.275 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestCls2PCQueue.Abort 2024-09-14T04:02:33.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:33 smithi094 ceph-mon[86597]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:02:33.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:33 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:33.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:33 smithi094 ceph-mon[86597]: osdmap e900: 8 total, 8 up, 8 in 2024-09-14T04:02:33.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:33 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:02:33.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:33 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:33.535 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:02:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:02:33] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.51.0" 2024-09-14T04:02:33.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:33 smithi070 ceph-mon[96704]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:02:33.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:33 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:33.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:33 smithi070 ceph-mon[96704]: osdmap e900: 8 total, 8 up, 8 in 2024-09-14T04:02:33.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:33 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:02:33.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:33 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:33.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:33 smithi070 ceph-mon[98640]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:02:33.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:33 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:33.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:33 smithi070 ceph-mon[98640]: osdmap e900: 8 total, 8 up, 8 in 2024-09-14T04:02:33.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:33 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:02:33.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:33 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:34.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:34 smithi094 ceph-mon[86597]: osdmap e901: 8 total, 8 up, 8 in 2024-09-14T04:02:34.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:34 smithi094 ceph-mon[86597]: pgmap v959: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:34.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:34 smithi070 ceph-mon[98640]: osdmap e901: 8 total, 8 up, 8 in 2024-09-14T04:02:34.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:34 smithi070 ceph-mon[98640]: pgmap v959: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:34.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:34 smithi070 ceph-mon[96704]: osdmap e901: 8 total, 8 up, 8 in 2024-09-14T04:02:34.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:34 smithi070 ceph-mon[96704]: pgmap v959: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:35.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:35 smithi070 ceph-mon[98640]: osdmap e902: 8 total, 8 up, 8 in 2024-09-14T04:02:35.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:35 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/1528860750' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:35.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:35 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:35.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:35 smithi070 ceph-mon[98640]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:02:35.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:35 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T04:02:35.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:35 smithi070 ceph-mon[96704]: osdmap e902: 8 total, 8 up, 8 in 2024-09-14T04:02:35.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:35 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/1528860750' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:35.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:35 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:35.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:35 smithi070 ceph-mon[96704]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:02:35.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:35 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T04:02:35.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:35 smithi094 ceph-mon[86597]: osdmap e902: 8 total, 8 up, 8 in 2024-09-14T04:02:35.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:35 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/1528860750' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:35.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:35 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:35.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:35 smithi094 ceph-mon[86597]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:02:35.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:35 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T04:02:36.295 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestCls2PCQueue.Abort (3021 ms) 2024-09-14T04:02:36.295 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestCls2PCQueue.ReserveError 2024-09-14T04:02:36.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:36 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:36.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:36 smithi094 ceph-mon[86597]: osdmap e903: 8 total, 8 up, 8 in 2024-09-14T04:02:36.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:36 smithi094 ceph-mon[86597]: pgmap v962: 137 pgs: 4 creating+peering, 24 unknown, 109 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:36.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:36 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:36.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:36 smithi070 ceph-mon[96704]: osdmap e903: 8 total, 8 up, 8 in 2024-09-14T04:02:36.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:36 smithi070 ceph-mon[96704]: pgmap v962: 137 pgs: 4 creating+peering, 24 unknown, 109 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:36.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:36 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:36.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:36 smithi070 ceph-mon[98640]: osdmap e903: 8 total, 8 up, 8 in 2024-09-14T04:02:36.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:36 smithi070 ceph-mon[98640]: pgmap v962: 137 pgs: 4 creating+peering, 24 unknown, 109 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:37.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:37 smithi094 ceph-mon[86597]: osdmap e904: 8 total, 8 up, 8 in 2024-09-14T04:02:37.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:37 smithi070 ceph-mon[98640]: osdmap e904: 8 total, 8 up, 8 in 2024-09-14T04:02:37.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:37 smithi070 ceph-mon[96704]: osdmap e904: 8 total, 8 up, 8 in 2024-09-14T04:02:38.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:38 smithi094 ceph-mon[86597]: osdmap e905: 8 total, 8 up, 8 in 2024-09-14T04:02:38.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:38 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/1576592358' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:38.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:38 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:38.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:38 smithi094 ceph-mon[86597]: pgmap v965: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:38.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:38 smithi070 ceph-mon[96704]: osdmap e905: 8 total, 8 up, 8 in 2024-09-14T04:02:38.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:38 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/1576592358' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:38.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:38 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:38.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:38 smithi070 ceph-mon[96704]: pgmap v965: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:38.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:38 smithi070 ceph-mon[98640]: osdmap e905: 8 total, 8 up, 8 in 2024-09-14T04:02:38.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:38 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/1576592358' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:38.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:38 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:38.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:38 smithi070 ceph-mon[98640]: pgmap v965: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:39.311 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestCls2PCQueue.ReserveError (3016 ms) 2024-09-14T04:02:39.312 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestCls2PCQueue.CommitError 2024-09-14T04:02:39.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:39 smithi094 ceph-mon[86597]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:02:39.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:39 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:39.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:39 smithi094 ceph-mon[86597]: osdmap e906: 8 total, 8 up, 8 in 2024-09-14T04:02:39.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:39 smithi070 ceph-mon[96704]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:02:39.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:39 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:39.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:39 smithi070 ceph-mon[96704]: osdmap e906: 8 total, 8 up, 8 in 2024-09-14T04:02:39.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:39 smithi070 ceph-mon[98640]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:02:39.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:39 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:39.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:39 smithi070 ceph-mon[98640]: osdmap e906: 8 total, 8 up, 8 in 2024-09-14T04:02:40.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:40 smithi094 ceph-mon[86597]: osdmap e907: 8 total, 8 up, 8 in 2024-09-14T04:02:40.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:40 smithi094 ceph-mon[86597]: pgmap v968: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:40.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:02:40 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:02:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:02:40.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:40 smithi070 ceph-mon[96704]: osdmap e907: 8 total, 8 up, 8 in 2024-09-14T04:02:40.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:40 smithi070 ceph-mon[96704]: pgmap v968: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:40.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:40 smithi070 ceph-mon[98640]: osdmap e907: 8 total, 8 up, 8 in 2024-09-14T04:02:40.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:40 smithi070 ceph-mon[98640]: pgmap v968: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:41.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:41 smithi094 ceph-mon[86597]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:02:41.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:41 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T04:02:41.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:41 smithi094 ceph-mon[86597]: osdmap e908: 8 total, 8 up, 8 in 2024-09-14T04:02:41.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:41 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/1411405818' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:41.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:41 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:41.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:41 smithi070 ceph-mon[96704]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:02:41.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:41 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T04:02:41.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:41 smithi070 ceph-mon[96704]: osdmap e908: 8 total, 8 up, 8 in 2024-09-14T04:02:41.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:41 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/1411405818' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:41.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:41 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:41.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:41 smithi070 ceph-mon[98640]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:02:41.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:41 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T04:02:41.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:41 smithi070 ceph-mon[98640]: osdmap e908: 8 total, 8 up, 8 in 2024-09-14T04:02:41.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:41 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/1411405818' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:41.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:41 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:42.332 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestCls2PCQueue.CommitError (3020 ms) 2024-09-14T04:02:42.332 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestCls2PCQueue.AbortError 2024-09-14T04:02:42.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:42 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:42.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:42 smithi094 ceph-mon[86597]: osdmap e909: 8 total, 8 up, 8 in 2024-09-14T04:02:42.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:42 smithi094 ceph-mon[86597]: pgmap v971: 137 pgs: 14 creating+peering, 16 unknown, 107 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:42.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:42 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:42.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:42 smithi070 ceph-mon[96704]: osdmap e909: 8 total, 8 up, 8 in 2024-09-14T04:02:42.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:42 smithi070 ceph-mon[96704]: pgmap v971: 137 pgs: 14 creating+peering, 16 unknown, 107 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:42.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:42 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:42.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:42 smithi070 ceph-mon[98640]: osdmap e909: 8 total, 8 up, 8 in 2024-09-14T04:02:42.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:42 smithi070 ceph-mon[98640]: pgmap v971: 137 pgs: 14 creating+peering, 16 unknown, 107 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:43.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:43 smithi094 ceph-mon[86597]: osdmap e910: 8 total, 8 up, 8 in 2024-09-14T04:02:43.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:43 smithi070 ceph-mon[98640]: osdmap e910: 8 total, 8 up, 8 in 2024-09-14T04:02:43.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:43 smithi070 ceph-mon[96704]: osdmap e910: 8 total, 8 up, 8 in 2024-09-14T04:02:43.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:02:43 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:02:43] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.51.0" 2024-09-14T04:02:44.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:44 smithi094 ceph-mon[86597]: osdmap e911: 8 total, 8 up, 8 in 2024-09-14T04:02:44.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:44 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/107576922' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:44.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:44 smithi094 ceph-mon[86597]: pgmap v974: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:44.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:44 smithi070 ceph-mon[96704]: osdmap e911: 8 total, 8 up, 8 in 2024-09-14T04:02:44.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:44 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/107576922' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:44.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:44 smithi070 ceph-mon[96704]: pgmap v974: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:44.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:44 smithi070 ceph-mon[98640]: osdmap e911: 8 total, 8 up, 8 in 2024-09-14T04:02:44.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:44 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/107576922' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:44.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:44 smithi070 ceph-mon[98640]: pgmap v974: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:45.350 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestCls2PCQueue.AbortError (3018 ms) 2024-09-14T04:02:45.350 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestCls2PCQueue.MultiReserve 2024-09-14T04:02:45.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:45 smithi094 ceph-mon[86597]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:02:45.780 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:45 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/107576922' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:45.780 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:45 smithi094 ceph-mon[86597]: osdmap e912: 8 total, 8 up, 8 in 2024-09-14T04:02:45.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:45 smithi070 ceph-mon[96704]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:02:45.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:45 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/107576922' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:45.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:45 smithi070 ceph-mon[96704]: osdmap e912: 8 total, 8 up, 8 in 2024-09-14T04:02:45.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:45 smithi070 ceph-mon[98640]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:02:45.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:45 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/107576922' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:45.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:45 smithi070 ceph-mon[98640]: osdmap e912: 8 total, 8 up, 8 in 2024-09-14T04:02:46.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:46 smithi094 ceph-mon[86597]: osdmap e913: 8 total, 8 up, 8 in 2024-09-14T04:02:46.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:46 smithi094 ceph-mon[86597]: pgmap v977: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:46.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:46 smithi070 ceph-mon[96704]: osdmap e913: 8 total, 8 up, 8 in 2024-09-14T04:02:46.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:46 smithi070 ceph-mon[96704]: pgmap v977: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:46.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:46 smithi070 ceph-mon[98640]: osdmap e913: 8 total, 8 up, 8 in 2024-09-14T04:02:46.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:46 smithi070 ceph-mon[98640]: pgmap v977: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:47.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:47 smithi094 ceph-mon[86597]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:02:47.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:47 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T04:02:47.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:47 smithi094 ceph-mon[86597]: osdmap e914: 8 total, 8 up, 8 in 2024-09-14T04:02:47.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:47 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/3140331955' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:47.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:47 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:02:47.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:47 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:47.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:47 smithi070 ceph-mon[96704]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:02:47.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:47 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T04:02:47.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:47 smithi070 ceph-mon[96704]: osdmap e914: 8 total, 8 up, 8 in 2024-09-14T04:02:47.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:47 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/3140331955' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:47.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:47 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:02:47.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:47 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:47.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:47 smithi070 ceph-mon[98640]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:02:47.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:47 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T04:02:47.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:47 smithi070 ceph-mon[98640]: osdmap e914: 8 total, 8 up, 8 in 2024-09-14T04:02:47.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:47 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/3140331955' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:47.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:47 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:02:47.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:47 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:48.368 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestCls2PCQueue.MultiReserve (3019 ms) 2024-09-14T04:02:48.368 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestCls2PCQueue.MultiCommit 2024-09-14T04:02:48.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:48 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/3140331955' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:48.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:48 smithi094 ceph-mon[86597]: osdmap e915: 8 total, 8 up, 8 in 2024-09-14T04:02:48.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:48 smithi094 ceph-mon[86597]: pgmap v980: 137 pgs: 8 creating+peering, 24 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:48.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:48 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/3140331955' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:48.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:48 smithi070 ceph-mon[96704]: osdmap e915: 8 total, 8 up, 8 in 2024-09-14T04:02:48.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:48 smithi070 ceph-mon[96704]: pgmap v980: 137 pgs: 8 creating+peering, 24 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:48.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:48 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/3140331955' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:48.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:48 smithi070 ceph-mon[98640]: osdmap e915: 8 total, 8 up, 8 in 2024-09-14T04:02:48.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:48 smithi070 ceph-mon[98640]: pgmap v980: 137 pgs: 8 creating+peering, 24 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:49.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:49 smithi094 ceph-mon[86597]: osdmap e916: 8 total, 8 up, 8 in 2024-09-14T04:02:49.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:49 smithi070 ceph-mon[96704]: osdmap e916: 8 total, 8 up, 8 in 2024-09-14T04:02:49.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:49 smithi070 ceph-mon[98640]: osdmap e916: 8 total, 8 up, 8 in 2024-09-14T04:02:50.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:02:50 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:02:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:02:50.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:50 smithi094 ceph-mon[86597]: osdmap e917: 8 total, 8 up, 8 in 2024-09-14T04:02:50.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:50 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/3846281711' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:50.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:50 smithi094 ceph-mon[86597]: pgmap v983: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:50.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:50 smithi070 ceph-mon[98640]: osdmap e917: 8 total, 8 up, 8 in 2024-09-14T04:02:50.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:50 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/3846281711' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:50.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:50 smithi070 ceph-mon[98640]: pgmap v983: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:50.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:50 smithi070 ceph-mon[96704]: osdmap e917: 8 total, 8 up, 8 in 2024-09-14T04:02:50.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:50 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/3846281711' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:50.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:50 smithi070 ceph-mon[96704]: pgmap v983: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:51.385 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestCls2PCQueue.MultiCommit (3016 ms) 2024-09-14T04:02:51.385 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestCls2PCQueue.MultiAbort 2024-09-14T04:02:51.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:51 smithi094 ceph-mon[86597]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:02:51.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:51 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/3846281711' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:51.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:51 smithi094 ceph-mon[86597]: osdmap e918: 8 total, 8 up, 8 in 2024-09-14T04:02:51.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:51 smithi070 ceph-mon[96704]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:02:51.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:51 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/3846281711' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:51.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:51 smithi070 ceph-mon[96704]: osdmap e918: 8 total, 8 up, 8 in 2024-09-14T04:02:51.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:51 smithi070 ceph-mon[98640]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:02:51.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:51 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/3846281711' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:51.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:51 smithi070 ceph-mon[98640]: osdmap e918: 8 total, 8 up, 8 in 2024-09-14T04:02:52.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:52 smithi094 ceph-mon[86597]: osdmap e919: 8 total, 8 up, 8 in 2024-09-14T04:02:52.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:52 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:02:52.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:52 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:02:52.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:52 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:02:52.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:52 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:02:52.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:52 smithi094 ceph-mon[86597]: pgmap v986: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:52.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:52 smithi070 ceph-mon[96704]: osdmap e919: 8 total, 8 up, 8 in 2024-09-14T04:02:52.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:52 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:02:52.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:52 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:02:52.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:52 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:02:52.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:52 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:02:52.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:52 smithi070 ceph-mon[96704]: pgmap v986: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:52.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:52 smithi070 ceph-mon[98640]: osdmap e919: 8 total, 8 up, 8 in 2024-09-14T04:02:52.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:52 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:02:52.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:52 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:02:52.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:52 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:02:52.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:52 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:02:52.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:52 smithi070 ceph-mon[98640]: pgmap v986: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:53.687 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:02:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:02:53] "GET /metrics HTTP/1.1" 200 36517 "" "Prometheus/2.51.0" 2024-09-14T04:02:53.687 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:53 smithi070 ceph-mon[96704]: osdmap e920: 8 total, 8 up, 8 in 2024-09-14T04:02:53.687 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:53 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/1248307039' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:53.687 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:53 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:53.688 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:53 smithi070 ceph-mon[96704]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:02:53.688 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:53 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T04:02:53.688 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:53 smithi070 ceph-mon[98640]: osdmap e920: 8 total, 8 up, 8 in 2024-09-14T04:02:53.688 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:53 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/1248307039' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:53.688 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:53 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:53.689 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:53 smithi070 ceph-mon[98640]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:02:53.689 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:53 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T04:02:53.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:53 smithi094 ceph-mon[86597]: osdmap e920: 8 total, 8 up, 8 in 2024-09-14T04:02:53.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:53 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/1248307039' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:53.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:53 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:53.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:53 smithi094 ceph-mon[86597]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:02:53.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:53 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T04:02:54.406 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestCls2PCQueue.MultiAbort (3022 ms) 2024-09-14T04:02:54.406 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestCls2PCQueue.ReserveCommit 2024-09-14T04:02:54.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:54 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:54.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:54 smithi094 ceph-mon[86597]: osdmap e921: 8 total, 8 up, 8 in 2024-09-14T04:02:54.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:54 smithi094 ceph-mon[86597]: pgmap v989: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:54.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:54 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:54.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:54 smithi070 ceph-mon[96704]: osdmap e921: 8 total, 8 up, 8 in 2024-09-14T04:02:54.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:54 smithi070 ceph-mon[96704]: pgmap v989: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:54.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:54 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:54.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:54 smithi070 ceph-mon[98640]: osdmap e921: 8 total, 8 up, 8 in 2024-09-14T04:02:54.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:54 smithi070 ceph-mon[98640]: pgmap v989: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:55.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:55 smithi094 ceph-mon[86597]: osdmap e922: 8 total, 8 up, 8 in 2024-09-14T04:02:55.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:55 smithi070 ceph-mon[96704]: osdmap e922: 8 total, 8 up, 8 in 2024-09-14T04:02:55.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:55 smithi070 ceph-mon[98640]: osdmap e922: 8 total, 8 up, 8 in 2024-09-14T04:02:56.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:56 smithi094 ceph-mon[86597]: osdmap e923: 8 total, 8 up, 8 in 2024-09-14T04:02:56.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:56 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/1504523448' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:56.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:56 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:56.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:56 smithi094 ceph-mon[86597]: pgmap v992: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:56.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:56 smithi070 ceph-mon[96704]: osdmap e923: 8 total, 8 up, 8 in 2024-09-14T04:02:56.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:56 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/1504523448' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:56.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:56 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:56.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:56 smithi070 ceph-mon[96704]: pgmap v992: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:56.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:56 smithi070 ceph-mon[98640]: osdmap e923: 8 total, 8 up, 8 in 2024-09-14T04:02:56.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:56 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/1504523448' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:56.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:56 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:56.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:56 smithi070 ceph-mon[98640]: pgmap v992: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:57.427 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestCls2PCQueue.ReserveCommit (3020 ms) 2024-09-14T04:02:57.427 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestCls2PCQueue.ReserveAbort 2024-09-14T04:02:57.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:57 smithi094 ceph-mon[86597]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:02:57.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:57 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:57.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:57 smithi094 ceph-mon[86597]: osdmap e924: 8 total, 8 up, 8 in 2024-09-14T04:02:57.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:57 smithi070 ceph-mon[96704]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:02:57.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:57 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:57.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:57 smithi070 ceph-mon[96704]: osdmap e924: 8 total, 8 up, 8 in 2024-09-14T04:02:57.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:57 smithi070 ceph-mon[98640]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:02:57.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:57 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:57.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:57 smithi070 ceph-mon[98640]: osdmap e924: 8 total, 8 up, 8 in 2024-09-14T04:02:58.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:58 smithi094 ceph-mon[86597]: osdmap e925: 8 total, 8 up, 8 in 2024-09-14T04:02:58.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:58 smithi094 ceph-mon[86597]: pgmap v995: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:58.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:58 smithi070 ceph-mon[96704]: osdmap e925: 8 total, 8 up, 8 in 2024-09-14T04:02:58.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:58 smithi070 ceph-mon[96704]: pgmap v995: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:58.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:58 smithi070 ceph-mon[98640]: osdmap e925: 8 total, 8 up, 8 in 2024-09-14T04:02:58.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:58 smithi070 ceph-mon[98640]: pgmap v995: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:02:59.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:59 smithi094 ceph-mon[86597]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:02:59.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:59 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T04:02:59.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:59 smithi094 ceph-mon[86597]: osdmap e926: 8 total, 8 up, 8 in 2024-09-14T04:02:59.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:59 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/3167707378' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:59.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:02:59 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:59.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:59 smithi070 ceph-mon[96704]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:02:59.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:59 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T04:02:59.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:59 smithi070 ceph-mon[96704]: osdmap e926: 8 total, 8 up, 8 in 2024-09-14T04:02:59.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:59 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/3167707378' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:59.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:02:59 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:59.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:59 smithi070 ceph-mon[98640]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:02:59.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:59 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T04:02:59.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:59 smithi070 ceph-mon[98640]: osdmap e926: 8 total, 8 up, 8 in 2024-09-14T04:02:59.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:59 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/3167707378' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:59.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:02:59 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:00.445 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestCls2PCQueue.ReserveAbort (3018 ms) 2024-09-14T04:03:00.445 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestCls2PCQueue.ManualCleanup 2024-09-14T04:03:00.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:00 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:00.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:00 smithi094 ceph-mon[86597]: osdmap e927: 8 total, 8 up, 8 in 2024-09-14T04:03:00.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:00 smithi094 ceph-mon[86597]: pgmap v998: 137 pgs: 6 creating+peering, 22 unknown, 109 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:03:00.528 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:03:00 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:03:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:03:00.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:00 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:00.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:00 smithi070 ceph-mon[96704]: osdmap e927: 8 total, 8 up, 8 in 2024-09-14T04:03:00.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:00 smithi070 ceph-mon[96704]: pgmap v998: 137 pgs: 6 creating+peering, 22 unknown, 109 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:03:00.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:00 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:00.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:00 smithi070 ceph-mon[98640]: osdmap e927: 8 total, 8 up, 8 in 2024-09-14T04:03:00.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:00 smithi070 ceph-mon[98640]: pgmap v998: 137 pgs: 6 creating+peering, 22 unknown, 109 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:03:01.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:01 smithi094 ceph-mon[86597]: osdmap e928: 8 total, 8 up, 8 in 2024-09-14T04:03:01.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:01 smithi070 ceph-mon[96704]: osdmap e928: 8 total, 8 up, 8 in 2024-09-14T04:03:01.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:01 smithi070 ceph-mon[98640]: osdmap e928: 8 total, 8 up, 8 in 2024-09-14T04:03:02.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:02 smithi094 ceph-mon[86597]: osdmap e929: 8 total, 8 up, 8 in 2024-09-14T04:03:02.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:02 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/3710810365' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:02.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:02 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:02.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:02 smithi094 ceph-mon[86597]: pgmap v1001: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:03:02.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:02 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:02.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:02 smithi070 ceph-mon[96704]: osdmap e929: 8 total, 8 up, 8 in 2024-09-14T04:03:02.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:02 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/3710810365' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:02.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:02 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:02.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:02 smithi070 ceph-mon[96704]: pgmap v1001: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:03:02.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:02 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:02.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:02 smithi070 ceph-mon[98640]: osdmap e929: 8 total, 8 up, 8 in 2024-09-14T04:03:02.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:02 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/3710810365' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:02.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:02 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:02.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:02 smithi070 ceph-mon[98640]: pgmap v1001: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:03:02.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:02 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:03.457 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestCls2PCQueue.ManualCleanup (3013 ms) 2024-09-14T04:03:03.458 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestCls2PCQueue.Cleanup 2024-09-14T04:03:03.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:03 smithi094 ceph-mon[86597]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:03.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:03 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:03.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:03 smithi094 ceph-mon[86597]: osdmap e930: 8 total, 8 up, 8 in 2024-09-14T04:03:03.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:03:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:03:03] "GET /metrics HTTP/1.1" 200 36520 "" "Prometheus/2.51.0" 2024-09-14T04:03:03.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:03 smithi070 ceph-mon[98640]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:03.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:03 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:03.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:03 smithi070 ceph-mon[98640]: osdmap e930: 8 total, 8 up, 8 in 2024-09-14T04:03:03.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:03 smithi070 ceph-mon[96704]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:03.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:03 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:03.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:03 smithi070 ceph-mon[96704]: osdmap e930: 8 total, 8 up, 8 in 2024-09-14T04:03:04.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:04 smithi094 ceph-mon[86597]: osdmap e931: 8 total, 8 up, 8 in 2024-09-14T04:03:04.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:04 smithi094 ceph-mon[86597]: pgmap v1004: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:03:04.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:04 smithi070 ceph-mon[96704]: osdmap e931: 8 total, 8 up, 8 in 2024-09-14T04:03:04.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:04 smithi070 ceph-mon[96704]: pgmap v1004: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:03:04.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:04 smithi070 ceph-mon[98640]: osdmap e931: 8 total, 8 up, 8 in 2024-09-14T04:03:04.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:04 smithi070 ceph-mon[98640]: pgmap v1004: 105 pgs: 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:03:05.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:05 smithi094 ceph-mon[86597]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:05.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:05 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T04:03:05.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:05 smithi094 ceph-mon[86597]: osdmap e932: 8 total, 8 up, 8 in 2024-09-14T04:03:05.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:05 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/2710998854' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:05.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:05 smithi070 ceph-mon[98640]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:05.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:05 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T04:03:05.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:05 smithi070 ceph-mon[98640]: osdmap e932: 8 total, 8 up, 8 in 2024-09-14T04:03:05.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:05 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/2710998854' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:05.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:05 smithi070 ceph-mon[96704]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:05.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:05 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T04:03:05.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:05 smithi070 ceph-mon[96704]: osdmap e932: 8 total, 8 up, 8 in 2024-09-14T04:03:05.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:05 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/2710998854' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:06.476 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestCls2PCQueue.Cleanup (3018 ms) 2024-09-14T04:03:06.476 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestCls2PCQueue.MultiProducer 2024-09-14T04:03:06.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:06 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/2710998854' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:06.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:06 smithi094 ceph-mon[86597]: osdmap e933: 8 total, 8 up, 8 in 2024-09-14T04:03:06.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:06 smithi094 ceph-mon[86597]: pgmap v1007: 137 pgs: 5 creating+peering, 23 unknown, 109 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:03:06.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:06 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/2710998854' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:06.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:06 smithi070 ceph-mon[96704]: osdmap e933: 8 total, 8 up, 8 in 2024-09-14T04:03:06.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:06 smithi070 ceph-mon[96704]: pgmap v1007: 137 pgs: 5 creating+peering, 23 unknown, 109 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:03:06.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:06 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/2710998854' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:06.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:06 smithi070 ceph-mon[98640]: osdmap e933: 8 total, 8 up, 8 in 2024-09-14T04:03:06.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:06 smithi070 ceph-mon[98640]: pgmap v1007: 137 pgs: 5 creating+peering, 23 unknown, 109 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:03:07.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:07 smithi094 ceph-mon[86597]: osdmap e934: 8 total, 8 up, 8 in 2024-09-14T04:03:07.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:07 smithi070 ceph-mon[96704]: osdmap e934: 8 total, 8 up, 8 in 2024-09-14T04:03:07.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:07 smithi070 ceph-mon[98640]: osdmap e934: 8 total, 8 up, 8 in 2024-09-14T04:03:08.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:08 smithi094 ceph-mon[86597]: osdmap e935: 8 total, 8 up, 8 in 2024-09-14T04:03:08.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:08 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/306653782' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:08.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:08 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:08.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:08 smithi094 ceph-mon[86597]: pgmap v1010: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:03:08.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:08 smithi070 ceph-mon[96704]: osdmap e935: 8 total, 8 up, 8 in 2024-09-14T04:03:08.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:08 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/306653782' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:08.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:08 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:08.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:08 smithi070 ceph-mon[96704]: pgmap v1010: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:03:08.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:08 smithi070 ceph-mon[98640]: osdmap e935: 8 total, 8 up, 8 in 2024-09-14T04:03:08.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:08 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/306653782' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:08.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:08 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:08.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:08 smithi070 ceph-mon[98640]: pgmap v1010: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:03:09.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:09 smithi094 ceph-mon[86597]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:09.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:09 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:09.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:09 smithi094 ceph-mon[86597]: osdmap e936: 8 total, 8 up, 8 in 2024-09-14T04:03:09.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:09 smithi070 ceph-mon[98640]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:09.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:09 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:09.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:09 smithi070 ceph-mon[98640]: osdmap e936: 8 total, 8 up, 8 in 2024-09-14T04:03:09.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:09 smithi070 ceph-mon[96704]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:09.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:09 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:09.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:09 smithi070 ceph-mon[96704]: osdmap e936: 8 total, 8 up, 8 in 2024-09-14T04:03:10.490 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:03:10 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:03:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:03:10.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:10 smithi094 ceph-mon[86597]: osdmap e937: 8 total, 8 up, 8 in 2024-09-14T04:03:10.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:10 smithi094 ceph-mon[86597]: pgmap v1013: 137 pgs: 10 creating+peering, 11 unknown, 116 active+clean; 851 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.6 MiB/s wr, 2.95k op/s 2024-09-14T04:03:10.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:10 smithi070 ceph-mon[96704]: osdmap e937: 8 total, 8 up, 8 in 2024-09-14T04:03:10.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:10 smithi070 ceph-mon[96704]: pgmap v1013: 137 pgs: 10 creating+peering, 11 unknown, 116 active+clean; 851 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.6 MiB/s wr, 2.95k op/s 2024-09-14T04:03:10.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:10 smithi070 ceph-mon[98640]: osdmap e937: 8 total, 8 up, 8 in 2024-09-14T04:03:10.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:10 smithi070 ceph-mon[98640]: pgmap v1013: 137 pgs: 10 creating+peering, 11 unknown, 116 active+clean; 851 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.6 MiB/s wr, 2.95k op/s 2024-09-14T04:03:11.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:11 smithi094 ceph-mon[86597]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:11.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:11 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T04:03:11.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:11 smithi070 ceph-mon[98640]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:11.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:11 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T04:03:11.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:11 smithi070 ceph-mon[96704]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:11.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:11 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T04:03:12.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:12 smithi094 ceph-mon[86597]: pgmap v1014: 137 pgs: 10 creating+peering, 127 active+clean; 851 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.8 MiB/s wr, 2.06k op/s 2024-09-14T04:03:12.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:12 smithi070 ceph-mon[96704]: pgmap v1014: 137 pgs: 10 creating+peering, 127 active+clean; 851 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.8 MiB/s wr, 2.06k op/s 2024-09-14T04:03:12.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:12 smithi070 ceph-mon[98640]: pgmap v1014: 137 pgs: 10 creating+peering, 127 active+clean; 851 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.8 MiB/s wr, 2.06k op/s 2024-09-14T04:03:13.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:03:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:03:13] "GET /metrics HTTP/1.1" 200 36520 "" "Prometheus/2.51.0" 2024-09-14T04:03:14.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:14 smithi094 ceph-mon[86597]: pgmap v1015: 137 pgs: 137 active+clean; 851 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.6 MiB/s wr, 1.76k op/s 2024-09-14T04:03:14.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:14 smithi070 ceph-mon[98640]: pgmap v1015: 137 pgs: 137 active+clean; 851 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.6 MiB/s wr, 1.76k op/s 2024-09-14T04:03:14.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:14 smithi070 ceph-mon[96704]: pgmap v1015: 137 pgs: 137 active+clean; 851 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.6 MiB/s wr, 1.76k op/s 2024-09-14T04:03:17.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:17 smithi094 ceph-mon[86597]: pgmap v1016: 137 pgs: 137 active+clean; 2.0 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 7.3 MiB/s wr, 8.24k op/s 2024-09-14T04:03:17.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:17 smithi070 ceph-mon[96704]: pgmap v1016: 137 pgs: 137 active+clean; 2.0 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 7.3 MiB/s wr, 8.24k op/s 2024-09-14T04:03:17.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:17 smithi070 ceph-mon[98640]: pgmap v1016: 137 pgs: 137 active+clean; 2.0 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 7.3 MiB/s wr, 8.24k op/s 2024-09-14T04:03:18.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:18 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:18.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:18 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:18.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:18 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:19.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:19 smithi094 ceph-mon[86597]: pgmap v1017: 137 pgs: 137 active+clean; 2.0 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 6.79k op/s 2024-09-14T04:03:19.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:19 smithi070 ceph-mon[96704]: pgmap v1017: 137 pgs: 137 active+clean; 2.0 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 6.79k op/s 2024-09-14T04:03:19.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:19 smithi070 ceph-mon[98640]: pgmap v1017: 137 pgs: 137 active+clean; 2.0 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 6.79k op/s 2024-09-14T04:03:20.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:03:20 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:03:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:03:21.261 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestCls2PCQueue.MultiProducer (14786 ms) 2024-09-14T04:03:21.261 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestCls2PCQueue.AsyncConsumer 2024-09-14T04:03:21.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:21 smithi094 ceph-mon[86597]: pgmap v1018: 137 pgs: 137 active+clean; 2.3 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 205 MiB/s rd, 6.0 MiB/s wr, 7.16k op/s 2024-09-14T04:03:21.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:21 smithi070 ceph-mon[96704]: pgmap v1018: 137 pgs: 137 active+clean; 2.3 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 205 MiB/s rd, 6.0 MiB/s wr, 7.16k op/s 2024-09-14T04:03:21.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:21 smithi070 ceph-mon[98640]: pgmap v1018: 137 pgs: 137 active+clean; 2.3 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 205 MiB/s rd, 6.0 MiB/s wr, 7.16k op/s 2024-09-14T04:03:22.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:22 smithi094 ceph-mon[86597]: osdmap e938: 8 total, 8 up, 8 in 2024-09-14T04:03:22.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:22 smithi070 ceph-mon[98640]: osdmap e938: 8 total, 8 up, 8 in 2024-09-14T04:03:22.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:22 smithi070 ceph-mon[96704]: osdmap e938: 8 total, 8 up, 8 in 2024-09-14T04:03:23.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:23 smithi094 ceph-mon[86597]: pgmap v1020: 105 pgs: 105 active+clean; 583 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:03:23.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:23 smithi094 ceph-mon[86597]: osdmap e939: 8 total, 8 up, 8 in 2024-09-14T04:03:23.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:23 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/1459003512' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:23.535 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:03:23 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:03:23] "GET /metrics HTTP/1.1" 200 36572 "" "Prometheus/2.51.0" 2024-09-14T04:03:23.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:23 smithi070 ceph-mon[96704]: pgmap v1020: 105 pgs: 105 active+clean; 583 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:03:23.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:23 smithi070 ceph-mon[96704]: osdmap e939: 8 total, 8 up, 8 in 2024-09-14T04:03:23.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:23 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/1459003512' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:23.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:23 smithi070 ceph-mon[98640]: pgmap v1020: 105 pgs: 105 active+clean; 583 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:03:23.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:23 smithi070 ceph-mon[98640]: osdmap e939: 8 total, 8 up, 8 in 2024-09-14T04:03:23.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:23 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/1459003512' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:24.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:24 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/1459003512' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:24.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:24 smithi094 ceph-mon[86597]: osdmap e940: 8 total, 8 up, 8 in 2024-09-14T04:03:24.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:24 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/1459003512' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:24.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:24 smithi070 ceph-mon[96704]: osdmap e940: 8 total, 8 up, 8 in 2024-09-14T04:03:24.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:24 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/1459003512' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:24.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:24 smithi070 ceph-mon[98640]: osdmap e940: 8 total, 8 up, 8 in 2024-09-14T04:03:25.281 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestCls2PCQueue.AsyncConsumer (4020 ms) 2024-09-14T04:03:25.281 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestCls2PCQueue.MultiProducerConsumer 2024-09-14T04:03:25.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:25 smithi070 ceph-mon[96704]: pgmap v1023: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:03:25.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:25 smithi070 ceph-mon[96704]: osdmap e941: 8 total, 8 up, 8 in 2024-09-14T04:03:25.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:25 smithi070 ceph-mon[98640]: pgmap v1023: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:03:25.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:25 smithi070 ceph-mon[98640]: osdmap e941: 8 total, 8 up, 8 in 2024-09-14T04:03:25.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:25 smithi094 ceph-mon[86597]: pgmap v1023: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:03:25.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:25 smithi094 ceph-mon[86597]: osdmap e941: 8 total, 8 up, 8 in 2024-09-14T04:03:26.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:26 smithi070 ceph-mon[98640]: osdmap e942: 8 total, 8 up, 8 in 2024-09-14T04:03:26.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:26 smithi070 ceph-mon[96704]: osdmap e942: 8 total, 8 up, 8 in 2024-09-14T04:03:26.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:26 smithi094 ceph-mon[86597]: osdmap e942: 8 total, 8 up, 8 in 2024-09-14T04:03:27.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:27 smithi094 ceph-mon[86597]: pgmap v1026: 105 pgs: 105 active+clean; 583 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:03:27.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:27 smithi094 ceph-mon[86597]: osdmap e943: 8 total, 8 up, 8 in 2024-09-14T04:03:27.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:27 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/4081920263' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:27.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:27 smithi070 ceph-mon[96704]: pgmap v1026: 105 pgs: 105 active+clean; 583 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:03:27.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:27 smithi070 ceph-mon[96704]: osdmap e943: 8 total, 8 up, 8 in 2024-09-14T04:03:27.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:27 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/4081920263' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:27.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:27 smithi070 ceph-mon[98640]: pgmap v1026: 105 pgs: 105 active+clean; 583 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:03:27.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:27 smithi070 ceph-mon[98640]: osdmap e943: 8 total, 8 up, 8 in 2024-09-14T04:03:27.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:27 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/4081920263' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:28.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:28 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/4081920263' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:28.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:28 smithi094 ceph-mon[86597]: osdmap e944: 8 total, 8 up, 8 in 2024-09-14T04:03:28.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:28 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/4081920263' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:28.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:28 smithi070 ceph-mon[98640]: osdmap e944: 8 total, 8 up, 8 in 2024-09-14T04:03:28.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:28 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/4081920263' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:28.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:28 smithi070 ceph-mon[96704]: osdmap e944: 8 total, 8 up, 8 in 2024-09-14T04:03:29.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:29 smithi094 ceph-mon[86597]: pgmap v1029: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:03:29.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:29 smithi094 ceph-mon[86597]: osdmap e945: 8 total, 8 up, 8 in 2024-09-14T04:03:29.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:29 smithi070 ceph-mon[98640]: pgmap v1029: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:03:29.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:29 smithi070 ceph-mon[98640]: osdmap e945: 8 total, 8 up, 8 in 2024-09-14T04:03:29.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:29 smithi070 ceph-mon[96704]: pgmap v1029: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:03:29.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:29 smithi070 ceph-mon[96704]: osdmap e945: 8 total, 8 up, 8 in 2024-09-14T04:03:30.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:03:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:03:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:03:31.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:31 smithi094 ceph-mon[86597]: pgmap v1031: 137 pgs: 6 creating+peering, 11 unknown, 120 active+clean; 1.1 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 30 MiB/s rd, 4.9 MiB/s wr, 5.80k op/s 2024-09-14T04:03:31.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:31 smithi070 ceph-mon[96704]: pgmap v1031: 137 pgs: 6 creating+peering, 11 unknown, 120 active+clean; 1.1 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 30 MiB/s rd, 4.9 MiB/s wr, 5.80k op/s 2024-09-14T04:03:31.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:31 smithi070 ceph-mon[98640]: pgmap v1031: 137 pgs: 6 creating+peering, 11 unknown, 120 active+clean; 1.1 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 30 MiB/s rd, 4.9 MiB/s wr, 5.80k op/s 2024-09-14T04:03:33.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:33 smithi094 ceph-mon[86597]: pgmap v1032: 137 pgs: 137 active+clean; 1.1 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 24 MiB/s rd, 4.0 MiB/s wr, 4.75k op/s 2024-09-14T04:03:33.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:33 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:33.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:03:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:03:33] "GET /metrics HTTP/1.1" 200 36569 "" "Prometheus/2.51.0" 2024-09-14T04:03:33.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:33 smithi070 ceph-mon[96704]: pgmap v1032: 137 pgs: 137 active+clean; 1.1 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 24 MiB/s rd, 4.0 MiB/s wr, 4.75k op/s 2024-09-14T04:03:33.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:33 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:33.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:33 smithi070 ceph-mon[98640]: pgmap v1032: 137 pgs: 137 active+clean; 1.1 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 24 MiB/s rd, 4.0 MiB/s wr, 4.75k op/s 2024-09-14T04:03:33.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:33 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:35.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:35 smithi094 ceph-mon[86597]: pgmap v1033: 137 pgs: 137 active+clean; 1.1 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 18 MiB/s rd, 3.0 MiB/s wr, 3.61k op/s 2024-09-14T04:03:35.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:35 smithi070 ceph-mon[96704]: pgmap v1033: 137 pgs: 137 active+clean; 1.1 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 18 MiB/s rd, 3.0 MiB/s wr, 3.61k op/s 2024-09-14T04:03:35.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:35 smithi070 ceph-mon[98640]: pgmap v1033: 137 pgs: 137 active+clean; 1.1 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 18 MiB/s rd, 3.0 MiB/s wr, 3.61k op/s 2024-09-14T04:03:37.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:37 smithi094 ceph-mon[86597]: pgmap v1034: 137 pgs: 137 active+clean; 1.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 75 MiB/s rd, 6.0 MiB/s wr, 7.19k op/s 2024-09-14T04:03:37.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:37 smithi070 ceph-mon[96704]: pgmap v1034: 137 pgs: 137 active+clean; 1.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 75 MiB/s rd, 6.0 MiB/s wr, 7.19k op/s 2024-09-14T04:03:37.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:37 smithi070 ceph-mon[98640]: pgmap v1034: 137 pgs: 137 active+clean; 1.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 75 MiB/s rd, 6.0 MiB/s wr, 7.19k op/s 2024-09-14T04:03:39.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:39 smithi094 ceph-mon[86597]: pgmap v1035: 137 pgs: 137 active+clean; 1.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 67 MiB/s rd, 5.4 MiB/s wr, 6.40k op/s 2024-09-14T04:03:39.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:39 smithi070 ceph-mon[96704]: pgmap v1035: 137 pgs: 137 active+clean; 1.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 67 MiB/s rd, 5.4 MiB/s wr, 6.40k op/s 2024-09-14T04:03:39.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:39 smithi070 ceph-mon[98640]: pgmap v1035: 137 pgs: 137 active+clean; 1.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 67 MiB/s rd, 5.4 MiB/s wr, 6.40k op/s 2024-09-14T04:03:40.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:03:40 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:03:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:03:41.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:41 smithi094 ceph-mon[86597]: pgmap v1036: 137 pgs: 137 active+clean; 1.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 83 MiB/s rd, 5.7 MiB/s wr, 6.87k op/s 2024-09-14T04:03:41.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:41 smithi070 ceph-mon[96704]: pgmap v1036: 137 pgs: 137 active+clean; 1.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 83 MiB/s rd, 5.7 MiB/s wr, 6.87k op/s 2024-09-14T04:03:41.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:41 smithi070 ceph-mon[98640]: pgmap v1036: 137 pgs: 137 active+clean; 1.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 83 MiB/s rd, 5.7 MiB/s wr, 6.87k op/s 2024-09-14T04:03:43.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:43 smithi094 ceph-mon[86597]: pgmap v1037: 137 pgs: 137 active+clean; 1.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 70 MiB/s rd, 3.6 MiB/s wr, 4.43k op/s 2024-09-14T04:03:43.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:43 smithi070 ceph-mon[98640]: pgmap v1037: 137 pgs: 137 active+clean; 1.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 70 MiB/s rd, 3.6 MiB/s wr, 4.43k op/s 2024-09-14T04:03:43.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:43 smithi070 ceph-mon[96704]: pgmap v1037: 137 pgs: 137 active+clean; 1.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 70 MiB/s rd, 3.6 MiB/s wr, 4.43k op/s 2024-09-14T04:03:43.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:03:43 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:03:43] "GET /metrics HTTP/1.1" 200 36569 "" "Prometheus/2.51.0" 2024-09-14T04:03:44.321 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestCls2PCQueue.MultiProducerConsumer (19040 ms) 2024-09-14T04:03:44.321 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestCls2PCQueue.ReserveSpillover 2024-09-14T04:03:45.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:45 smithi094 ceph-mon[86597]: pgmap v1038: 137 pgs: 137 active+clean; 1.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 70 MiB/s rd, 3.6 MiB/s wr, 4.43k op/s 2024-09-14T04:03:45.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:45 smithi094 ceph-mon[86597]: osdmap e946: 8 total, 8 up, 8 in 2024-09-14T04:03:45.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:45 smithi070 ceph-mon[96704]: pgmap v1038: 137 pgs: 137 active+clean; 1.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 70 MiB/s rd, 3.6 MiB/s wr, 4.43k op/s 2024-09-14T04:03:45.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:45 smithi070 ceph-mon[96704]: osdmap e946: 8 total, 8 up, 8 in 2024-09-14T04:03:45.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:45 smithi070 ceph-mon[98640]: pgmap v1038: 137 pgs: 137 active+clean; 1.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 70 MiB/s rd, 3.6 MiB/s wr, 4.43k op/s 2024-09-14T04:03:45.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:45 smithi070 ceph-mon[98640]: osdmap e946: 8 total, 8 up, 8 in 2024-09-14T04:03:46.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:46 smithi094 ceph-mon[86597]: osdmap e947: 8 total, 8 up, 8 in 2024-09-14T04:03:46.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:46 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/3087175846' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:46.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:46 smithi070 ceph-mon[96704]: osdmap e947: 8 total, 8 up, 8 in 2024-09-14T04:03:46.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:46 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/3087175846' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:46.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:46 smithi070 ceph-mon[98640]: osdmap e947: 8 total, 8 up, 8 in 2024-09-14T04:03:46.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:46 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/3087175846' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:47.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:47 smithi094 ceph-mon[86597]: pgmap v1041: 137 pgs: 11 creating+peering, 21 unknown, 105 active+clean; 583 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:03:47.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:47 smithi094 ceph-mon[86597]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:47.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:47 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/3087175846' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:47.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:47 smithi094 ceph-mon[86597]: osdmap e948: 8 total, 8 up, 8 in 2024-09-14T04:03:47.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:47 smithi070 ceph-mon[96704]: pgmap v1041: 137 pgs: 11 creating+peering, 21 unknown, 105 active+clean; 583 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:03:47.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:47 smithi070 ceph-mon[96704]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:47.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:47 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/3087175846' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:47.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:47 smithi070 ceph-mon[96704]: osdmap e948: 8 total, 8 up, 8 in 2024-09-14T04:03:47.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:47 smithi070 ceph-mon[98640]: pgmap v1041: 137 pgs: 11 creating+peering, 21 unknown, 105 active+clean; 583 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:03:47.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:47 smithi070 ceph-mon[98640]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:47.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:47 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/3087175846' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:47.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:47 smithi070 ceph-mon[98640]: osdmap e948: 8 total, 8 up, 8 in 2024-09-14T04:03:48.344 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestCls2PCQueue.ReserveSpillover (4022 ms) 2024-09-14T04:03:48.344 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestCls2PCQueue.CommitSpillover 2024-09-14T04:03:48.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:48 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:03:48.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:48 smithi094 ceph-mon[86597]: osdmap e949: 8 total, 8 up, 8 in 2024-09-14T04:03:48.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:48 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:48.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:48 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:03:48.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:48 smithi070 ceph-mon[98640]: osdmap e949: 8 total, 8 up, 8 in 2024-09-14T04:03:48.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:48 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:48.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:48 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:03:48.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:48 smithi070 ceph-mon[96704]: osdmap e949: 8 total, 8 up, 8 in 2024-09-14T04:03:48.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:48 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:49.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:49 smithi094 ceph-mon[86597]: pgmap v1044: 137 pgs: 13 creating+peering, 16 unknown, 108 active+clean; 583 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:03:49.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:49 smithi094 ceph-mon[86597]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:49.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:49 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T04:03:49.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:49 smithi094 ceph-mon[86597]: osdmap e950: 8 total, 8 up, 8 in 2024-09-14T04:03:49.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:49 smithi070 ceph-mon[96704]: pgmap v1044: 137 pgs: 13 creating+peering, 16 unknown, 108 active+clean; 583 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:03:49.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:49 smithi070 ceph-mon[96704]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:49.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:49 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T04:03:49.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:49 smithi070 ceph-mon[96704]: osdmap e950: 8 total, 8 up, 8 in 2024-09-14T04:03:49.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:49 smithi070 ceph-mon[98640]: pgmap v1044: 137 pgs: 13 creating+peering, 16 unknown, 108 active+clean; 583 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:03:49.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:49 smithi070 ceph-mon[98640]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:49.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:49 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T04:03:49.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:49 smithi070 ceph-mon[98640]: osdmap e950: 8 total, 8 up, 8 in 2024-09-14T04:03:50.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:50 smithi094 ceph-mon[86597]: osdmap e951: 8 total, 8 up, 8 in 2024-09-14T04:03:50.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:50 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/4248020819' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:50.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:50 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:50.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:03:50 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:03:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:03:50.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:50 smithi070 ceph-mon[98640]: osdmap e951: 8 total, 8 up, 8 in 2024-09-14T04:03:50.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:50 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/4248020819' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:50.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:50 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:50.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:50 smithi070 ceph-mon[96704]: osdmap e951: 8 total, 8 up, 8 in 2024-09-14T04:03:50.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:50 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/4248020819' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:50.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:50 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:51.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:51 smithi094 ceph-mon[86597]: pgmap v1047: 137 pgs: 11 creating+peering, 21 unknown, 105 active+clean; 583 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:03:51.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:51 smithi094 ceph-mon[86597]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:51.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:51 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:51.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:51 smithi094 ceph-mon[86597]: osdmap e952: 8 total, 8 up, 8 in 2024-09-14T04:03:51.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:51 smithi070 ceph-mon[96704]: pgmap v1047: 137 pgs: 11 creating+peering, 21 unknown, 105 active+clean; 583 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:03:51.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:51 smithi070 ceph-mon[96704]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:51.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:51 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:51.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:51 smithi070 ceph-mon[96704]: osdmap e952: 8 total, 8 up, 8 in 2024-09-14T04:03:51.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:51 smithi070 ceph-mon[98640]: pgmap v1047: 137 pgs: 11 creating+peering, 21 unknown, 105 active+clean; 583 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:03:51.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:51 smithi070 ceph-mon[98640]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:51.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:51 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:51.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:51 smithi070 ceph-mon[98640]: osdmap e952: 8 total, 8 up, 8 in 2024-09-14T04:03:52.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:52 smithi094 ceph-mon[86597]: osdmap e953: 8 total, 8 up, 8 in 2024-09-14T04:03:52.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:52 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:03:52.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:52 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:03:52.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:52 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:03:52.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:52 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:03:52.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:52 smithi070 ceph-mon[96704]: osdmap e953: 8 total, 8 up, 8 in 2024-09-14T04:03:52.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:52 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:03:52.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:52 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:03:52.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:52 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:03:52.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:52 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:03:52.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:52 smithi070 ceph-mon[98640]: osdmap e953: 8 total, 8 up, 8 in 2024-09-14T04:03:52.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:52 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:03:52.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:52 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:03:52.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:52 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:03:52.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:52 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:03:53.686 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:03:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:03:53] "GET /metrics HTTP/1.1" 200 36513 "" "Prometheus/2.51.0" 2024-09-14T04:03:53.687 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:53 smithi070 ceph-mon[98640]: pgmap v1050: 137 pgs: 11 creating+peering, 7 unknown, 119 active+clean; 583 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:03:53.687 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:53 smithi070 ceph-mon[98640]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:53.687 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:53 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T04:03:53.687 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:53 smithi070 ceph-mon[96704]: pgmap v1050: 137 pgs: 11 creating+peering, 7 unknown, 119 active+clean; 583 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:03:53.687 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:53 smithi070 ceph-mon[96704]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:53.687 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:53 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T04:03:53.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:53 smithi094 ceph-mon[86597]: pgmap v1050: 137 pgs: 11 creating+peering, 7 unknown, 119 active+clean; 583 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:03:53.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:53 smithi094 ceph-mon[86597]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:53.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:53 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T04:03:54.371 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestCls2PCQueue.CommitSpillover (6027 ms) 2024-09-14T04:03:54.371 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestCls2PCQueue.AbortSpillover 2024-09-14T04:03:55.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:55 smithi094 ceph-mon[86597]: pgmap v1051: 137 pgs: 11 creating+peering, 126 active+clean; 657 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 4.7 MiB/s rd, 5.0 MiB/s wr, 1.75k op/s 2024-09-14T04:03:55.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:55 smithi094 ceph-mon[86597]: osdmap e954: 8 total, 8 up, 8 in 2024-09-14T04:03:55.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:55 smithi070 ceph-mon[96704]: pgmap v1051: 137 pgs: 11 creating+peering, 126 active+clean; 657 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 4.7 MiB/s rd, 5.0 MiB/s wr, 1.75k op/s 2024-09-14T04:03:55.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:55 smithi070 ceph-mon[96704]: osdmap e954: 8 total, 8 up, 8 in 2024-09-14T04:03:55.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:55 smithi070 ceph-mon[98640]: pgmap v1051: 137 pgs: 11 creating+peering, 126 active+clean; 657 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 4.7 MiB/s rd, 5.0 MiB/s wr, 1.75k op/s 2024-09-14T04:03:55.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:55 smithi070 ceph-mon[98640]: osdmap e954: 8 total, 8 up, 8 in 2024-09-14T04:03:56.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:56 smithi094 ceph-mon[86597]: osdmap e955: 8 total, 8 up, 8 in 2024-09-14T04:03:56.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:56 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/769204247' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:56.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:56 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:56.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:56 smithi070 ceph-mon[96704]: osdmap e955: 8 total, 8 up, 8 in 2024-09-14T04:03:56.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:56 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/769204247' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:56.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:56 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:56.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:56 smithi070 ceph-mon[98640]: osdmap e955: 8 total, 8 up, 8 in 2024-09-14T04:03:56.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:56 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/769204247' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:56.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:56 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:57.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:57 smithi094 ceph-mon[86597]: pgmap v1054: 137 pgs: 8 creating+peering, 24 unknown, 105 active+clean; 583 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:03:57.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:57 smithi094 ceph-mon[86597]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:57.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:57 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:57.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:57 smithi094 ceph-mon[86597]: osdmap e956: 8 total, 8 up, 8 in 2024-09-14T04:03:57.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:57 smithi070 ceph-mon[96704]: pgmap v1054: 137 pgs: 8 creating+peering, 24 unknown, 105 active+clean; 583 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:03:57.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:57 smithi070 ceph-mon[96704]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:57.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:57 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:57.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:57 smithi070 ceph-mon[96704]: osdmap e956: 8 total, 8 up, 8 in 2024-09-14T04:03:57.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:57 smithi070 ceph-mon[98640]: pgmap v1054: 137 pgs: 8 creating+peering, 24 unknown, 105 active+clean; 583 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:03:57.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:57 smithi070 ceph-mon[98640]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:57.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:57 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-128437-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:57.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:57 smithi070 ceph-mon[98640]: osdmap e956: 8 total, 8 up, 8 in 2024-09-14T04:03:58.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:58 smithi094 ceph-mon[86597]: osdmap e957: 8 total, 8 up, 8 in 2024-09-14T04:03:58.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:58 smithi070 ceph-mon[96704]: osdmap e957: 8 total, 8 up, 8 in 2024-09-14T04:03:58.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:58 smithi070 ceph-mon[98640]: osdmap e957: 8 total, 8 up, 8 in 2024-09-14T04:03:59.395 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestCls2PCQueue.AbortSpillover (5024 ms) 2024-09-14T04:03:59.395 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] 22 tests from TestCls2PCQueue (100840 ms total) 2024-09-14T04:03:59.395 INFO:tasks.workunit.client.0.smithi070.stdout: 2024-09-14T04:03:59.395 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] Global test environment tear-down 2024-09-14T04:03:59.395 INFO:tasks.workunit.client.0.smithi070.stdout:[==========] 22 tests from 1 test suite ran. (100840 ms total) 2024-09-14T04:03:59.395 INFO:tasks.workunit.client.0.smithi070.stdout:[ PASSED ] 22 tests. 2024-09-14T04:03:59.397 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-14T04:03:59.398 DEBUG:teuthology.orchestra.run.smithi070:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-14T04:03:59.426 INFO:tasks.workunit:Running workunit cls/test_cls_cas.sh... 2024-09-14T04:03:59.426 DEBUG:teuthology.orchestra.run.smithi070:workunit test cls/test_cls_cas.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_cas.sh 2024-09-14T04:03:59.493 INFO:tasks.workunit.client.0.smithi070.stdout:Running main() from gmock_main.cc 2024-09-14T04:03:59.493 INFO:tasks.workunit.client.0.smithi070.stdout:[==========] Running 6 tests from 2 test suites. 2024-09-14T04:03:59.493 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] Global test environment set-up. 2024-09-14T04:03:59.494 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] 5 tests from cls_cas 2024-09-14T04:03:59.494 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] cls_cas.get_put 2024-09-14T04:03:59.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:59 smithi094 ceph-mon[86597]: pgmap v1057: 137 pgs: 12 creating+peering, 15 unknown, 110 active+clean; 583 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:03:59.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:59 smithi094 ceph-mon[86597]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:59.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:03:59 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T04:03:59.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:59 smithi070 ceph-mon[96704]: pgmap v1057: 137 pgs: 12 creating+peering, 15 unknown, 110 active+clean; 583 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:03:59.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:59 smithi070 ceph-mon[96704]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:59.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:03:59 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T04:03:59.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:59 smithi070 ceph-mon[98640]: pgmap v1057: 137 pgs: 12 creating+peering, 15 unknown, 110 active+clean; 583 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:03:59.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:59 smithi070 ceph-mon[98640]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:59.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:03:59 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T04:04:00.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:00 smithi094 ceph-mon[86597]: osdmap e958: 8 total, 8 up, 8 in 2024-09-14T04:04:00.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:04:00 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:04:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:04:00.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:00 smithi070 ceph-mon[96704]: osdmap e958: 8 total, 8 up, 8 in 2024-09-14T04:04:00.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:00 smithi070 ceph-mon[98640]: osdmap e958: 8 total, 8 up, 8 in 2024-09-14T04:04:01.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:01 smithi094 ceph-mon[86597]: pgmap v1059: 105 pgs: 105 active+clean; 583 KiB data, 4.3 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:01.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:01 smithi094 ceph-mon[86597]: osdmap e959: 8 total, 8 up, 8 in 2024-09-14T04:04:01.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:01 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/1618771964' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:01.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:01 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:01.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:01 smithi070 ceph-mon[96704]: pgmap v1059: 105 pgs: 105 active+clean; 583 KiB data, 4.3 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:01.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:01 smithi070 ceph-mon[96704]: osdmap e959: 8 total, 8 up, 8 in 2024-09-14T04:04:01.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:01 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/1618771964' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:01.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:01 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:01.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:01 smithi070 ceph-mon[98640]: pgmap v1059: 105 pgs: 105 active+clean; 583 KiB data, 4.3 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:01.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:01 smithi070 ceph-mon[98640]: osdmap e959: 8 total, 8 up, 8 in 2024-09-14T04:04:01.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:01 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/1618771964' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:01.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:01 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:02.415 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] cls_cas.get_put (2922 ms) 2024-09-14T04:04:02.415 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] cls_cas.wrong_put 2024-09-14T04:04:02.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:02 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:02.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:02 smithi094 ceph-mon[86597]: osdmap e960: 8 total, 8 up, 8 in 2024-09-14T04:04:02.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:02 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:04:02.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:02 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:02.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:02 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:02.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:02 smithi070 ceph-mon[96704]: osdmap e960: 8 total, 8 up, 8 in 2024-09-14T04:04:02.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:02 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:04:02.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:02 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:02.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:02 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:02.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:02 smithi070 ceph-mon[98640]: osdmap e960: 8 total, 8 up, 8 in 2024-09-14T04:04:02.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:02 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:04:02.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:02 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:03.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:03 smithi094 ceph-mon[86597]: pgmap v1062: 137 pgs: 15 creating+peering, 15 unknown, 107 active+clean; 583 KiB data, 4.3 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:03.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:03 smithi094 ceph-mon[86597]: osdmap e961: 8 total, 8 up, 8 in 2024-09-14T04:04:03.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:04:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:04:03] "GET /metrics HTTP/1.1" 200 36518 "" "Prometheus/2.51.0" 2024-09-14T04:04:03.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:03 smithi070 ceph-mon[98640]: pgmap v1062: 137 pgs: 15 creating+peering, 15 unknown, 107 active+clean; 583 KiB data, 4.3 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:03.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:03 smithi070 ceph-mon[98640]: osdmap e961: 8 total, 8 up, 8 in 2024-09-14T04:04:03.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:03 smithi070 ceph-mon[96704]: pgmap v1062: 137 pgs: 15 creating+peering, 15 unknown, 107 active+clean; 583 KiB data, 4.3 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:03.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:03 smithi070 ceph-mon[96704]: osdmap e961: 8 total, 8 up, 8 in 2024-09-14T04:04:04.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:04 smithi094 ceph-mon[86597]: osdmap e962: 8 total, 8 up, 8 in 2024-09-14T04:04:04.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:04 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/1726097899' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:04.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:04 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:04.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:04 smithi070 ceph-mon[96704]: osdmap e962: 8 total, 8 up, 8 in 2024-09-14T04:04:04.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:04 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/1726097899' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:04.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:04 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:04.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:04 smithi070 ceph-mon[98640]: osdmap e962: 8 total, 8 up, 8 in 2024-09-14T04:04:04.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:04 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/1726097899' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:04.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:04 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:05.429 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] cls_cas.wrong_put (3014 ms) 2024-09-14T04:04:05.430 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] cls_cas.dup_get 2024-09-14T04:04:05.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:05 smithi094 ceph-mon[86597]: pgmap v1065: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:05.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:05 smithi094 ceph-mon[86597]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:05.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:05 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:05.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:05 smithi094 ceph-mon[86597]: osdmap e963: 8 total, 8 up, 8 in 2024-09-14T04:04:05.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:05 smithi070 ceph-mon[96704]: pgmap v1065: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:05.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:05 smithi070 ceph-mon[96704]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:05.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:05 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:05.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:05 smithi070 ceph-mon[96704]: osdmap e963: 8 total, 8 up, 8 in 2024-09-14T04:04:05.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:05 smithi070 ceph-mon[98640]: pgmap v1065: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:05.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:05 smithi070 ceph-mon[98640]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:05.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:05 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:05.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:05 smithi070 ceph-mon[98640]: osdmap e963: 8 total, 8 up, 8 in 2024-09-14T04:04:06.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:06 smithi094 ceph-mon[86597]: osdmap e964: 8 total, 8 up, 8 in 2024-09-14T04:04:06.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:06 smithi070 ceph-mon[96704]: osdmap e964: 8 total, 8 up, 8 in 2024-09-14T04:04:06.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:06 smithi070 ceph-mon[98640]: osdmap e964: 8 total, 8 up, 8 in 2024-09-14T04:04:07.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:07 smithi094 ceph-mon[86597]: pgmap v1068: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:07.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:07 smithi094 ceph-mon[86597]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:07.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:07 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T04:04:07.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:07 smithi094 ceph-mon[86597]: osdmap e965: 8 total, 8 up, 8 in 2024-09-14T04:04:07.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:07 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/4198227391' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:07.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:07 smithi070 ceph-mon[96704]: pgmap v1068: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:07.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:07 smithi070 ceph-mon[96704]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:07.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:07 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T04:04:07.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:07 smithi070 ceph-mon[96704]: osdmap e965: 8 total, 8 up, 8 in 2024-09-14T04:04:07.788 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:07 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/4198227391' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:07.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:07 smithi070 ceph-mon[98640]: pgmap v1068: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:07.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:07 smithi070 ceph-mon[98640]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:07.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:07 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T04:04:07.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:07 smithi070 ceph-mon[98640]: osdmap e965: 8 total, 8 up, 8 in 2024-09-14T04:04:07.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:07 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/4198227391' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:08.448 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] cls_cas.dup_get (3018 ms) 2024-09-14T04:04:08.448 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] cls_cas.dup_put 2024-09-14T04:04:08.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:08 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/4198227391' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:08.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:08 smithi094 ceph-mon[86597]: osdmap e966: 8 total, 8 up, 8 in 2024-09-14T04:04:08.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:08 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/4198227391' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:08.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:08 smithi070 ceph-mon[96704]: osdmap e966: 8 total, 8 up, 8 in 2024-09-14T04:04:08.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:08 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/4198227391' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:08.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:08 smithi070 ceph-mon[98640]: osdmap e966: 8 total, 8 up, 8 in 2024-09-14T04:04:09.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:09 smithi094 ceph-mon[86597]: pgmap v1071: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:09.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:09 smithi094 ceph-mon[86597]: osdmap e967: 8 total, 8 up, 8 in 2024-09-14T04:04:09.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:09 smithi070 ceph-mon[96704]: pgmap v1071: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:09.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:09 smithi070 ceph-mon[96704]: osdmap e967: 8 total, 8 up, 8 in 2024-09-14T04:04:09.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:09 smithi070 ceph-mon[98640]: pgmap v1071: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:09.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:09 smithi070 ceph-mon[98640]: osdmap e967: 8 total, 8 up, 8 in 2024-09-14T04:04:10.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:10 smithi094 ceph-mon[86597]: osdmap e968: 8 total, 8 up, 8 in 2024-09-14T04:04:10.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:10 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/1925829102' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:10.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:04:10 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:04:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:04:10.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:10 smithi070 ceph-mon[96704]: osdmap e968: 8 total, 8 up, 8 in 2024-09-14T04:04:10.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:10 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/1925829102' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:10.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:10 smithi070 ceph-mon[98640]: osdmap e968: 8 total, 8 up, 8 in 2024-09-14T04:04:10.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:10 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/1925829102' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:11.467 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] cls_cas.dup_put (3019 ms) 2024-09-14T04:04:11.467 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] cls_cas.get_wrong_data 2024-09-14T04:04:11.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:11 smithi094 ceph-mon[86597]: pgmap v1074: 137 pgs: 9 creating+peering, 23 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:11.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:11 smithi094 ceph-mon[86597]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:11.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:11 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/1925829102' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:11.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:11 smithi094 ceph-mon[86597]: osdmap e969: 8 total, 8 up, 8 in 2024-09-14T04:04:11.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:11 smithi070 ceph-mon[96704]: pgmap v1074: 137 pgs: 9 creating+peering, 23 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:11.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:11 smithi070 ceph-mon[96704]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:11.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:11 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/1925829102' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:11.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:11 smithi070 ceph-mon[96704]: osdmap e969: 8 total, 8 up, 8 in 2024-09-14T04:04:11.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:11 smithi070 ceph-mon[98640]: pgmap v1074: 137 pgs: 9 creating+peering, 23 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:11.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:11 smithi070 ceph-mon[98640]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:11.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:11 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/1925829102' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:11.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:11 smithi070 ceph-mon[98640]: osdmap e969: 8 total, 8 up, 8 in 2024-09-14T04:04:12.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:12 smithi094 ceph-mon[86597]: osdmap e970: 8 total, 8 up, 8 in 2024-09-14T04:04:12.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:12 smithi070 ceph-mon[96704]: osdmap e970: 8 total, 8 up, 8 in 2024-09-14T04:04:12.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:12 smithi070 ceph-mon[98640]: osdmap e970: 8 total, 8 up, 8 in 2024-09-14T04:04:13.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:13 smithi094 ceph-mon[86597]: pgmap v1077: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:13.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:13 smithi094 ceph-mon[86597]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:13.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:13 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T04:04:13.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:13 smithi094 ceph-mon[86597]: osdmap e971: 8 total, 8 up, 8 in 2024-09-14T04:04:13.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:13 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/2743092047' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:13.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:13 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:13.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:04:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:04:13] "GET /metrics HTTP/1.1" 200 36518 "" "Prometheus/2.51.0" 2024-09-14T04:04:13.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:13 smithi070 ceph-mon[96704]: pgmap v1077: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:13.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:13 smithi070 ceph-mon[96704]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:13.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:13 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T04:04:13.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:13 smithi070 ceph-mon[96704]: osdmap e971: 8 total, 8 up, 8 in 2024-09-14T04:04:13.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:13 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/2743092047' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:13.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:13 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:13.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:13 smithi070 ceph-mon[98640]: pgmap v1077: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:13.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:13 smithi070 ceph-mon[98640]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:13.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:13 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T04:04:13.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:13 smithi070 ceph-mon[98640]: osdmap e971: 8 total, 8 up, 8 in 2024-09-14T04:04:13.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:13 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/2743092047' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:13.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:13 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:14.478 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] cls_cas.get_wrong_data (3012 ms) 2024-09-14T04:04:14.479 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] 5 tests from cls_cas (14985 ms total) 2024-09-14T04:04:14.479 INFO:tasks.workunit.client.0.smithi070.stdout: 2024-09-14T04:04:14.479 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] 1 test from chunk_refs_t 2024-09-14T04:04:14.479 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] chunk_refs_t.size 2024-09-14T04:04:14.479 INFO:tasks.workunit.client.0.smithi070.stdout:1 60 by_object 2024-09-14T04:04:14.479 INFO:tasks.workunit.client.0.smithi070.stdout:2 103 by_object 2024-09-14T04:04:14.479 INFO:tasks.workunit.client.0.smithi070.stdout:4 189 by_object 2024-09-14T04:04:14.479 INFO:tasks.workunit.client.0.smithi070.stdout:8 361 by_object 2024-09-14T04:04:14.479 INFO:tasks.workunit.client.0.smithi070.stdout:16 112 by_hash(32 bits) 2024-09-14T04:04:14.479 INFO:tasks.workunit.client.0.smithi070.stdout:32 208 by_hash(32 bits) 2024-09-14T04:04:14.479 INFO:tasks.workunit.client.0.smithi070.stdout:64 400 by_hash(32 bits) 2024-09-14T04:04:14.480 INFO:tasks.workunit.client.0.smithi070.stdout:128 509 by_hash(10 bits) 2024-09-14T04:04:14.480 INFO:tasks.workunit.client.0.smithi070.stdout:256 499 by_hash(7 bits) 2024-09-14T04:04:14.480 INFO:tasks.workunit.client.0.smithi070.stdout:512 462 by_hash(5 bits) 2024-09-14T04:04:14.481 INFO:tasks.workunit.client.0.smithi070.stdout:1024 466 by_hash(5 bits) 2024-09-14T04:04:14.481 INFO:tasks.workunit.client.0.smithi070.stdout:2048 466 by_hash(5 bits) 2024-09-14T04:04:14.481 INFO:tasks.workunit.client.0.smithi070.stdout:4096 470 by_hash(5 bits) 2024-09-14T04:04:14.481 INFO:tasks.workunit.client.0.smithi070.stdout:8192 498 by_hash(4 bits) 2024-09-14T04:04:14.482 INFO:tasks.workunit.client.0.smithi070.stdout:16384 278 by_hash(3 bits) 2024-09-14T04:04:14.485 INFO:tasks.workunit.client.0.smithi070.stdout:32768 337 by_hash(2 bits) 2024-09-14T04:04:14.489 INFO:tasks.workunit.client.0.smithi070.stdout:65536 367 by_hash(2 bits) 2024-09-14T04:04:14.497 INFO:tasks.workunit.client.0.smithi070.stdout:131072 387 by_hash(1 bits) 2024-09-14T04:04:14.511 INFO:tasks.workunit.client.0.smithi070.stdout:262144 331 by_pool 2024-09-14T04:04:14.541 INFO:tasks.workunit.client.0.smithi070.stdout:524288 21 count 2024-09-14T04:04:14.590 INFO:tasks.workunit.client.0.smithi070.stdout:1048576 21 count 2024-09-14T04:04:14.695 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] chunk_refs_t.size (216 ms) 2024-09-14T04:04:14.695 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] 1 test from chunk_refs_t (216 ms total) 2024-09-14T04:04:14.695 INFO:tasks.workunit.client.0.smithi070.stdout: 2024-09-14T04:04:14.695 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] Global test environment tear-down 2024-09-14T04:04:14.695 INFO:tasks.workunit.client.0.smithi070.stdout:[==========] 6 tests from 2 test suites ran. (15201 ms total) 2024-09-14T04:04:14.695 INFO:tasks.workunit.client.0.smithi070.stdout:[ PASSED ] 6 tests. 2024-09-14T04:04:14.697 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-14T04:04:14.697 DEBUG:teuthology.orchestra.run.smithi070:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-14T04:04:14.759 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:14 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:14.759 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:14 smithi070 ceph-mon[96704]: osdmap e972: 8 total, 8 up, 8 in 2024-09-14T04:04:14.759 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:14 smithi070 ceph-mon[96704]: osdmap e973: 8 total, 8 up, 8 in 2024-09-14T04:04:14.759 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:14 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:14.759 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:14 smithi070 ceph-mon[98640]: osdmap e972: 8 total, 8 up, 8 in 2024-09-14T04:04:14.760 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:14 smithi070 ceph-mon[98640]: osdmap e973: 8 total, 8 up, 8 in 2024-09-14T04:04:14.763 INFO:tasks.workunit:Running workunit cls/test_cls_cmpomap.sh... 2024-09-14T04:04:14.763 DEBUG:teuthology.orchestra.run.smithi070:workunit test cls/test_cls_cmpomap.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_cmpomap.sh 2024-09-14T04:04:14.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:14 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129139-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:14.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:14 smithi094 ceph-mon[86597]: osdmap e972: 8 total, 8 up, 8 in 2024-09-14T04:04:14.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:14 smithi094 ceph-mon[86597]: osdmap e973: 8 total, 8 up, 8 in 2024-09-14T04:04:14.830 INFO:tasks.workunit.client.0.smithi070.stdout:Running main() from gmock_main.cc 2024-09-14T04:04:14.830 INFO:tasks.workunit.client.0.smithi070.stdout:[==========] Running 31 tests from 1 test suite. 2024-09-14T04:04:14.831 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] Global test environment set-up. 2024-09-14T04:04:15.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:15 smithi094 ceph-mon[86597]: pgmap v1080: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:15.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:15 smithi070 ceph-mon[98640]: pgmap v1080: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:15.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:15 smithi070 ceph-mon[96704]: pgmap v1080: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:16.503 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] 31 tests from CmpOmap 2024-09-14T04:04:16.503 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_str 2024-09-14T04:04:16.518 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] CmpOmap.cmp_vals_noexist_str (16 ms) 2024-09-14T04:04:16.518 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_str_default 2024-09-14T04:04:16.523 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] CmpOmap.cmp_vals_noexist_str_default (5 ms) 2024-09-14T04:04:16.524 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_u64 2024-09-14T04:04:16.527 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] CmpOmap.cmp_vals_noexist_u64 (3 ms) 2024-09-14T04:04:16.527 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_u64_default 2024-09-14T04:04:16.529 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] CmpOmap.cmp_vals_noexist_u64_default (3 ms) 2024-09-14T04:04:16.530 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] CmpOmap.cmp_vals_str 2024-09-14T04:04:16.535 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] CmpOmap.cmp_vals_str (5 ms) 2024-09-14T04:04:16.535 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] CmpOmap.cmp_vals_u64 2024-09-14T04:04:16.541 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] CmpOmap.cmp_vals_u64 (6 ms) 2024-09-14T04:04:16.541 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] CmpOmap.cmp_vals_u64_invalid_input 2024-09-14T04:04:16.544 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] CmpOmap.cmp_vals_u64_invalid_input (3 ms) 2024-09-14T04:04:16.544 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] CmpOmap.cmp_vals_u64_invalid_default 2024-09-14T04:04:16.546 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] CmpOmap.cmp_vals_u64_invalid_default (3 ms) 2024-09-14T04:04:16.547 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] CmpOmap.cmp_vals_u64_empty_default 2024-09-14T04:04:16.549 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] CmpOmap.cmp_vals_u64_empty_default (3 ms) 2024-09-14T04:04:16.549 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] CmpOmap.cmp_vals_u64_invalid_value 2024-09-14T04:04:16.552 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] CmpOmap.cmp_vals_u64_invalid_value (3 ms) 2024-09-14T04:04:16.553 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] CmpOmap.cmp_vals_at_max_keys 2024-09-14T04:04:16.553 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] CmpOmap.cmp_vals_at_max_keys (1 ms) 2024-09-14T04:04:16.553 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] CmpOmap.cmp_vals_over_max_keys 2024-09-14T04:04:16.554 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] CmpOmap.cmp_vals_over_max_keys (1 ms) 2024-09-14T04:04:16.554 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_str 2024-09-14T04:04:16.560 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_str (5 ms) 2024-09-14T04:04:16.560 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_str_default 2024-09-14T04:04:16.565 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_str_default (6 ms) 2024-09-14T04:04:16.566 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_u64 2024-09-14T04:04:16.571 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_u64 (5 ms) 2024-09-14T04:04:16.571 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_u64_default 2024-09-14T04:04:16.578 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_u64_default (7 ms) 2024-09-14T04:04:16.578 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] CmpOmap.cmp_set_vals_str 2024-09-14T04:04:16.584 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] CmpOmap.cmp_set_vals_str (7 ms) 2024-09-14T04:04:16.584 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] CmpOmap.cmp_set_vals_u64 2024-09-14T04:04:16.593 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] CmpOmap.cmp_set_vals_u64 (6 ms) 2024-09-14T04:04:16.593 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] CmpOmap.cmp_set_vals_u64_einval 2024-09-14T04:04:16.594 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] CmpOmap.cmp_set_vals_u64_einval (2 ms) 2024-09-14T04:04:16.594 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] CmpOmap.cmp_set_vals_u64_eio 2024-09-14T04:04:16.594 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] CmpOmap.cmp_set_vals_u64_eio (2 ms) 2024-09-14T04:04:16.594 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] CmpOmap.cmp_set_vals_at_max_keys 2024-09-14T04:04:16.595 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] CmpOmap.cmp_set_vals_at_max_keys (1 ms) 2024-09-14T04:04:16.595 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] CmpOmap.cmp_set_vals_over_max_keys 2024-09-14T04:04:16.596 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] CmpOmap.cmp_set_vals_over_max_keys (1 ms) 2024-09-14T04:04:16.596 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] CmpOmap.cmp_rm_keys_noexist_str 2024-09-14T04:04:16.602 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] CmpOmap.cmp_rm_keys_noexist_str (6 ms) 2024-09-14T04:04:16.602 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] CmpOmap.cmp_rm_keys_noexist_u64 2024-09-14T04:04:16.608 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] CmpOmap.cmp_rm_keys_noexist_u64 (5 ms) 2024-09-14T04:04:16.608 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] CmpOmap.cmp_rm_keys_str 2024-09-14T04:04:16.614 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] CmpOmap.cmp_rm_keys_str (6 ms) 2024-09-14T04:04:16.614 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64 2024-09-14T04:04:16.621 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64 (8 ms) 2024-09-14T04:04:16.621 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64_einval 2024-09-14T04:04:16.622 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64_einval (1 ms) 2024-09-14T04:04:16.622 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64_eio 2024-09-14T04:04:16.624 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64_eio (2 ms) 2024-09-14T04:04:16.625 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] CmpOmap.cmp_rm_keys_at_max_keys 2024-09-14T04:04:16.625 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] CmpOmap.cmp_rm_keys_at_max_keys (1 ms) 2024-09-14T04:04:16.626 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] CmpOmap.cmp_rm_keys_over_max_keys 2024-09-14T04:04:16.626 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] CmpOmap.cmp_rm_keys_over_max_keys (1 ms) 2024-09-14T04:04:16.626 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64_empty 2024-09-14T04:04:16.634 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64_empty (8 ms) 2024-09-14T04:04:16.634 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] 31 tests from CmpOmap (132 ms total) 2024-09-14T04:04:16.634 INFO:tasks.workunit.client.0.smithi070.stdout: 2024-09-14T04:04:16.634 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] Global test environment tear-down 2024-09-14T04:04:16.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:16 smithi094 ceph-mon[86597]: osdmap e974: 8 total, 8 up, 8 in 2024-09-14T04:04:16.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:16 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/1234298596' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129262-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:16.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:16 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129262-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:16.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:16 smithi070 ceph-mon[96704]: osdmap e974: 8 total, 8 up, 8 in 2024-09-14T04:04:16.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:16 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/1234298596' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129262-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:16.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:16 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129262-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:16.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:16 smithi070 ceph-mon[98640]: osdmap e974: 8 total, 8 up, 8 in 2024-09-14T04:04:16.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:16 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/1234298596' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129262-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:16.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:16 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129262-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:17.509 INFO:tasks.workunit.client.0.smithi070.stdout:[==========] 31 tests from 1 test suite ran. (2680 ms total) 2024-09-14T04:04:17.509 INFO:tasks.workunit.client.0.smithi070.stdout:[ PASSED ] 31 tests. 2024-09-14T04:04:17.512 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-14T04:04:17.512 DEBUG:teuthology.orchestra.run.smithi070:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-14T04:04:17.581 INFO:tasks.workunit:Running workunit cls/test_cls_hello.sh... 2024-09-14T04:04:17.581 DEBUG:teuthology.orchestra.run.smithi070:workunit test cls/test_cls_hello.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_hello.sh 2024-09-14T04:04:17.652 INFO:tasks.workunit.client.0.smithi070.stdout:Running main() from gmock_main.cc 2024-09-14T04:04:17.652 INFO:tasks.workunit.client.0.smithi070.stdout:[==========] Running 6 tests from 1 test suite. 2024-09-14T04:04:17.652 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] Global test environment set-up. 2024-09-14T04:04:17.652 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] 6 tests from ClsHello 2024-09-14T04:04:17.652 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] ClsHello.SayHello 2024-09-14T04:04:17.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:17 smithi094 ceph-mon[86597]: pgmap v1083: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:17.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:17 smithi094 ceph-mon[86597]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:17.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:17 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129262-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:17.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:17 smithi094 ceph-mon[86597]: osdmap e975: 8 total, 8 up, 8 in 2024-09-14T04:04:17.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:17 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:04:17.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:17 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:17.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:17 smithi070 ceph-mon[96704]: pgmap v1083: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:17.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:17 smithi070 ceph-mon[96704]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:17.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:17 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129262-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:17.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:17 smithi070 ceph-mon[96704]: osdmap e975: 8 total, 8 up, 8 in 2024-09-14T04:04:17.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:17 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:04:17.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:17 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:17.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:17 smithi070 ceph-mon[98640]: pgmap v1083: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:17.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:17 smithi070 ceph-mon[98640]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:17.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:17 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129262-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:17.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:17 smithi070 ceph-mon[98640]: osdmap e975: 8 total, 8 up, 8 in 2024-09-14T04:04:17.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:17 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:04:17.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:17 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:18.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:18 smithi094 ceph-mon[86597]: osdmap e976: 8 total, 8 up, 8 in 2024-09-14T04:04:18.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:18 smithi070 ceph-mon[96704]: osdmap e976: 8 total, 8 up, 8 in 2024-09-14T04:04:18.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:18 smithi070 ceph-mon[98640]: osdmap e976: 8 total, 8 up, 8 in 2024-09-14T04:04:19.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:19 smithi094 ceph-mon[86597]: pgmap v1086: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:19.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:19 smithi094 ceph-mon[86597]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:19.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:19 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T04:04:19.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:19 smithi094 ceph-mon[86597]: osdmap e977: 8 total, 8 up, 8 in 2024-09-14T04:04:19.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:19 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/2653498572' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:19.778 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:19 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:19.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:19 smithi070 ceph-mon[96704]: pgmap v1086: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:19.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:19 smithi070 ceph-mon[96704]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:19.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:19 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T04:04:19.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:19 smithi070 ceph-mon[96704]: osdmap e977: 8 total, 8 up, 8 in 2024-09-14T04:04:19.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:19 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/2653498572' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:19.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:19 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:19.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:19 smithi070 ceph-mon[98640]: pgmap v1086: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:19.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:19 smithi070 ceph-mon[98640]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:19.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:19 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T04:04:19.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:19 smithi070 ceph-mon[98640]: osdmap e977: 8 total, 8 up, 8 in 2024-09-14T04:04:19.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:19 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/2653498572' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:19.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:19 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:20.523 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:04:20 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:04:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:04:20.529 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] ClsHello.SayHello (2876 ms) 2024-09-14T04:04:20.529 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] ClsHello.RecordHello 2024-09-14T04:04:20.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:20 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:20.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:20 smithi094 ceph-mon[86597]: osdmap e978: 8 total, 8 up, 8 in 2024-09-14T04:04:20.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:20 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:20.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:20 smithi070 ceph-mon[96704]: osdmap e978: 8 total, 8 up, 8 in 2024-09-14T04:04:20.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:20 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:20.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:20 smithi070 ceph-mon[98640]: osdmap e978: 8 total, 8 up, 8 in 2024-09-14T04:04:21.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:21 smithi094 ceph-mon[86597]: pgmap v1089: 137 pgs: 24 unknown, 113 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:21.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:21 smithi094 ceph-mon[86597]: osdmap e979: 8 total, 8 up, 8 in 2024-09-14T04:04:21.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:21 smithi070 ceph-mon[96704]: pgmap v1089: 137 pgs: 24 unknown, 113 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:21.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:21 smithi070 ceph-mon[96704]: osdmap e979: 8 total, 8 up, 8 in 2024-09-14T04:04:21.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:21 smithi070 ceph-mon[98640]: pgmap v1089: 137 pgs: 24 unknown, 113 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:21.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:21 smithi070 ceph-mon[98640]: osdmap e979: 8 total, 8 up, 8 in 2024-09-14T04:04:23.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:22 smithi094 ceph-mon[86597]: osdmap e980: 8 total, 8 up, 8 in 2024-09-14T04:04:23.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:22 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/125150631' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:23.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:22 smithi070 ceph-mon[96704]: osdmap e980: 8 total, 8 up, 8 in 2024-09-14T04:04:23.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:22 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/125150631' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:23.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:22 smithi070 ceph-mon[98640]: osdmap e980: 8 total, 8 up, 8 in 2024-09-14T04:04:23.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:22 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/125150631' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:23.546 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] ClsHello.RecordHello (3018 ms) 2024-09-14T04:04:23.547 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] ClsHello.WriteReturnData 2024-09-14T04:04:23.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:04:23 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:04:23] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.51.0" 2024-09-14T04:04:23.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:23 smithi070 ceph-mon[98640]: pgmap v1092: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:23.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:23 smithi070 ceph-mon[98640]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:23.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:23 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/125150631' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:23.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:23 smithi070 ceph-mon[98640]: osdmap e981: 8 total, 8 up, 8 in 2024-09-14T04:04:23.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:23 smithi070 ceph-mon[96704]: pgmap v1092: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:23.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:23 smithi070 ceph-mon[96704]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:23.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:23 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/125150631' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:23.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:23 smithi070 ceph-mon[96704]: osdmap e981: 8 total, 8 up, 8 in 2024-09-14T04:04:24.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:23 smithi094 ceph-mon[86597]: pgmap v1092: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:24.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:23 smithi094 ceph-mon[86597]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:24.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:23 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/125150631' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:24.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:23 smithi094 ceph-mon[86597]: osdmap e981: 8 total, 8 up, 8 in 2024-09-14T04:04:25.008 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:24 smithi070 ceph-mon[96704]: osdmap e982: 8 total, 8 up, 8 in 2024-09-14T04:04:25.008 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:24 smithi070 ceph-mon[98640]: osdmap e982: 8 total, 8 up, 8 in 2024-09-14T04:04:25.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:24 smithi094 ceph-mon[86597]: osdmap e982: 8 total, 8 up, 8 in 2024-09-14T04:04:25.559 INFO:tasks.workunit.client.0.smithi070.stdout:require_osd_release = squid 2024-09-14T04:04:25.567 INFO:tasks.workunit.client.0.smithi070.stdout:0 2024-09-14T04:04:25.823 INFO:tasks.workunit.client.0.smithi070.stdout:1 2024-09-14T04:04:25.824 INFO:tasks.workunit.client.0.smithi070.stdout:2 2024-09-14T04:04:25.825 INFO:tasks.workunit.client.0.smithi070.stdout:3 2024-09-14T04:04:25.826 INFO:tasks.workunit.client.0.smithi070.stdout:4 2024-09-14T04:04:25.827 INFO:tasks.workunit.client.0.smithi070.stdout:5 2024-09-14T04:04:25.828 INFO:tasks.workunit.client.0.smithi070.stdout:6 2024-09-14T04:04:25.829 INFO:tasks.workunit.client.0.smithi070.stdout:7 2024-09-14T04:04:25.830 INFO:tasks.workunit.client.0.smithi070.stdout:8 2024-09-14T04:04:25.831 INFO:tasks.workunit.client.0.smithi070.stdout:9 2024-09-14T04:04:25.833 INFO:tasks.workunit.client.0.smithi070.stdout:10 2024-09-14T04:04:25.834 INFO:tasks.workunit.client.0.smithi070.stdout:11 2024-09-14T04:04:25.835 INFO:tasks.workunit.client.0.smithi070.stdout:12 2024-09-14T04:04:25.836 INFO:tasks.workunit.client.0.smithi070.stdout:13 2024-09-14T04:04:25.837 INFO:tasks.workunit.client.0.smithi070.stdout:14 2024-09-14T04:04:25.838 INFO:tasks.workunit.client.0.smithi070.stdout:15 2024-09-14T04:04:25.839 INFO:tasks.workunit.client.0.smithi070.stdout:16 2024-09-14T04:04:25.840 INFO:tasks.workunit.client.0.smithi070.stdout:17 2024-09-14T04:04:25.842 INFO:tasks.workunit.client.0.smithi070.stdout:18 2024-09-14T04:04:25.844 INFO:tasks.workunit.client.0.smithi070.stdout:19 2024-09-14T04:04:25.846 INFO:tasks.workunit.client.0.smithi070.stdout:20 2024-09-14T04:04:25.847 INFO:tasks.workunit.client.0.smithi070.stdout:21 2024-09-14T04:04:25.848 INFO:tasks.workunit.client.0.smithi070.stdout:22 2024-09-14T04:04:25.849 INFO:tasks.workunit.client.0.smithi070.stdout:23 2024-09-14T04:04:25.850 INFO:tasks.workunit.client.0.smithi070.stdout:24 2024-09-14T04:04:25.851 INFO:tasks.workunit.client.0.smithi070.stdout:25 2024-09-14T04:04:25.852 INFO:tasks.workunit.client.0.smithi070.stdout:26 2024-09-14T04:04:25.853 INFO:tasks.workunit.client.0.smithi070.stdout:27 2024-09-14T04:04:25.854 INFO:tasks.workunit.client.0.smithi070.stdout:28 2024-09-14T04:04:25.855 INFO:tasks.workunit.client.0.smithi070.stdout:29 2024-09-14T04:04:25.856 INFO:tasks.workunit.client.0.smithi070.stdout:30 2024-09-14T04:04:25.857 INFO:tasks.workunit.client.0.smithi070.stdout:31 2024-09-14T04:04:25.857 INFO:tasks.workunit.client.0.smithi070.stdout:32 2024-09-14T04:04:25.858 INFO:tasks.workunit.client.0.smithi070.stdout:33 2024-09-14T04:04:25.859 INFO:tasks.workunit.client.0.smithi070.stdout:34 2024-09-14T04:04:25.860 INFO:tasks.workunit.client.0.smithi070.stdout:35 2024-09-14T04:04:25.861 INFO:tasks.workunit.client.0.smithi070.stdout:36 2024-09-14T04:04:25.862 INFO:tasks.workunit.client.0.smithi070.stdout:37 2024-09-14T04:04:25.863 INFO:tasks.workunit.client.0.smithi070.stdout:38 2024-09-14T04:04:25.864 INFO:tasks.workunit.client.0.smithi070.stdout:39 2024-09-14T04:04:25.865 INFO:tasks.workunit.client.0.smithi070.stdout:40 2024-09-14T04:04:25.866 INFO:tasks.workunit.client.0.smithi070.stdout:41 2024-09-14T04:04:25.867 INFO:tasks.workunit.client.0.smithi070.stdout:42 2024-09-14T04:04:25.867 INFO:tasks.workunit.client.0.smithi070.stdout:43 2024-09-14T04:04:25.868 INFO:tasks.workunit.client.0.smithi070.stdout:44 2024-09-14T04:04:25.869 INFO:tasks.workunit.client.0.smithi070.stdout:45 2024-09-14T04:04:25.870 INFO:tasks.workunit.client.0.smithi070.stdout:46 2024-09-14T04:04:25.871 INFO:tasks.workunit.client.0.smithi070.stdout:47 2024-09-14T04:04:25.872 INFO:tasks.workunit.client.0.smithi070.stdout:48 2024-09-14T04:04:25.873 INFO:tasks.workunit.client.0.smithi070.stdout:49 2024-09-14T04:04:25.874 INFO:tasks.workunit.client.0.smithi070.stdout:50 2024-09-14T04:04:25.875 INFO:tasks.workunit.client.0.smithi070.stdout:51 2024-09-14T04:04:25.876 INFO:tasks.workunit.client.0.smithi070.stdout:52 2024-09-14T04:04:25.876 INFO:tasks.workunit.client.0.smithi070.stdout:53 2024-09-14T04:04:25.877 INFO:tasks.workunit.client.0.smithi070.stdout:54 2024-09-14T04:04:25.878 INFO:tasks.workunit.client.0.smithi070.stdout:55 2024-09-14T04:04:25.879 INFO:tasks.workunit.client.0.smithi070.stdout:56 2024-09-14T04:04:25.880 INFO:tasks.workunit.client.0.smithi070.stdout:57 2024-09-14T04:04:25.881 INFO:tasks.workunit.client.0.smithi070.stdout:58 2024-09-14T04:04:25.882 INFO:tasks.workunit.client.0.smithi070.stdout:59 2024-09-14T04:04:25.883 INFO:tasks.workunit.client.0.smithi070.stdout:60 2024-09-14T04:04:25.883 INFO:tasks.workunit.client.0.smithi070.stdout:61 2024-09-14T04:04:25.884 INFO:tasks.workunit.client.0.smithi070.stdout:62 2024-09-14T04:04:25.885 INFO:tasks.workunit.client.0.smithi070.stdout:63 2024-09-14T04:04:25.886 INFO:tasks.workunit.client.0.smithi070.stdout:64 2024-09-14T04:04:25.887 INFO:tasks.workunit.client.0.smithi070.stdout:65 2024-09-14T04:04:25.888 INFO:tasks.workunit.client.0.smithi070.stdout:66 2024-09-14T04:04:25.889 INFO:tasks.workunit.client.0.smithi070.stdout:67 2024-09-14T04:04:25.889 INFO:tasks.workunit.client.0.smithi070.stdout:68 2024-09-14T04:04:25.890 INFO:tasks.workunit.client.0.smithi070.stdout:69 2024-09-14T04:04:25.891 INFO:tasks.workunit.client.0.smithi070.stdout:70 2024-09-14T04:04:25.892 INFO:tasks.workunit.client.0.smithi070.stdout:71 2024-09-14T04:04:25.893 INFO:tasks.workunit.client.0.smithi070.stdout:72 2024-09-14T04:04:25.894 INFO:tasks.workunit.client.0.smithi070.stdout:73 2024-09-14T04:04:25.895 INFO:tasks.workunit.client.0.smithi070.stdout:74 2024-09-14T04:04:25.895 INFO:tasks.workunit.client.0.smithi070.stdout:75 2024-09-14T04:04:25.896 INFO:tasks.workunit.client.0.smithi070.stdout:76 2024-09-14T04:04:25.897 INFO:tasks.workunit.client.0.smithi070.stdout:77 2024-09-14T04:04:25.898 INFO:tasks.workunit.client.0.smithi070.stdout:78 2024-09-14T04:04:25.899 INFO:tasks.workunit.client.0.smithi070.stdout:79 2024-09-14T04:04:25.900 INFO:tasks.workunit.client.0.smithi070.stdout:80 2024-09-14T04:04:25.901 INFO:tasks.workunit.client.0.smithi070.stdout:81 2024-09-14T04:04:25.902 INFO:tasks.workunit.client.0.smithi070.stdout:82 2024-09-14T04:04:25.902 INFO:tasks.workunit.client.0.smithi070.stdout:83 2024-09-14T04:04:25.903 INFO:tasks.workunit.client.0.smithi070.stdout:84 2024-09-14T04:04:25.904 INFO:tasks.workunit.client.0.smithi070.stdout:85 2024-09-14T04:04:25.905 INFO:tasks.workunit.client.0.smithi070.stdout:86 2024-09-14T04:04:25.906 INFO:tasks.workunit.client.0.smithi070.stdout:87 2024-09-14T04:04:25.907 INFO:tasks.workunit.client.0.smithi070.stdout:88 2024-09-14T04:04:25.908 INFO:tasks.workunit.client.0.smithi070.stdout:89 2024-09-14T04:04:25.909 INFO:tasks.workunit.client.0.smithi070.stdout:90 2024-09-14T04:04:25.909 INFO:tasks.workunit.client.0.smithi070.stdout:91 2024-09-14T04:04:25.910 INFO:tasks.workunit.client.0.smithi070.stdout:92 2024-09-14T04:04:25.911 INFO:tasks.workunit.client.0.smithi070.stdout:93 2024-09-14T04:04:25.912 INFO:tasks.workunit.client.0.smithi070.stdout:94 2024-09-14T04:04:25.913 INFO:tasks.workunit.client.0.smithi070.stdout:95 2024-09-14T04:04:25.914 INFO:tasks.workunit.client.0.smithi070.stdout:96 2024-09-14T04:04:25.915 INFO:tasks.workunit.client.0.smithi070.stdout:97 2024-09-14T04:04:25.916 INFO:tasks.workunit.client.0.smithi070.stdout:98 2024-09-14T04:04:25.916 INFO:tasks.workunit.client.0.smithi070.stdout:99 2024-09-14T04:04:25.917 INFO:tasks.workunit.client.0.smithi070.stdout:100 2024-09-14T04:04:25.918 INFO:tasks.workunit.client.0.smithi070.stdout:101 2024-09-14T04:04:25.919 INFO:tasks.workunit.client.0.smithi070.stdout:102 2024-09-14T04:04:25.920 INFO:tasks.workunit.client.0.smithi070.stdout:103 2024-09-14T04:04:25.921 INFO:tasks.workunit.client.0.smithi070.stdout:104 2024-09-14T04:04:25.922 INFO:tasks.workunit.client.0.smithi070.stdout:105 2024-09-14T04:04:25.922 INFO:tasks.workunit.client.0.smithi070.stdout:106 2024-09-14T04:04:25.923 INFO:tasks.workunit.client.0.smithi070.stdout:107 2024-09-14T04:04:25.924 INFO:tasks.workunit.client.0.smithi070.stdout:108 2024-09-14T04:04:25.925 INFO:tasks.workunit.client.0.smithi070.stdout:109 2024-09-14T04:04:25.926 INFO:tasks.workunit.client.0.smithi070.stdout:110 2024-09-14T04:04:25.926 INFO:tasks.workunit.client.0.smithi070.stdout:111 2024-09-14T04:04:25.927 INFO:tasks.workunit.client.0.smithi070.stdout:112 2024-09-14T04:04:25.928 INFO:tasks.workunit.client.0.smithi070.stdout:113 2024-09-14T04:04:25.929 INFO:tasks.workunit.client.0.smithi070.stdout:114 2024-09-14T04:04:25.930 INFO:tasks.workunit.client.0.smithi070.stdout:115 2024-09-14T04:04:25.930 INFO:tasks.workunit.client.0.smithi070.stdout:116 2024-09-14T04:04:25.931 INFO:tasks.workunit.client.0.smithi070.stdout:117 2024-09-14T04:04:25.932 INFO:tasks.workunit.client.0.smithi070.stdout:118 2024-09-14T04:04:25.933 INFO:tasks.workunit.client.0.smithi070.stdout:119 2024-09-14T04:04:25.934 INFO:tasks.workunit.client.0.smithi070.stdout:120 2024-09-14T04:04:25.934 INFO:tasks.workunit.client.0.smithi070.stdout:121 2024-09-14T04:04:25.935 INFO:tasks.workunit.client.0.smithi070.stdout:122 2024-09-14T04:04:25.936 INFO:tasks.workunit.client.0.smithi070.stdout:123 2024-09-14T04:04:25.937 INFO:tasks.workunit.client.0.smithi070.stdout:124 2024-09-14T04:04:25.938 INFO:tasks.workunit.client.0.smithi070.stdout:125 2024-09-14T04:04:25.938 INFO:tasks.workunit.client.0.smithi070.stdout:126 2024-09-14T04:04:25.939 INFO:tasks.workunit.client.0.smithi070.stdout:127 2024-09-14T04:04:25.940 INFO:tasks.workunit.client.0.smithi070.stdout:128 2024-09-14T04:04:25.941 INFO:tasks.workunit.client.0.smithi070.stdout:129 2024-09-14T04:04:25.942 INFO:tasks.workunit.client.0.smithi070.stdout:130 2024-09-14T04:04:25.943 INFO:tasks.workunit.client.0.smithi070.stdout:131 2024-09-14T04:04:25.943 INFO:tasks.workunit.client.0.smithi070.stdout:132 2024-09-14T04:04:25.944 INFO:tasks.workunit.client.0.smithi070.stdout:133 2024-09-14T04:04:25.945 INFO:tasks.workunit.client.0.smithi070.stdout:134 2024-09-14T04:04:25.945 INFO:tasks.workunit.client.0.smithi070.stdout:135 2024-09-14T04:04:25.946 INFO:tasks.workunit.client.0.smithi070.stdout:136 2024-09-14T04:04:25.947 INFO:tasks.workunit.client.0.smithi070.stdout:137 2024-09-14T04:04:25.948 INFO:tasks.workunit.client.0.smithi070.stdout:138 2024-09-14T04:04:25.949 INFO:tasks.workunit.client.0.smithi070.stdout:139 2024-09-14T04:04:25.950 INFO:tasks.workunit.client.0.smithi070.stdout:140 2024-09-14T04:04:25.950 INFO:tasks.workunit.client.0.smithi070.stdout:141 2024-09-14T04:04:25.951 INFO:tasks.workunit.client.0.smithi070.stdout:142 2024-09-14T04:04:25.952 INFO:tasks.workunit.client.0.smithi070.stdout:143 2024-09-14T04:04:25.953 INFO:tasks.workunit.client.0.smithi070.stdout:144 2024-09-14T04:04:25.954 INFO:tasks.workunit.client.0.smithi070.stdout:145 2024-09-14T04:04:25.955 INFO:tasks.workunit.client.0.smithi070.stdout:146 2024-09-14T04:04:25.956 INFO:tasks.workunit.client.0.smithi070.stdout:147 2024-09-14T04:04:25.956 INFO:tasks.workunit.client.0.smithi070.stdout:148 2024-09-14T04:04:25.957 INFO:tasks.workunit.client.0.smithi070.stdout:149 2024-09-14T04:04:25.958 INFO:tasks.workunit.client.0.smithi070.stdout:150 2024-09-14T04:04:25.959 INFO:tasks.workunit.client.0.smithi070.stdout:151 2024-09-14T04:04:25.960 INFO:tasks.workunit.client.0.smithi070.stdout:152 2024-09-14T04:04:25.961 INFO:tasks.workunit.client.0.smithi070.stdout:153 2024-09-14T04:04:25.962 INFO:tasks.workunit.client.0.smithi070.stdout:154 2024-09-14T04:04:25.963 INFO:tasks.workunit.client.0.smithi070.stdout:155 2024-09-14T04:04:25.963 INFO:tasks.workunit.client.0.smithi070.stdout:156 2024-09-14T04:04:25.964 INFO:tasks.workunit.client.0.smithi070.stdout:157 2024-09-14T04:04:25.965 INFO:tasks.workunit.client.0.smithi070.stdout:158 2024-09-14T04:04:25.966 INFO:tasks.workunit.client.0.smithi070.stdout:159 2024-09-14T04:04:25.967 INFO:tasks.workunit.client.0.smithi070.stdout:160 2024-09-14T04:04:25.968 INFO:tasks.workunit.client.0.smithi070.stdout:161 2024-09-14T04:04:25.969 INFO:tasks.workunit.client.0.smithi070.stdout:162 2024-09-14T04:04:25.970 INFO:tasks.workunit.client.0.smithi070.stdout:163 2024-09-14T04:04:25.971 INFO:tasks.workunit.client.0.smithi070.stdout:164 2024-09-14T04:04:25.972 INFO:tasks.workunit.client.0.smithi070.stdout:165 2024-09-14T04:04:25.972 INFO:tasks.workunit.client.0.smithi070.stdout:166 2024-09-14T04:04:25.973 INFO:tasks.workunit.client.0.smithi070.stdout:167 2024-09-14T04:04:25.974 INFO:tasks.workunit.client.0.smithi070.stdout:168 2024-09-14T04:04:25.975 INFO:tasks.workunit.client.0.smithi070.stdout:169 2024-09-14T04:04:25.976 INFO:tasks.workunit.client.0.smithi070.stdout:170 2024-09-14T04:04:25.977 INFO:tasks.workunit.client.0.smithi070.stdout:171 2024-09-14T04:04:25.977 INFO:tasks.workunit.client.0.smithi070.stdout:172 2024-09-14T04:04:25.978 INFO:tasks.workunit.client.0.smithi070.stdout:173 2024-09-14T04:04:25.979 INFO:tasks.workunit.client.0.smithi070.stdout:174 2024-09-14T04:04:25.980 INFO:tasks.workunit.client.0.smithi070.stdout:175 2024-09-14T04:04:25.981 INFO:tasks.workunit.client.0.smithi070.stdout:176 2024-09-14T04:04:25.982 INFO:tasks.workunit.client.0.smithi070.stdout:177 2024-09-14T04:04:25.983 INFO:tasks.workunit.client.0.smithi070.stdout:178 2024-09-14T04:04:25.983 INFO:tasks.workunit.client.0.smithi070.stdout:179 2024-09-14T04:04:25.984 INFO:tasks.workunit.client.0.smithi070.stdout:180 2024-09-14T04:04:25.985 INFO:tasks.workunit.client.0.smithi070.stdout:181 2024-09-14T04:04:25.986 INFO:tasks.workunit.client.0.smithi070.stdout:182 2024-09-14T04:04:25.987 INFO:tasks.workunit.client.0.smithi070.stdout:183 2024-09-14T04:04:25.987 INFO:tasks.workunit.client.0.smithi070.stdout:184 2024-09-14T04:04:25.988 INFO:tasks.workunit.client.0.smithi070.stdout:185 2024-09-14T04:04:25.989 INFO:tasks.workunit.client.0.smithi070.stdout:186 2024-09-14T04:04:25.990 INFO:tasks.workunit.client.0.smithi070.stdout:187 2024-09-14T04:04:25.991 INFO:tasks.workunit.client.0.smithi070.stdout:188 2024-09-14T04:04:25.991 INFO:tasks.workunit.client.0.smithi070.stdout:189 2024-09-14T04:04:25.992 INFO:tasks.workunit.client.0.smithi070.stdout:190 2024-09-14T04:04:25.996 INFO:tasks.workunit.client.0.smithi070.stdout:191 2024-09-14T04:04:25.996 INFO:tasks.workunit.client.0.smithi070.stdout:192 2024-09-14T04:04:25.996 INFO:tasks.workunit.client.0.smithi070.stdout:193 2024-09-14T04:04:25.996 INFO:tasks.workunit.client.0.smithi070.stdout:194 2024-09-14T04:04:25.996 INFO:tasks.workunit.client.0.smithi070.stdout:195 2024-09-14T04:04:25.997 INFO:tasks.workunit.client.0.smithi070.stdout:196 2024-09-14T04:04:25.998 INFO:tasks.workunit.client.0.smithi070.stdout:197 2024-09-14T04:04:25.999 INFO:tasks.workunit.client.0.smithi070.stdout:198 2024-09-14T04:04:26.000 INFO:tasks.workunit.client.0.smithi070.stdout:199 2024-09-14T04:04:26.000 INFO:tasks.workunit.client.0.smithi070.stdout:200 2024-09-14T04:04:26.001 INFO:tasks.workunit.client.0.smithi070.stdout:201 2024-09-14T04:04:26.002 INFO:tasks.workunit.client.0.smithi070.stdout:202 2024-09-14T04:04:26.003 INFO:tasks.workunit.client.0.smithi070.stdout:203 2024-09-14T04:04:26.004 INFO:tasks.workunit.client.0.smithi070.stdout:204 2024-09-14T04:04:26.005 INFO:tasks.workunit.client.0.smithi070.stdout:205 2024-09-14T04:04:26.006 INFO:tasks.workunit.client.0.smithi070.stdout:206 2024-09-14T04:04:26.007 INFO:tasks.workunit.client.0.smithi070.stdout:207 2024-09-14T04:04:26.007 INFO:tasks.workunit.client.0.smithi070.stdout:208 2024-09-14T04:04:26.008 INFO:tasks.workunit.client.0.smithi070.stdout:209 2024-09-14T04:04:26.009 INFO:tasks.workunit.client.0.smithi070.stdout:210 2024-09-14T04:04:26.010 INFO:tasks.workunit.client.0.smithi070.stdout:211 2024-09-14T04:04:26.011 INFO:tasks.workunit.client.0.smithi070.stdout:212 2024-09-14T04:04:26.012 INFO:tasks.workunit.client.0.smithi070.stdout:213 2024-09-14T04:04:26.013 INFO:tasks.workunit.client.0.smithi070.stdout:214 2024-09-14T04:04:26.014 INFO:tasks.workunit.client.0.smithi070.stdout:215 2024-09-14T04:04:26.015 INFO:tasks.workunit.client.0.smithi070.stdout:216 2024-09-14T04:04:26.015 INFO:tasks.workunit.client.0.smithi070.stdout:217 2024-09-14T04:04:26.016 INFO:tasks.workunit.client.0.smithi070.stdout:218 2024-09-14T04:04:26.017 INFO:tasks.workunit.client.0.smithi070.stdout:219 2024-09-14T04:04:26.018 INFO:tasks.workunit.client.0.smithi070.stdout:220 2024-09-14T04:04:26.019 INFO:tasks.workunit.client.0.smithi070.stdout:221 2024-09-14T04:04:26.020 INFO:tasks.workunit.client.0.smithi070.stdout:222 2024-09-14T04:04:26.020 INFO:tasks.workunit.client.0.smithi070.stdout:223 2024-09-14T04:04:26.021 INFO:tasks.workunit.client.0.smithi070.stdout:224 2024-09-14T04:04:26.022 INFO:tasks.workunit.client.0.smithi070.stdout:225 2024-09-14T04:04:26.023 INFO:tasks.workunit.client.0.smithi070.stdout:226 2024-09-14T04:04:26.024 INFO:tasks.workunit.client.0.smithi070.stdout:227 2024-09-14T04:04:26.024 INFO:tasks.workunit.client.0.smithi070.stdout:228 2024-09-14T04:04:26.025 INFO:tasks.workunit.client.0.smithi070.stdout:229 2024-09-14T04:04:26.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:25 smithi094 ceph-mon[86597]: pgmap v1095: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:26.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:25 smithi094 ceph-mon[86597]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:26.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:25 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T04:04:26.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:25 smithi094 ceph-mon[86597]: osdmap e983: 8 total, 8 up, 8 in 2024-09-14T04:04:26.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:25 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/776475119' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:26.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:25 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:26.027 INFO:tasks.workunit.client.0.smithi070.stdout:230 2024-09-14T04:04:26.027 INFO:tasks.workunit.client.0.smithi070.stdout:231 2024-09-14T04:04:26.028 INFO:tasks.workunit.client.0.smithi070.stdout:232 2024-09-14T04:04:26.029 INFO:tasks.workunit.client.0.smithi070.stdout:233 2024-09-14T04:04:26.030 INFO:tasks.workunit.client.0.smithi070.stdout:234 2024-09-14T04:04:26.030 INFO:tasks.workunit.client.0.smithi070.stdout:235 2024-09-14T04:04:26.031 INFO:tasks.workunit.client.0.smithi070.stdout:236 2024-09-14T04:04:26.032 INFO:tasks.workunit.client.0.smithi070.stdout:237 2024-09-14T04:04:26.033 INFO:tasks.workunit.client.0.smithi070.stdout:238 2024-09-14T04:04:26.033 INFO:tasks.workunit.client.0.smithi070.stdout:239 2024-09-14T04:04:26.034 INFO:tasks.workunit.client.0.smithi070.stdout:240 2024-09-14T04:04:26.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:25 smithi070 ceph-mon[96704]: pgmap v1095: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:26.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:25 smithi070 ceph-mon[96704]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:26.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:25 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T04:04:26.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:25 smithi070 ceph-mon[96704]: osdmap e983: 8 total, 8 up, 8 in 2024-09-14T04:04:26.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:25 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/776475119' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:26.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:25 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:26.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:25 smithi070 ceph-mon[98640]: pgmap v1095: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:26.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:25 smithi070 ceph-mon[98640]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:26.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:25 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T04:04:26.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:25 smithi070 ceph-mon[98640]: osdmap e983: 8 total, 8 up, 8 in 2024-09-14T04:04:26.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:25 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/776475119' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:26.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:25 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:26.037 INFO:tasks.workunit.client.0.smithi070.stdout:241 2024-09-14T04:04:26.037 INFO:tasks.workunit.client.0.smithi070.stdout:242 2024-09-14T04:04:26.037 INFO:tasks.workunit.client.0.smithi070.stdout:243 2024-09-14T04:04:26.038 INFO:tasks.workunit.client.0.smithi070.stdout:244 2024-09-14T04:04:26.038 INFO:tasks.workunit.client.0.smithi070.stdout:245 2024-09-14T04:04:26.039 INFO:tasks.workunit.client.0.smithi070.stdout:246 2024-09-14T04:04:26.040 INFO:tasks.workunit.client.0.smithi070.stdout:247 2024-09-14T04:04:26.041 INFO:tasks.workunit.client.0.smithi070.stdout:248 2024-09-14T04:04:26.042 INFO:tasks.workunit.client.0.smithi070.stdout:249 2024-09-14T04:04:26.043 INFO:tasks.workunit.client.0.smithi070.stdout:250 2024-09-14T04:04:26.043 INFO:tasks.workunit.client.0.smithi070.stdout:251 2024-09-14T04:04:26.044 INFO:tasks.workunit.client.0.smithi070.stdout:252 2024-09-14T04:04:26.045 INFO:tasks.workunit.client.0.smithi070.stdout:253 2024-09-14T04:04:26.046 INFO:tasks.workunit.client.0.smithi070.stdout:254 2024-09-14T04:04:26.047 INFO:tasks.workunit.client.0.smithi070.stdout:255 2024-09-14T04:04:26.048 INFO:tasks.workunit.client.0.smithi070.stdout:256 2024-09-14T04:04:26.048 INFO:tasks.workunit.client.0.smithi070.stdout:257 2024-09-14T04:04:26.049 INFO:tasks.workunit.client.0.smithi070.stdout:258 2024-09-14T04:04:26.050 INFO:tasks.workunit.client.0.smithi070.stdout:259 2024-09-14T04:04:26.051 INFO:tasks.workunit.client.0.smithi070.stdout:260 2024-09-14T04:04:26.052 INFO:tasks.workunit.client.0.smithi070.stdout:261 2024-09-14T04:04:26.052 INFO:tasks.workunit.client.0.smithi070.stdout:262 2024-09-14T04:04:26.053 INFO:tasks.workunit.client.0.smithi070.stdout:263 2024-09-14T04:04:26.054 INFO:tasks.workunit.client.0.smithi070.stdout:264 2024-09-14T04:04:26.055 INFO:tasks.workunit.client.0.smithi070.stdout:265 2024-09-14T04:04:26.056 INFO:tasks.workunit.client.0.smithi070.stdout:266 2024-09-14T04:04:26.056 INFO:tasks.workunit.client.0.smithi070.stdout:267 2024-09-14T04:04:26.057 INFO:tasks.workunit.client.0.smithi070.stdout:268 2024-09-14T04:04:26.058 INFO:tasks.workunit.client.0.smithi070.stdout:269 2024-09-14T04:04:26.059 INFO:tasks.workunit.client.0.smithi070.stdout:270 2024-09-14T04:04:26.060 INFO:tasks.workunit.client.0.smithi070.stdout:271 2024-09-14T04:04:26.061 INFO:tasks.workunit.client.0.smithi070.stdout:272 2024-09-14T04:04:26.061 INFO:tasks.workunit.client.0.smithi070.stdout:273 2024-09-14T04:04:26.062 INFO:tasks.workunit.client.0.smithi070.stdout:274 2024-09-14T04:04:26.063 INFO:tasks.workunit.client.0.smithi070.stdout:275 2024-09-14T04:04:26.064 INFO:tasks.workunit.client.0.smithi070.stdout:276 2024-09-14T04:04:26.065 INFO:tasks.workunit.client.0.smithi070.stdout:277 2024-09-14T04:04:26.066 INFO:tasks.workunit.client.0.smithi070.stdout:278 2024-09-14T04:04:26.067 INFO:tasks.workunit.client.0.smithi070.stdout:279 2024-09-14T04:04:26.068 INFO:tasks.workunit.client.0.smithi070.stdout:280 2024-09-14T04:04:26.068 INFO:tasks.workunit.client.0.smithi070.stdout:281 2024-09-14T04:04:26.069 INFO:tasks.workunit.client.0.smithi070.stdout:282 2024-09-14T04:04:26.070 INFO:tasks.workunit.client.0.smithi070.stdout:283 2024-09-14T04:04:26.071 INFO:tasks.workunit.client.0.smithi070.stdout:284 2024-09-14T04:04:26.072 INFO:tasks.workunit.client.0.smithi070.stdout:285 2024-09-14T04:04:26.073 INFO:tasks.workunit.client.0.smithi070.stdout:286 2024-09-14T04:04:26.074 INFO:tasks.workunit.client.0.smithi070.stdout:287 2024-09-14T04:04:26.075 INFO:tasks.workunit.client.0.smithi070.stdout:288 2024-09-14T04:04:26.076 INFO:tasks.workunit.client.0.smithi070.stdout:289 2024-09-14T04:04:26.077 INFO:tasks.workunit.client.0.smithi070.stdout:290 2024-09-14T04:04:26.078 INFO:tasks.workunit.client.0.smithi070.stdout:291 2024-09-14T04:04:26.079 INFO:tasks.workunit.client.0.smithi070.stdout:292 2024-09-14T04:04:26.080 INFO:tasks.workunit.client.0.smithi070.stdout:293 2024-09-14T04:04:26.081 INFO:tasks.workunit.client.0.smithi070.stdout:294 2024-09-14T04:04:26.082 INFO:tasks.workunit.client.0.smithi070.stdout:295 2024-09-14T04:04:26.083 INFO:tasks.workunit.client.0.smithi070.stdout:296 2024-09-14T04:04:26.083 INFO:tasks.workunit.client.0.smithi070.stdout:297 2024-09-14T04:04:26.084 INFO:tasks.workunit.client.0.smithi070.stdout:298 2024-09-14T04:04:26.085 INFO:tasks.workunit.client.0.smithi070.stdout:299 2024-09-14T04:04:26.086 INFO:tasks.workunit.client.0.smithi070.stdout:300 2024-09-14T04:04:26.087 INFO:tasks.workunit.client.0.smithi070.stdout:301 2024-09-14T04:04:26.088 INFO:tasks.workunit.client.0.smithi070.stdout:302 2024-09-14T04:04:26.088 INFO:tasks.workunit.client.0.smithi070.stdout:303 2024-09-14T04:04:26.089 INFO:tasks.workunit.client.0.smithi070.stdout:304 2024-09-14T04:04:26.090 INFO:tasks.workunit.client.0.smithi070.stdout:305 2024-09-14T04:04:26.091 INFO:tasks.workunit.client.0.smithi070.stdout:306 2024-09-14T04:04:26.092 INFO:tasks.workunit.client.0.smithi070.stdout:307 2024-09-14T04:04:26.093 INFO:tasks.workunit.client.0.smithi070.stdout:308 2024-09-14T04:04:26.094 INFO:tasks.workunit.client.0.smithi070.stdout:309 2024-09-14T04:04:26.095 INFO:tasks.workunit.client.0.smithi070.stdout:310 2024-09-14T04:04:26.096 INFO:tasks.workunit.client.0.smithi070.stdout:311 2024-09-14T04:04:26.096 INFO:tasks.workunit.client.0.smithi070.stdout:312 2024-09-14T04:04:26.097 INFO:tasks.workunit.client.0.smithi070.stdout:313 2024-09-14T04:04:26.098 INFO:tasks.workunit.client.0.smithi070.stdout:314 2024-09-14T04:04:26.099 INFO:tasks.workunit.client.0.smithi070.stdout:315 2024-09-14T04:04:26.099 INFO:tasks.workunit.client.0.smithi070.stdout:316 2024-09-14T04:04:26.100 INFO:tasks.workunit.client.0.smithi070.stdout:317 2024-09-14T04:04:26.101 INFO:tasks.workunit.client.0.smithi070.stdout:318 2024-09-14T04:04:26.102 INFO:tasks.workunit.client.0.smithi070.stdout:319 2024-09-14T04:04:26.103 INFO:tasks.workunit.client.0.smithi070.stdout:320 2024-09-14T04:04:26.103 INFO:tasks.workunit.client.0.smithi070.stdout:321 2024-09-14T04:04:26.104 INFO:tasks.workunit.client.0.smithi070.stdout:322 2024-09-14T04:04:26.105 INFO:tasks.workunit.client.0.smithi070.stdout:323 2024-09-14T04:04:26.106 INFO:tasks.workunit.client.0.smithi070.stdout:324 2024-09-14T04:04:26.106 INFO:tasks.workunit.client.0.smithi070.stdout:325 2024-09-14T04:04:26.107 INFO:tasks.workunit.client.0.smithi070.stdout:326 2024-09-14T04:04:26.108 INFO:tasks.workunit.client.0.smithi070.stdout:327 2024-09-14T04:04:26.109 INFO:tasks.workunit.client.0.smithi070.stdout:328 2024-09-14T04:04:26.110 INFO:tasks.workunit.client.0.smithi070.stdout:329 2024-09-14T04:04:26.111 INFO:tasks.workunit.client.0.smithi070.stdout:330 2024-09-14T04:04:26.112 INFO:tasks.workunit.client.0.smithi070.stdout:331 2024-09-14T04:04:26.112 INFO:tasks.workunit.client.0.smithi070.stdout:332 2024-09-14T04:04:26.113 INFO:tasks.workunit.client.0.smithi070.stdout:333 2024-09-14T04:04:26.114 INFO:tasks.workunit.client.0.smithi070.stdout:334 2024-09-14T04:04:26.115 INFO:tasks.workunit.client.0.smithi070.stdout:335 2024-09-14T04:04:26.116 INFO:tasks.workunit.client.0.smithi070.stdout:336 2024-09-14T04:04:26.117 INFO:tasks.workunit.client.0.smithi070.stdout:337 2024-09-14T04:04:26.117 INFO:tasks.workunit.client.0.smithi070.stdout:338 2024-09-14T04:04:26.118 INFO:tasks.workunit.client.0.smithi070.stdout:339 2024-09-14T04:04:26.119 INFO:tasks.workunit.client.0.smithi070.stdout:340 2024-09-14T04:04:26.120 INFO:tasks.workunit.client.0.smithi070.stdout:341 2024-09-14T04:04:26.121 INFO:tasks.workunit.client.0.smithi070.stdout:342 2024-09-14T04:04:26.121 INFO:tasks.workunit.client.0.smithi070.stdout:343 2024-09-14T04:04:26.122 INFO:tasks.workunit.client.0.smithi070.stdout:344 2024-09-14T04:04:26.123 INFO:tasks.workunit.client.0.smithi070.stdout:345 2024-09-14T04:04:26.124 INFO:tasks.workunit.client.0.smithi070.stdout:346 2024-09-14T04:04:26.125 INFO:tasks.workunit.client.0.smithi070.stdout:347 2024-09-14T04:04:26.125 INFO:tasks.workunit.client.0.smithi070.stdout:348 2024-09-14T04:04:26.126 INFO:tasks.workunit.client.0.smithi070.stdout:349 2024-09-14T04:04:26.127 INFO:tasks.workunit.client.0.smithi070.stdout:350 2024-09-14T04:04:26.128 INFO:tasks.workunit.client.0.smithi070.stdout:351 2024-09-14T04:04:26.129 INFO:tasks.workunit.client.0.smithi070.stdout:352 2024-09-14T04:04:26.130 INFO:tasks.workunit.client.0.smithi070.stdout:353 2024-09-14T04:04:26.130 INFO:tasks.workunit.client.0.smithi070.stdout:354 2024-09-14T04:04:26.131 INFO:tasks.workunit.client.0.smithi070.stdout:355 2024-09-14T04:04:26.132 INFO:tasks.workunit.client.0.smithi070.stdout:356 2024-09-14T04:04:26.133 INFO:tasks.workunit.client.0.smithi070.stdout:357 2024-09-14T04:04:26.134 INFO:tasks.workunit.client.0.smithi070.stdout:358 2024-09-14T04:04:26.135 INFO:tasks.workunit.client.0.smithi070.stdout:359 2024-09-14T04:04:26.135 INFO:tasks.workunit.client.0.smithi070.stdout:360 2024-09-14T04:04:26.136 INFO:tasks.workunit.client.0.smithi070.stdout:361 2024-09-14T04:04:26.137 INFO:tasks.workunit.client.0.smithi070.stdout:362 2024-09-14T04:04:26.138 INFO:tasks.workunit.client.0.smithi070.stdout:363 2024-09-14T04:04:26.139 INFO:tasks.workunit.client.0.smithi070.stdout:364 2024-09-14T04:04:26.140 INFO:tasks.workunit.client.0.smithi070.stdout:365 2024-09-14T04:04:26.140 INFO:tasks.workunit.client.0.smithi070.stdout:366 2024-09-14T04:04:26.141 INFO:tasks.workunit.client.0.smithi070.stdout:367 2024-09-14T04:04:26.142 INFO:tasks.workunit.client.0.smithi070.stdout:368 2024-09-14T04:04:26.143 INFO:tasks.workunit.client.0.smithi070.stdout:369 2024-09-14T04:04:26.144 INFO:tasks.workunit.client.0.smithi070.stdout:370 2024-09-14T04:04:26.145 INFO:tasks.workunit.client.0.smithi070.stdout:371 2024-09-14T04:04:26.146 INFO:tasks.workunit.client.0.smithi070.stdout:372 2024-09-14T04:04:26.147 INFO:tasks.workunit.client.0.smithi070.stdout:373 2024-09-14T04:04:26.148 INFO:tasks.workunit.client.0.smithi070.stdout:374 2024-09-14T04:04:26.148 INFO:tasks.workunit.client.0.smithi070.stdout:375 2024-09-14T04:04:26.149 INFO:tasks.workunit.client.0.smithi070.stdout:376 2024-09-14T04:04:26.150 INFO:tasks.workunit.client.0.smithi070.stdout:377 2024-09-14T04:04:26.151 INFO:tasks.workunit.client.0.smithi070.stdout:378 2024-09-14T04:04:26.152 INFO:tasks.workunit.client.0.smithi070.stdout:379 2024-09-14T04:04:26.153 INFO:tasks.workunit.client.0.smithi070.stdout:380 2024-09-14T04:04:26.154 INFO:tasks.workunit.client.0.smithi070.stdout:381 2024-09-14T04:04:26.154 INFO:tasks.workunit.client.0.smithi070.stdout:382 2024-09-14T04:04:26.155 INFO:tasks.workunit.client.0.smithi070.stdout:383 2024-09-14T04:04:26.156 INFO:tasks.workunit.client.0.smithi070.stdout:384 2024-09-14T04:04:26.157 INFO:tasks.workunit.client.0.smithi070.stdout:385 2024-09-14T04:04:26.158 INFO:tasks.workunit.client.0.smithi070.stdout:386 2024-09-14T04:04:26.159 INFO:tasks.workunit.client.0.smithi070.stdout:387 2024-09-14T04:04:26.159 INFO:tasks.workunit.client.0.smithi070.stdout:388 2024-09-14T04:04:26.162 INFO:tasks.workunit.client.0.smithi070.stdout:389 2024-09-14T04:04:26.162 INFO:tasks.workunit.client.0.smithi070.stdout:390 2024-09-14T04:04:26.162 INFO:tasks.workunit.client.0.smithi070.stdout:391 2024-09-14T04:04:26.163 INFO:tasks.workunit.client.0.smithi070.stdout:392 2024-09-14T04:04:26.164 INFO:tasks.workunit.client.0.smithi070.stdout:393 2024-09-14T04:04:26.165 INFO:tasks.workunit.client.0.smithi070.stdout:394 2024-09-14T04:04:26.165 INFO:tasks.workunit.client.0.smithi070.stdout:395 2024-09-14T04:04:26.166 INFO:tasks.workunit.client.0.smithi070.stdout:396 2024-09-14T04:04:26.167 INFO:tasks.workunit.client.0.smithi070.stdout:397 2024-09-14T04:04:26.168 INFO:tasks.workunit.client.0.smithi070.stdout:398 2024-09-14T04:04:26.169 INFO:tasks.workunit.client.0.smithi070.stdout:399 2024-09-14T04:04:26.170 INFO:tasks.workunit.client.0.smithi070.stdout:400 2024-09-14T04:04:26.171 INFO:tasks.workunit.client.0.smithi070.stdout:401 2024-09-14T04:04:26.172 INFO:tasks.workunit.client.0.smithi070.stdout:402 2024-09-14T04:04:26.172 INFO:tasks.workunit.client.0.smithi070.stdout:403 2024-09-14T04:04:26.173 INFO:tasks.workunit.client.0.smithi070.stdout:404 2024-09-14T04:04:26.174 INFO:tasks.workunit.client.0.smithi070.stdout:405 2024-09-14T04:04:26.175 INFO:tasks.workunit.client.0.smithi070.stdout:406 2024-09-14T04:04:26.176 INFO:tasks.workunit.client.0.smithi070.stdout:407 2024-09-14T04:04:26.177 INFO:tasks.workunit.client.0.smithi070.stdout:408 2024-09-14T04:04:26.177 INFO:tasks.workunit.client.0.smithi070.stdout:409 2024-09-14T04:04:26.178 INFO:tasks.workunit.client.0.smithi070.stdout:410 2024-09-14T04:04:26.179 INFO:tasks.workunit.client.0.smithi070.stdout:411 2024-09-14T04:04:26.180 INFO:tasks.workunit.client.0.smithi070.stdout:412 2024-09-14T04:04:26.181 INFO:tasks.workunit.client.0.smithi070.stdout:413 2024-09-14T04:04:26.182 INFO:tasks.workunit.client.0.smithi070.stdout:414 2024-09-14T04:04:26.183 INFO:tasks.workunit.client.0.smithi070.stdout:415 2024-09-14T04:04:26.183 INFO:tasks.workunit.client.0.smithi070.stdout:416 2024-09-14T04:04:26.184 INFO:tasks.workunit.client.0.smithi070.stdout:417 2024-09-14T04:04:26.185 INFO:tasks.workunit.client.0.smithi070.stdout:418 2024-09-14T04:04:26.186 INFO:tasks.workunit.client.0.smithi070.stdout:419 2024-09-14T04:04:26.187 INFO:tasks.workunit.client.0.smithi070.stdout:420 2024-09-14T04:04:26.188 INFO:tasks.workunit.client.0.smithi070.stdout:421 2024-09-14T04:04:26.190 INFO:tasks.workunit.client.0.smithi070.stdout:422 2024-09-14T04:04:26.190 INFO:tasks.workunit.client.0.smithi070.stdout:423 2024-09-14T04:04:26.190 INFO:tasks.workunit.client.0.smithi070.stdout:424 2024-09-14T04:04:26.191 INFO:tasks.workunit.client.0.smithi070.stdout:425 2024-09-14T04:04:26.192 INFO:tasks.workunit.client.0.smithi070.stdout:426 2024-09-14T04:04:26.193 INFO:tasks.workunit.client.0.smithi070.stdout:427 2024-09-14T04:04:26.194 INFO:tasks.workunit.client.0.smithi070.stdout:428 2024-09-14T04:04:26.195 INFO:tasks.workunit.client.0.smithi070.stdout:429 2024-09-14T04:04:26.195 INFO:tasks.workunit.client.0.smithi070.stdout:430 2024-09-14T04:04:26.196 INFO:tasks.workunit.client.0.smithi070.stdout:431 2024-09-14T04:04:26.197 INFO:tasks.workunit.client.0.smithi070.stdout:432 2024-09-14T04:04:26.198 INFO:tasks.workunit.client.0.smithi070.stdout:433 2024-09-14T04:04:26.199 INFO:tasks.workunit.client.0.smithi070.stdout:434 2024-09-14T04:04:26.200 INFO:tasks.workunit.client.0.smithi070.stdout:435 2024-09-14T04:04:26.201 INFO:tasks.workunit.client.0.smithi070.stdout:436 2024-09-14T04:04:26.202 INFO:tasks.workunit.client.0.smithi070.stdout:437 2024-09-14T04:04:26.203 INFO:tasks.workunit.client.0.smithi070.stdout:438 2024-09-14T04:04:26.203 INFO:tasks.workunit.client.0.smithi070.stdout:439 2024-09-14T04:04:26.204 INFO:tasks.workunit.client.0.smithi070.stdout:440 2024-09-14T04:04:26.205 INFO:tasks.workunit.client.0.smithi070.stdout:441 2024-09-14T04:04:26.206 INFO:tasks.workunit.client.0.smithi070.stdout:442 2024-09-14T04:04:26.207 INFO:tasks.workunit.client.0.smithi070.stdout:443 2024-09-14T04:04:26.208 INFO:tasks.workunit.client.0.smithi070.stdout:444 2024-09-14T04:04:26.209 INFO:tasks.workunit.client.0.smithi070.stdout:445 2024-09-14T04:04:26.210 INFO:tasks.workunit.client.0.smithi070.stdout:446 2024-09-14T04:04:26.211 INFO:tasks.workunit.client.0.smithi070.stdout:447 2024-09-14T04:04:26.212 INFO:tasks.workunit.client.0.smithi070.stdout:448 2024-09-14T04:04:26.212 INFO:tasks.workunit.client.0.smithi070.stdout:449 2024-09-14T04:04:26.213 INFO:tasks.workunit.client.0.smithi070.stdout:450 2024-09-14T04:04:26.214 INFO:tasks.workunit.client.0.smithi070.stdout:451 2024-09-14T04:04:26.215 INFO:tasks.workunit.client.0.smithi070.stdout:452 2024-09-14T04:04:26.216 INFO:tasks.workunit.client.0.smithi070.stdout:453 2024-09-14T04:04:26.217 INFO:tasks.workunit.client.0.smithi070.stdout:454 2024-09-14T04:04:26.218 INFO:tasks.workunit.client.0.smithi070.stdout:455 2024-09-14T04:04:26.219 INFO:tasks.workunit.client.0.smithi070.stdout:456 2024-09-14T04:04:26.219 INFO:tasks.workunit.client.0.smithi070.stdout:457 2024-09-14T04:04:26.220 INFO:tasks.workunit.client.0.smithi070.stdout:458 2024-09-14T04:04:26.221 INFO:tasks.workunit.client.0.smithi070.stdout:459 2024-09-14T04:04:26.222 INFO:tasks.workunit.client.0.smithi070.stdout:460 2024-09-14T04:04:26.223 INFO:tasks.workunit.client.0.smithi070.stdout:461 2024-09-14T04:04:26.224 INFO:tasks.workunit.client.0.smithi070.stdout:462 2024-09-14T04:04:26.225 INFO:tasks.workunit.client.0.smithi070.stdout:463 2024-09-14T04:04:26.226 INFO:tasks.workunit.client.0.smithi070.stdout:464 2024-09-14T04:04:26.227 INFO:tasks.workunit.client.0.smithi070.stdout:465 2024-09-14T04:04:26.227 INFO:tasks.workunit.client.0.smithi070.stdout:466 2024-09-14T04:04:26.228 INFO:tasks.workunit.client.0.smithi070.stdout:467 2024-09-14T04:04:26.229 INFO:tasks.workunit.client.0.smithi070.stdout:468 2024-09-14T04:04:26.230 INFO:tasks.workunit.client.0.smithi070.stdout:469 2024-09-14T04:04:26.231 INFO:tasks.workunit.client.0.smithi070.stdout:470 2024-09-14T04:04:26.232 INFO:tasks.workunit.client.0.smithi070.stdout:471 2024-09-14T04:04:26.233 INFO:tasks.workunit.client.0.smithi070.stdout:472 2024-09-14T04:04:26.234 INFO:tasks.workunit.client.0.smithi070.stdout:473 2024-09-14T04:04:26.234 INFO:tasks.workunit.client.0.smithi070.stdout:474 2024-09-14T04:04:26.236 INFO:tasks.workunit.client.0.smithi070.stdout:475 2024-09-14T04:04:26.236 INFO:tasks.workunit.client.0.smithi070.stdout:476 2024-09-14T04:04:26.237 INFO:tasks.workunit.client.0.smithi070.stdout:477 2024-09-14T04:04:26.238 INFO:tasks.workunit.client.0.smithi070.stdout:478 2024-09-14T04:04:26.239 INFO:tasks.workunit.client.0.smithi070.stdout:479 2024-09-14T04:04:26.240 INFO:tasks.workunit.client.0.smithi070.stdout:480 2024-09-14T04:04:26.241 INFO:tasks.workunit.client.0.smithi070.stdout:481 2024-09-14T04:04:26.242 INFO:tasks.workunit.client.0.smithi070.stdout:482 2024-09-14T04:04:26.243 INFO:tasks.workunit.client.0.smithi070.stdout:483 2024-09-14T04:04:26.243 INFO:tasks.workunit.client.0.smithi070.stdout:484 2024-09-14T04:04:26.244 INFO:tasks.workunit.client.0.smithi070.stdout:485 2024-09-14T04:04:26.245 INFO:tasks.workunit.client.0.smithi070.stdout:486 2024-09-14T04:04:26.246 INFO:tasks.workunit.client.0.smithi070.stdout:487 2024-09-14T04:04:26.247 INFO:tasks.workunit.client.0.smithi070.stdout:488 2024-09-14T04:04:26.248 INFO:tasks.workunit.client.0.smithi070.stdout:489 2024-09-14T04:04:26.249 INFO:tasks.workunit.client.0.smithi070.stdout:490 2024-09-14T04:04:26.250 INFO:tasks.workunit.client.0.smithi070.stdout:491 2024-09-14T04:04:26.251 INFO:tasks.workunit.client.0.smithi070.stdout:492 2024-09-14T04:04:26.251 INFO:tasks.workunit.client.0.smithi070.stdout:493 2024-09-14T04:04:26.252 INFO:tasks.workunit.client.0.smithi070.stdout:494 2024-09-14T04:04:26.253 INFO:tasks.workunit.client.0.smithi070.stdout:495 2024-09-14T04:04:26.254 INFO:tasks.workunit.client.0.smithi070.stdout:496 2024-09-14T04:04:26.255 INFO:tasks.workunit.client.0.smithi070.stdout:497 2024-09-14T04:04:26.256 INFO:tasks.workunit.client.0.smithi070.stdout:498 2024-09-14T04:04:26.257 INFO:tasks.workunit.client.0.smithi070.stdout:499 2024-09-14T04:04:26.258 INFO:tasks.workunit.client.0.smithi070.stdout:500 2024-09-14T04:04:26.259 INFO:tasks.workunit.client.0.smithi070.stdout:501 2024-09-14T04:04:26.260 INFO:tasks.workunit.client.0.smithi070.stdout:502 2024-09-14T04:04:26.261 INFO:tasks.workunit.client.0.smithi070.stdout:503 2024-09-14T04:04:26.262 INFO:tasks.workunit.client.0.smithi070.stdout:504 2024-09-14T04:04:26.262 INFO:tasks.workunit.client.0.smithi070.stdout:505 2024-09-14T04:04:26.263 INFO:tasks.workunit.client.0.smithi070.stdout:506 2024-09-14T04:04:26.264 INFO:tasks.workunit.client.0.smithi070.stdout:507 2024-09-14T04:04:26.265 INFO:tasks.workunit.client.0.smithi070.stdout:508 2024-09-14T04:04:26.266 INFO:tasks.workunit.client.0.smithi070.stdout:509 2024-09-14T04:04:26.268 INFO:tasks.workunit.client.0.smithi070.stdout:510 2024-09-14T04:04:26.269 INFO:tasks.workunit.client.0.smithi070.stdout:511 2024-09-14T04:04:26.270 INFO:tasks.workunit.client.0.smithi070.stdout:512 2024-09-14T04:04:26.271 INFO:tasks.workunit.client.0.smithi070.stdout:513 2024-09-14T04:04:26.272 INFO:tasks.workunit.client.0.smithi070.stdout:514 2024-09-14T04:04:26.273 INFO:tasks.workunit.client.0.smithi070.stdout:515 2024-09-14T04:04:26.274 INFO:tasks.workunit.client.0.smithi070.stdout:516 2024-09-14T04:04:26.275 INFO:tasks.workunit.client.0.smithi070.stdout:517 2024-09-14T04:04:26.276 INFO:tasks.workunit.client.0.smithi070.stdout:518 2024-09-14T04:04:26.277 INFO:tasks.workunit.client.0.smithi070.stdout:519 2024-09-14T04:04:26.278 INFO:tasks.workunit.client.0.smithi070.stdout:520 2024-09-14T04:04:26.279 INFO:tasks.workunit.client.0.smithi070.stdout:521 2024-09-14T04:04:26.280 INFO:tasks.workunit.client.0.smithi070.stdout:522 2024-09-14T04:04:26.281 INFO:tasks.workunit.client.0.smithi070.stdout:523 2024-09-14T04:04:26.282 INFO:tasks.workunit.client.0.smithi070.stdout:524 2024-09-14T04:04:26.284 INFO:tasks.workunit.client.0.smithi070.stdout:525 2024-09-14T04:04:26.285 INFO:tasks.workunit.client.0.smithi070.stdout:526 2024-09-14T04:04:26.286 INFO:tasks.workunit.client.0.smithi070.stdout:527 2024-09-14T04:04:26.287 INFO:tasks.workunit.client.0.smithi070.stdout:528 2024-09-14T04:04:26.288 INFO:tasks.workunit.client.0.smithi070.stdout:529 2024-09-14T04:04:26.289 INFO:tasks.workunit.client.0.smithi070.stdout:530 2024-09-14T04:04:26.291 INFO:tasks.workunit.client.0.smithi070.stdout:531 2024-09-14T04:04:26.291 INFO:tasks.workunit.client.0.smithi070.stdout:532 2024-09-14T04:04:26.293 INFO:tasks.workunit.client.0.smithi070.stdout:533 2024-09-14T04:04:26.293 INFO:tasks.workunit.client.0.smithi070.stdout:534 2024-09-14T04:04:26.294 INFO:tasks.workunit.client.0.smithi070.stdout:535 2024-09-14T04:04:26.295 INFO:tasks.workunit.client.0.smithi070.stdout:536 2024-09-14T04:04:26.296 INFO:tasks.workunit.client.0.smithi070.stdout:537 2024-09-14T04:04:26.297 INFO:tasks.workunit.client.0.smithi070.stdout:538 2024-09-14T04:04:26.298 INFO:tasks.workunit.client.0.smithi070.stdout:539 2024-09-14T04:04:26.299 INFO:tasks.workunit.client.0.smithi070.stdout:540 2024-09-14T04:04:26.300 INFO:tasks.workunit.client.0.smithi070.stdout:541 2024-09-14T04:04:26.301 INFO:tasks.workunit.client.0.smithi070.stdout:542 2024-09-14T04:04:26.302 INFO:tasks.workunit.client.0.smithi070.stdout:543 2024-09-14T04:04:26.303 INFO:tasks.workunit.client.0.smithi070.stdout:544 2024-09-14T04:04:26.305 INFO:tasks.workunit.client.0.smithi070.stdout:545 2024-09-14T04:04:26.306 INFO:tasks.workunit.client.0.smithi070.stdout:546 2024-09-14T04:04:26.307 INFO:tasks.workunit.client.0.smithi070.stdout:547 2024-09-14T04:04:26.308 INFO:tasks.workunit.client.0.smithi070.stdout:548 2024-09-14T04:04:26.309 INFO:tasks.workunit.client.0.smithi070.stdout:549 2024-09-14T04:04:26.310 INFO:tasks.workunit.client.0.smithi070.stdout:550 2024-09-14T04:04:26.311 INFO:tasks.workunit.client.0.smithi070.stdout:551 2024-09-14T04:04:26.312 INFO:tasks.workunit.client.0.smithi070.stdout:552 2024-09-14T04:04:26.313 INFO:tasks.workunit.client.0.smithi070.stdout:553 2024-09-14T04:04:26.314 INFO:tasks.workunit.client.0.smithi070.stdout:554 2024-09-14T04:04:26.315 INFO:tasks.workunit.client.0.smithi070.stdout:555 2024-09-14T04:04:26.316 INFO:tasks.workunit.client.0.smithi070.stdout:556 2024-09-14T04:04:26.317 INFO:tasks.workunit.client.0.smithi070.stdout:557 2024-09-14T04:04:26.318 INFO:tasks.workunit.client.0.smithi070.stdout:558 2024-09-14T04:04:26.319 INFO:tasks.workunit.client.0.smithi070.stdout:559 2024-09-14T04:04:26.320 INFO:tasks.workunit.client.0.smithi070.stdout:560 2024-09-14T04:04:26.321 INFO:tasks.workunit.client.0.smithi070.stdout:561 2024-09-14T04:04:26.322 INFO:tasks.workunit.client.0.smithi070.stdout:562 2024-09-14T04:04:26.323 INFO:tasks.workunit.client.0.smithi070.stdout:563 2024-09-14T04:04:26.323 INFO:tasks.workunit.client.0.smithi070.stdout:564 2024-09-14T04:04:26.325 INFO:tasks.workunit.client.0.smithi070.stdout:565 2024-09-14T04:04:26.325 INFO:tasks.workunit.client.0.smithi070.stdout:566 2024-09-14T04:04:26.326 INFO:tasks.workunit.client.0.smithi070.stdout:567 2024-09-14T04:04:26.327 INFO:tasks.workunit.client.0.smithi070.stdout:568 2024-09-14T04:04:26.328 INFO:tasks.workunit.client.0.smithi070.stdout:569 2024-09-14T04:04:26.329 INFO:tasks.workunit.client.0.smithi070.stdout:570 2024-09-14T04:04:26.330 INFO:tasks.workunit.client.0.smithi070.stdout:571 2024-09-14T04:04:26.331 INFO:tasks.workunit.client.0.smithi070.stdout:572 2024-09-14T04:04:26.332 INFO:tasks.workunit.client.0.smithi070.stdout:573 2024-09-14T04:04:26.333 INFO:tasks.workunit.client.0.smithi070.stdout:574 2024-09-14T04:04:26.334 INFO:tasks.workunit.client.0.smithi070.stdout:575 2024-09-14T04:04:26.335 INFO:tasks.workunit.client.0.smithi070.stdout:576 2024-09-14T04:04:26.336 INFO:tasks.workunit.client.0.smithi070.stdout:577 2024-09-14T04:04:26.337 INFO:tasks.workunit.client.0.smithi070.stdout:578 2024-09-14T04:04:26.338 INFO:tasks.workunit.client.0.smithi070.stdout:579 2024-09-14T04:04:26.339 INFO:tasks.workunit.client.0.smithi070.stdout:580 2024-09-14T04:04:26.340 INFO:tasks.workunit.client.0.smithi070.stdout:581 2024-09-14T04:04:26.341 INFO:tasks.workunit.client.0.smithi070.stdout:582 2024-09-14T04:04:26.342 INFO:tasks.workunit.client.0.smithi070.stdout:583 2024-09-14T04:04:26.343 INFO:tasks.workunit.client.0.smithi070.stdout:584 2024-09-14T04:04:26.344 INFO:tasks.workunit.client.0.smithi070.stdout:585 2024-09-14T04:04:26.345 INFO:tasks.workunit.client.0.smithi070.stdout:586 2024-09-14T04:04:26.346 INFO:tasks.workunit.client.0.smithi070.stdout:587 2024-09-14T04:04:26.347 INFO:tasks.workunit.client.0.smithi070.stdout:588 2024-09-14T04:04:26.348 INFO:tasks.workunit.client.0.smithi070.stdout:589 2024-09-14T04:04:26.349 INFO:tasks.workunit.client.0.smithi070.stdout:590 2024-09-14T04:04:26.350 INFO:tasks.workunit.client.0.smithi070.stdout:591 2024-09-14T04:04:26.351 INFO:tasks.workunit.client.0.smithi070.stdout:592 2024-09-14T04:04:26.352 INFO:tasks.workunit.client.0.smithi070.stdout:593 2024-09-14T04:04:26.353 INFO:tasks.workunit.client.0.smithi070.stdout:594 2024-09-14T04:04:26.354 INFO:tasks.workunit.client.0.smithi070.stdout:595 2024-09-14T04:04:26.355 INFO:tasks.workunit.client.0.smithi070.stdout:596 2024-09-14T04:04:26.356 INFO:tasks.workunit.client.0.smithi070.stdout:597 2024-09-14T04:04:26.357 INFO:tasks.workunit.client.0.smithi070.stdout:598 2024-09-14T04:04:26.358 INFO:tasks.workunit.client.0.smithi070.stdout:599 2024-09-14T04:04:26.359 INFO:tasks.workunit.client.0.smithi070.stdout:600 2024-09-14T04:04:26.359 INFO:tasks.workunit.client.0.smithi070.stdout:601 2024-09-14T04:04:26.361 INFO:tasks.workunit.client.0.smithi070.stdout:602 2024-09-14T04:04:26.362 INFO:tasks.workunit.client.0.smithi070.stdout:603 2024-09-14T04:04:26.363 INFO:tasks.workunit.client.0.smithi070.stdout:604 2024-09-14T04:04:26.364 INFO:tasks.workunit.client.0.smithi070.stdout:605 2024-09-14T04:04:26.365 INFO:tasks.workunit.client.0.smithi070.stdout:606 2024-09-14T04:04:26.366 INFO:tasks.workunit.client.0.smithi070.stdout:607 2024-09-14T04:04:26.367 INFO:tasks.workunit.client.0.smithi070.stdout:608 2024-09-14T04:04:26.368 INFO:tasks.workunit.client.0.smithi070.stdout:609 2024-09-14T04:04:26.369 INFO:tasks.workunit.client.0.smithi070.stdout:610 2024-09-14T04:04:26.370 INFO:tasks.workunit.client.0.smithi070.stdout:611 2024-09-14T04:04:26.371 INFO:tasks.workunit.client.0.smithi070.stdout:612 2024-09-14T04:04:26.372 INFO:tasks.workunit.client.0.smithi070.stdout:613 2024-09-14T04:04:26.373 INFO:tasks.workunit.client.0.smithi070.stdout:614 2024-09-14T04:04:26.374 INFO:tasks.workunit.client.0.smithi070.stdout:615 2024-09-14T04:04:26.375 INFO:tasks.workunit.client.0.smithi070.stdout:616 2024-09-14T04:04:26.376 INFO:tasks.workunit.client.0.smithi070.stdout:617 2024-09-14T04:04:26.377 INFO:tasks.workunit.client.0.smithi070.stdout:618 2024-09-14T04:04:26.378 INFO:tasks.workunit.client.0.smithi070.stdout:619 2024-09-14T04:04:26.379 INFO:tasks.workunit.client.0.smithi070.stdout:620 2024-09-14T04:04:26.380 INFO:tasks.workunit.client.0.smithi070.stdout:621 2024-09-14T04:04:26.381 INFO:tasks.workunit.client.0.smithi070.stdout:622 2024-09-14T04:04:26.382 INFO:tasks.workunit.client.0.smithi070.stdout:623 2024-09-14T04:04:26.383 INFO:tasks.workunit.client.0.smithi070.stdout:624 2024-09-14T04:04:26.384 INFO:tasks.workunit.client.0.smithi070.stdout:625 2024-09-14T04:04:26.385 INFO:tasks.workunit.client.0.smithi070.stdout:626 2024-09-14T04:04:26.386 INFO:tasks.workunit.client.0.smithi070.stdout:627 2024-09-14T04:04:26.387 INFO:tasks.workunit.client.0.smithi070.stdout:628 2024-09-14T04:04:26.388 INFO:tasks.workunit.client.0.smithi070.stdout:629 2024-09-14T04:04:26.389 INFO:tasks.workunit.client.0.smithi070.stdout:630 2024-09-14T04:04:26.390 INFO:tasks.workunit.client.0.smithi070.stdout:631 2024-09-14T04:04:26.391 INFO:tasks.workunit.client.0.smithi070.stdout:632 2024-09-14T04:04:26.391 INFO:tasks.workunit.client.0.smithi070.stdout:633 2024-09-14T04:04:26.393 INFO:tasks.workunit.client.0.smithi070.stdout:634 2024-09-14T04:04:26.394 INFO:tasks.workunit.client.0.smithi070.stdout:635 2024-09-14T04:04:26.395 INFO:tasks.workunit.client.0.smithi070.stdout:636 2024-09-14T04:04:26.396 INFO:tasks.workunit.client.0.smithi070.stdout:637 2024-09-14T04:04:26.397 INFO:tasks.workunit.client.0.smithi070.stdout:638 2024-09-14T04:04:26.399 INFO:tasks.workunit.client.0.smithi070.stdout:639 2024-09-14T04:04:26.399 INFO:tasks.workunit.client.0.smithi070.stdout:640 2024-09-14T04:04:26.400 INFO:tasks.workunit.client.0.smithi070.stdout:641 2024-09-14T04:04:26.401 INFO:tasks.workunit.client.0.smithi070.stdout:642 2024-09-14T04:04:26.402 INFO:tasks.workunit.client.0.smithi070.stdout:643 2024-09-14T04:04:26.403 INFO:tasks.workunit.client.0.smithi070.stdout:644 2024-09-14T04:04:26.404 INFO:tasks.workunit.client.0.smithi070.stdout:645 2024-09-14T04:04:26.405 INFO:tasks.workunit.client.0.smithi070.stdout:646 2024-09-14T04:04:26.406 INFO:tasks.workunit.client.0.smithi070.stdout:647 2024-09-14T04:04:26.408 INFO:tasks.workunit.client.0.smithi070.stdout:648 2024-09-14T04:04:26.408 INFO:tasks.workunit.client.0.smithi070.stdout:649 2024-09-14T04:04:26.409 INFO:tasks.workunit.client.0.smithi070.stdout:650 2024-09-14T04:04:26.410 INFO:tasks.workunit.client.0.smithi070.stdout:651 2024-09-14T04:04:26.411 INFO:tasks.workunit.client.0.smithi070.stdout:652 2024-09-14T04:04:26.412 INFO:tasks.workunit.client.0.smithi070.stdout:653 2024-09-14T04:04:26.413 INFO:tasks.workunit.client.0.smithi070.stdout:654 2024-09-14T04:04:26.414 INFO:tasks.workunit.client.0.smithi070.stdout:655 2024-09-14T04:04:26.415 INFO:tasks.workunit.client.0.smithi070.stdout:656 2024-09-14T04:04:26.416 INFO:tasks.workunit.client.0.smithi070.stdout:657 2024-09-14T04:04:26.416 INFO:tasks.workunit.client.0.smithi070.stdout:658 2024-09-14T04:04:26.417 INFO:tasks.workunit.client.0.smithi070.stdout:659 2024-09-14T04:04:26.418 INFO:tasks.workunit.client.0.smithi070.stdout:660 2024-09-14T04:04:26.419 INFO:tasks.workunit.client.0.smithi070.stdout:661 2024-09-14T04:04:26.420 INFO:tasks.workunit.client.0.smithi070.stdout:662 2024-09-14T04:04:26.422 INFO:tasks.workunit.client.0.smithi070.stdout:663 2024-09-14T04:04:26.423 INFO:tasks.workunit.client.0.smithi070.stdout:664 2024-09-14T04:04:26.423 INFO:tasks.workunit.client.0.smithi070.stdout:665 2024-09-14T04:04:26.424 INFO:tasks.workunit.client.0.smithi070.stdout:666 2024-09-14T04:04:26.426 INFO:tasks.workunit.client.0.smithi070.stdout:667 2024-09-14T04:04:26.426 INFO:tasks.workunit.client.0.smithi070.stdout:668 2024-09-14T04:04:26.428 INFO:tasks.workunit.client.0.smithi070.stdout:669 2024-09-14T04:04:26.429 INFO:tasks.workunit.client.0.smithi070.stdout:670 2024-09-14T04:04:26.429 INFO:tasks.workunit.client.0.smithi070.stdout:671 2024-09-14T04:04:26.431 INFO:tasks.workunit.client.0.smithi070.stdout:672 2024-09-14T04:04:26.432 INFO:tasks.workunit.client.0.smithi070.stdout:673 2024-09-14T04:04:26.433 INFO:tasks.workunit.client.0.smithi070.stdout:674 2024-09-14T04:04:26.434 INFO:tasks.workunit.client.0.smithi070.stdout:675 2024-09-14T04:04:26.435 INFO:tasks.workunit.client.0.smithi070.stdout:676 2024-09-14T04:04:26.436 INFO:tasks.workunit.client.0.smithi070.stdout:677 2024-09-14T04:04:26.437 INFO:tasks.workunit.client.0.smithi070.stdout:678 2024-09-14T04:04:26.438 INFO:tasks.workunit.client.0.smithi070.stdout:679 2024-09-14T04:04:26.439 INFO:tasks.workunit.client.0.smithi070.stdout:680 2024-09-14T04:04:26.440 INFO:tasks.workunit.client.0.smithi070.stdout:681 2024-09-14T04:04:26.441 INFO:tasks.workunit.client.0.smithi070.stdout:682 2024-09-14T04:04:26.442 INFO:tasks.workunit.client.0.smithi070.stdout:683 2024-09-14T04:04:26.443 INFO:tasks.workunit.client.0.smithi070.stdout:684 2024-09-14T04:04:26.444 INFO:tasks.workunit.client.0.smithi070.stdout:685 2024-09-14T04:04:26.445 INFO:tasks.workunit.client.0.smithi070.stdout:686 2024-09-14T04:04:26.446 INFO:tasks.workunit.client.0.smithi070.stdout:687 2024-09-14T04:04:26.447 INFO:tasks.workunit.client.0.smithi070.stdout:688 2024-09-14T04:04:26.448 INFO:tasks.workunit.client.0.smithi070.stdout:689 2024-09-14T04:04:26.449 INFO:tasks.workunit.client.0.smithi070.stdout:690 2024-09-14T04:04:26.450 INFO:tasks.workunit.client.0.smithi070.stdout:691 2024-09-14T04:04:26.451 INFO:tasks.workunit.client.0.smithi070.stdout:692 2024-09-14T04:04:26.452 INFO:tasks.workunit.client.0.smithi070.stdout:693 2024-09-14T04:04:26.453 INFO:tasks.workunit.client.0.smithi070.stdout:694 2024-09-14T04:04:26.454 INFO:tasks.workunit.client.0.smithi070.stdout:695 2024-09-14T04:04:26.455 INFO:tasks.workunit.client.0.smithi070.stdout:696 2024-09-14T04:04:26.456 INFO:tasks.workunit.client.0.smithi070.stdout:697 2024-09-14T04:04:26.457 INFO:tasks.workunit.client.0.smithi070.stdout:698 2024-09-14T04:04:26.458 INFO:tasks.workunit.client.0.smithi070.stdout:699 2024-09-14T04:04:26.459 INFO:tasks.workunit.client.0.smithi070.stdout:700 2024-09-14T04:04:26.460 INFO:tasks.workunit.client.0.smithi070.stdout:701 2024-09-14T04:04:26.461 INFO:tasks.workunit.client.0.smithi070.stdout:702 2024-09-14T04:04:26.462 INFO:tasks.workunit.client.0.smithi070.stdout:703 2024-09-14T04:04:26.463 INFO:tasks.workunit.client.0.smithi070.stdout:704 2024-09-14T04:04:26.464 INFO:tasks.workunit.client.0.smithi070.stdout:705 2024-09-14T04:04:26.465 INFO:tasks.workunit.client.0.smithi070.stdout:706 2024-09-14T04:04:26.466 INFO:tasks.workunit.client.0.smithi070.stdout:707 2024-09-14T04:04:26.467 INFO:tasks.workunit.client.0.smithi070.stdout:708 2024-09-14T04:04:26.468 INFO:tasks.workunit.client.0.smithi070.stdout:709 2024-09-14T04:04:26.469 INFO:tasks.workunit.client.0.smithi070.stdout:710 2024-09-14T04:04:26.470 INFO:tasks.workunit.client.0.smithi070.stdout:711 2024-09-14T04:04:26.471 INFO:tasks.workunit.client.0.smithi070.stdout:712 2024-09-14T04:04:26.472 INFO:tasks.workunit.client.0.smithi070.stdout:713 2024-09-14T04:04:26.473 INFO:tasks.workunit.client.0.smithi070.stdout:714 2024-09-14T04:04:26.474 INFO:tasks.workunit.client.0.smithi070.stdout:715 2024-09-14T04:04:26.475 INFO:tasks.workunit.client.0.smithi070.stdout:716 2024-09-14T04:04:26.476 INFO:tasks.workunit.client.0.smithi070.stdout:717 2024-09-14T04:04:26.477 INFO:tasks.workunit.client.0.smithi070.stdout:718 2024-09-14T04:04:26.478 INFO:tasks.workunit.client.0.smithi070.stdout:719 2024-09-14T04:04:26.480 INFO:tasks.workunit.client.0.smithi070.stdout:720 2024-09-14T04:04:26.481 INFO:tasks.workunit.client.0.smithi070.stdout:721 2024-09-14T04:04:26.482 INFO:tasks.workunit.client.0.smithi070.stdout:722 2024-09-14T04:04:26.483 INFO:tasks.workunit.client.0.smithi070.stdout:723 2024-09-14T04:04:26.484 INFO:tasks.workunit.client.0.smithi070.stdout:724 2024-09-14T04:04:26.485 INFO:tasks.workunit.client.0.smithi070.stdout:725 2024-09-14T04:04:26.486 INFO:tasks.workunit.client.0.smithi070.stdout:726 2024-09-14T04:04:26.487 INFO:tasks.workunit.client.0.smithi070.stdout:727 2024-09-14T04:04:26.488 INFO:tasks.workunit.client.0.smithi070.stdout:728 2024-09-14T04:04:26.490 INFO:tasks.workunit.client.0.smithi070.stdout:729 2024-09-14T04:04:26.491 INFO:tasks.workunit.client.0.smithi070.stdout:730 2024-09-14T04:04:26.492 INFO:tasks.workunit.client.0.smithi070.stdout:731 2024-09-14T04:04:26.493 INFO:tasks.workunit.client.0.smithi070.stdout:732 2024-09-14T04:04:26.495 INFO:tasks.workunit.client.0.smithi070.stdout:733 2024-09-14T04:04:26.496 INFO:tasks.workunit.client.0.smithi070.stdout:734 2024-09-14T04:04:26.498 INFO:tasks.workunit.client.0.smithi070.stdout:735 2024-09-14T04:04:26.499 INFO:tasks.workunit.client.0.smithi070.stdout:736 2024-09-14T04:04:26.500 INFO:tasks.workunit.client.0.smithi070.stdout:737 2024-09-14T04:04:26.501 INFO:tasks.workunit.client.0.smithi070.stdout:738 2024-09-14T04:04:26.502 INFO:tasks.workunit.client.0.smithi070.stdout:739 2024-09-14T04:04:26.504 INFO:tasks.workunit.client.0.smithi070.stdout:740 2024-09-14T04:04:26.505 INFO:tasks.workunit.client.0.smithi070.stdout:741 2024-09-14T04:04:26.506 INFO:tasks.workunit.client.0.smithi070.stdout:742 2024-09-14T04:04:26.507 INFO:tasks.workunit.client.0.smithi070.stdout:743 2024-09-14T04:04:26.508 INFO:tasks.workunit.client.0.smithi070.stdout:744 2024-09-14T04:04:26.509 INFO:tasks.workunit.client.0.smithi070.stdout:745 2024-09-14T04:04:26.510 INFO:tasks.workunit.client.0.smithi070.stdout:746 2024-09-14T04:04:26.511 INFO:tasks.workunit.client.0.smithi070.stdout:747 2024-09-14T04:04:26.512 INFO:tasks.workunit.client.0.smithi070.stdout:748 2024-09-14T04:04:26.514 INFO:tasks.workunit.client.0.smithi070.stdout:749 2024-09-14T04:04:26.515 INFO:tasks.workunit.client.0.smithi070.stdout:750 2024-09-14T04:04:26.516 INFO:tasks.workunit.client.0.smithi070.stdout:751 2024-09-14T04:04:26.517 INFO:tasks.workunit.client.0.smithi070.stdout:752 2024-09-14T04:04:26.518 INFO:tasks.workunit.client.0.smithi070.stdout:753 2024-09-14T04:04:26.519 INFO:tasks.workunit.client.0.smithi070.stdout:754 2024-09-14T04:04:26.521 INFO:tasks.workunit.client.0.smithi070.stdout:755 2024-09-14T04:04:26.522 INFO:tasks.workunit.client.0.smithi070.stdout:756 2024-09-14T04:04:26.523 INFO:tasks.workunit.client.0.smithi070.stdout:757 2024-09-14T04:04:26.524 INFO:tasks.workunit.client.0.smithi070.stdout:758 2024-09-14T04:04:26.525 INFO:tasks.workunit.client.0.smithi070.stdout:759 2024-09-14T04:04:26.526 INFO:tasks.workunit.client.0.smithi070.stdout:760 2024-09-14T04:04:26.527 INFO:tasks.workunit.client.0.smithi070.stdout:761 2024-09-14T04:04:26.528 INFO:tasks.workunit.client.0.smithi070.stdout:762 2024-09-14T04:04:26.529 INFO:tasks.workunit.client.0.smithi070.stdout:763 2024-09-14T04:04:26.530 INFO:tasks.workunit.client.0.smithi070.stdout:764 2024-09-14T04:04:26.531 INFO:tasks.workunit.client.0.smithi070.stdout:765 2024-09-14T04:04:26.532 INFO:tasks.workunit.client.0.smithi070.stdout:766 2024-09-14T04:04:26.533 INFO:tasks.workunit.client.0.smithi070.stdout:767 2024-09-14T04:04:26.534 INFO:tasks.workunit.client.0.smithi070.stdout:768 2024-09-14T04:04:26.535 INFO:tasks.workunit.client.0.smithi070.stdout:769 2024-09-14T04:04:26.536 INFO:tasks.workunit.client.0.smithi070.stdout:770 2024-09-14T04:04:26.537 INFO:tasks.workunit.client.0.smithi070.stdout:771 2024-09-14T04:04:26.539 INFO:tasks.workunit.client.0.smithi070.stdout:772 2024-09-14T04:04:26.540 INFO:tasks.workunit.client.0.smithi070.stdout:773 2024-09-14T04:04:26.542 INFO:tasks.workunit.client.0.smithi070.stdout:774 2024-09-14T04:04:26.543 INFO:tasks.workunit.client.0.smithi070.stdout:775 2024-09-14T04:04:26.544 INFO:tasks.workunit.client.0.smithi070.stdout:776 2024-09-14T04:04:26.545 INFO:tasks.workunit.client.0.smithi070.stdout:777 2024-09-14T04:04:26.546 INFO:tasks.workunit.client.0.smithi070.stdout:778 2024-09-14T04:04:26.547 INFO:tasks.workunit.client.0.smithi070.stdout:779 2024-09-14T04:04:26.548 INFO:tasks.workunit.client.0.smithi070.stdout:780 2024-09-14T04:04:26.549 INFO:tasks.workunit.client.0.smithi070.stdout:781 2024-09-14T04:04:26.550 INFO:tasks.workunit.client.0.smithi070.stdout:782 2024-09-14T04:04:26.552 INFO:tasks.workunit.client.0.smithi070.stdout:783 2024-09-14T04:04:26.553 INFO:tasks.workunit.client.0.smithi070.stdout:784 2024-09-14T04:04:26.554 INFO:tasks.workunit.client.0.smithi070.stdout:785 2024-09-14T04:04:26.555 INFO:tasks.workunit.client.0.smithi070.stdout:786 2024-09-14T04:04:26.556 INFO:tasks.workunit.client.0.smithi070.stdout:787 2024-09-14T04:04:26.558 INFO:tasks.workunit.client.0.smithi070.stdout:788 2024-09-14T04:04:26.559 INFO:tasks.workunit.client.0.smithi070.stdout:789 2024-09-14T04:04:26.561 INFO:tasks.workunit.client.0.smithi070.stdout:790 2024-09-14T04:04:26.562 INFO:tasks.workunit.client.0.smithi070.stdout:791 2024-09-14T04:04:26.564 INFO:tasks.workunit.client.0.smithi070.stdout:792 2024-09-14T04:04:26.565 INFO:tasks.workunit.client.0.smithi070.stdout:793 2024-09-14T04:04:26.566 INFO:tasks.workunit.client.0.smithi070.stdout:794 2024-09-14T04:04:26.567 INFO:tasks.workunit.client.0.smithi070.stdout:795 2024-09-14T04:04:26.568 INFO:tasks.workunit.client.0.smithi070.stdout:796 2024-09-14T04:04:26.569 INFO:tasks.workunit.client.0.smithi070.stdout:797 2024-09-14T04:04:26.570 INFO:tasks.workunit.client.0.smithi070.stdout:798 2024-09-14T04:04:26.571 INFO:tasks.workunit.client.0.smithi070.stdout:799 2024-09-14T04:04:26.572 INFO:tasks.workunit.client.0.smithi070.stdout:800 2024-09-14T04:04:26.573 INFO:tasks.workunit.client.0.smithi070.stdout:801 2024-09-14T04:04:26.574 INFO:tasks.workunit.client.0.smithi070.stdout:802 2024-09-14T04:04:26.575 INFO:tasks.workunit.client.0.smithi070.stdout:803 2024-09-14T04:04:26.576 INFO:tasks.workunit.client.0.smithi070.stdout:804 2024-09-14T04:04:26.577 INFO:tasks.workunit.client.0.smithi070.stdout:805 2024-09-14T04:04:26.578 INFO:tasks.workunit.client.0.smithi070.stdout:806 2024-09-14T04:04:26.579 INFO:tasks.workunit.client.0.smithi070.stdout:807 2024-09-14T04:04:26.580 INFO:tasks.workunit.client.0.smithi070.stdout:808 2024-09-14T04:04:26.581 INFO:tasks.workunit.client.0.smithi070.stdout:809 2024-09-14T04:04:26.582 INFO:tasks.workunit.client.0.smithi070.stdout:810 2024-09-14T04:04:26.583 INFO:tasks.workunit.client.0.smithi070.stdout:811 2024-09-14T04:04:26.584 INFO:tasks.workunit.client.0.smithi070.stdout:812 2024-09-14T04:04:26.585 INFO:tasks.workunit.client.0.smithi070.stdout:813 2024-09-14T04:04:26.586 INFO:tasks.workunit.client.0.smithi070.stdout:814 2024-09-14T04:04:26.587 INFO:tasks.workunit.client.0.smithi070.stdout:815 2024-09-14T04:04:26.588 INFO:tasks.workunit.client.0.smithi070.stdout:816 2024-09-14T04:04:26.589 INFO:tasks.workunit.client.0.smithi070.stdout:817 2024-09-14T04:04:26.590 INFO:tasks.workunit.client.0.smithi070.stdout:818 2024-09-14T04:04:26.591 INFO:tasks.workunit.client.0.smithi070.stdout:819 2024-09-14T04:04:26.592 INFO:tasks.workunit.client.0.smithi070.stdout:820 2024-09-14T04:04:26.594 INFO:tasks.workunit.client.0.smithi070.stdout:821 2024-09-14T04:04:26.595 INFO:tasks.workunit.client.0.smithi070.stdout:822 2024-09-14T04:04:26.596 INFO:tasks.workunit.client.0.smithi070.stdout:823 2024-09-14T04:04:26.597 INFO:tasks.workunit.client.0.smithi070.stdout:824 2024-09-14T04:04:26.598 INFO:tasks.workunit.client.0.smithi070.stdout:825 2024-09-14T04:04:26.599 INFO:tasks.workunit.client.0.smithi070.stdout:826 2024-09-14T04:04:26.600 INFO:tasks.workunit.client.0.smithi070.stdout:827 2024-09-14T04:04:26.601 INFO:tasks.workunit.client.0.smithi070.stdout:828 2024-09-14T04:04:26.602 INFO:tasks.workunit.client.0.smithi070.stdout:829 2024-09-14T04:04:26.603 INFO:tasks.workunit.client.0.smithi070.stdout:830 2024-09-14T04:04:26.604 INFO:tasks.workunit.client.0.smithi070.stdout:831 2024-09-14T04:04:26.605 INFO:tasks.workunit.client.0.smithi070.stdout:832 2024-09-14T04:04:26.606 INFO:tasks.workunit.client.0.smithi070.stdout:833 2024-09-14T04:04:26.607 INFO:tasks.workunit.client.0.smithi070.stdout:834 2024-09-14T04:04:26.608 INFO:tasks.workunit.client.0.smithi070.stdout:835 2024-09-14T04:04:26.609 INFO:tasks.workunit.client.0.smithi070.stdout:836 2024-09-14T04:04:26.610 INFO:tasks.workunit.client.0.smithi070.stdout:837 2024-09-14T04:04:26.611 INFO:tasks.workunit.client.0.smithi070.stdout:838 2024-09-14T04:04:26.612 INFO:tasks.workunit.client.0.smithi070.stdout:839 2024-09-14T04:04:26.613 INFO:tasks.workunit.client.0.smithi070.stdout:840 2024-09-14T04:04:26.614 INFO:tasks.workunit.client.0.smithi070.stdout:841 2024-09-14T04:04:26.615 INFO:tasks.workunit.client.0.smithi070.stdout:842 2024-09-14T04:04:26.616 INFO:tasks.workunit.client.0.smithi070.stdout:843 2024-09-14T04:04:26.617 INFO:tasks.workunit.client.0.smithi070.stdout:844 2024-09-14T04:04:26.618 INFO:tasks.workunit.client.0.smithi070.stdout:845 2024-09-14T04:04:26.620 INFO:tasks.workunit.client.0.smithi070.stdout:846 2024-09-14T04:04:26.621 INFO:tasks.workunit.client.0.smithi070.stdout:847 2024-09-14T04:04:26.622 INFO:tasks.workunit.client.0.smithi070.stdout:848 2024-09-14T04:04:26.623 INFO:tasks.workunit.client.0.smithi070.stdout:849 2024-09-14T04:04:26.624 INFO:tasks.workunit.client.0.smithi070.stdout:850 2024-09-14T04:04:26.625 INFO:tasks.workunit.client.0.smithi070.stdout:851 2024-09-14T04:04:26.626 INFO:tasks.workunit.client.0.smithi070.stdout:852 2024-09-14T04:04:26.627 INFO:tasks.workunit.client.0.smithi070.stdout:853 2024-09-14T04:04:26.628 INFO:tasks.workunit.client.0.smithi070.stdout:854 2024-09-14T04:04:26.629 INFO:tasks.workunit.client.0.smithi070.stdout:855 2024-09-14T04:04:26.630 INFO:tasks.workunit.client.0.smithi070.stdout:856 2024-09-14T04:04:26.631 INFO:tasks.workunit.client.0.smithi070.stdout:857 2024-09-14T04:04:26.632 INFO:tasks.workunit.client.0.smithi070.stdout:858 2024-09-14T04:04:26.633 INFO:tasks.workunit.client.0.smithi070.stdout:859 2024-09-14T04:04:26.634 INFO:tasks.workunit.client.0.smithi070.stdout:860 2024-09-14T04:04:26.635 INFO:tasks.workunit.client.0.smithi070.stdout:861 2024-09-14T04:04:26.637 INFO:tasks.workunit.client.0.smithi070.stdout:862 2024-09-14T04:04:26.638 INFO:tasks.workunit.client.0.smithi070.stdout:863 2024-09-14T04:04:26.639 INFO:tasks.workunit.client.0.smithi070.stdout:864 2024-09-14T04:04:26.640 INFO:tasks.workunit.client.0.smithi070.stdout:865 2024-09-14T04:04:26.641 INFO:tasks.workunit.client.0.smithi070.stdout:866 2024-09-14T04:04:26.642 INFO:tasks.workunit.client.0.smithi070.stdout:867 2024-09-14T04:04:26.643 INFO:tasks.workunit.client.0.smithi070.stdout:868 2024-09-14T04:04:26.644 INFO:tasks.workunit.client.0.smithi070.stdout:869 2024-09-14T04:04:26.645 INFO:tasks.workunit.client.0.smithi070.stdout:870 2024-09-14T04:04:26.646 INFO:tasks.workunit.client.0.smithi070.stdout:871 2024-09-14T04:04:26.647 INFO:tasks.workunit.client.0.smithi070.stdout:872 2024-09-14T04:04:26.648 INFO:tasks.workunit.client.0.smithi070.stdout:873 2024-09-14T04:04:26.649 INFO:tasks.workunit.client.0.smithi070.stdout:874 2024-09-14T04:04:26.650 INFO:tasks.workunit.client.0.smithi070.stdout:875 2024-09-14T04:04:26.651 INFO:tasks.workunit.client.0.smithi070.stdout:876 2024-09-14T04:04:26.653 INFO:tasks.workunit.client.0.smithi070.stdout:877 2024-09-14T04:04:26.654 INFO:tasks.workunit.client.0.smithi070.stdout:878 2024-09-14T04:04:26.655 INFO:tasks.workunit.client.0.smithi070.stdout:879 2024-09-14T04:04:26.656 INFO:tasks.workunit.client.0.smithi070.stdout:880 2024-09-14T04:04:26.657 INFO:tasks.workunit.client.0.smithi070.stdout:881 2024-09-14T04:04:26.658 INFO:tasks.workunit.client.0.smithi070.stdout:882 2024-09-14T04:04:26.659 INFO:tasks.workunit.client.0.smithi070.stdout:883 2024-09-14T04:04:26.660 INFO:tasks.workunit.client.0.smithi070.stdout:884 2024-09-14T04:04:26.661 INFO:tasks.workunit.client.0.smithi070.stdout:885 2024-09-14T04:04:26.662 INFO:tasks.workunit.client.0.smithi070.stdout:886 2024-09-14T04:04:26.664 INFO:tasks.workunit.client.0.smithi070.stdout:887 2024-09-14T04:04:26.665 INFO:tasks.workunit.client.0.smithi070.stdout:888 2024-09-14T04:04:26.666 INFO:tasks.workunit.client.0.smithi070.stdout:889 2024-09-14T04:04:26.667 INFO:tasks.workunit.client.0.smithi070.stdout:890 2024-09-14T04:04:26.668 INFO:tasks.workunit.client.0.smithi070.stdout:891 2024-09-14T04:04:26.669 INFO:tasks.workunit.client.0.smithi070.stdout:892 2024-09-14T04:04:26.670 INFO:tasks.workunit.client.0.smithi070.stdout:893 2024-09-14T04:04:26.671 INFO:tasks.workunit.client.0.smithi070.stdout:894 2024-09-14T04:04:26.672 INFO:tasks.workunit.client.0.smithi070.stdout:895 2024-09-14T04:04:26.673 INFO:tasks.workunit.client.0.smithi070.stdout:896 2024-09-14T04:04:26.674 INFO:tasks.workunit.client.0.smithi070.stdout:897 2024-09-14T04:04:26.675 INFO:tasks.workunit.client.0.smithi070.stdout:898 2024-09-14T04:04:26.676 INFO:tasks.workunit.client.0.smithi070.stdout:899 2024-09-14T04:04:26.677 INFO:tasks.workunit.client.0.smithi070.stdout:900 2024-09-14T04:04:26.678 INFO:tasks.workunit.client.0.smithi070.stdout:901 2024-09-14T04:04:26.679 INFO:tasks.workunit.client.0.smithi070.stdout:902 2024-09-14T04:04:26.680 INFO:tasks.workunit.client.0.smithi070.stdout:903 2024-09-14T04:04:26.681 INFO:tasks.workunit.client.0.smithi070.stdout:904 2024-09-14T04:04:26.683 INFO:tasks.workunit.client.0.smithi070.stdout:905 2024-09-14T04:04:26.685 INFO:tasks.workunit.client.0.smithi070.stdout:906 2024-09-14T04:04:26.686 INFO:tasks.workunit.client.0.smithi070.stdout:907 2024-09-14T04:04:26.687 INFO:tasks.workunit.client.0.smithi070.stdout:908 2024-09-14T04:04:26.688 INFO:tasks.workunit.client.0.smithi070.stdout:909 2024-09-14T04:04:26.689 INFO:tasks.workunit.client.0.smithi070.stdout:910 2024-09-14T04:04:26.690 INFO:tasks.workunit.client.0.smithi070.stdout:911 2024-09-14T04:04:26.691 INFO:tasks.workunit.client.0.smithi070.stdout:912 2024-09-14T04:04:26.692 INFO:tasks.workunit.client.0.smithi070.stdout:913 2024-09-14T04:04:26.693 INFO:tasks.workunit.client.0.smithi070.stdout:914 2024-09-14T04:04:26.694 INFO:tasks.workunit.client.0.smithi070.stdout:915 2024-09-14T04:04:26.695 INFO:tasks.workunit.client.0.smithi070.stdout:916 2024-09-14T04:04:26.696 INFO:tasks.workunit.client.0.smithi070.stdout:917 2024-09-14T04:04:26.697 INFO:tasks.workunit.client.0.smithi070.stdout:918 2024-09-14T04:04:26.698 INFO:tasks.workunit.client.0.smithi070.stdout:919 2024-09-14T04:04:26.699 INFO:tasks.workunit.client.0.smithi070.stdout:920 2024-09-14T04:04:26.701 INFO:tasks.workunit.client.0.smithi070.stdout:921 2024-09-14T04:04:26.702 INFO:tasks.workunit.client.0.smithi070.stdout:922 2024-09-14T04:04:26.703 INFO:tasks.workunit.client.0.smithi070.stdout:923 2024-09-14T04:04:26.704 INFO:tasks.workunit.client.0.smithi070.stdout:924 2024-09-14T04:04:26.705 INFO:tasks.workunit.client.0.smithi070.stdout:925 2024-09-14T04:04:26.707 INFO:tasks.workunit.client.0.smithi070.stdout:926 2024-09-14T04:04:26.708 INFO:tasks.workunit.client.0.smithi070.stdout:927 2024-09-14T04:04:26.709 INFO:tasks.workunit.client.0.smithi070.stdout:928 2024-09-14T04:04:26.710 INFO:tasks.workunit.client.0.smithi070.stdout:929 2024-09-14T04:04:26.711 INFO:tasks.workunit.client.0.smithi070.stdout:930 2024-09-14T04:04:26.712 INFO:tasks.workunit.client.0.smithi070.stdout:931 2024-09-14T04:04:26.713 INFO:tasks.workunit.client.0.smithi070.stdout:932 2024-09-14T04:04:26.714 INFO:tasks.workunit.client.0.smithi070.stdout:933 2024-09-14T04:04:26.715 INFO:tasks.workunit.client.0.smithi070.stdout:934 2024-09-14T04:04:26.716 INFO:tasks.workunit.client.0.smithi070.stdout:935 2024-09-14T04:04:26.717 INFO:tasks.workunit.client.0.smithi070.stdout:936 2024-09-14T04:04:26.718 INFO:tasks.workunit.client.0.smithi070.stdout:937 2024-09-14T04:04:26.719 INFO:tasks.workunit.client.0.smithi070.stdout:938 2024-09-14T04:04:26.720 INFO:tasks.workunit.client.0.smithi070.stdout:939 2024-09-14T04:04:26.721 INFO:tasks.workunit.client.0.smithi070.stdout:940 2024-09-14T04:04:26.722 INFO:tasks.workunit.client.0.smithi070.stdout:941 2024-09-14T04:04:26.723 INFO:tasks.workunit.client.0.smithi070.stdout:942 2024-09-14T04:04:26.724 INFO:tasks.workunit.client.0.smithi070.stdout:943 2024-09-14T04:04:26.725 INFO:tasks.workunit.client.0.smithi070.stdout:944 2024-09-14T04:04:26.726 INFO:tasks.workunit.client.0.smithi070.stdout:945 2024-09-14T04:04:26.727 INFO:tasks.workunit.client.0.smithi070.stdout:946 2024-09-14T04:04:26.728 INFO:tasks.workunit.client.0.smithi070.stdout:947 2024-09-14T04:04:26.730 INFO:tasks.workunit.client.0.smithi070.stdout:948 2024-09-14T04:04:26.731 INFO:tasks.workunit.client.0.smithi070.stdout:949 2024-09-14T04:04:26.732 INFO:tasks.workunit.client.0.smithi070.stdout:950 2024-09-14T04:04:26.733 INFO:tasks.workunit.client.0.smithi070.stdout:951 2024-09-14T04:04:26.734 INFO:tasks.workunit.client.0.smithi070.stdout:952 2024-09-14T04:04:26.735 INFO:tasks.workunit.client.0.smithi070.stdout:953 2024-09-14T04:04:26.736 INFO:tasks.workunit.client.0.smithi070.stdout:954 2024-09-14T04:04:26.737 INFO:tasks.workunit.client.0.smithi070.stdout:955 2024-09-14T04:04:26.738 INFO:tasks.workunit.client.0.smithi070.stdout:956 2024-09-14T04:04:26.739 INFO:tasks.workunit.client.0.smithi070.stdout:957 2024-09-14T04:04:26.740 INFO:tasks.workunit.client.0.smithi070.stdout:958 2024-09-14T04:04:26.742 INFO:tasks.workunit.client.0.smithi070.stdout:959 2024-09-14T04:04:26.743 INFO:tasks.workunit.client.0.smithi070.stdout:960 2024-09-14T04:04:26.744 INFO:tasks.workunit.client.0.smithi070.stdout:961 2024-09-14T04:04:26.745 INFO:tasks.workunit.client.0.smithi070.stdout:962 2024-09-14T04:04:26.747 INFO:tasks.workunit.client.0.smithi070.stdout:963 2024-09-14T04:04:26.748 INFO:tasks.workunit.client.0.smithi070.stdout:964 2024-09-14T04:04:26.749 INFO:tasks.workunit.client.0.smithi070.stdout:965 2024-09-14T04:04:26.750 INFO:tasks.workunit.client.0.smithi070.stdout:966 2024-09-14T04:04:26.751 INFO:tasks.workunit.client.0.smithi070.stdout:967 2024-09-14T04:04:26.752 INFO:tasks.workunit.client.0.smithi070.stdout:968 2024-09-14T04:04:26.754 INFO:tasks.workunit.client.0.smithi070.stdout:969 2024-09-14T04:04:26.755 INFO:tasks.workunit.client.0.smithi070.stdout:970 2024-09-14T04:04:26.756 INFO:tasks.workunit.client.0.smithi070.stdout:971 2024-09-14T04:04:26.757 INFO:tasks.workunit.client.0.smithi070.stdout:972 2024-09-14T04:04:26.759 INFO:tasks.workunit.client.0.smithi070.stdout:973 2024-09-14T04:04:26.760 INFO:tasks.workunit.client.0.smithi070.stdout:974 2024-09-14T04:04:26.761 INFO:tasks.workunit.client.0.smithi070.stdout:975 2024-09-14T04:04:26.762 INFO:tasks.workunit.client.0.smithi070.stdout:976 2024-09-14T04:04:26.764 INFO:tasks.workunit.client.0.smithi070.stdout:977 2024-09-14T04:04:26.765 INFO:tasks.workunit.client.0.smithi070.stdout:978 2024-09-14T04:04:26.766 INFO:tasks.workunit.client.0.smithi070.stdout:979 2024-09-14T04:04:26.767 INFO:tasks.workunit.client.0.smithi070.stdout:980 2024-09-14T04:04:26.768 INFO:tasks.workunit.client.0.smithi070.stdout:981 2024-09-14T04:04:26.769 INFO:tasks.workunit.client.0.smithi070.stdout:982 2024-09-14T04:04:26.770 INFO:tasks.workunit.client.0.smithi070.stdout:983 2024-09-14T04:04:26.772 INFO:tasks.workunit.client.0.smithi070.stdout:984 2024-09-14T04:04:26.773 INFO:tasks.workunit.client.0.smithi070.stdout:985 2024-09-14T04:04:26.775 INFO:tasks.workunit.client.0.smithi070.stdout:986 2024-09-14T04:04:26.776 INFO:tasks.workunit.client.0.smithi070.stdout:987 2024-09-14T04:04:26.777 INFO:tasks.workunit.client.0.smithi070.stdout:988 2024-09-14T04:04:26.778 INFO:tasks.workunit.client.0.smithi070.stdout:989 2024-09-14T04:04:26.779 INFO:tasks.workunit.client.0.smithi070.stdout:990 2024-09-14T04:04:26.780 INFO:tasks.workunit.client.0.smithi070.stdout:991 2024-09-14T04:04:26.782 INFO:tasks.workunit.client.0.smithi070.stdout:992 2024-09-14T04:04:26.783 INFO:tasks.workunit.client.0.smithi070.stdout:993 2024-09-14T04:04:26.784 INFO:tasks.workunit.client.0.smithi070.stdout:994 2024-09-14T04:04:26.785 INFO:tasks.workunit.client.0.smithi070.stdout:995 2024-09-14T04:04:26.786 INFO:tasks.workunit.client.0.smithi070.stdout:996 2024-09-14T04:04:26.788 INFO:tasks.workunit.client.0.smithi070.stdout:997 2024-09-14T04:04:26.789 INFO:tasks.workunit.client.0.smithi070.stdout:998 2024-09-14T04:04:26.790 INFO:tasks.workunit.client.0.smithi070.stdout:999 2024-09-14T04:04:27.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:26 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:27.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:26 smithi094 ceph-mon[86597]: osdmap e984: 8 total, 8 up, 8 in 2024-09-14T04:04:27.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:26 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/776475119' entity='client.admin' cmd={"prefix": "osd dump","format":"json"} : dispatch 2024-09-14T04:04:27.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:26 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:27.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:26 smithi070 ceph-mon[96704]: osdmap e984: 8 total, 8 up, 8 in 2024-09-14T04:04:27.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:26 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/776475119' entity='client.admin' cmd={"prefix": "osd dump","format":"json"} : dispatch 2024-09-14T04:04:27.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:26 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:27.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:26 smithi070 ceph-mon[98640]: osdmap e984: 8 total, 8 up, 8 in 2024-09-14T04:04:27.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:26 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/776475119' entity='client.admin' cmd={"prefix": "osd dump","format":"json"} : dispatch 2024-09-14T04:04:27.566 INFO:tasks.workunit.client.0.smithi070.stdout:00000000 79 6f 75 20 6d 69 67 68 74 20 73 65 65 20 74 68 |you might see th| 2024-09-14T04:04:27.566 INFO:tasks.workunit.client.0.smithi070.stdout:00000010 69 73 |is| 2024-09-14T04:04:27.567 INFO:tasks.workunit.client.0.smithi070.stdout:00000012 2024-09-14T04:04:27.567 INFO:tasks.workunit.client.0.smithi070.stdout:00000000 79 6f 75 20 6d 69 67 68 74 20 73 65 65 20 74 68 |you might see th| 2024-09-14T04:04:27.567 INFO:tasks.workunit.client.0.smithi070.stdout:00000010 69 73 |is| 2024-09-14T04:04:27.567 INFO:tasks.workunit.client.0.smithi070.stdout:00000012 2024-09-14T04:04:27.567 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] ClsHello.WriteReturnData (4020 ms) 2024-09-14T04:04:27.567 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] ClsHello.Loud 2024-09-14T04:04:28.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:27 smithi094 ceph-mon[86597]: pgmap v1098: 137 pgs: 5 creating+peering, 18 unknown, 114 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:28.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:27 smithi094 ceph-mon[86597]: osdmap e985: 8 total, 8 up, 8 in 2024-09-14T04:04:28.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:27 smithi070 ceph-mon[96704]: pgmap v1098: 137 pgs: 5 creating+peering, 18 unknown, 114 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:28.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:27 smithi070 ceph-mon[96704]: osdmap e985: 8 total, 8 up, 8 in 2024-09-14T04:04:28.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:27 smithi070 ceph-mon[98640]: pgmap v1098: 137 pgs: 5 creating+peering, 18 unknown, 114 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:28.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:27 smithi070 ceph-mon[98640]: osdmap e985: 8 total, 8 up, 8 in 2024-09-14T04:04:29.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:28 smithi094 ceph-mon[86597]: osdmap e986: 8 total, 8 up, 8 in 2024-09-14T04:04:29.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:28 smithi070 ceph-mon[96704]: osdmap e986: 8 total, 8 up, 8 in 2024-09-14T04:04:29.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:28 smithi070 ceph-mon[98640]: osdmap e986: 8 total, 8 up, 8 in 2024-09-14T04:04:30.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:29 smithi094 ceph-mon[86597]: pgmap v1101: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:30.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:29 smithi094 ceph-mon[86597]: osdmap e987: 8 total, 8 up, 8 in 2024-09-14T04:04:30.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:29 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/2236148826' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:30.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:29 smithi070 ceph-mon[96704]: pgmap v1101: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:30.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:29 smithi070 ceph-mon[96704]: osdmap e987: 8 total, 8 up, 8 in 2024-09-14T04:04:30.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:29 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/2236148826' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:30.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:29 smithi070 ceph-mon[98640]: pgmap v1101: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:30.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:29 smithi070 ceph-mon[98640]: osdmap e987: 8 total, 8 up, 8 in 2024-09-14T04:04:30.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:29 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/2236148826' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:30.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:04:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:04:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:04:30.588 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] ClsHello.Loud (3021 ms) 2024-09-14T04:04:30.588 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] ClsHello.BadMethods 2024-09-14T04:04:31.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:30 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/2236148826' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:31.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:30 smithi094 ceph-mon[86597]: osdmap e988: 8 total, 8 up, 8 in 2024-09-14T04:04:31.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:30 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/2236148826' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:31.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:30 smithi070 ceph-mon[96704]: osdmap e988: 8 total, 8 up, 8 in 2024-09-14T04:04:31.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:30 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/2236148826' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:31.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:30 smithi070 ceph-mon[98640]: osdmap e988: 8 total, 8 up, 8 in 2024-09-14T04:04:32.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:31 smithi094 ceph-mon[86597]: pgmap v1104: 137 pgs: 24 unknown, 113 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:32.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:31 smithi094 ceph-mon[86597]: osdmap e989: 8 total, 8 up, 8 in 2024-09-14T04:04:32.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:31 smithi070 ceph-mon[96704]: pgmap v1104: 137 pgs: 24 unknown, 113 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:32.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:31 smithi070 ceph-mon[96704]: osdmap e989: 8 total, 8 up, 8 in 2024-09-14T04:04:32.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:31 smithi070 ceph-mon[98640]: pgmap v1104: 137 pgs: 24 unknown, 113 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:32.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:31 smithi070 ceph-mon[98640]: osdmap e989: 8 total, 8 up, 8 in 2024-09-14T04:04:33.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:32 smithi094 ceph-mon[86597]: osdmap e990: 8 total, 8 up, 8 in 2024-09-14T04:04:33.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:32 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/1291324162' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:33.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:32 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:04:33.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:32 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:33.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:32 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/1291324162' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:33.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:32 smithi094 ceph-mon[86597]: osdmap e991: 8 total, 8 up, 8 in 2024-09-14T04:04:33.035 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 04:04:32 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3[117637]: 2024-09-14T04:04:32.608+0000 7f4b1d2cd640 -1 osd.3 pg_epoch: 991 pg[38.2( v 991'1 (0'0,991'1] local-lis/les=990/991 n=1 ec=990/990 lis/c=990/990 les/c/f=991/991/0 sis=990) [3,7,5] r=0 lpr=990 crt=991'1 lcod 0'0 mlcod 0'0 active+clean] method hello.bad_reader tried to read object but is not marked RD 2024-09-14T04:04:33.035 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 04:04:32 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3[117637]: 2024-09-14T04:04:32.609+0000 7f4b212d5640 -1 osd.3 pg_epoch: 991 pg[38.2( v 991'2 (0'0,991'2] local-lis/les=990/991 n=1 ec=990/990 lis/c=990/990 les/c/f=991/991/0 sis=990) [3,7,5] r=0 lpr=990 crt=991'1 lcod 991'1 mlcod 991'1 active+clean] method hello.bad_writer tried to update object but is not marked WR 2024-09-14T04:04:33.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:32 smithi070 ceph-mon[98640]: osdmap e990: 8 total, 8 up, 8 in 2024-09-14T04:04:33.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:32 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/1291324162' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:33.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:32 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:04:33.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:32 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:33.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:32 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/1291324162' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:33.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:32 smithi070 ceph-mon[98640]: osdmap e991: 8 total, 8 up, 8 in 2024-09-14T04:04:33.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:32 smithi070 ceph-mon[96704]: osdmap e990: 8 total, 8 up, 8 in 2024-09-14T04:04:33.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:32 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/1291324162' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:33.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:32 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:04:33.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:32 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:33.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:32 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/1291324162' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:33.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:32 smithi070 ceph-mon[96704]: osdmap e991: 8 total, 8 up, 8 in 2024-09-14T04:04:33.604 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] ClsHello.BadMethods (3016 ms) 2024-09-14T04:04:33.604 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] ClsHello.Filter 2024-09-14T04:04:33.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:04:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:04:33] "GET /metrics HTTP/1.1" 200 36518 "" "Prometheus/2.51.0" 2024-09-14T04:04:33.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:33 smithi070 ceph-mon[96704]: pgmap v1107: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:33.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:33 smithi070 ceph-mon[96704]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:33.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:33 smithi070 ceph-mon[96704]: osdmap e992: 8 total, 8 up, 8 in 2024-09-14T04:04:33.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:33 smithi070 ceph-mon[98640]: pgmap v1107: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:33.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:33 smithi070 ceph-mon[98640]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:33.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:33 smithi070 ceph-mon[98640]: osdmap e992: 8 total, 8 up, 8 in 2024-09-14T04:04:34.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:33 smithi094 ceph-mon[86597]: pgmap v1107: 137 pgs: 2 creating+peering, 30 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:34.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:33 smithi094 ceph-mon[86597]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:34.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:33 smithi094 ceph-mon[86597]: osdmap e992: 8 total, 8 up, 8 in 2024-09-14T04:04:34.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:34 smithi070 ceph-mon[96704]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:34.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:34 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T04:04:34.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:34 smithi070 ceph-mon[96704]: osdmap e993: 8 total, 8 up, 8 in 2024-09-14T04:04:34.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:34 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/2715726776' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:34.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:34 smithi070 ceph-mon[98640]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:34.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:34 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T04:04:34.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:34 smithi070 ceph-mon[98640]: osdmap e993: 8 total, 8 up, 8 in 2024-09-14T04:04:34.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:34 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/2715726776' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:35.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:34 smithi094 ceph-mon[86597]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:35.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:34 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T04:04:35.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:34 smithi094 ceph-mon[86597]: osdmap e993: 8 total, 8 up, 8 in 2024-09-14T04:04:35.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:34 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/2715726776' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:36.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:35 smithi094 ceph-mon[86597]: pgmap v1110: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:36.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:35 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/2715726776' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:36.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:35 smithi094 ceph-mon[86597]: osdmap e994: 8 total, 8 up, 8 in 2024-09-14T04:04:36.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:35 smithi070 ceph-mon[96704]: pgmap v1110: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:36.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:35 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/2715726776' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:36.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:35 smithi070 ceph-mon[96704]: osdmap e994: 8 total, 8 up, 8 in 2024-09-14T04:04:36.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:35 smithi070 ceph-mon[98640]: pgmap v1110: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:36.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:35 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/2715726776' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129317-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:36.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:35 smithi070 ceph-mon[98640]: osdmap e994: 8 total, 8 up, 8 in 2024-09-14T04:04:36.619 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] ClsHello.Filter (3015 ms) 2024-09-14T04:04:36.619 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] 6 tests from ClsHello (18966 ms total) 2024-09-14T04:04:36.619 INFO:tasks.workunit.client.0.smithi070.stdout: 2024-09-14T04:04:36.619 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] Global test environment tear-down 2024-09-14T04:04:36.619 INFO:tasks.workunit.client.0.smithi070.stdout:[==========] 6 tests from 1 test suite ran. (18966 ms total) 2024-09-14T04:04:36.619 INFO:tasks.workunit.client.0.smithi070.stdout:[ PASSED ] 6 tests. 2024-09-14T04:04:36.621 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-14T04:04:36.621 DEBUG:teuthology.orchestra.run.smithi070:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-14T04:04:36.690 INFO:tasks.workunit:Running workunit cls/test_cls_journal.sh... 2024-09-14T04:04:36.691 DEBUG:teuthology.orchestra.run.smithi070:workunit test cls/test_cls_journal.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_journal.sh 2024-09-14T04:04:36.760 INFO:tasks.workunit.client.0.smithi070.stdout:Running main() from gmock_main.cc 2024-09-14T04:04:36.760 INFO:tasks.workunit.client.0.smithi070.stdout:[==========] Running 27 tests from 1 test suite. 2024-09-14T04:04:36.760 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] Global test environment set-up. 2024-09-14T04:04:36.760 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] 27 tests from TestClsJournal 2024-09-14T04:04:38.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:37 smithi094 ceph-mon[86597]: pgmap v1113: 137 pgs: 11 creating+peering, 13 unknown, 113 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T04:04:38.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:37 smithi094 ceph-mon[86597]: osdmap e995: 8 total, 8 up, 8 in 2024-09-14T04:04:38.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:37 smithi070 ceph-mon[96704]: pgmap v1113: 137 pgs: 11 creating+peering, 13 unknown, 113 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T04:04:38.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:37 smithi070 ceph-mon[96704]: osdmap e995: 8 total, 8 up, 8 in 2024-09-14T04:04:38.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:37 smithi070 ceph-mon[98640]: pgmap v1113: 137 pgs: 11 creating+peering, 13 unknown, 113 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T04:04:38.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:37 smithi070 ceph-mon[98640]: osdmap e995: 8 total, 8 up, 8 in 2024-09-14T04:04:38.671 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsJournal.Create 2024-09-14T04:04:38.688 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsJournal.Create (16 ms) 2024-09-14T04:04:38.688 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsJournal.MinimumSet 2024-09-14T04:04:38.693 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsJournal.MinimumSet (5 ms) 2024-09-14T04:04:38.693 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsJournal.MinimumSetStale 2024-09-14T04:04:38.698 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsJournal.MinimumSetStale (5 ms) 2024-09-14T04:04:38.698 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsJournal.MinimumSetOrderConstraint 2024-09-14T04:04:38.701 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsJournal.MinimumSetOrderConstraint (4 ms) 2024-09-14T04:04:38.701 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsJournal.ActiveSet 2024-09-14T04:04:38.703 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsJournal.ActiveSet (2 ms) 2024-09-14T04:04:38.704 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsJournal.ActiveSetStale 2024-09-14T04:04:38.707 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsJournal.ActiveSetStale (4 ms) 2024-09-14T04:04:38.707 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsJournal.CreateDuplicate 2024-09-14T04:04:38.709 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsJournal.CreateDuplicate (2 ms) 2024-09-14T04:04:38.710 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsJournal.GetClient 2024-09-14T04:04:38.712 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsJournal.GetClient (3 ms) 2024-09-14T04:04:38.712 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsJournal.ClientRegister 2024-09-14T04:04:38.715 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsJournal.ClientRegister (3 ms) 2024-09-14T04:04:38.715 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsJournal.ClientRegisterDuplicate 2024-09-14T04:04:38.718 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsJournal.ClientRegisterDuplicate (3 ms) 2024-09-14T04:04:38.718 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsJournal.ClientUpdateData 2024-09-14T04:04:38.722 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsJournal.ClientUpdateData (4 ms) 2024-09-14T04:04:38.722 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsJournal.ClientUpdateState 2024-09-14T04:04:38.727 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsJournal.ClientUpdateState (4 ms) 2024-09-14T04:04:38.727 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsJournal.ClientUnregister 2024-09-14T04:04:38.729 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsJournal.ClientUnregister (3 ms) 2024-09-14T04:04:38.730 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsJournal.ClientUnregisterDNE 2024-09-14T04:04:38.733 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsJournal.ClientUnregisterDNE (4 ms) 2024-09-14T04:04:38.734 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsJournal.ClientUnregisterPruneTags 2024-09-14T04:04:38.826 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsJournal.ClientUnregisterPruneTags (93 ms) 2024-09-14T04:04:38.826 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsJournal.ClientCommit 2024-09-14T04:04:38.829 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsJournal.ClientCommit (3 ms) 2024-09-14T04:04:38.829 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsJournal.ClientCommitInvalid 2024-09-14T04:04:38.832 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsJournal.ClientCommitInvalid (2 ms) 2024-09-14T04:04:38.832 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsJournal.ClientCommitDNE 2024-09-14T04:04:38.833 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsJournal.ClientCommitDNE (1 ms) 2024-09-14T04:04:38.833 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsJournal.ClientList 2024-09-14T04:04:38.869 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsJournal.ClientList (36 ms) 2024-09-14T04:04:38.869 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsJournal.GetNextTagTid 2024-09-14T04:04:38.872 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsJournal.GetNextTagTid (4 ms) 2024-09-14T04:04:38.872 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsJournal.TagCreate 2024-09-14T04:04:38.881 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsJournal.TagCreate (9 ms) 2024-09-14T04:04:38.881 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsJournal.TagCreatePrunesTags 2024-09-14T04:04:38.887 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsJournal.TagCreatePrunesTags (6 ms) 2024-09-14T04:04:38.888 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsJournal.TagList 2024-09-14T04:04:38.973 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsJournal.TagList (85 ms) 2024-09-14T04:04:38.973 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsJournal.GuardAppend 2024-09-14T04:04:38.975 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsJournal.GuardAppend (2 ms) 2024-09-14T04:04:38.975 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsJournal.GuardAppendDNE 2024-09-14T04:04:38.976 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsJournal.GuardAppendDNE (1 ms) 2024-09-14T04:04:38.976 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsJournal.GuardAppendOverflow 2024-09-14T04:04:38.977 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsJournal.GuardAppendOverflow (2 ms) 2024-09-14T04:04:38.978 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsJournal.Append 2024-09-14T04:04:38.983 INFO:tasks.workunit.client.0.smithi070.stdout:padding is not enabled 2024-09-14T04:04:38.983 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsJournal.Append (6 ms) 2024-09-14T04:04:39.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:38 smithi094 ceph-mon[86597]: osdmap e996: 8 total, 8 up, 8 in 2024-09-14T04:04:39.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:38 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/3876201903' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129469-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:39.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:38 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129469-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:39.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:38 smithi070 ceph-mon[96704]: osdmap e996: 8 total, 8 up, 8 in 2024-09-14T04:04:39.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:38 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/3876201903' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129469-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:39.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:38 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129469-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:39.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:38 smithi070 ceph-mon[98640]: osdmap e996: 8 total, 8 up, 8 in 2024-09-14T04:04:39.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:38 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/3876201903' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129469-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:39.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:38 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129469-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:39.678 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] 27 tests from TestClsJournal (312 ms total) 2024-09-14T04:04:39.678 INFO:tasks.workunit.client.0.smithi070.stdout: 2024-09-14T04:04:39.678 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] Global test environment tear-down 2024-09-14T04:04:39.678 INFO:tasks.workunit.client.0.smithi070.stdout:[==========] 27 tests from 1 test suite ran. (2918 ms total) 2024-09-14T04:04:39.678 INFO:tasks.workunit.client.0.smithi070.stdout:[ PASSED ] 27 tests. 2024-09-14T04:04:39.680 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-14T04:04:39.680 DEBUG:teuthology.orchestra.run.smithi070:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-14T04:04:39.748 INFO:tasks.workunit:Running workunit cls/test_cls_lock.sh... 2024-09-14T04:04:39.748 DEBUG:teuthology.orchestra.run.smithi070:workunit test cls/test_cls_lock.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_lock.sh 2024-09-14T04:04:39.818 INFO:tasks.workunit.client.0.smithi070.stdout:Running main() from gmock_main.cc 2024-09-14T04:04:39.818 INFO:tasks.workunit.client.0.smithi070.stdout:[==========] Running 11 tests from 1 test suite. 2024-09-14T04:04:39.818 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] Global test environment set-up. 2024-09-14T04:04:39.818 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] 11 tests from ClsLock 2024-09-14T04:04:39.818 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] ClsLock.TestMultiLocking 2024-09-14T04:04:40.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:39 smithi094 ceph-mon[86597]: pgmap v1116: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:40.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:39 smithi094 ceph-mon[86597]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:40.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:39 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129469-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:40.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:39 smithi094 ceph-mon[86597]: osdmap e997: 8 total, 8 up, 8 in 2024-09-14T04:04:40.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:39 smithi070 ceph-mon[96704]: pgmap v1116: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:40.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:39 smithi070 ceph-mon[96704]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:40.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:39 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129469-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:40.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:39 smithi070 ceph-mon[96704]: osdmap e997: 8 total, 8 up, 8 in 2024-09-14T04:04:40.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:39 smithi070 ceph-mon[98640]: pgmap v1116: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:40.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:39 smithi070 ceph-mon[98640]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:40.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:39 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129469-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:40.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:39 smithi070 ceph-mon[98640]: osdmap e997: 8 total, 8 up, 8 in 2024-09-14T04:04:40.528 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:04:40 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:04:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:04:41.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:40 smithi094 ceph-mon[86597]: osdmap e998: 8 total, 8 up, 8 in 2024-09-14T04:04:41.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:40 smithi070 ceph-mon[98640]: osdmap e998: 8 total, 8 up, 8 in 2024-09-14T04:04:41.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:40 smithi070 ceph-mon[96704]: osdmap e998: 8 total, 8 up, 8 in 2024-09-14T04:04:41.720 INFO:tasks.workunit.client.0.smithi070.stdout:lock: mylock 2024-09-14T04:04:41.721 INFO:tasks.workunit.client.0.smithi070.stdout: lock_type: exclusive 2024-09-14T04:04:41.721 INFO:tasks.workunit.client.0.smithi070.stdout: tag: 2024-09-14T04:04:41.721 INFO:tasks.workunit.client.0.smithi070.stdout: lockers: 2024-09-14T04:04:41.721 INFO:tasks.workunit.client.0.smithi070.stdout: client.44744 expiration=2024-09-14T04:06:41.716582+0000 addr=v1:172.21.15.70:0/3889633370 cookie= 2024-09-14T04:04:41.724 INFO:tasks.workunit.client.0.smithi070.stdout:lock: mylock 2024-09-14T04:04:41.724 INFO:tasks.workunit.client.0.smithi070.stdout: lock_type: shared 2024-09-14T04:04:41.724 INFO:tasks.workunit.client.0.smithi070.stdout: tag: 2024-09-14T04:04:41.725 INFO:tasks.workunit.client.0.smithi070.stdout: lockers: 2024-09-14T04:04:41.725 INFO:tasks.workunit.client.0.smithi070.stdout: client.34833 expiration=0.000000 addr=v1:172.21.15.70:0/1212299641 cookie= 2024-09-14T04:04:41.725 INFO:tasks.workunit.client.0.smithi070.stdout: client.44744 expiration=2024-09-14T04:06:41.723094+0000 addr=v1:172.21.15.70:0/3889633370 cookie= 2024-09-14T04:04:41.726 INFO:tasks.workunit.client.0.smithi070.stdout:lock: mylock 2024-09-14T04:04:41.726 INFO:tasks.workunit.client.0.smithi070.stdout: lock_type: shared 2024-09-14T04:04:41.726 INFO:tasks.workunit.client.0.smithi070.stdout: tag: 2024-09-14T04:04:41.726 INFO:tasks.workunit.client.0.smithi070.stdout: lockers: 2024-09-14T04:04:41.726 INFO:tasks.workunit.client.0.smithi070.stdout: client.34833 expiration=0.000000 addr=v1:172.21.15.70:0/1212299641 cookie= 2024-09-14T04:04:42.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:41 smithi094 ceph-mon[86597]: pgmap v1119: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:42.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:41 smithi094 ceph-mon[86597]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:42.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:41 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T04:04:42.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:41 smithi094 ceph-mon[86597]: osdmap e999: 8 total, 8 up, 8 in 2024-09-14T04:04:42.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:41 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/3889633370' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:42.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:41 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:42.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:41 smithi070 ceph-mon[96704]: pgmap v1119: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:42.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:41 smithi070 ceph-mon[96704]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:42.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:41 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T04:04:42.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:41 smithi070 ceph-mon[96704]: osdmap e999: 8 total, 8 up, 8 in 2024-09-14T04:04:42.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:41 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/3889633370' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:42.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:41 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:42.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:41 smithi070 ceph-mon[98640]: pgmap v1119: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:42.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:41 smithi070 ceph-mon[98640]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:42.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:41 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T04:04:42.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:41 smithi070 ceph-mon[98640]: osdmap e999: 8 total, 8 up, 8 in 2024-09-14T04:04:42.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:41 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/3889633370' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:42.037 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:41 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:42.703 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] ClsLock.TestMultiLocking (2885 ms) 2024-09-14T04:04:42.703 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] ClsLock.TestMeta 2024-09-14T04:04:43.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:42 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:43.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:42 smithi094 ceph-mon[86597]: osdmap e1000: 8 total, 8 up, 8 in 2024-09-14T04:04:43.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:42 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:43.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:42 smithi070 ceph-mon[96704]: osdmap e1000: 8 total, 8 up, 8 in 2024-09-14T04:04:43.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:42 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:43.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:42 smithi070 ceph-mon[98640]: osdmap e1000: 8 total, 8 up, 8 in 2024-09-14T04:04:43.696 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:04:43 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:04:43] "GET /metrics HTTP/1.1" 200 36518 "" "Prometheus/2.51.0" 2024-09-14T04:04:44.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:43 smithi094 ceph-mon[86597]: pgmap v1122: 137 pgs: 10 creating+peering, 20 unknown, 107 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:44.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:43 smithi094 ceph-mon[86597]: osdmap e1001: 8 total, 8 up, 8 in 2024-09-14T04:04:44.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:43 smithi070 ceph-mon[96704]: pgmap v1122: 137 pgs: 10 creating+peering, 20 unknown, 107 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:44.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:43 smithi070 ceph-mon[96704]: osdmap e1001: 8 total, 8 up, 8 in 2024-09-14T04:04:44.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:43 smithi070 ceph-mon[98640]: pgmap v1122: 137 pgs: 10 creating+peering, 20 unknown, 107 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:44.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:43 smithi070 ceph-mon[98640]: osdmap e1001: 8 total, 8 up, 8 in 2024-09-14T04:04:44.741 INFO:tasks.workunit.client.0.smithi070.stdout:lock: mylock 2024-09-14T04:04:44.742 INFO:tasks.workunit.client.0.smithi070.stdout: lock_type: shared 2024-09-14T04:04:44.742 INFO:tasks.workunit.client.0.smithi070.stdout: tag: 2024-09-14T04:04:44.742 INFO:tasks.workunit.client.0.smithi070.stdout: lockers: 2024-09-14T04:04:44.742 INFO:tasks.workunit.client.0.smithi070.stdout: client.34845 expiration=0.000000 addr=v1:172.21.15.70:0/1496891216 cookie= 2024-09-14T04:04:44.743 INFO:tasks.workunit.client.0.smithi070.stdout:lock: mylock 2024-09-14T04:04:44.743 INFO:tasks.workunit.client.0.smithi070.stdout: lock_type: exclusive 2024-09-14T04:04:44.743 INFO:tasks.workunit.client.0.smithi070.stdout: tag: new_tag 2024-09-14T04:04:44.744 INFO:tasks.workunit.client.0.smithi070.stdout: lockers: 2024-09-14T04:04:44.744 INFO:tasks.workunit.client.0.smithi070.stdout: client.34839 expiration=0.000000 addr=v1:172.21.15.70:0/1962300618 cookie= 2024-09-14T04:04:45.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:44 smithi094 ceph-mon[86597]: osdmap e1002: 8 total, 8 up, 8 in 2024-09-14T04:04:45.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:44 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/1962300618' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:45.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:44 smithi070 ceph-mon[96704]: osdmap e1002: 8 total, 8 up, 8 in 2024-09-14T04:04:45.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:44 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/1962300618' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:45.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:44 smithi070 ceph-mon[98640]: osdmap e1002: 8 total, 8 up, 8 in 2024-09-14T04:04:45.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:44 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/1962300618' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:45.721 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] ClsLock.TestMeta (3019 ms) 2024-09-14T04:04:45.721 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] ClsLock.TestCookie 2024-09-14T04:04:46.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:45 smithi094 ceph-mon[86597]: pgmap v1125: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:46.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:45 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/1962300618' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:46.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:45 smithi094 ceph-mon[86597]: osdmap e1003: 8 total, 8 up, 8 in 2024-09-14T04:04:46.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:45 smithi094 ceph-mon[86597]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:46.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:45 smithi070 ceph-mon[96704]: pgmap v1125: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:46.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:45 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/1962300618' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:46.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:45 smithi070 ceph-mon[96704]: osdmap e1003: 8 total, 8 up, 8 in 2024-09-14T04:04:46.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:45 smithi070 ceph-mon[96704]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:46.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:45 smithi070 ceph-mon[98640]: pgmap v1125: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:46.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:45 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/1962300618' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:46.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:45 smithi070 ceph-mon[98640]: osdmap e1003: 8 total, 8 up, 8 in 2024-09-14T04:04:46.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:45 smithi070 ceph-mon[98640]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:47.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:46 smithi094 ceph-mon[86597]: osdmap e1004: 8 total, 8 up, 8 in 2024-09-14T04:04:47.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:46 smithi070 ceph-mon[96704]: osdmap e1004: 8 total, 8 up, 8 in 2024-09-14T04:04:47.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:46 smithi070 ceph-mon[98640]: osdmap e1004: 8 total, 8 up, 8 in 2024-09-14T04:04:48.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:47 smithi094 ceph-mon[86597]: pgmap v1128: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:48.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:47 smithi094 ceph-mon[86597]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:48.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:47 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T04:04:48.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:47 smithi094 ceph-mon[86597]: osdmap e1005: 8 total, 8 up, 8 in 2024-09-14T04:04:48.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:47 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/1313394833' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:48.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:47 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:48.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:47 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:48.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:47 smithi070 ceph-mon[98640]: pgmap v1128: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:48.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:47 smithi070 ceph-mon[98640]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:48.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:47 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T04:04:48.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:47 smithi070 ceph-mon[98640]: osdmap e1005: 8 total, 8 up, 8 in 2024-09-14T04:04:48.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:47 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/1313394833' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:48.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:47 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:48.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:47 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:48.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:47 smithi070 ceph-mon[96704]: pgmap v1128: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:48.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:47 smithi070 ceph-mon[96704]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:48.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:47 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T04:04:48.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:47 smithi070 ceph-mon[96704]: osdmap e1005: 8 total, 8 up, 8 in 2024-09-14T04:04:48.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:47 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/1313394833' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:48.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:47 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:48.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:47 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:48.340 INFO:tasks.workunit.client.0.smithi070.stdout:lock: mylock 2024-09-14T04:04:48.340 INFO:tasks.workunit.client.0.smithi070.stdout: lock_type: exclusive 2024-09-14T04:04:48.340 INFO:tasks.workunit.client.0.smithi070.stdout: tag: 2024-09-14T04:04:48.340 INFO:tasks.workunit.client.0.smithi070.stdout: lockers: 2024-09-14T04:04:48.342 INFO:tasks.workunit.client.0.smithi070.stdout:lock: mylock 2024-09-14T04:04:48.343 INFO:tasks.workunit.client.0.smithi070.stdout: lock_type: shared 2024-09-14T04:04:48.343 INFO:tasks.workunit.client.0.smithi070.stdout: tag: 2024-09-14T04:04:48.343 INFO:tasks.workunit.client.0.smithi070.stdout: lockers: 2024-09-14T04:04:48.343 INFO:tasks.workunit.client.0.smithi070.stdout: client.44768 expiration=0.000000 addr=v1:172.21.15.70:0/1313394833 cookie= 2024-09-14T04:04:48.343 INFO:tasks.workunit.client.0.smithi070.stdout: client.44768 expiration=0.000000 addr=v1:172.21.15.70:0/1313394833 cookie=new cookie 2024-09-14T04:04:48.740 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] ClsLock.TestCookie (3018 ms) 2024-09-14T04:04:48.740 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] ClsLock.TestMultipleLocks 2024-09-14T04:04:49.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:48 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:49.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:48 smithi094 ceph-mon[86597]: osdmap e1006: 8 total, 8 up, 8 in 2024-09-14T04:04:49.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:48 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:49.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:48 smithi070 ceph-mon[98640]: osdmap e1006: 8 total, 8 up, 8 in 2024-09-14T04:04:49.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:48 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:49.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:48 smithi070 ceph-mon[96704]: osdmap e1006: 8 total, 8 up, 8 in 2024-09-14T04:04:50.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:49 smithi094 ceph-mon[86597]: pgmap v1131: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:50.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:49 smithi094 ceph-mon[86597]: osdmap e1007: 8 total, 8 up, 8 in 2024-09-14T04:04:50.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:49 smithi070 ceph-mon[98640]: pgmap v1131: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:50.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:49 smithi070 ceph-mon[98640]: osdmap e1007: 8 total, 8 up, 8 in 2024-09-14T04:04:50.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:49 smithi070 ceph-mon[96704]: pgmap v1131: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:50.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:49 smithi070 ceph-mon[96704]: osdmap e1007: 8 total, 8 up, 8 in 2024-09-14T04:04:50.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:04:50 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:04:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:04:51.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:50 smithi094 ceph-mon[86597]: osdmap e1008: 8 total, 8 up, 8 in 2024-09-14T04:04:51.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:50 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/565923247' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:51.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:50 smithi070 ceph-mon[98640]: osdmap e1008: 8 total, 8 up, 8 in 2024-09-14T04:04:51.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:50 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/565923247' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:51.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:50 smithi070 ceph-mon[96704]: osdmap e1008: 8 total, 8 up, 8 in 2024-09-14T04:04:51.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:50 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/565923247' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:51.755 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] ClsLock.TestMultipleLocks (3016 ms) 2024-09-14T04:04:51.756 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] ClsLock.TestLockDuration 2024-09-14T04:04:52.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:51 smithi094 ceph-mon[86597]: pgmap v1134: 137 pgs: 8 creating+peering, 24 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:52.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:51 smithi094 ceph-mon[86597]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:52.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:51 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/565923247' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:52.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:51 smithi094 ceph-mon[86597]: osdmap e1009: 8 total, 8 up, 8 in 2024-09-14T04:04:52.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:51 smithi070 ceph-mon[98640]: pgmap v1134: 137 pgs: 8 creating+peering, 24 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:52.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:51 smithi070 ceph-mon[98640]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:52.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:51 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/565923247' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:52.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:51 smithi070 ceph-mon[98640]: osdmap e1009: 8 total, 8 up, 8 in 2024-09-14T04:04:52.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:51 smithi070 ceph-mon[96704]: pgmap v1134: 137 pgs: 8 creating+peering, 24 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:52.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:51 smithi070 ceph-mon[96704]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:52.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:51 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/565923247' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:52.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:51 smithi070 ceph-mon[96704]: osdmap e1009: 8 total, 8 up, 8 in 2024-09-14T04:04:53.028 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:52 smithi094 ceph-mon[86597]: osdmap e1010: 8 total, 8 up, 8 in 2024-09-14T04:04:53.029 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:52 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:04:53.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:52 smithi070 ceph-mon[96704]: osdmap e1010: 8 total, 8 up, 8 in 2024-09-14T04:04:53.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:52 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:04:53.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:52 smithi070 ceph-mon[98640]: osdmap e1010: 8 total, 8 up, 8 in 2024-09-14T04:04:53.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:52 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:04:53.756 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:04:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:04:53] "GET /metrics HTTP/1.1" 200 36514 "" "Prometheus/2.51.0" 2024-09-14T04:04:54.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:53 smithi094 ceph-mon[86597]: pgmap v1137: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:54.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:53 smithi094 ceph-mon[86597]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:54.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:53 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T04:04:54.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:53 smithi094 ceph-mon[86597]: osdmap e1011: 8 total, 8 up, 8 in 2024-09-14T04:04:54.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:53 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/535480482' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:54.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:53 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:04:54.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:53 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:04:54.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:53 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:04:54.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:53 smithi070 ceph-mon[98640]: pgmap v1137: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:54.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:53 smithi070 ceph-mon[98640]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:54.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:53 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T04:04:54.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:53 smithi070 ceph-mon[98640]: osdmap e1011: 8 total, 8 up, 8 in 2024-09-14T04:04:54.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:53 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/535480482' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:54.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:53 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:04:54.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:53 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:04:54.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:53 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:04:54.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:53 smithi070 ceph-mon[96704]: pgmap v1137: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:54.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:53 smithi070 ceph-mon[96704]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:54.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:53 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T04:04:54.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:53 smithi070 ceph-mon[96704]: osdmap e1011: 8 total, 8 up, 8 in 2024-09-14T04:04:54.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:53 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/535480482' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:54.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:53 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:04:54.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:53 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:04:54.037 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:53 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:04:55.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:54 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/535480482' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:55.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:54 smithi094 ceph-mon[86597]: osdmap e1012: 8 total, 8 up, 8 in 2024-09-14T04:04:55.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:54 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/535480482' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:55.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:54 smithi070 ceph-mon[96704]: osdmap e1012: 8 total, 8 up, 8 in 2024-09-14T04:04:55.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:54 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/535480482' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:55.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:54 smithi070 ceph-mon[98640]: osdmap e1012: 8 total, 8 up, 8 in 2024-09-14T04:04:56.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:55 smithi094 ceph-mon[86597]: pgmap v1140: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:56.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:55 smithi094 ceph-mon[86597]: osdmap e1013: 8 total, 8 up, 8 in 2024-09-14T04:04:56.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:55 smithi070 ceph-mon[96704]: pgmap v1140: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:56.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:55 smithi070 ceph-mon[96704]: osdmap e1013: 8 total, 8 up, 8 in 2024-09-14T04:04:56.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:55 smithi070 ceph-mon[98640]: pgmap v1140: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:04:56.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:55 smithi070 ceph-mon[98640]: osdmap e1013: 8 total, 8 up, 8 in 2024-09-14T04:04:58.027 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:57 smithi094 ceph-mon[86597]: pgmap v1142: 137 pgs: 4 creating+peering, 7 unknown, 126 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:04:58.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:57 smithi070 ceph-mon[96704]: pgmap v1142: 137 pgs: 4 creating+peering, 7 unknown, 126 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:04:58.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:57 smithi070 ceph-mon[98640]: pgmap v1142: 137 pgs: 4 creating+peering, 7 unknown, 126 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:04:59.784 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] ClsLock.TestLockDuration (8028 ms) 2024-09-14T04:04:59.784 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] ClsLock.TestAssertLocked 2024-09-14T04:05:00.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:04:59 smithi094 ceph-mon[86597]: pgmap v1143: 137 pgs: 4 creating+peering, 133 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:05:00.035 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:04:59 smithi070 ceph-mon[96704]: pgmap v1143: 137 pgs: 4 creating+peering, 133 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:05:00.036 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:04:59 smithi070 ceph-mon[98640]: pgmap v1143: 137 pgs: 4 creating+peering, 133 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:05:00.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:05:00 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:05:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:05:01.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:00 smithi070 ceph-mon[98640]: osdmap e1014: 8 total, 8 up, 8 in 2024-09-14T04:05:01.036 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:00 smithi070 ceph-mon[96704]: osdmap e1014: 8 total, 8 up, 8 in 2024-09-14T04:05:01.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:00 smithi094 ceph-mon[86597]: osdmap e1014: 8 total, 8 up, 8 in 2024-09-14T04:05:02.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:01 smithi094 ceph-mon[86597]: pgmap v1145: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:02.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:01 smithi094 ceph-mon[86597]: osdmap e1015: 8 total, 8 up, 8 in 2024-09-14T04:05:02.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:01 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/2096372780' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:02.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:01 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:02.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:01 smithi070 ceph-mon[96704]: pgmap v1145: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:02.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:01 smithi070 ceph-mon[96704]: osdmap e1015: 8 total, 8 up, 8 in 2024-09-14T04:05:02.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:01 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/2096372780' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:02.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:01 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:02.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:01 smithi070 ceph-mon[98640]: pgmap v1145: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:02.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:01 smithi070 ceph-mon[98640]: osdmap e1015: 8 total, 8 up, 8 in 2024-09-14T04:05:02.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:01 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/2096372780' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:02.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:01 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:02.806 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] ClsLock.TestAssertLocked (3022 ms) 2024-09-14T04:05:02.806 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] ClsLock.TestSetCookie 2024-09-14T04:05:03.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:02 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:05:03.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:02 smithi094 ceph-mon[86597]: osdmap e1016: 8 total, 8 up, 8 in 2024-09-14T04:05:03.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:02 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:03.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:02 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:05:03.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:02 smithi070 ceph-mon[96704]: osdmap e1016: 8 total, 8 up, 8 in 2024-09-14T04:05:03.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:02 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:03.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:02 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:05:03.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:02 smithi070 ceph-mon[98640]: osdmap e1016: 8 total, 8 up, 8 in 2024-09-14T04:05:03.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:02 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:03.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:05:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:05:03] "GET /metrics HTTP/1.1" 200 36518 "" "Prometheus/2.51.0" 2024-09-14T04:05:04.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:03 smithi094 ceph-mon[86597]: pgmap v1148: 137 pgs: 15 creating+peering, 11 unknown, 111 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:04.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:03 smithi094 ceph-mon[86597]: osdmap e1017: 8 total, 8 up, 8 in 2024-09-14T04:05:04.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:03 smithi070 ceph-mon[98640]: pgmap v1148: 137 pgs: 15 creating+peering, 11 unknown, 111 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:04.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:03 smithi070 ceph-mon[98640]: osdmap e1017: 8 total, 8 up, 8 in 2024-09-14T04:05:04.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:03 smithi070 ceph-mon[96704]: pgmap v1148: 137 pgs: 15 creating+peering, 11 unknown, 111 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:04.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:03 smithi070 ceph-mon[96704]: osdmap e1017: 8 total, 8 up, 8 in 2024-09-14T04:05:05.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:04 smithi094 ceph-mon[86597]: osdmap e1018: 8 total, 8 up, 8 in 2024-09-14T04:05:05.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:04 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/2060649929' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:05.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:04 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:05.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:04 smithi070 ceph-mon[96704]: osdmap e1018: 8 total, 8 up, 8 in 2024-09-14T04:05:05.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:04 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/2060649929' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:05.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:04 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:05.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:04 smithi070 ceph-mon[98640]: osdmap e1018: 8 total, 8 up, 8 in 2024-09-14T04:05:05.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:04 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/2060649929' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:05.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:04 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:05.817 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] ClsLock.TestSetCookie (3012 ms) 2024-09-14T04:05:05.817 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] ClsLock.TestRenew 2024-09-14T04:05:06.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:05 smithi094 ceph-mon[86597]: pgmap v1151: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:06.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:05 smithi094 ceph-mon[86597]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:05:06.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:05 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:05:06.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:05 smithi094 ceph-mon[86597]: osdmap e1019: 8 total, 8 up, 8 in 2024-09-14T04:05:06.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:05 smithi070 ceph-mon[96704]: pgmap v1151: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:06.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:05 smithi070 ceph-mon[96704]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:05:06.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:05 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:05:06.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:05 smithi070 ceph-mon[96704]: osdmap e1019: 8 total, 8 up, 8 in 2024-09-14T04:05:06.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:05 smithi070 ceph-mon[98640]: pgmap v1151: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:06.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:05 smithi070 ceph-mon[98640]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:05:06.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:05 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:05:06.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:05 smithi070 ceph-mon[98640]: osdmap e1019: 8 total, 8 up, 8 in 2024-09-14T04:05:07.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:06 smithi094 ceph-mon[86597]: osdmap e1020: 8 total, 8 up, 8 in 2024-09-14T04:05:07.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:06 smithi070 ceph-mon[96704]: osdmap e1020: 8 total, 8 up, 8 in 2024-09-14T04:05:07.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:06 smithi070 ceph-mon[98640]: osdmap e1020: 8 total, 8 up, 8 in 2024-09-14T04:05:08.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:07 smithi094 ceph-mon[86597]: pgmap v1154: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:08.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:07 smithi094 ceph-mon[86597]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:05:08.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:07 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T04:05:08.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:07 smithi094 ceph-mon[86597]: osdmap e1021: 8 total, 8 up, 8 in 2024-09-14T04:05:08.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:07 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/1493464379' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:08.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:07 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:08.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:07 smithi070 ceph-mon[96704]: pgmap v1154: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:08.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:07 smithi070 ceph-mon[96704]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:05:08.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:07 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T04:05:08.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:07 smithi070 ceph-mon[96704]: osdmap e1021: 8 total, 8 up, 8 in 2024-09-14T04:05:08.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:07 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/1493464379' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:08.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:07 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:08.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:07 smithi070 ceph-mon[98640]: pgmap v1154: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:08.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:07 smithi070 ceph-mon[98640]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:05:08.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:07 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T04:05:08.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:07 smithi070 ceph-mon[98640]: osdmap e1021: 8 total, 8 up, 8 in 2024-09-14T04:05:08.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:07 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/1493464379' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:08.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:07 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:09.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:08 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:05:09.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:08 smithi094 ceph-mon[86597]: osdmap e1022: 8 total, 8 up, 8 in 2024-09-14T04:05:09.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:08 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:05:09.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:08 smithi070 ceph-mon[96704]: osdmap e1022: 8 total, 8 up, 8 in 2024-09-14T04:05:09.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:08 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:05:09.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:08 smithi070 ceph-mon[98640]: osdmap e1022: 8 total, 8 up, 8 in 2024-09-14T04:05:10.178 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:09 smithi094 ceph-mon[86597]: pgmap v1157: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:10.178 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:09 smithi094 ceph-mon[86597]: osdmap e1023: 8 total, 8 up, 8 in 2024-09-14T04:05:10.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:09 smithi070 ceph-mon[96704]: pgmap v1157: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:10.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:09 smithi070 ceph-mon[96704]: osdmap e1023: 8 total, 8 up, 8 in 2024-09-14T04:05:10.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:09 smithi070 ceph-mon[98640]: pgmap v1157: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:10.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:09 smithi070 ceph-mon[98640]: osdmap e1023: 8 total, 8 up, 8 in 2024-09-14T04:05:10.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:05:10 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:05:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:05:12.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:11 smithi094 ceph-mon[86597]: pgmap v1159: 137 pgs: 4 creating+peering, 12 unknown, 121 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:12.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:11 smithi070 ceph-mon[96704]: pgmap v1159: 137 pgs: 4 creating+peering, 12 unknown, 121 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:12.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:11 smithi070 ceph-mon[98640]: pgmap v1159: 137 pgs: 4 creating+peering, 12 unknown, 121 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:13.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:05:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:05:13] "GET /metrics HTTP/1.1" 200 36518 "" "Prometheus/2.51.0" 2024-09-14T04:05:14.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:13 smithi094 ceph-mon[86597]: pgmap v1160: 137 pgs: 137 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:05:14.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:13 smithi070 ceph-mon[96704]: pgmap v1160: 137 pgs: 137 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:05:14.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:13 smithi070 ceph-mon[98640]: pgmap v1160: 137 pgs: 137 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:05:16.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:15 smithi094 ceph-mon[86597]: pgmap v1161: 137 pgs: 137 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 138 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:05:16.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:15 smithi070 ceph-mon[96704]: pgmap v1161: 137 pgs: 137 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 138 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:05:16.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:15 smithi070 ceph-mon[98640]: pgmap v1161: 137 pgs: 137 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 138 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:05:18.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:17 smithi094 ceph-mon[86597]: pgmap v1162: 137 pgs: 137 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 122 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:05:18.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:17 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:18.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:17 smithi070 ceph-mon[96704]: pgmap v1162: 137 pgs: 137 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 122 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:05:18.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:17 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:18.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:17 smithi070 ceph-mon[98640]: pgmap v1162: 137 pgs: 137 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 122 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:05:18.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:17 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:20.213 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:19 smithi094 ceph-mon[86597]: pgmap v1163: 137 pgs: 137 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 102 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:05:20.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:19 smithi070 ceph-mon[96704]: pgmap v1163: 137 pgs: 137 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 102 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:05:20.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:19 smithi070 ceph-mon[98640]: pgmap v1163: 137 pgs: 137 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 102 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:05:20.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:05:20 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:05:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:05:22.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:21 smithi094 ceph-mon[86597]: pgmap v1164: 137 pgs: 137 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 89 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:05:22.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:21 smithi070 ceph-mon[96704]: pgmap v1164: 137 pgs: 137 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 89 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:05:22.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:21 smithi070 ceph-mon[98640]: pgmap v1164: 137 pgs: 137 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 89 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:05:23.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:05:23 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:05:23] "GET /metrics HTTP/1.1" 200 36517 "" "Prometheus/2.51.0" 2024-09-14T04:05:24.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:23 smithi094 ceph-mon[86597]: pgmap v1165: 137 pgs: 137 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 B/s wr, 1 op/s 2024-09-14T04:05:24.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:23 smithi070 ceph-mon[98640]: pgmap v1165: 137 pgs: 137 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 B/s wr, 1 op/s 2024-09-14T04:05:24.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:23 smithi070 ceph-mon[96704]: pgmap v1165: 137 pgs: 137 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 B/s wr, 1 op/s 2024-09-14T04:05:26.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:25 smithi094 ceph-mon[86597]: pgmap v1166: 137 pgs: 137 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:05:26.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:25 smithi070 ceph-mon[96704]: pgmap v1166: 137 pgs: 137 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:05:26.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:25 smithi070 ceph-mon[98640]: pgmap v1166: 137 pgs: 137 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:05:26.869 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] ClsLock.TestRenew (21051 ms) 2024-09-14T04:05:26.869 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] ClsLock.TestExclusiveEphemeralBasic 2024-09-14T04:05:28.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:27 smithi094 ceph-mon[86597]: pgmap v1167: 137 pgs: 137 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:05:28.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:27 smithi094 ceph-mon[86597]: osdmap e1024: 8 total, 8 up, 8 in 2024-09-14T04:05:28.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:27 smithi070 ceph-mon[96704]: pgmap v1167: 137 pgs: 137 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:05:28.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:27 smithi070 ceph-mon[96704]: osdmap e1024: 8 total, 8 up, 8 in 2024-09-14T04:05:28.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:27 smithi070 ceph-mon[98640]: pgmap v1167: 137 pgs: 137 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:05:28.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:27 smithi070 ceph-mon[98640]: osdmap e1024: 8 total, 8 up, 8 in 2024-09-14T04:05:29.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:28 smithi094 ceph-mon[86597]: osdmap e1025: 8 total, 8 up, 8 in 2024-09-14T04:05:29.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:28 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/1330976091' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:29.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:28 smithi070 ceph-mon[96704]: osdmap e1025: 8 total, 8 up, 8 in 2024-09-14T04:05:29.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:28 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/1330976091' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:29.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:28 smithi070 ceph-mon[98640]: osdmap e1025: 8 total, 8 up, 8 in 2024-09-14T04:05:29.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:28 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/1330976091' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:30.214 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:29 smithi094 ceph-mon[86597]: pgmap v1170: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:30.214 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:29 smithi094 ceph-mon[86597]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:05:30.214 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:29 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/1330976091' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:05:30.214 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:29 smithi094 ceph-mon[86597]: osdmap e1026: 8 total, 8 up, 8 in 2024-09-14T04:05:30.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:29 smithi070 ceph-mon[96704]: pgmap v1170: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:30.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:29 smithi070 ceph-mon[96704]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:05:30.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:29 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/1330976091' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:05:30.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:29 smithi070 ceph-mon[96704]: osdmap e1026: 8 total, 8 up, 8 in 2024-09-14T04:05:30.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:29 smithi070 ceph-mon[98640]: pgmap v1170: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:30.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:29 smithi070 ceph-mon[98640]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:05:30.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:29 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/1330976091' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:05:30.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:29 smithi070 ceph-mon[98640]: osdmap e1026: 8 total, 8 up, 8 in 2024-09-14T04:05:30.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:05:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:05:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:05:31.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:30 smithi094 ceph-mon[86597]: osdmap e1027: 8 total, 8 up, 8 in 2024-09-14T04:05:31.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:30 smithi070 ceph-mon[96704]: osdmap e1027: 8 total, 8 up, 8 in 2024-09-14T04:05:31.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:30 smithi070 ceph-mon[98640]: osdmap e1027: 8 total, 8 up, 8 in 2024-09-14T04:05:32.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:31 smithi094 ceph-mon[86597]: pgmap v1173: 137 pgs: 8 creating+peering, 16 unknown, 113 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:32.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:31 smithi094 ceph-mon[86597]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:05:32.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:31 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T04:05:32.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:31 smithi070 ceph-mon[96704]: pgmap v1173: 137 pgs: 8 creating+peering, 16 unknown, 113 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:32.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:31 smithi070 ceph-mon[96704]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:05:32.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:31 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T04:05:32.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:31 smithi070 ceph-mon[98640]: pgmap v1173: 137 pgs: 8 creating+peering, 16 unknown, 113 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:32.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:31 smithi070 ceph-mon[98640]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:05:32.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:31 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T04:05:33.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:32 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:33.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:32 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:33.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:32 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:33.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:05:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:05:33] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.51.0" 2024-09-14T04:05:33.902 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] ClsLock.TestExclusiveEphemeralBasic (7034 ms) 2024-09-14T04:05:33.903 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] ClsLock.TestExclusiveEphemeralStealEphemeral 2024-09-14T04:05:34.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:33 smithi094 ceph-mon[86597]: pgmap v1174: 137 pgs: 8 creating+peering, 129 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:34.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:33 smithi070 ceph-mon[96704]: pgmap v1174: 137 pgs: 8 creating+peering, 129 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:34.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:33 smithi070 ceph-mon[98640]: pgmap v1174: 137 pgs: 8 creating+peering, 129 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:35.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:34 smithi094 ceph-mon[86597]: osdmap e1028: 8 total, 8 up, 8 in 2024-09-14T04:05:35.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:34 smithi070 ceph-mon[96704]: osdmap e1028: 8 total, 8 up, 8 in 2024-09-14T04:05:35.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:34 smithi070 ceph-mon[98640]: osdmap e1028: 8 total, 8 up, 8 in 2024-09-14T04:05:36.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:35 smithi094 ceph-mon[86597]: pgmap v1176: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:36.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:35 smithi094 ceph-mon[86597]: osdmap e1029: 8 total, 8 up, 8 in 2024-09-14T04:05:36.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:35 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/131067192' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:36.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:35 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:36.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:35 smithi070 ceph-mon[98640]: pgmap v1176: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:36.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:35 smithi070 ceph-mon[98640]: osdmap e1029: 8 total, 8 up, 8 in 2024-09-14T04:05:36.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:35 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/131067192' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:36.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:35 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:36.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:35 smithi070 ceph-mon[96704]: pgmap v1176: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:36.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:35 smithi070 ceph-mon[96704]: osdmap e1029: 8 total, 8 up, 8 in 2024-09-14T04:05:36.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:35 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/131067192' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:36.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:35 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:37.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:36 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:05:37.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:36 smithi094 ceph-mon[86597]: osdmap e1030: 8 total, 8 up, 8 in 2024-09-14T04:05:37.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:36 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:05:37.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:36 smithi070 ceph-mon[98640]: osdmap e1030: 8 total, 8 up, 8 in 2024-09-14T04:05:37.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:36 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:05:37.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:36 smithi070 ceph-mon[96704]: osdmap e1030: 8 total, 8 up, 8 in 2024-09-14T04:05:38.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:37 smithi094 ceph-mon[86597]: pgmap v1179: 137 pgs: 4 creating+peering, 20 unknown, 113 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:38.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:37 smithi094 ceph-mon[86597]: osdmap e1031: 8 total, 8 up, 8 in 2024-09-14T04:05:38.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:37 smithi070 ceph-mon[96704]: pgmap v1179: 137 pgs: 4 creating+peering, 20 unknown, 113 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:38.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:37 smithi070 ceph-mon[96704]: osdmap e1031: 8 total, 8 up, 8 in 2024-09-14T04:05:38.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:37 smithi070 ceph-mon[98640]: pgmap v1179: 137 pgs: 4 creating+peering, 20 unknown, 113 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:38.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:37 smithi070 ceph-mon[98640]: osdmap e1031: 8 total, 8 up, 8 in 2024-09-14T04:05:40.214 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:39 smithi094 ceph-mon[86597]: pgmap v1181: 137 pgs: 4 creating+peering, 10 unknown, 123 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:40.289 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:39 smithi070 ceph-mon[96704]: pgmap v1181: 137 pgs: 4 creating+peering, 10 unknown, 123 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:40.290 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:39 smithi070 ceph-mon[98640]: pgmap v1181: 137 pgs: 4 creating+peering, 10 unknown, 123 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:40.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:05:40 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:05:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:05:41.939 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] ClsLock.TestExclusiveEphemeralStealEphemeral (8036 ms) 2024-09-14T04:05:41.939 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] ClsLock.TestExclusiveEphemeralStealExclusive 2024-09-14T04:05:42.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:41 smithi094 ceph-mon[86597]: pgmap v1182: 137 pgs: 137 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:05:42.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:41 smithi070 ceph-mon[96704]: pgmap v1182: 137 pgs: 137 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:05:42.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:41 smithi070 ceph-mon[98640]: pgmap v1182: 137 pgs: 137 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:05:43.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:42 smithi094 ceph-mon[86597]: osdmap e1032: 8 total, 8 up, 8 in 2024-09-14T04:05:43.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:42 smithi070 ceph-mon[96704]: osdmap e1032: 8 total, 8 up, 8 in 2024-09-14T04:05:43.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:42 smithi070 ceph-mon[98640]: osdmap e1032: 8 total, 8 up, 8 in 2024-09-14T04:05:43.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:05:43 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:05:43] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.51.0" 2024-09-14T04:05:44.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:43 smithi094 ceph-mon[86597]: pgmap v1184: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:44.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:43 smithi094 ceph-mon[86597]: osdmap e1033: 8 total, 8 up, 8 in 2024-09-14T04:05:44.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:43 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/3745200569' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:44.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:43 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:44.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:43 smithi070 ceph-mon[96704]: pgmap v1184: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:44.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:43 smithi070 ceph-mon[96704]: osdmap e1033: 8 total, 8 up, 8 in 2024-09-14T04:05:44.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:43 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/3745200569' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:44.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:43 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:44.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:43 smithi070 ceph-mon[98640]: pgmap v1184: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:44.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:43 smithi070 ceph-mon[98640]: osdmap e1033: 8 total, 8 up, 8 in 2024-09-14T04:05:44.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:43 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/3745200569' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:44.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:43 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:45.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:44 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:05:45.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:44 smithi094 ceph-mon[86597]: osdmap e1034: 8 total, 8 up, 8 in 2024-09-14T04:05:45.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:44 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:05:45.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:44 smithi070 ceph-mon[96704]: osdmap e1034: 8 total, 8 up, 8 in 2024-09-14T04:05:45.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:44 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129524-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:05:45.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:44 smithi070 ceph-mon[98640]: osdmap e1034: 8 total, 8 up, 8 in 2024-09-14T04:05:46.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:45 smithi094 ceph-mon[86597]: pgmap v1187: 137 pgs: 8 creating+peering, 24 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:46.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:45 smithi094 ceph-mon[86597]: osdmap e1035: 8 total, 8 up, 8 in 2024-09-14T04:05:46.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:45 smithi070 ceph-mon[98640]: pgmap v1187: 137 pgs: 8 creating+peering, 24 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:46.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:45 smithi070 ceph-mon[98640]: osdmap e1035: 8 total, 8 up, 8 in 2024-09-14T04:05:46.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:45 smithi070 ceph-mon[96704]: pgmap v1187: 137 pgs: 8 creating+peering, 24 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:46.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:45 smithi070 ceph-mon[96704]: osdmap e1035: 8 total, 8 up, 8 in 2024-09-14T04:05:48.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:47 smithi094 ceph-mon[86597]: pgmap v1189: 137 pgs: 8 creating+peering, 7 unknown, 122 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:05:48.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:47 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:48.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:47 smithi070 ceph-mon[96704]: pgmap v1189: 137 pgs: 8 creating+peering, 7 unknown, 122 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:05:48.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:47 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:48.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:47 smithi070 ceph-mon[98640]: pgmap v1189: 137 pgs: 8 creating+peering, 7 unknown, 122 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:05:48.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:47 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:49.970 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] ClsLock.TestExclusiveEphemeralStealExclusive (8031 ms) 2024-09-14T04:05:49.970 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] 11 tests from ClsLock (70152 ms total) 2024-09-14T04:05:49.970 INFO:tasks.workunit.client.0.smithi070.stdout: 2024-09-14T04:05:49.970 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] Global test environment tear-down 2024-09-14T04:05:49.970 INFO:tasks.workunit.client.0.smithi070.stdout:[==========] 11 tests from 1 test suite ran. (70152 ms total) 2024-09-14T04:05:49.970 INFO:tasks.workunit.client.0.smithi070.stdout:[ PASSED ] 11 tests. 2024-09-14T04:05:49.972 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-14T04:05:49.973 DEBUG:teuthology.orchestra.run.smithi070:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-14T04:05:50.044 INFO:tasks.workunit:Running workunit cls/test_cls_log.sh... 2024-09-14T04:05:50.044 DEBUG:teuthology.orchestra.run.smithi070:workunit test cls/test_cls_log.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_log.sh 2024-09-14T04:05:50.116 INFO:tasks.workunit.client.0.smithi070.stdout:Running main() from gmock_main.cc 2024-09-14T04:05:50.117 INFO:tasks.workunit.client.0.smithi070.stdout:[==========] Running 4 tests from 1 test suite. 2024-09-14T04:05:50.117 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] Global test environment set-up. 2024-09-14T04:05:50.117 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] 4 tests from cls_log 2024-09-14T04:05:50.117 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] cls_log.test_log_add_same_time 2024-09-14T04:05:50.213 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:49 smithi094 ceph-mon[86597]: pgmap v1190: 137 pgs: 8 creating+peering, 129 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:05:50.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:49 smithi070 ceph-mon[96704]: pgmap v1190: 137 pgs: 8 creating+peering, 129 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:05:50.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:49 smithi070 ceph-mon[98640]: pgmap v1190: 137 pgs: 8 creating+peering, 129 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:05:50.526 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:05:50 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:05:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:05:51.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:50 smithi094 ceph-mon[86597]: osdmap e1036: 8 total, 8 up, 8 in 2024-09-14T04:05:51.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:50 smithi070 ceph-mon[98640]: osdmap e1036: 8 total, 8 up, 8 in 2024-09-14T04:05:51.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:50 smithi070 ceph-mon[96704]: osdmap e1036: 8 total, 8 up, 8 in 2024-09-14T04:05:52.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:51 smithi094 ceph-mon[86597]: pgmap v1192: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:52.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:51 smithi094 ceph-mon[86597]: osdmap e1037: 8 total, 8 up, 8 in 2024-09-14T04:05:52.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:51 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/4071641333' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:52.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:51 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:52.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:51 smithi070 ceph-mon[98640]: pgmap v1192: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:52.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:51 smithi070 ceph-mon[98640]: osdmap e1037: 8 total, 8 up, 8 in 2024-09-14T04:05:52.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:51 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/4071641333' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:52.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:51 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:52.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:51 smithi070 ceph-mon[96704]: pgmap v1192: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:52.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:51 smithi070 ceph-mon[96704]: osdmap e1037: 8 total, 8 up, 8 in 2024-09-14T04:05:52.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:51 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/4071641333' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:52.287 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:51 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:52.988 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] cls_log.test_log_add_same_time (2872 ms) 2024-09-14T04:05:52.988 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] cls_log.test_log_add_different_time 2024-09-14T04:05:53.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:52 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:05:53.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:52 smithi094 ceph-mon[86597]: osdmap e1038: 8 total, 8 up, 8 in 2024-09-14T04:05:53.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:52 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:05:53.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:52 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:05:53.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:52 smithi070 ceph-mon[96704]: osdmap e1038: 8 total, 8 up, 8 in 2024-09-14T04:05:53.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:52 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:05:53.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:52 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:05:53.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:52 smithi070 ceph-mon[98640]: osdmap e1038: 8 total, 8 up, 8 in 2024-09-14T04:05:53.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:52 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:05:53.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:05:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:05:53] "GET /metrics HTTP/1.1" 200 36517 "" "Prometheus/2.51.0" 2024-09-14T04:05:54.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:53 smithi094 ceph-mon[86597]: pgmap v1195: 137 pgs: 12 creating+peering, 18 unknown, 107 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:54.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:53 smithi094 ceph-mon[86597]: osdmap e1039: 8 total, 8 up, 8 in 2024-09-14T04:05:54.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:53 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:05:54.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:53 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:05:54.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:53 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:05:54.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:53 smithi070 ceph-mon[98640]: pgmap v1195: 137 pgs: 12 creating+peering, 18 unknown, 107 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:54.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:53 smithi070 ceph-mon[98640]: osdmap e1039: 8 total, 8 up, 8 in 2024-09-14T04:05:54.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:53 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:05:54.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:53 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:05:54.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:53 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:05:54.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:53 smithi070 ceph-mon[96704]: pgmap v1195: 137 pgs: 12 creating+peering, 18 unknown, 107 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:54.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:53 smithi070 ceph-mon[96704]: osdmap e1039: 8 total, 8 up, 8 in 2024-09-14T04:05:54.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:53 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:05:54.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:53 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:05:54.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:53 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:05:55.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:54 smithi094 ceph-mon[86597]: osdmap e1040: 8 total, 8 up, 8 in 2024-09-14T04:05:55.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:54 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/3292992244' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:55.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:54 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:55.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:54 smithi070 ceph-mon[96704]: osdmap e1040: 8 total, 8 up, 8 in 2024-09-14T04:05:55.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:54 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/3292992244' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:55.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:54 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:55.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:54 smithi070 ceph-mon[98640]: osdmap e1040: 8 total, 8 up, 8 in 2024-09-14T04:05:55.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:54 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/3292992244' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:55.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:54 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:56.004 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] cls_log.test_log_add_different_time (3015 ms) 2024-09-14T04:05:56.004 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] cls_log.trim_by_time 2024-09-14T04:05:56.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:55 smithi094 ceph-mon[86597]: pgmap v1198: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:56.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:55 smithi094 ceph-mon[86597]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:05:56.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:55 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:05:56.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:55 smithi094 ceph-mon[86597]: osdmap e1041: 8 total, 8 up, 8 in 2024-09-14T04:05:56.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:55 smithi070 ceph-mon[96704]: pgmap v1198: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:56.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:55 smithi070 ceph-mon[96704]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:05:56.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:55 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:05:56.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:55 smithi070 ceph-mon[96704]: osdmap e1041: 8 total, 8 up, 8 in 2024-09-14T04:05:56.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:55 smithi070 ceph-mon[98640]: pgmap v1198: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:56.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:55 smithi070 ceph-mon[98640]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:05:56.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:55 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:05:56.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:55 smithi070 ceph-mon[98640]: osdmap e1041: 8 total, 8 up, 8 in 2024-09-14T04:05:57.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:57 smithi094 ceph-mon[86597]: osdmap e1042: 8 total, 8 up, 8 in 2024-09-14T04:05:57.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:57 smithi070 ceph-mon[96704]: osdmap e1042: 8 total, 8 up, 8 in 2024-09-14T04:05:57.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:57 smithi070 ceph-mon[98640]: osdmap e1042: 8 total, 8 up, 8 in 2024-09-14T04:05:58.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:58 smithi094 ceph-mon[86597]: pgmap v1201: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:58.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:58 smithi094 ceph-mon[86597]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:05:58.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:58 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T04:05:58.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:58 smithi094 ceph-mon[86597]: osdmap e1043: 8 total, 8 up, 8 in 2024-09-14T04:05:58.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:58 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/3595048255' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:58.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:58 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:58.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:58 smithi070 ceph-mon[96704]: pgmap v1201: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:58.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:58 smithi070 ceph-mon[96704]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:05:58.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:58 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T04:05:58.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:58 smithi070 ceph-mon[96704]: osdmap e1043: 8 total, 8 up, 8 in 2024-09-14T04:05:58.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:58 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/3595048255' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:58.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:58 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:58.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:58 smithi070 ceph-mon[98640]: pgmap v1201: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:05:58.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:58 smithi070 ceph-mon[98640]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:05:58.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:58 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T04:05:58.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:58 smithi070 ceph-mon[98640]: osdmap e1043: 8 total, 8 up, 8 in 2024-09-14T04:05:58.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:58 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/3595048255' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:58.287 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:58 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:59.025 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] cls_log.trim_by_time (3022 ms) 2024-09-14T04:05:59.025 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] cls_log.trim_by_marker 2024-09-14T04:05:59.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:59 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:05:59.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:05:59 smithi094 ceph-mon[86597]: osdmap e1044: 8 total, 8 up, 8 in 2024-09-14T04:05:59.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:59 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:05:59.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:05:59 smithi070 ceph-mon[96704]: osdmap e1044: 8 total, 8 up, 8 in 2024-09-14T04:05:59.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:59 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:05:59.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:05:59 smithi070 ceph-mon[98640]: osdmap e1044: 8 total, 8 up, 8 in 2024-09-14T04:06:00.276 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:00 smithi094 ceph-mon[86597]: pgmap v1204: 137 pgs: 1 creating+peering, 31 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:00.277 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:00 smithi094 ceph-mon[86597]: osdmap e1045: 8 total, 8 up, 8 in 2024-09-14T04:06:00.277 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:06:00 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:06:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:06:00.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:00 smithi070 ceph-mon[96704]: pgmap v1204: 137 pgs: 1 creating+peering, 31 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:00.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:00 smithi070 ceph-mon[96704]: osdmap e1045: 8 total, 8 up, 8 in 2024-09-14T04:06:00.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:00 smithi070 ceph-mon[98640]: pgmap v1204: 137 pgs: 1 creating+peering, 31 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:00.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:00 smithi070 ceph-mon[98640]: osdmap e1045: 8 total, 8 up, 8 in 2024-09-14T04:06:01.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:01 smithi070 ceph-mon[96704]: osdmap e1046: 8 total, 8 up, 8 in 2024-09-14T04:06:01.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:01 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/1550322052' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:01.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:01 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:01.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:01 smithi070 ceph-mon[98640]: osdmap e1046: 8 total, 8 up, 8 in 2024-09-14T04:06:01.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:01 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/1550322052' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:01.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:01 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:01.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:01 smithi094 ceph-mon[86597]: osdmap e1046: 8 total, 8 up, 8 in 2024-09-14T04:06:01.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:01 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/1550322052' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:01.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:01 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:02.042 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] cls_log.trim_by_marker (3017 ms) 2024-09-14T04:06:02.042 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] 4 tests from cls_log (11926 ms total) 2024-09-14T04:06:02.042 INFO:tasks.workunit.client.0.smithi070.stdout: 2024-09-14T04:06:02.043 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] Global test environment tear-down 2024-09-14T04:06:02.043 INFO:tasks.workunit.client.0.smithi070.stdout:[==========] 4 tests from 1 test suite ran. (11926 ms total) 2024-09-14T04:06:02.043 INFO:tasks.workunit.client.0.smithi070.stdout:[ PASSED ] 4 tests. 2024-09-14T04:06:02.045 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-14T04:06:02.045 DEBUG:teuthology.orchestra.run.smithi070:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-14T04:06:02.115 INFO:tasks.workunit:Running workunit cls/test_cls_numops.sh... 2024-09-14T04:06:02.115 DEBUG:teuthology.orchestra.run.smithi070:workunit test cls/test_cls_numops.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_numops.sh 2024-09-14T04:06:02.185 INFO:tasks.workunit.client.0.smithi070.stdout:Running main() from gmock_main.cc 2024-09-14T04:06:02.186 INFO:tasks.workunit.client.0.smithi070.stdout:[==========] Running 4 tests from 1 test suite. 2024-09-14T04:06:02.186 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] Global test environment set-up. 2024-09-14T04:06:02.186 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] 4 tests from ClsNumOps 2024-09-14T04:06:02.186 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] ClsNumOps.Add 2024-09-14T04:06:02.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:02 smithi070 ceph-mon[96704]: pgmap v1207: 137 pgs: 9 creating+peering, 23 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:02.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:02 smithi070 ceph-mon[96704]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:06:02.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:02 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:06:02.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:02 smithi070 ceph-mon[96704]: osdmap e1047: 8 total, 8 up, 8 in 2024-09-14T04:06:02.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:02 smithi070 ceph-mon[98640]: pgmap v1207: 137 pgs: 9 creating+peering, 23 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:02.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:02 smithi070 ceph-mon[98640]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:06:02.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:02 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:06:02.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:02 smithi070 ceph-mon[98640]: osdmap e1047: 8 total, 8 up, 8 in 2024-09-14T04:06:02.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:02 smithi094 ceph-mon[86597]: pgmap v1207: 137 pgs: 9 creating+peering, 23 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:02.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:02 smithi094 ceph-mon[86597]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:06:02.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:02 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-129860-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:06:02.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:02 smithi094 ceph-mon[86597]: osdmap e1047: 8 total, 8 up, 8 in 2024-09-14T04:06:03.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:03 smithi070 ceph-mon[96704]: osdmap e1048: 8 total, 8 up, 8 in 2024-09-14T04:06:03.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:03 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:06:03.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:03 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:06:03.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:03 smithi070 ceph-mon[98640]: osdmap e1048: 8 total, 8 up, 8 in 2024-09-14T04:06:03.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:03 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:06:03.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:03 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:06:03.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:03 smithi094 ceph-mon[86597]: osdmap e1048: 8 total, 8 up, 8 in 2024-09-14T04:06:03.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:03 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:06:03.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:03 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:06:03.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:06:03 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:06:03] "GET /metrics HTTP/1.1" 200 34149 "" "Prometheus/2.51.0" 2024-09-14T04:06:04.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:04 smithi094 ceph-mon[86597]: pgmap v1210: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:04.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:04 smithi094 ceph-mon[86597]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:06:04.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:04 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T04:06:04.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:04 smithi094 ceph-mon[86597]: osdmap e1049: 8 total, 8 up, 8 in 2024-09-14T04:06:04.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:04 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/1540761806' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130047-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:04.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:04 smithi070 ceph-mon[98640]: pgmap v1210: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:04.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:04 smithi070 ceph-mon[98640]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:06:04.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:04 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T04:06:04.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:04 smithi070 ceph-mon[98640]: osdmap e1049: 8 total, 8 up, 8 in 2024-09-14T04:06:04.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:04 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/1540761806' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130047-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:04.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:04 smithi070 ceph-mon[96704]: pgmap v1210: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:04.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:04 smithi070 ceph-mon[96704]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:06:04.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:04 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T04:06:04.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:04 smithi070 ceph-mon[96704]: osdmap e1049: 8 total, 8 up, 8 in 2024-09-14T04:06:04.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:04 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/1540761806' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130047-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:05.065 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] ClsNumOps.Add (2879 ms) 2024-09-14T04:06:05.065 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] ClsNumOps.Sub 2024-09-14T04:06:05.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:05 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/1540761806' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130047-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:06:05.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:05 smithi094 ceph-mon[86597]: osdmap e1050: 8 total, 8 up, 8 in 2024-09-14T04:06:05.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:05 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/1540761806' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130047-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:06:05.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:05 smithi070 ceph-mon[96704]: osdmap e1050: 8 total, 8 up, 8 in 2024-09-14T04:06:05.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:05 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/1540761806' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130047-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:06:05.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:05 smithi070 ceph-mon[98640]: osdmap e1050: 8 total, 8 up, 8 in 2024-09-14T04:06:06.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:06 smithi094 ceph-mon[86597]: pgmap v1213: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:06.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:06 smithi094 ceph-mon[86597]: osdmap e1051: 8 total, 8 up, 8 in 2024-09-14T04:06:06.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:06 smithi070 ceph-mon[96704]: pgmap v1213: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:06.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:06 smithi070 ceph-mon[96704]: osdmap e1051: 8 total, 8 up, 8 in 2024-09-14T04:06:06.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:06 smithi070 ceph-mon[98640]: pgmap v1213: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:06.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:06 smithi070 ceph-mon[98640]: osdmap e1051: 8 total, 8 up, 8 in 2024-09-14T04:06:07.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:07 smithi094 ceph-mon[86597]: osdmap e1052: 8 total, 8 up, 8 in 2024-09-14T04:06:07.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:07 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/3091639341' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130047-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:07.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:07 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130047-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:07.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:07 smithi070 ceph-mon[96704]: osdmap e1052: 8 total, 8 up, 8 in 2024-09-14T04:06:07.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:07 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/3091639341' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130047-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:07.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:07 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130047-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:07.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:07 smithi070 ceph-mon[98640]: osdmap e1052: 8 total, 8 up, 8 in 2024-09-14T04:06:07.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:07 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/3091639341' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130047-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:07.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:07 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130047-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:08.081 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] ClsNumOps.Sub (3016 ms) 2024-09-14T04:06:08.081 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] ClsNumOps.Mul 2024-09-14T04:06:08.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:08 smithi094 ceph-mon[86597]: pgmap v1216: 137 pgs: 9 creating+peering, 23 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:08.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:08 smithi094 ceph-mon[86597]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:06:08.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:08 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130047-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:06:08.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:08 smithi094 ceph-mon[86597]: osdmap e1053: 8 total, 8 up, 8 in 2024-09-14T04:06:08.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:08 smithi070 ceph-mon[96704]: pgmap v1216: 137 pgs: 9 creating+peering, 23 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:08.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:08 smithi070 ceph-mon[96704]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:06:08.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:08 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130047-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:06:08.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:08 smithi070 ceph-mon[96704]: osdmap e1053: 8 total, 8 up, 8 in 2024-09-14T04:06:08.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:08 smithi070 ceph-mon[98640]: pgmap v1216: 137 pgs: 9 creating+peering, 23 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:08.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:08 smithi070 ceph-mon[98640]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:06:08.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:08 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130047-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:06:08.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:08 smithi070 ceph-mon[98640]: osdmap e1053: 8 total, 8 up, 8 in 2024-09-14T04:06:09.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:09 smithi094 ceph-mon[86597]: osdmap e1054: 8 total, 8 up, 8 in 2024-09-14T04:06:09.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:09 smithi070 ceph-mon[96704]: osdmap e1054: 8 total, 8 up, 8 in 2024-09-14T04:06:09.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:09 smithi070 ceph-mon[98640]: osdmap e1054: 8 total, 8 up, 8 in 2024-09-14T04:06:10.428 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:06:10 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:06:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:06:10.429 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:10 smithi094 ceph-mon[86597]: pgmap v1219: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:10.429 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:10 smithi094 ceph-mon[86597]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:06:10.429 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:10 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T04:06:10.429 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:10 smithi094 ceph-mon[86597]: osdmap e1055: 8 total, 8 up, 8 in 2024-09-14T04:06:10.429 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:10 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/454232172' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130047-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:10.429 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:10 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130047-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:10.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:10 smithi070 ceph-mon[98640]: pgmap v1219: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:10.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:10 smithi070 ceph-mon[98640]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:06:10.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:10 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T04:06:10.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:10 smithi070 ceph-mon[98640]: osdmap e1055: 8 total, 8 up, 8 in 2024-09-14T04:06:10.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:10 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/454232172' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130047-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:10.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:10 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130047-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:10.540 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:10 smithi070 ceph-mon[96704]: pgmap v1219: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:10.540 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:10 smithi070 ceph-mon[96704]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:06:10.540 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:10 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T04:06:10.540 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:10 smithi070 ceph-mon[96704]: osdmap e1055: 8 total, 8 up, 8 in 2024-09-14T04:06:10.540 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:10 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/454232172' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130047-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:10.540 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:10 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130047-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:11.099 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] ClsNumOps.Mul (3018 ms) 2024-09-14T04:06:11.099 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] ClsNumOps.Div 2024-09-14T04:06:11.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:11 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130047-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:06:11.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:11 smithi094 ceph-mon[86597]: osdmap e1056: 8 total, 8 up, 8 in 2024-09-14T04:06:11.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:11 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130047-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:06:11.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:11 smithi070 ceph-mon[98640]: osdmap e1056: 8 total, 8 up, 8 in 2024-09-14T04:06:11.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:11 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130047-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:06:11.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:11 smithi070 ceph-mon[96704]: osdmap e1056: 8 total, 8 up, 8 in 2024-09-14T04:06:12.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:12 smithi094 ceph-mon[86597]: pgmap v1222: 137 pgs: 7 creating+peering, 21 unknown, 109 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:12.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:12 smithi094 ceph-mon[86597]: osdmap e1057: 8 total, 8 up, 8 in 2024-09-14T04:06:12.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:12 smithi070 ceph-mon[98640]: pgmap v1222: 137 pgs: 7 creating+peering, 21 unknown, 109 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:12.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:12 smithi070 ceph-mon[98640]: osdmap e1057: 8 total, 8 up, 8 in 2024-09-14T04:06:12.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:12 smithi070 ceph-mon[96704]: pgmap v1222: 137 pgs: 7 creating+peering, 21 unknown, 109 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:12.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:12 smithi070 ceph-mon[96704]: osdmap e1057: 8 total, 8 up, 8 in 2024-09-14T04:06:13.406 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:13 smithi070 ceph-mon[96704]: osdmap e1058: 8 total, 8 up, 8 in 2024-09-14T04:06:13.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:13 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/275486999' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130047-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:13.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:13 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130047-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:13.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:13 smithi070 ceph-mon[98640]: osdmap e1058: 8 total, 8 up, 8 in 2024-09-14T04:06:13.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:13 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/275486999' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130047-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:13.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:13 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130047-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:13.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:13 smithi094 ceph-mon[86597]: osdmap e1058: 8 total, 8 up, 8 in 2024-09-14T04:06:13.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:13 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/275486999' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130047-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:13.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:13 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130047-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:13.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:06:13 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:06:13] "GET /metrics HTTP/1.1" 200 34149 "" "Prometheus/2.51.0" 2024-09-14T04:06:14.114 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] ClsNumOps.Div (3015 ms) 2024-09-14T04:06:14.114 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] 4 tests from ClsNumOps (11928 ms total) 2024-09-14T04:06:14.114 INFO:tasks.workunit.client.0.smithi070.stdout: 2024-09-14T04:06:14.114 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] Global test environment tear-down 2024-09-14T04:06:14.114 INFO:tasks.workunit.client.0.smithi070.stdout:[==========] 4 tests from 1 test suite ran. (11928 ms total) 2024-09-14T04:06:14.114 INFO:tasks.workunit.client.0.smithi070.stdout:[ PASSED ] 4 tests. 2024-09-14T04:06:14.116 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-14T04:06:14.116 DEBUG:teuthology.orchestra.run.smithi070:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-14T04:06:14.184 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2024-09-14T04:06:14.185 DEBUG:teuthology.orchestra.run.smithi070:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2024-09-14T04:06:14.255 INFO:tasks.workunit.client.0.smithi070.stdout:Running main() from gmock_main.cc 2024-09-14T04:06:14.255 INFO:tasks.workunit.client.0.smithi070.stdout:[==========] Running 70 tests from 1 test suite. 2024-09-14T04:06:14.255 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] Global test environment set-up. 2024-09-14T04:06:14.255 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] 70 tests from TestClsRbd 2024-09-14T04:06:14.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:14 smithi094 ceph-mon[86597]: pgmap v1225: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:14.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:14 smithi094 ceph-mon[86597]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:06:14.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:14 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130047-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:06:14.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:14 smithi094 ceph-mon[86597]: osdmap e1059: 8 total, 8 up, 8 in 2024-09-14T04:06:14.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:14 smithi070 ceph-mon[96704]: pgmap v1225: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:14.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:14 smithi070 ceph-mon[96704]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:06:14.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:14 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130047-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:06:14.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:14 smithi070 ceph-mon[96704]: osdmap e1059: 8 total, 8 up, 8 in 2024-09-14T04:06:14.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:14 smithi070 ceph-mon[98640]: pgmap v1225: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:14.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:14 smithi070 ceph-mon[98640]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:06:14.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:14 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130047-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:06:14.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:14 smithi070 ceph-mon[98640]: osdmap e1059: 8 total, 8 up, 8 in 2024-09-14T04:06:15.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:15 smithi094 ceph-mon[86597]: osdmap e1060: 8 total, 8 up, 8 in 2024-09-14T04:06:15.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:15 smithi070 ceph-mon[96704]: osdmap e1060: 8 total, 8 up, 8 in 2024-09-14T04:06:15.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:15 smithi070 ceph-mon[98640]: osdmap e1060: 8 total, 8 up, 8 in 2024-09-14T04:06:16.125 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.get_all_features 2024-09-14T04:06:16.137 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.get_all_features (12 ms) 2024-09-14T04:06:16.137 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.copyup 2024-09-14T04:06:16.313 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.copyup (176 ms) 2024-09-14T04:06:16.314 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.sparse_copyup 2024-09-14T04:06:16.323 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.sparse_copyup (10 ms) 2024-09-14T04:06:16.323 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.get_and_set_id 2024-09-14T04:06:16.330 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.get_and_set_id (7 ms) 2024-09-14T04:06:16.330 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.add_remove_child 2024-09-14T04:06:16.340 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.add_remove_child (10 ms) 2024-09-14T04:06:16.340 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.directory_methods 2024-09-14T04:06:16.364 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.directory_methods (24 ms) 2024-09-14T04:06:16.364 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.create 2024-09-14T04:06:16.376 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.create (12 ms) 2024-09-14T04:06:16.377 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.get_features 2024-09-14T04:06:16.378 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.get_features (2 ms) 2024-09-14T04:06:16.378 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.get_object_prefix 2024-09-14T04:06:16.379 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.get_object_prefix (1 ms) 2024-09-14T04:06:16.380 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2024-09-14T04:06:16.381 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.get_create_timestamp (2 ms) 2024-09-14T04:06:16.381 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2024-09-14T04:06:16.383 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.get_access_timestamp (2 ms) 2024-09-14T04:06:16.383 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2024-09-14T04:06:16.384 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.get_modify_timestamp (1 ms) 2024-09-14T04:06:16.384 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.get_data_pool 2024-09-14T04:06:16.387 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.get_data_pool (3 ms) 2024-09-14T04:06:16.387 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.get_size 2024-09-14T04:06:16.391 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.get_size (4 ms) 2024-09-14T04:06:16.391 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.set_size 2024-09-14T04:06:16.395 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.set_size (4 ms) 2024-09-14T04:06:16.395 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.protection_status 2024-09-14T04:06:16.411 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.protection_status (16 ms) 2024-09-14T04:06:16.411 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.snapshot_limits 2024-09-14T04:06:16.418 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.snapshot_limits (6 ms) 2024-09-14T04:06:16.418 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.parents_v1 2024-09-14T04:06:16.456 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.parents_v1 (39 ms) 2024-09-14T04:06:16.456 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.parents_v2 2024-09-14T04:06:16.485 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.parents_v2 (29 ms) 2024-09-14T04:06:16.485 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.snapshots 2024-09-14T04:06:16.505 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.snapshots (20 ms) 2024-09-14T04:06:16.505 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.snapid_race 2024-09-14T04:06:16.508 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.snapid_race (3 ms) 2024-09-14T04:06:16.509 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.stripingv2 2024-09-14T04:06:16.517 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.stripingv2 (9 ms) 2024-09-14T04:06:16.517 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.object_map_save 2024-09-14T04:06:16.518 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.object_map_save (1 ms) 2024-09-14T04:06:16.518 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.object_map_resize 2024-09-14T04:06:16.522 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.object_map_resize (4 ms) 2024-09-14T04:06:16.523 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.object_map_update 2024-09-14T04:06:16.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:16 smithi094 ceph-mon[86597]: pgmap v1228: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:16.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:16 smithi094 ceph-mon[86597]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:06:16.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:16 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T04:06:16.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:16 smithi094 ceph-mon[86597]: osdmap e1061: 8 total, 8 up, 8 in 2024-09-14T04:06:16.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:16 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/373313597' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130153-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:16.528 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.object_map_update (5 ms) 2024-09-14T04:06:16.528 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2024-09-14T04:06:16.528 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2024-09-14T04:06:16.528 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2024-09-14T04:06:16.532 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.object_map_snap_add (4 ms) 2024-09-14T04:06:16.532 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2024-09-14T04:06:16.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:16 smithi070 ceph-mon[98640]: pgmap v1228: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:16.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:16 smithi070 ceph-mon[98640]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:06:16.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:16 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T04:06:16.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:16 smithi070 ceph-mon[98640]: osdmap e1061: 8 total, 8 up, 8 in 2024-09-14T04:06:16.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:16 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/373313597' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130153-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:16.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:16 smithi070 ceph-mon[96704]: pgmap v1228: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:16.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:16 smithi070 ceph-mon[96704]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:06:16.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:16 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T04:06:16.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:16 smithi070 ceph-mon[96704]: osdmap e1061: 8 total, 8 up, 8 in 2024-09-14T04:06:16.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:16 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/373313597' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130153-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:16.537 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.object_map_snap_remove (4 ms) 2024-09-14T04:06:16.537 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.flags 2024-09-14T04:06:16.540 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.flags (4 ms) 2024-09-14T04:06:16.540 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.metadata 2024-09-14T04:06:16.564 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.metadata (24 ms) 2024-09-14T04:06:16.564 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.set_features 2024-09-14T04:06:16.570 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.set_features (7 ms) 2024-09-14T04:06:16.570 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.mirror 2024-09-14T04:06:16.600 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.mirror (29 ms) 2024-09-14T04:06:16.600 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.mirror_image 2024-09-14T04:06:16.612 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.mirror_image (13 ms) 2024-09-14T04:06:16.612 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.mirror_image_status 2024-09-14T04:06:17.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:17 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/373313597' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130153-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:06:17.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:17 smithi094 ceph-mon[86597]: osdmap e1062: 8 total, 8 up, 8 in 2024-09-14T04:06:17.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:17 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/373313597' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130153-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:06:17.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:17 smithi070 ceph-mon[96704]: osdmap e1062: 8 total, 8 up, 8 in 2024-09-14T04:06:17.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:17 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/373313597' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi070-130153-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:06:17.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:17 smithi070 ceph-mon[98640]: osdmap e1062: 8 total, 8 up, 8 in 2024-09-14T04:06:18.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:18 smithi094 ceph-mon[86597]: pgmap v1231: 137 pgs: 1 creating+peering, 21 unknown, 115 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:06:18.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:18 smithi094 ceph-mon[86597]: osdmap e1063: 8 total, 8 up, 8 in 2024-09-14T04:06:18.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:18 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:06:18.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:18 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:06:18.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:18 smithi070 ceph-mon[96704]: pgmap v1231: 137 pgs: 1 creating+peering, 21 unknown, 115 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:06:18.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:18 smithi070 ceph-mon[96704]: osdmap e1063: 8 total, 8 up, 8 in 2024-09-14T04:06:18.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:18 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:06:18.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:18 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:06:18.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:18 smithi070 ceph-mon[98640]: pgmap v1231: 137 pgs: 1 creating+peering, 21 unknown, 115 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-14T04:06:18.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:18 smithi070 ceph-mon[98640]: osdmap e1063: 8 total, 8 up, 8 in 2024-09-14T04:06:18.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:18 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' 2024-09-14T04:06:18.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:18 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:06:18.712 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.mirror_image_status (2100 ms) 2024-09-14T04:06:18.713 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.mirror_image_map 2024-09-14T04:06:18.787 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.mirror_image_map (75 ms) 2024-09-14T04:06:18.787 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.mirror_instances 2024-09-14T04:06:18.793 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.mirror_instances (6 ms) 2024-09-14T04:06:18.794 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2024-09-14T04:06:18.804 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.mirror_snapshot (11 ms) 2024-09-14T04:06:18.804 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.group_dir_list 2024-09-14T04:06:18.806 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.group_dir_list (2 ms) 2024-09-14T04:06:18.806 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.group_dir_add 2024-09-14T04:06:18.808 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.group_dir_add (2 ms) 2024-09-14T04:06:18.808 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2024-09-14T04:06:18.811 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.dir_add_already_existing (2 ms) 2024-09-14T04:06:18.811 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.group_dir_rename 2024-09-14T04:06:18.816 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.group_dir_rename (6 ms) 2024-09-14T04:06:18.816 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.group_dir_remove 2024-09-14T04:06:18.819 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.group_dir_remove (3 ms) 2024-09-14T04:06:18.819 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2024-09-14T04:06:18.823 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (4 ms) 2024-09-14T04:06:18.823 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.group_image_add 2024-09-14T04:06:18.825 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.group_image_add (2 ms) 2024-09-14T04:06:18.825 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.group_image_remove 2024-09-14T04:06:18.828 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.group_image_remove (2 ms) 2024-09-14T04:06:18.828 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.group_image_list 2024-09-14T04:06:18.830 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.group_image_list (3 ms) 2024-09-14T04:06:18.830 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.group_image_clean 2024-09-14T04:06:18.834 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.group_image_clean (4 ms) 2024-09-14T04:06:18.834 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.image_group_add 2024-09-14T04:06:18.836 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.image_group_add (2 ms) 2024-09-14T04:06:18.836 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.image_group_remove 2024-09-14T04:06:18.839 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.image_group_remove (3 ms) 2024-09-14T04:06:18.839 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.image_group_get 2024-09-14T04:06:18.841 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.image_group_get (2 ms) 2024-09-14T04:06:18.841 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2024-09-14T04:06:18.843 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2024-09-14T04:06:18.843 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2024-09-14T04:06:18.845 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (1 ms) 2024-09-14T04:06:18.845 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2024-09-14T04:06:18.847 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (3 ms) 2024-09-14T04:06:18.847 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2024-09-14T04:06:18.851 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (3 ms) 2024-09-14T04:06:18.851 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.group_snap_set 2024-09-14T04:06:18.852 INFO:tasks.workunit.client.0.smithi070.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/17.2.7-1629-geb16b97b/rpm/el9/BUILD/ceph-17.2.7-1629-geb16b97b/src/test/cls_rbd/test_cls_rbd.cc:2681: Failure 2024-09-14T04:06:18.852 INFO:tasks.workunit.client.0.smithi070.stdout:Expected equality of these values: 2024-09-14T04:06:18.852 INFO:tasks.workunit.client.0.smithi070.stdout: 1U 2024-09-14T04:06:18.852 INFO:tasks.workunit.client.0.smithi070.stdout: Which is: 1 2024-09-14T04:06:18.853 INFO:tasks.workunit.client.0.smithi070.stdout: keys.size() 2024-09-14T04:06:18.853 INFO:tasks.workunit.client.0.smithi070.stdout: Which is: 3 2024-09-14T04:06:18.853 INFO:tasks.workunit.client.0.smithi070.stdout:[ FAILED ] TestClsRbd.group_snap_set (2 ms) 2024-09-14T04:06:18.853 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.group_snap_list 2024-09-14T04:06:18.855 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.group_snap_list (3 ms) 2024-09-14T04:06:18.855 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2024-09-14T04:06:18.872 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (16 ms) 2024-09-14T04:06:18.872 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2024-09-14T04:06:19.066 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (195 ms) 2024-09-14T04:06:19.066 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.group_snap_remove 2024-09-14T04:06:19.068 INFO:tasks.workunit.client.0.smithi070.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/17.2.7-1629-geb16b97b/rpm/el9/BUILD/ceph-17.2.7-1629-geb16b97b/src/test/cls_rbd/test_cls_rbd.cc:2792: Failure 2024-09-14T04:06:19.068 INFO:tasks.workunit.client.0.smithi070.stdout:Expected equality of these values: 2024-09-14T04:06:19.069 INFO:tasks.workunit.client.0.smithi070.stdout: 1U 2024-09-14T04:06:19.069 INFO:tasks.workunit.client.0.smithi070.stdout: Which is: 1 2024-09-14T04:06:19.069 INFO:tasks.workunit.client.0.smithi070.stdout: keys.size() 2024-09-14T04:06:19.069 INFO:tasks.workunit.client.0.smithi070.stdout: Which is: 3 2024-09-14T04:06:19.069 INFO:tasks.workunit.client.0.smithi070.stdout:[ FAILED ] TestClsRbd.group_snap_remove (2 ms) 2024-09-14T04:06:19.069 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2024-09-14T04:06:19.070 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2024-09-14T04:06:19.070 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.trash_methods 2024-09-14T04:06:19.077 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.trash_methods (6 ms) 2024-09-14T04:06:19.077 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.op_features 2024-09-14T04:06:19.082 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.op_features (6 ms) 2024-09-14T04:06:19.083 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.clone_parent 2024-09-14T04:06:19.101 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.clone_parent (18 ms) 2024-09-14T04:06:19.101 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2024-09-14T04:06:19.109 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.clone_parent_ns (9 ms) 2024-09-14T04:06:19.109 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.clone_child 2024-09-14T04:06:19.119 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.clone_child (10 ms) 2024-09-14T04:06:19.119 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.namespace_methods 2024-09-14T04:06:19.125 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.namespace_methods (6 ms) 2024-09-14T04:06:19.126 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.migration 2024-09-14T04:06:19.135 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.migration (10 ms) 2024-09-14T04:06:19.135 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.migration_v1 2024-09-14T04:06:19.143 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.migration_v1 (8 ms) 2024-09-14T04:06:19.143 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2024-09-14T04:06:20.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:20 smithi094 ceph-mon[86597]: pgmap v1233: 137 pgs: 1 creating+peering, 13 unknown, 123 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 370 KiB/s rd, 222 KiB/s wr, 620 op/s 2024-09-14T04:06:20.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:06:20 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:06:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:06:20.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:20 smithi070 ceph-mon[96704]: pgmap v1233: 137 pgs: 1 creating+peering, 13 unknown, 123 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 370 KiB/s rd, 222 KiB/s wr, 620 op/s 2024-09-14T04:06:20.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:20 smithi070 ceph-mon[98640]: pgmap v1233: 137 pgs: 1 creating+peering, 13 unknown, 123 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 370 KiB/s rd, 222 KiB/s wr, 620 op/s 2024-09-14T04:06:21.141 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1998 ms) 2024-09-14T04:06:21.142 INFO:tasks.workunit.client.0.smithi070.stdout:[ RUN ] TestClsRbd.sparsify 2024-09-14T04:06:21.184 INFO:tasks.workunit.client.0.smithi070.stdout:[ OK ] TestClsRbd.sparsify (43 ms) 2024-09-14T04:06:21.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:21 smithi094 ceph-mon[86597]: osdmap e1064: 8 total, 8 up, 8 in 2024-09-14T04:06:21.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:21 smithi070 ceph-mon[96704]: osdmap e1064: 8 total, 8 up, 8 in 2024-09-14T04:06:21.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:21 smithi070 ceph-mon[98640]: osdmap e1064: 8 total, 8 up, 8 in 2024-09-14T04:06:22.148 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] 70 tests from TestClsRbd (5059 ms total) 2024-09-14T04:06:22.149 INFO:tasks.workunit.client.0.smithi070.stdout: 2024-09-14T04:06:22.149 INFO:tasks.workunit.client.0.smithi070.stdout:[----------] Global test environment tear-down 2024-09-14T04:06:22.149 INFO:tasks.workunit.client.0.smithi070.stdout:[==========] 70 tests from 1 test suite ran. (7894 ms total) 2024-09-14T04:06:22.149 INFO:tasks.workunit.client.0.smithi070.stdout:[ PASSED ] 68 tests. 2024-09-14T04:06:22.149 INFO:tasks.workunit.client.0.smithi070.stdout:[ FAILED ] 2 tests, listed below: 2024-09-14T04:06:22.149 INFO:tasks.workunit.client.0.smithi070.stdout:[ FAILED ] TestClsRbd.group_snap_set 2024-09-14T04:06:22.149 INFO:tasks.workunit.client.0.smithi070.stdout:[ FAILED ] TestClsRbd.group_snap_remove 2024-09-14T04:06:22.149 INFO:tasks.workunit.client.0.smithi070.stdout: 2024-09-14T04:06:22.149 INFO:tasks.workunit.client.0.smithi070.stdout: 2 FAILED TESTS 2024-09-14T04:06:22.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:06:22.151 INFO:tasks.workunit:Stopping ['cls'] on client.0... 2024-09-14T04:06:22.151 DEBUG:teuthology.orchestra.run.smithi070:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-09-14T04:06:22.406 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:22 smithi070 ceph-mon[96704]: pgmap v1235: 137 pgs: 137 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 323 KiB/s rd, 188 KiB/s wr, 539 op/s 2024-09-14T04:06:22.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:22 smithi070 ceph-mon[96704]: osdmap e1065: 8 total, 8 up, 8 in 2024-09-14T04:06:22.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:22 smithi070 ceph-mon[98640]: pgmap v1235: 137 pgs: 137 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 323 KiB/s rd, 188 KiB/s wr, 539 op/s 2024-09-14T04:06:22.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:22 smithi070 ceph-mon[98640]: osdmap e1065: 8 total, 8 up, 8 in 2024-09-14T04:06:22.408 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/full_sequential.py", line 30, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_1c941a66c1b420259b8fc5ac19e792dbbf60aded/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_1c941a66c1b420259b8fc5ac19e792dbbf60aded/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi070 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-09-14T04:06:22.649 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=175b3f66ed9049f686ef4fb3001af135 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/full_sequential.py", line 30, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_1c941a66c1b420259b8fc5ac19e792dbbf60aded/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_1c941a66c1b420259b8fc5ac19e792dbbf60aded/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi070 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-09-14T04:06:22.652 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-14T04:06:22.664 INFO:tasks.cephadm:Teardown begin 2024-09-14T04:06:22.664 INFO:tasks.cephfs.filesystem:Destroying file system cephfs and related pools 2024-09-14T04:06:22.664 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph fs dump --format=json 2024-09-14T04:06:22.669 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:22 smithi094 ceph-mon[86597]: pgmap v1235: 137 pgs: 137 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 323 KiB/s rd, 188 KiB/s wr, 539 op/s 2024-09-14T04:06:22.670 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:22 smithi094 ceph-mon[86597]: osdmap e1065: 8 total, 8 up, 8 in 2024-09-14T04:06:22.916 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:06:23.343 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:23 smithi070 ceph-mon[98640]: osdmap e1066: 8 total, 8 up, 8 in 2024-09-14T04:06:23.343 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:23 smithi070 ceph-mon[96704]: osdmap e1066: 8 total, 8 up, 8 in 2024-09-14T04:06:23.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:23 smithi094 ceph-mon[86597]: osdmap e1066: 8 total, 8 up, 8 in 2024-09-14T04:06:23.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:06:23 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:06:23] "GET /metrics HTTP/1.1" 200 36517 "" "Prometheus/2.51.0" 2024-09-14T04:06:24.505 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:24 smithi070 ceph-mon[96704]: pgmap v1238: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:24.506 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:24 smithi070 ceph-mon[98640]: pgmap v1238: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:24.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:24 smithi094 ceph-mon[86597]: pgmap v1238: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:25.276 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T04:06:25.277 INFO:teuthology.orchestra.run.smithi070.stdout:{"epoch":20,"btime":"2024-09-14T03:57:17:084158+0000","default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":54364,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.94:6833/1666439704","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6832","nonce":1666439704},{"type":"v1","addr":"172.21.15.94:6833","nonce":1666439704}]},"join_fscid":-1,"export_targets":[],"features":4540719139924082687,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":20}],"filesystems":[{"mdsmap":{"epoch":19,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-14T03:44:03.568995+0000","modified":"2024-09-14T03:57:10.283118+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":596,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":1,"in":[0],"up":{"mds_0":34497},"failed":[],"damaged":[],"stopped":[],"info":{"gid_34497":{"gid":34497,"name":"a","rank":0,"incarnation":16,"state":"up:active","state_seq":6,"addr":"172.21.15.70:6835/2011419410","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6834","nonce":2011419410},{"type":"v1","addr":"172.21.15.70:6835","nonce":2011419410}]},"join_fscid":-1,"export_targets":[],"features":4540719139924082687,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":34497,"qdb_cluster":[34497]},"id":1}]} 2024-09-14T04:06:25.277 INFO:teuthology.orchestra.run.smithi070.stderr:dumped fsmap epoch 20 2024-09-14T04:06:25.886 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph fs dump --format=json 2024-09-14T04:06:26.130 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:06:26.405 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:26 smithi070 ceph-mon[98640]: pgmap v1239: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:26.406 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:26 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/865014417' entity='client.admin' cmd={"prefix": "fs dump", "format": "json"} : dispatch 2024-09-14T04:06:26.406 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:26 smithi070 ceph-mon[96704]: pgmap v1239: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:26.406 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:26 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/865014417' entity='client.admin' cmd={"prefix": "fs dump", "format": "json"} : dispatch 2024-09-14T04:06:26.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:26 smithi094 ceph-mon[86597]: pgmap v1239: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:26.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:26 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/865014417' entity='client.admin' cmd={"prefix": "fs dump", "format": "json"} : dispatch 2024-09-14T04:06:27.958 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T04:06:27.959 INFO:teuthology.orchestra.run.smithi070.stdout:{"epoch":20,"btime":"2024-09-14T03:57:17:084158+0000","default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":54364,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.94:6833/1666439704","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6832","nonce":1666439704},{"type":"v1","addr":"172.21.15.94:6833","nonce":1666439704}]},"join_fscid":-1,"export_targets":[],"features":4540719139924082687,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":20}],"filesystems":[{"mdsmap":{"epoch":19,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-14T03:44:03.568995+0000","modified":"2024-09-14T03:57:10.283118+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":596,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":1,"in":[0],"up":{"mds_0":34497},"failed":[],"damaged":[],"stopped":[],"info":{"gid_34497":{"gid":34497,"name":"a","rank":0,"incarnation":16,"state":"up:active","state_seq":6,"addr":"172.21.15.70:6835/2011419410","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6834","nonce":2011419410},{"type":"v1","addr":"172.21.15.70:6835","nonce":2011419410}]},"join_fscid":-1,"export_targets":[],"features":4540719139924082687,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":34497,"qdb_cluster":[34497]},"id":1}]} 2024-09-14T04:06:27.959 INFO:teuthology.orchestra.run.smithi070.stderr:dumped fsmap epoch 20 2024-09-14T04:06:28.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:28 smithi070 ceph-mon[98640]: pgmap v1240: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:28.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:28 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/3848976094' entity='client.admin' cmd={"prefix": "fs dump", "format": "json"} : dispatch 2024-09-14T04:06:28.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:28 smithi070 ceph-mon[96704]: pgmap v1240: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:28.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:28 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/3848976094' entity='client.admin' cmd={"prefix": "fs dump", "format": "json"} : dispatch 2024-09-14T04:06:28.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:28 smithi094 ceph-mon[86597]: pgmap v1240: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:28.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:28 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/3848976094' entity='client.admin' cmd={"prefix": "fs dump", "format": "json"} : dispatch 2024-09-14T04:06:28.651 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T04:06:28.898 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:06:30.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:30 smithi070 ceph-mon[98640]: pgmap v1241: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:30.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:30 smithi070 ceph-mon[96704]: pgmap v1241: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:30.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:30 smithi094 ceph-mon[86597]: pgmap v1241: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:30.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:06:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:06:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:06:30.729 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-09-14T04:06:30.729 INFO:teuthology.orchestra.run.smithi070.stdout:{"epoch":1066,"fsid":"b6380d42-724a-11ef-bceb-c7b262605968","created":"2024-09-14T03:38:29.406762+0000","modified":"2024-09-14T04:06:22.137232+0000","last_up_change":"2024-09-14T03:56:42.433445+0000","last_in_change":"2024-09-14T03:43:17.652311+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":60,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:41:38.745294+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-14T03:43:57.402183+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-14T03:44:00.803078+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"rbd","create_time":"2024-09-14T03:45:08.059178+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":67,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":3,"score_stable":3,"optimal_score":1,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"a7fabda1-9e60-42f2-af60-eec626181688","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":333,"up_from":346,"up_thru":1061,"down_at":334,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6802","nonce":1152327285},{"type":"v1","addr":"172.21.15.70:6803","nonce":1152327285}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6804","nonce":1152327285},{"type":"v1","addr":"172.21.15.70:6805","nonce":1152327285}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6808","nonce":1152327285},{"type":"v1","addr":"172.21.15.70:6809","nonce":1152327285}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6806","nonce":1152327285},{"type":"v1","addr":"172.21.15.70:6807","nonce":1152327285}]},"public_addr":"172.21.15.70:6803/1152327285","cluster_addr":"172.21.15.70:6805/1152327285","heartbeat_back_addr":"172.21.15.70:6809/1152327285","heartbeat_front_addr":"172.21.15.70:6807/1152327285","state":["exists","up"]},{"osd":1,"uuid":"f5b9ef91-59af-4ef0-a101-1c00ba6007ad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":366,"up_from":379,"up_thru":1061,"down_at":367,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6810","nonce":478220551},{"type":"v1","addr":"172.21.15.70:6811","nonce":478220551}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6812","nonce":478220551},{"type":"v1","addr":"172.21.15.70:6813","nonce":478220551}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6816","nonce":478220551},{"type":"v1","addr":"172.21.15.70:6817","nonce":478220551}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6814","nonce":478220551},{"type":"v1","addr":"172.21.15.70:6815","nonce":478220551}]},"public_addr":"172.21.15.70:6811/478220551","cluster_addr":"172.21.15.70:6813/478220551","heartbeat_back_addr":"172.21.15.70:6817/478220551","heartbeat_front_addr":"172.21.15.70:6815/478220551","state":["exists","up"]},{"osd":2,"uuid":"0dc07495-c6a5-4762-8f63-575eb0fe1671","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":399,"up_from":411,"up_thru":1061,"down_at":400,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6818","nonce":2093874787},{"type":"v1","addr":"172.21.15.70:6819","nonce":2093874787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6820","nonce":2093874787},{"type":"v1","addr":"172.21.15.70:6821","nonce":2093874787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6824","nonce":2093874787},{"type":"v1","addr":"172.21.15.70:6825","nonce":2093874787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6822","nonce":2093874787},{"type":"v1","addr":"172.21.15.70:6823","nonce":2093874787}]},"public_addr":"172.21.15.70:6819/2093874787","cluster_addr":"172.21.15.70:6821/2093874787","heartbeat_back_addr":"172.21.15.70:6825/2093874787","heartbeat_front_addr":"172.21.15.70:6823/2093874787","state":["exists","up"]},{"osd":3,"uuid":"583e5165-ea10-4d03-b731-36cfb1108362","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":431,"up_from":444,"up_thru":1061,"down_at":432,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6826","nonce":3491567009},{"type":"v1","addr":"172.21.15.70:6827","nonce":3491567009}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6828","nonce":3491567009},{"type":"v1","addr":"172.21.15.70:6829","nonce":3491567009}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6832","nonce":3491567009},{"type":"v1","addr":"172.21.15.70:6833","nonce":3491567009}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6830","nonce":3491567009},{"type":"v1","addr":"172.21.15.70:6831","nonce":3491567009}]},"public_addr":"172.21.15.70:6827/3491567009","cluster_addr":"172.21.15.70:6829/3491567009","heartbeat_back_addr":"172.21.15.70:6833/3491567009","heartbeat_front_addr":"172.21.15.70:6831/3491567009","state":["exists","up"]},{"osd":4,"uuid":"33177119-9632-431c-9180-9d27e7bc0c42","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":31,"last_clean_end":461,"up_from":474,"up_thru":1061,"down_at":462,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6800","nonce":930498880},{"type":"v1","addr":"172.21.15.94:6801","nonce":930498880}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6802","nonce":930498880},{"type":"v1","addr":"172.21.15.94:6803","nonce":930498880}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6806","nonce":930498880},{"type":"v1","addr":"172.21.15.94:6807","nonce":930498880}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6804","nonce":930498880},{"type":"v1","addr":"172.21.15.94:6805","nonce":930498880}]},"public_addr":"172.21.15.94:6801/930498880","cluster_addr":"172.21.15.94:6803/930498880","heartbeat_back_addr":"172.21.15.94:6807/930498880","heartbeat_front_addr":"172.21.15.94:6805/930498880","state":["exists","up"]},{"osd":5,"uuid":"eb037130-5a05-42e0-947e-09bfba8fac22","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":37,"last_clean_end":492,"up_from":505,"up_thru":1061,"down_at":493,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6808","nonce":1267079794},{"type":"v1","addr":"172.21.15.94:6809","nonce":1267079794}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6810","nonce":1267079794},{"type":"v1","addr":"172.21.15.94:6811","nonce":1267079794}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6814","nonce":1267079794},{"type":"v1","addr":"172.21.15.94:6815","nonce":1267079794}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6812","nonce":1267079794},{"type":"v1","addr":"172.21.15.94:6813","nonce":1267079794}]},"public_addr":"172.21.15.94:6809/1267079794","cluster_addr":"172.21.15.94:6811/1267079794","heartbeat_back_addr":"172.21.15.94:6815/1267079794","heartbeat_front_addr":"172.21.15.94:6813/1267079794","state":["exists","up"]},{"osd":6,"uuid":"f5dca4b5-8906-4eac-84e4-fcb7d1a29674","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":43,"last_clean_end":525,"up_from":536,"up_thru":1061,"down_at":526,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6816","nonce":1180082390},{"type":"v1","addr":"172.21.15.94:6817","nonce":1180082390}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6818","nonce":1180082390},{"type":"v1","addr":"172.21.15.94:6819","nonce":1180082390}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6822","nonce":1180082390},{"type":"v1","addr":"172.21.15.94:6823","nonce":1180082390}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6820","nonce":1180082390},{"type":"v1","addr":"172.21.15.94:6821","nonce":1180082390}]},"public_addr":"172.21.15.94:6817/1180082390","cluster_addr":"172.21.15.94:6819/1180082390","heartbeat_back_addr":"172.21.15.94:6823/1180082390","heartbeat_front_addr":"172.21.15.94:6821/1180082390","state":["exists","up"]},{"osd":7,"uuid":"86e3ef7d-6ab8-47c3-a4bb-db923af2b49b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":49,"last_clean_end":557,"up_from":571,"up_thru":1061,"down_at":558,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6824","nonce":1152477702},{"type":"v1","addr":"172.21.15.94:6825","nonce":1152477702}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6826","nonce":1152477702},{"type":"v1","addr":"172.21.15.94:6827","nonce":1152477702}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6830","nonce":1152477702},{"type":"v1","addr":"172.21.15.94:6831","nonce":1152477702}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.94:6828","nonce":1152477702},{"type":"v1","addr":"172.21.15.94:6829","nonce":1152477702}]},"public_addr":"172.21.15.94:6825/1152477702","cluster_addr":"172.21.15.94:6827/1152477702","heartbeat_back_addr":"172.21.15.94:6831/1152477702","heartbeat_front_addr":"172.21.15.94:6829/1152477702","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:40:47.681447+0000","dead_epoch":333},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:41:11.801118+0000","dead_epoch":366},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:41:35.764497+0000","dead_epoch":399},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:42:00.424802+0000","dead_epoch":431},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:42:21.618849+0000","dead_epoch":461},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:42:45.384437+0000","dead_epoch":492},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:43:09.350263+0000","dead_epoch":525},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:43:32.472815+0000","dead_epoch":557}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.18","mappings":[{"from":7,"to":2}]},{"pgid":"3.3","mappings":[{"from":0,"to":1}]},{"pgid":"3.a","mappings":[{"from":6,"to":2}]},{"pgid":"3.b","mappings":[{"from":0,"to":2},{"from":4,"to":1}]},{"pgid":"3.26","mappings":[{"from":4,"to":3}]},{"pgid":"3.36","mappings":[{"from":5,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.94:6833/3620660683":"2024-09-15T03:57:07.271721+0000","172.21.15.94:6832/3620660683":"2024-09-15T03:57:07.271721+0000","172.21.15.70:6835/421917097":"2024-09-15T03:56:47.027591+0000","172.21.15.70:6834/421917097":"2024-09-15T03:56:47.027591+0000","172.21.15.70:0/22543101":"2024-09-15T03:39:01.589983+0000","172.21.15.70:0/1338085373":"2024-09-15T03:51:31.997365+0000","172.21.15.70:0/2239676154":"2024-09-15T03:51:31.997365+0000","172.21.15.94:0/3953959655":"2024-09-15T03:50:51.201434+0000","172.21.15.70:6800/2555772132":"2024-09-15T03:51:31.997365+0000","172.21.15.70:6800/2541724886":"2024-09-15T03:49:34.505287+0000","172.21.15.70:0/2339853520":"2024-09-15T03:45:06.535242+0000","172.21.15.70:0/1654192586":"2024-09-15T03:49:34.505287+0000","172.21.15.70:6801/673916264":"2024-09-15T03:39:24.448616+0000","172.21.15.70:0/3005166882":"2024-09-15T03:39:24.448616+0000","172.21.15.94:0/2129663473":"2024-09-15T03:50:51.201434+0000","172.21.15.70:0/4130351200":"2024-09-15T03:51:31.997365+0000","172.21.15.70:0/3134643770":"2024-09-15T03:39:24.448616+0000","172.21.15.70:6801/2254107694":"2024-09-15T03:39:01.589983+0000","172.21.15.70:6800/2254107694":"2024-09-15T03:39:01.589983+0000","172.21.15.70:0/4203647763":"2024-09-15T03:39:01.589983+0000","172.21.15.70:0/3515912540":"2024-09-15T03:51:31.997365+0000","172.21.15.70:0/2892154750":"2024-09-15T03:49:34.505287+0000","172.21.15.70:0/869883831":"2024-09-15T03:49:34.505287+0000","172.21.15.94:6834/4056574908":"2024-09-15T03:50:51.201434+0000","172.21.15.70:0/2978576061":"2024-09-15T03:39:24.448616+0000","172.21.15.70:0/3407824580":"2024-09-15T03:51:31.997365+0000","172.21.15.70:0/3199751552":"2024-09-15T03:45:06.535242+0000","172.21.15.94:0/3098594889":"2024-09-15T03:50:51.201434+0000","172.21.15.70:0/2324380307":"2024-09-15T03:49:34.505287+0000","172.21.15.70:0/2447056811":"2024-09-15T03:39:01.589983+0000","172.21.15.70:0/2818410174":"2024-09-15T03:51:31.997365+0000","172.21.15.70:0/3204520938":"2024-09-15T03:51:31.997365+0000","172.21.15.70:6800/3164926893":"2024-09-15T03:45:06.535242+0000","172.21.15.94:0/2404500339":"2024-09-15T03:50:51.201434+0000","172.21.15.70:0/3067035520":"2024-09-15T03:45:06.535242+0000","172.21.15.70:6801/3164926893":"2024-09-15T03:45:06.535242+0000","172.21.15.70:6801/2555772132":"2024-09-15T03:51:31.997365+0000","172.21.15.70:6801/2541724886":"2024-09-15T03:49:34.505287+0000","172.21.15.94:0/1940159411":"2024-09-15T03:50:51.201434+0000","172.21.15.94:0/3503149243":"2024-09-15T03:50:51.201434+0000","172.21.15.70:6800/673916264":"2024-09-15T03:39:24.448616+0000","172.21.15.94:0/3283822429":"2024-09-15T03:50:51.201434+0000","172.21.15.94:6835/4056574908":"2024-09-15T03:50:51.201434+0000","172.21.15.70:0/4275249763":"2024-09-15T03:45:06.535242+0000","172.21.15.70:0/2993355975":"2024-09-15T03:49:34.505287+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"},"teuthologyprofile":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van","w":"8"}},"removed_snaps_queue":[{"pool":60,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T04:06:31.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:31 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/446240901' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T04:06:31.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:31 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/446240901' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T04:06:31.347 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph fs fail cephfs 2024-09-14T04:06:31.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:31 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/446240901' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T04:06:31.592 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:06:32.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:32 smithi070 ceph-mon[96704]: pgmap v1242: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:32.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:32 smithi070 ceph-mon[98640]: pgmap v1242: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:32.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:32 smithi094 ceph-mon[86597]: pgmap v1242: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:33.286 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:33 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:06:33.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:33 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:06:33.459 INFO:teuthology.orchestra.run.smithi070.stderr:cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. 2024-09-14T04:06:33.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:33 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:06:33.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-a[96681]: 2024-09-14T04:06:33.453+0000 7f512eabf640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-14T04:06:33.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:06:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:06:33] "GET /metrics HTTP/1.1" 200 34147 "" "Prometheus/2.51.0" 2024-09-14T04:06:33.787 INFO:journalctl@ceph.mds.a.smithi070.stdout:Sep 14 04:06:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-a[122489]: -4657> 2024-09-14T03:57:12.094+0000 7f7f2891e640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-09-14T04:06:33.787 INFO:journalctl@ceph.mds.a.smithi070.stdout:Sep 14 04:06:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-a[122489]: ignoring --setuser ceph since I am not root 2024-09-14T04:06:33.787 INFO:journalctl@ceph.mds.a.smithi070.stdout:Sep 14 04:06:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-a[122489]: ignoring --setgroup ceph since I am not root 2024-09-14T04:06:33.787 INFO:journalctl@ceph.mds.a.smithi070.stdout:Sep 14 04:06:33 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-a[122489]: starting mds.a at 2024-09-14T04:06:34.079 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph fs rm cephfs --yes-i-really-mean-it 2024-09-14T04:06:34.331 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:06:34.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:34 smithi070 ceph-mon[96704]: pgmap v1243: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:34.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:34 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/3641782791' entity='client.admin' cmd={"prefix": "fs fail", "fs_name": "cephfs"} : dispatch 2024-09-14T04:06:34.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:34 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "fs fail", "fs_name": "cephfs"} : dispatch 2024-09-14T04:06:34.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:34 smithi070 ceph-mon[96704]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-14T04:06:34.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:34 smithi070 ceph-mon[96704]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-14T04:06:34.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:34 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs fail", "fs_name": "cephfs"}]': finished 2024-09-14T04:06:34.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:34 smithi070 ceph-mon[96704]: osdmap e1067: 8 total, 8 up, 8 in 2024-09-14T04:06:34.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:34 smithi070 ceph-mon[96704]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-14T04:06:34.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:34 smithi070 ceph-mon[98640]: pgmap v1243: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:34.454 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:34 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/3641782791' entity='client.admin' cmd={"prefix": "fs fail", "fs_name": "cephfs"} : dispatch 2024-09-14T04:06:34.454 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:34 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "fs fail", "fs_name": "cephfs"} : dispatch 2024-09-14T04:06:34.454 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:34 smithi070 ceph-mon[98640]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-14T04:06:34.454 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:34 smithi070 ceph-mon[98640]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-14T04:06:34.454 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:34 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs fail", "fs_name": "cephfs"}]': finished 2024-09-14T04:06:34.454 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:34 smithi070 ceph-mon[98640]: osdmap e1067: 8 total, 8 up, 8 in 2024-09-14T04:06:34.454 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:34 smithi070 ceph-mon[98640]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-14T04:06:34.526 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:34 smithi094 ceph-mon[86597]: pgmap v1243: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:34.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:34 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/3641782791' entity='client.admin' cmd={"prefix": "fs fail", "fs_name": "cephfs"} : dispatch 2024-09-14T04:06:34.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:34 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "fs fail", "fs_name": "cephfs"} : dispatch 2024-09-14T04:06:34.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:34 smithi094 ceph-mon[86597]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-14T04:06:34.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:34 smithi094 ceph-mon[86597]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-14T04:06:34.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:34 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs fail", "fs_name": "cephfs"}]': finished 2024-09-14T04:06:34.527 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:34 smithi094 ceph-mon[86597]: osdmap e1067: 8 total, 8 up, 8 in 2024-09-14T04:06:34.528 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:34 smithi094 ceph-mon[86597]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-14T04:06:35.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:35 smithi070 ceph-mon[96704]: pgmap v1245: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:35.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:35 smithi070 ceph-mon[96704]: mds.? [v2:172.21.15.70:6834/1633282838,v1:172.21.15.70:6835/1633282838] up:boot 2024-09-14T04:06:35.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:35 smithi070 ceph-mon[96704]: fsmap cephfs:0/1 2 up:standby, 1 failed 2024-09-14T04:06:35.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:35 smithi070 ceph-mon[96704]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-14T04:06:35.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:35 smithi070 ceph-mon[98640]: pgmap v1245: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:35.536 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:35 smithi070 ceph-mon[98640]: mds.? [v2:172.21.15.70:6834/1633282838,v1:172.21.15.70:6835/1633282838] up:boot 2024-09-14T04:06:35.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:35 smithi070 ceph-mon[98640]: fsmap cephfs:0/1 2 up:standby, 1 failed 2024-09-14T04:06:35.537 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:35 smithi070 ceph-mon[98640]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-14T04:06:35.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:35 smithi094 ceph-mon[86597]: pgmap v1245: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:35.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:35 smithi094 ceph-mon[86597]: mds.? [v2:172.21.15.70:6834/1633282838,v1:172.21.15.70:6835/1633282838] up:boot 2024-09-14T04:06:35.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:35 smithi094 ceph-mon[86597]: fsmap cephfs:0/1 2 up:standby, 1 failed 2024-09-14T04:06:35.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:35 smithi094 ceph-mon[86597]: from='mgr.34107 172.21.15.70:0/2819738506' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-14T04:06:36.776 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:36 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/1401188536' entity='client.admin' cmd={"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:36.776 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:36 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd={"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:36.777 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:36 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/1401188536' entity='client.admin' cmd={"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:36.777 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:36 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd={"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:36.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:36 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/1401188536' entity='client.admin' cmd={"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:36.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:36 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd={"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:06:37.086 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph osd pool rm cephfs_metadata cephfs_metadata --yes-i-really-really-mean-it 2024-09-14T04:06:37.333 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:06:37.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:37 smithi070 ceph-mon[96704]: pgmap v1246: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:37.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:37 smithi070 ceph-mon[96704]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-14T04:06:37.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:37 smithi070 ceph-mon[96704]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-14T04:06:37.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:37 smithi070 ceph-mon[96704]: Cluster is now healthy 2024-09-14T04:06:37.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:37 smithi070 ceph-mon[96704]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:06:37.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:37 smithi070 ceph-mon[96704]: fsmap 2 up:standby 2024-09-14T04:06:37.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:37 smithi070 ceph-mon[98640]: pgmap v1246: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:37.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:37 smithi070 ceph-mon[98640]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-14T04:06:37.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:37 smithi070 ceph-mon[98640]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-14T04:06:37.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:37 smithi070 ceph-mon[98640]: Cluster is now healthy 2024-09-14T04:06:37.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:37 smithi070 ceph-mon[98640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:06:37.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:37 smithi070 ceph-mon[98640]: fsmap 2 up:standby 2024-09-14T04:06:37.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:37 smithi094 ceph-mon[86597]: pgmap v1246: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:37.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:37 smithi094 ceph-mon[86597]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-14T04:06:37.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:37 smithi094 ceph-mon[86597]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-14T04:06:37.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:37 smithi094 ceph-mon[86597]: Cluster is now healthy 2024-09-14T04:06:37.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:37 smithi094 ceph-mon[86597]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:06:37.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:37 smithi094 ceph-mon[86597]: fsmap 2 up:standby 2024-09-14T04:06:39.468 INFO:teuthology.orchestra.run.smithi070.stderr:pool 'cephfs_metadata' removed 2024-09-14T04:06:39.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:39 smithi094 ceph-mon[86597]: pgmap v1247: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:39.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:39 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/3055396102' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:06:39.781 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:39 smithi070 ceph-mon[96704]: pgmap v1247: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:39.781 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:39 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/3055396102' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:06:39.781 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:39 smithi070 ceph-mon[98640]: pgmap v1247: 105 pgs: 105 active+clean; 583 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:39.781 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:39 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/3055396102' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:06:40.089 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b6380d42-724a-11ef-bceb-c7b262605968 -- ceph osd pool rm cephfs_data cephfs_data --yes-i-really-really-mean-it 2024-09-14T04:06:40.333 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:06:40.472 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:06:40 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:06:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:06:40.741 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:40 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/3055396102' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:06:40.742 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:40 smithi070 ceph-mon[96704]: osdmap e1068: 8 total, 8 up, 8 in 2024-09-14T04:06:40.742 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:40 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/3055396102' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:06:40.742 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:40 smithi070 ceph-mon[98640]: osdmap e1068: 8 total, 8 up, 8 in 2024-09-14T04:06:40.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:40 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/3055396102' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:06:40.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:40 smithi094 ceph-mon[86597]: osdmap e1068: 8 total, 8 up, 8 in 2024-09-14T04:06:41.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:41 smithi094 ceph-mon[86597]: pgmap v1249: 73 pgs: 73 active+clean; 577 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:41.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:41 smithi070 ceph-mon[98640]: pgmap v1249: 73 pgs: 73 active+clean; 577 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:41.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:41 smithi070 ceph-mon[96704]: pgmap v1249: 73 pgs: 73 active+clean; 577 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:42.480 INFO:teuthology.orchestra.run.smithi070.stderr:pool 'cephfs_data' removed 2024-09-14T04:06:42.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:42 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/1472753052' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:06:42.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:42 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/1472753052' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:06:42.785 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:42 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/1472753052' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:06:43.055 DEBUG:teuthology.orchestra.run.smithi070:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-14T04:06:43.084 DEBUG:teuthology.orchestra.run.smithi094:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-14T04:06:43.118 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-14T04:06:43.118 DEBUG:teuthology.orchestra.run.smithi070:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-14T04:06:43.142 DEBUG:teuthology.orchestra.run.smithi094:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-14T04:06:43.176 INFO:tasks.cephadm:Stopping all daemons... 2024-09-14T04:06:43.177 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-09-14T04:06:43.177 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ceph-b6380d42-724a-11ef-bceb-c7b262605968@mon.a 2024-09-14T04:06:43.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:43 smithi070 systemd[1]: Stopping Ceph mon.a for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T04:06:43.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:43 smithi094 ceph-mon[86597]: pgmap v1250: 73 pgs: 73 active+clean; 577 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:43.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:43 smithi094 ceph-mon[86597]: from='client.? 172.21.15.70:0/1472753052' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:06:43.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:43 smithi094 ceph-mon[86597]: osdmap e1069: 8 total, 8 up, 8 in 2024-09-14T04:06:43.785 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:06:43 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: ::ffff:172.21.15.94 - - [14/Sep/2024:04:06:43] "GET /metrics HTTP/1.1" 200 34147 "" "Prometheus/2.51.0" 2024-09-14T04:06:43.786 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:06:43 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: [14/Sep/2024:04:06:43] ENGINE Bus STOPPING 2024-09-14T04:06:43.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:43 smithi070 ceph-mon[98640]: pgmap v1250: 73 pgs: 73 active+clean; 577 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:43.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:43 smithi070 ceph-mon[98640]: from='client.? 172.21.15.70:0/1472753052' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:06:43.786 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:43 smithi070 ceph-mon[98640]: osdmap e1069: 8 total, 8 up, 8 in 2024-09-14T04:06:43.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:43 smithi070 ceph-mon[96704]: pgmap v1250: 73 pgs: 73 active+clean; 577 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:06:43.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:43 smithi070 ceph-mon[96704]: from='client.? 172.21.15.70:0/1472753052' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:06:43.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:43 smithi070 ceph-mon[96704]: osdmap e1069: 8 total, 8 up, 8 in 2024-09-14T04:06:43.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:43 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-a[96681]: 2024-09-14T04:06:43.505+0000 7f51342ca640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:06:43.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:43 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-a[96681]: 2024-09-14T04:06:43.505+0000 7f51342ca640 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-09-14T04:06:43.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:43 smithi070 podman[131731]: 2024-09-14 04:06:43.537116292 +0000 UTC m=+0.163356185 container died 96542de31cac9c93f77613ee79c8ff2fe4ea7431d2292a5bb822697b7019365b (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-a, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , RELEASE=main-a278231, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-14T04:06:44.198 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:43 smithi070 podman[131731]: 2024-09-14 04:06:43.921119447 +0000 UTC m=+0.547359341 container cleanup 96542de31cac9c93f77613ee79c8ff2fe4ea7431d2292a5bb822697b7019365b (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-a, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-14T04:06:44.198 INFO:journalctl@ceph.mon.a.smithi070.stdout:Sep 14 04:06:43 smithi070 bash[131731]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-a 2024-09-14T04:06:44.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:06:43 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: [14/Sep/2024:04:06:43] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T04:06:44.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:06:43 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: [14/Sep/2024:04:06:43] ENGINE Bus STOPPED 2024-09-14T04:06:44.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:06:43 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: [14/Sep/2024:04:06:43] ENGINE Bus STARTING 2024-09-14T04:06:44.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:06:44 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: [14/Sep/2024:04:06:44] ENGINE Serving on http://:::9283 2024-09-14T04:06:44.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:06:44 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: [14/Sep/2024:04:06:44] ENGINE Bus STARTED 2024-09-14T04:06:44.338 DEBUG:teuthology.orchestra.run.smithi070:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@mon.a.service' 2024-09-14T04:06:44.403 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T04:06:44.404 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-09-14T04:06:44.404 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-09-14T04:06:44.404 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ceph-b6380d42-724a-11ef-bceb-c7b262605968@mon.c 2024-09-14T04:06:44.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:44 smithi070 systemd[1]: Stopping Ceph mon.c for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T04:06:45.002 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:44 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-c[98598]: 2024-09-14T04:06:44.685+0000 7f475f82c640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:06:45.002 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:44 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-c[98598]: 2024-09-14T04:06:44.685+0000 7f475f82c640 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-09-14T04:06:45.002 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:44 smithi070 podman[131876]: 2024-09-14 04:06:44.737343163 +0000 UTC m=+0.155577321 container died 5741e4fcf4b689cbaec23247721f6e1bb963f63973edd270d98aec5e1d955239 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-c, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=main-a278231, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-14T04:06:45.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:45 smithi070 podman[131876]: 2024-09-14 04:06:45.158982699 +0000 UTC m=+0.577216854 container cleanup 5741e4fcf4b689cbaec23247721f6e1bb963f63973edd270d98aec5e1d955239 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-c, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=main-a278231, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-14T04:06:45.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:45 smithi070 bash[131876]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-c 2024-09-14T04:06:45.635 DEBUG:teuthology.orchestra.run.smithi070:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@mon.c.service' 2024-09-14T04:06:45.657 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:45 smithi070 podman[131893]: 2024-09-14 04:06:45.385418854 +0000 UTC m=+0.643348301 container remove 5741e4fcf4b689cbaec23247721f6e1bb963f63973edd270d98aec5e1d955239 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-c, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-14T04:06:45.657 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:45 smithi070 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@mon.c.service: Deactivated successfully. 2024-09-14T04:06:45.657 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:45 smithi070 systemd[1]: Stopped Ceph mon.c for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T04:06:45.657 INFO:journalctl@ceph.mon.c.smithi070.stdout:Sep 14 04:06:45 smithi070 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@mon.c.service: Consumed 11.743s CPU time. 2024-09-14T04:06:45.669 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T04:06:45.669 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-09-14T04:06:45.669 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-09-14T04:06:45.670 DEBUG:teuthology.orchestra.run.smithi094:> sudo systemctl stop ceph-b6380d42-724a-11ef-bceb-c7b262605968@mon.b 2024-09-14T04:06:46.026 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:45 smithi094 systemd[1]: Stopping Ceph mon.b for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T04:06:46.472 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:46 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-b[86574]: 2024-09-14T04:06:46.080+0000 7f3ed04fd640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:06:46.473 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:46 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-b[86574]: 2024-09-14T04:06:46.080+0000 7f3ed04fd640 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-09-14T04:06:46.473 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:46 smithi094 podman[118806]: 2024-09-14 04:06:46.193666059 +0000 UTC m=+0.329630214 container died 86827be986590181cb74d8eebe7278c077280c14de375cb751fbdbf912ab1df7 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-b, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, io.buildah.version=1.37.2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T04:06:46.776 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:46 smithi094 podman[118806]: 2024-09-14 04:06:46.662782206 +0000 UTC m=+0.798746417 container cleanup 86827be986590181cb74d8eebe7278c077280c14de375cb751fbdbf912ab1df7 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-b, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=main-a278231, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T04:06:46.777 INFO:journalctl@ceph.mon.b.smithi094.stdout:Sep 14 04:06:46 smithi094 bash[118806]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-mon-b 2024-09-14T04:06:47.096 DEBUG:teuthology.orchestra.run.smithi094:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@mon.b.service' 2024-09-14T04:06:47.145 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T04:06:47.145 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-09-14T04:06:47.145 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-09-14T04:06:47.146 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ceph-b6380d42-724a-11ef-bceb-c7b262605968@mgr.y 2024-09-14T04:06:47.433 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:06:47 smithi070 systemd[1]: Stopping Ceph mgr.y for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T04:06:47.756 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:06:47 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y[88616]: 2024-09-14T04:06:47.431+0000 7fa50d855640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:06:47.756 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:06:47 smithi070 podman[132008]: 2024-09-14 04:06:47.463317219 +0000 UTC m=+0.134740920 container died 0ef4d8e0cdbeb034eb69f6ae58fd51178439b15cb034db2198d5ce8b3e5c39b4 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-14T04:06:48.035 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:06:47 smithi070 podman[132008]: 2024-09-14 04:06:47.8466105 +0000 UTC m=+0.518034200 container cleanup 0ef4d8e0cdbeb034eb69f6ae58fd51178439b15cb034db2198d5ce8b3e5c39b4 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y, RELEASE=main-a278231, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-14T04:06:48.036 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Sep 14 04:06:47 smithi070 bash[132008]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-y 2024-09-14T04:06:48.222 DEBUG:teuthology.orchestra.run.smithi070:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@mgr.y.service' 2024-09-14T04:06:48.256 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T04:06:48.257 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-09-14T04:06:48.257 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-09-14T04:06:48.257 DEBUG:teuthology.orchestra.run.smithi094:> sudo systemctl stop ceph-b6380d42-724a-11ef-bceb-c7b262605968@mgr.x 2024-09-14T04:06:48.602 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 04:06:48 smithi094 systemd[1]: Stopping Ceph mgr.x for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T04:06:48.887 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 04:06:48 smithi094 podman[118940]: 2024-09-14 04:06:48.600780866 +0000 UTC m=+0.151890893 container died 569d9c7e0f060ad8d6b06f075081d11c326448eac0ff973a390fd0df55b4c0da (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, RELEASE=main-a278231, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-14T04:06:49.253 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 04:06:48 smithi094 podman[118940]: 2024-09-14 04:06:48.959779419 +0000 UTC m=+0.510889445 container cleanup 569d9c7e0f060ad8d6b06f075081d11c326448eac0ff973a390fd0df55b4c0da (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=main-a278231) 2024-09-14T04:06:49.254 INFO:journalctl@ceph.mgr.x.smithi094.stdout:Sep 14 04:06:48 smithi094 bash[118940]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-mgr-x 2024-09-14T04:06:49.437 DEBUG:teuthology.orchestra.run.smithi094:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@mgr.x.service' 2024-09-14T04:06:49.473 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T04:06:49.473 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-09-14T04:06:49.474 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-14T04:06:49.474 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.0 2024-09-14T04:06:49.785 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 04:06:49 smithi070 systemd[1]: Stopping Ceph osd.0 for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T04:06:50.285 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 04:06:50 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0[103808]: 2024-09-14T04:06:50.009+0000 7f4ae1f2c640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:06:50.286 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 04:06:50 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0[103808]: 2024-09-14T04:06:50.009+0000 7f4ae1f2c640 -1 osd.0 1069 *** Got signal Terminated *** 2024-09-14T04:06:50.286 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 04:06:50 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0[103808]: 2024-09-14T04:06:50.009+0000 7f4ae1f2c640 -1 osd.0 1069 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T04:06:50.527 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:06:50 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:06:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"b6380d42-724a-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-14T04:06:55.692 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 04:06:55 smithi070 podman[132137]: 2024-09-14 04:06:55.42736157 +0000 UTC m=+5.760844240 container died 2345ccac208538d9c77f6cbd822cb5c03442a76571331d7a384011581cede572 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, RELEASE=main-a278231) 2024-09-14T04:06:56.036 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 04:06:55 smithi070 podman[132137]: 2024-09-14 04:06:55.824235166 +0000 UTC m=+6.157717826 container cleanup 2345ccac208538d9c77f6cbd822cb5c03442a76571331d7a384011581cede572 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=main-a278231, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-14T04:06:56.037 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 04:06:55 smithi070 bash[132137]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0 2024-09-14T04:06:56.402 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 04:06:56 smithi070 podman[132157]: 2024-09-14 04:06:56.084589186 +0000 UTC m=+0.654058596 container remove 2345ccac208538d9c77f6cbd822cb5c03442a76571331d7a384011581cede572 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=main-a278231) 2024-09-14T04:06:56.680 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 04:06:56 smithi070 podman[132253]: 2024-09-14 04:06:56.304774708 +0000 UTC m=+0.015775842 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:06:56.680 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 04:06:56 smithi070 podman[132253]: 2024-09-14 04:06:56.452104914 +0000 UTC m=+0.163106035 container create d5ba1168f3102a1cf9b6532c5e33642543c5b27d1d479638546bf46694428adb (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=main-a278231, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-14T04:06:56.680 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 04:06:56 smithi070 podman[132253]: 2024-09-14 04:06:56.679630888 +0000 UTC m=+0.390632014 container init d5ba1168f3102a1cf9b6532c5e33642543c5b27d1d479638546bf46694428adb (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-09-14T04:06:57.035 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 04:06:56 smithi070 podman[132253]: 2024-09-14 04:06:56.68434597 +0000 UTC m=+0.395347092 container start d5ba1168f3102a1cf9b6532c5e33642543c5b27d1d479638546bf46694428adb (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-14T04:06:57.036 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 04:06:56 smithi070 podman[132253]: 2024-09-14 04:06:56.779528229 +0000 UTC m=+0.490529362 container attach d5ba1168f3102a1cf9b6532c5e33642543c5b27d1d479638546bf46694428adb (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, RELEASE=main-a278231, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-14T04:06:57.036 INFO:journalctl@ceph.osd.0.smithi070.stdout:Sep 14 04:06:56 smithi070 podman[132253]: 2024-09-14 04:06:56.866358667 +0000 UTC m=+0.577359796 container died d5ba1168f3102a1cf9b6532c5e33642543c5b27d1d479638546bf46694428adb (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-14T04:06:57.436 DEBUG:teuthology.orchestra.run.smithi070:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.0.service' 2024-09-14T04:06:57.477 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T04:06:57.478 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-14T04:06:57.478 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-14T04:06:57.478 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.1 2024-09-14T04:06:57.727 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 04:06:57 smithi070 systemd[1]: Stopping Ceph osd.1 for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T04:06:58.035 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 04:06:57 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1[108413]: 2024-09-14T04:06:57.854+0000 7fde50db0640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:06:58.036 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 04:06:57 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1[108413]: 2024-09-14T04:06:57.854+0000 7fde50db0640 -1 osd.1 1069 *** Got signal Terminated *** 2024-09-14T04:06:58.036 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 04:06:57 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1[108413]: 2024-09-14T04:06:57.854+0000 7fde50db0640 -1 osd.1 1069 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T04:07:03.687 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 04:07:03 smithi070 podman[132381]: 2024-09-14 04:07:03.394833109 +0000 UTC m=+5.687071397 container died 53868856512a30fe44c93e5115ca093444ae01f9821f3d11b1376d762ed75798 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=main-a278231, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-14T04:07:04.035 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 04:07:03 smithi070 podman[132381]: 2024-09-14 04:07:03.86030591 +0000 UTC m=+6.152544194 container cleanup 53868856512a30fe44c93e5115ca093444ae01f9821f3d11b1376d762ed75798 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-14T04:07:04.036 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 04:07:03 smithi070 bash[132381]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1 2024-09-14T04:07:04.477 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 04:07:04 smithi070 podman[132412]: 2024-09-14 04:07:04.128565841 +0000 UTC m=+0.734050270 container remove 53868856512a30fe44c93e5115ca093444ae01f9821f3d11b1376d762ed75798 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1, RELEASE=main-a278231, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-14T04:07:04.766 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 04:07:04 smithi070 podman[132506]: 2024-09-14 04:07:04.379816312 +0000 UTC m=+0.015592410 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:07:04.766 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 04:07:04 smithi070 podman[132506]: 2024-09-14 04:07:04.499401032 +0000 UTC m=+0.135177166 container create f1a6d5eb65648092931d21e016ce2fe959bac156684d5658ffcb87d7fdcbbffc (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, RELEASE=main-a278231, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-14T04:07:04.766 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 04:07:04 smithi070 podman[132506]: 2024-09-14 04:07:04.765287061 +0000 UTC m=+0.401063156 container init f1a6d5eb65648092931d21e016ce2fe959bac156684d5658ffcb87d7fdcbbffc (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=main-a278231, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True) 2024-09-14T04:07:05.036 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 04:07:04 smithi070 podman[132506]: 2024-09-14 04:07:04.77002113 +0000 UTC m=+0.405797230 container start f1a6d5eb65648092931d21e016ce2fe959bac156684d5658ffcb87d7fdcbbffc (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1-deactivate, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-14T04:07:05.037 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 04:07:04 smithi070 podman[132506]: 2024-09-14 04:07:04.848485635 +0000 UTC m=+0.484261739 container attach f1a6d5eb65648092931d21e016ce2fe959bac156684d5658ffcb87d7fdcbbffc (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1-deactivate, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-14T04:07:05.037 INFO:journalctl@ceph.osd.1.smithi070.stdout:Sep 14 04:07:04 smithi070 podman[132506]: 2024-09-14 04:07:04.958443974 +0000 UTC m=+0.594220071 container died f1a6d5eb65648092931d21e016ce2fe959bac156684d5658ffcb87d7fdcbbffc (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=main-a278231, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-14T04:07:05.681 DEBUG:teuthology.orchestra.run.smithi070:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.1.service' 2024-09-14T04:07:05.725 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T04:07:05.725 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-14T04:07:05.725 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-14T04:07:05.725 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.2 2024-09-14T04:07:05.777 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:07:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:07:05.379Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nfs msg="Unable to refresh target groups" err="Get \"http://172.21.15.70:8765/sd/prometheus/sd-config?service=nfs\": dial tcp 172.21.15.70:8765: connect: connection refused" 2024-09-14T04:07:05.777 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:07:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:07:05.379Z caller=refresh.go:90 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.70:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.70:8765: connect: connection refused" 2024-09-14T04:07:05.777 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:07:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:07:05.379Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=smb msg="Unable to refresh target groups" err="Get \"http://172.21.15.70:8765/sd/prometheus/sd-config?service=smb\": dial tcp 172.21.15.70:8765: connect: connection refused" 2024-09-14T04:07:05.777 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:07:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:07:05.380Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.70:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.70:8765: connect: connection refused" 2024-09-14T04:07:05.777 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:07:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:07:05.383Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.70:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.70:8765: connect: connection refused" 2024-09-14T04:07:05.778 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:07:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:07:05.383Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.70:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.70:8765: connect: connection refused" 2024-09-14T04:07:05.778 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:07:05 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:07:05.383Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.70:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.70:8765: connect: connection refused" 2024-09-14T04:07:06.035 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 04:07:05 smithi070 systemd[1]: Stopping Ceph osd.2 for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T04:07:06.535 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 04:07:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2[113018]: 2024-09-14T04:07:06.090+0000 7f5ca36d0640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:07:06.536 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 04:07:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2[113018]: 2024-09-14T04:07:06.090+0000 7f5ca36d0640 -1 osd.2 1069 *** Got signal Terminated *** 2024-09-14T04:07:06.536 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 04:07:06 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2[113018]: 2024-09-14T04:07:06.090+0000 7f5ca36d0640 -1 osd.2 1069 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T04:07:11.865 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 04:07:11 smithi070 podman[132632]: 2024-09-14 04:07:11.587364301 +0000 UTC m=+5.638425337 container died 39ef876d2fffa79521c362315dc0028c2e2e8d9d97efa6c1bca3813e494aa7b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, ceph=True, RELEASE=main-a278231, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T04:07:12.285 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 04:07:12 smithi070 podman[132632]: 2024-09-14 04:07:12.054985274 +0000 UTC m=+6.106046309 container cleanup 39ef876d2fffa79521c362315dc0028c2e2e8d9d97efa6c1bca3813e494aa7b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True) 2024-09-14T04:07:12.285 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 04:07:12 smithi070 bash[132632]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2 2024-09-14T04:07:12.707 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 04:07:12 smithi070 podman[132653]: 2024-09-14 04:07:12.390107396 +0000 UTC m=+0.806668382 container remove 39ef876d2fffa79521c362315dc0028c2e2e8d9d97efa6c1bca3813e494aa7b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=main-a278231, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T04:07:13.019 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 04:07:12 smithi070 podman[132746]: 2024-09-14 04:07:12.60984141 +0000 UTC m=+0.015830975 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:07:13.019 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 04:07:12 smithi070 podman[132746]: 2024-09-14 04:07:12.715820788 +0000 UTC m=+0.121810337 container create 054171616c707d1c38f8fbbd67de58cce7674766b77dd772346e29f3191709f5 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=main-a278231, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-14T04:07:13.019 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 04:07:12 smithi070 podman[132746]: 2024-09-14 04:07:12.918240078 +0000 UTC m=+0.324229632 container init 054171616c707d1c38f8fbbd67de58cce7674766b77dd772346e29f3191709f5 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=main-a278231, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T04:07:13.019 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 04:07:12 smithi070 podman[132746]: 2024-09-14 04:07:12.923466798 +0000 UTC m=+0.329456349 container start 054171616c707d1c38f8fbbd67de58cce7674766b77dd772346e29f3191709f5 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-14T04:07:13.019 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 04:07:13 smithi070 podman[132746]: 2024-09-14 04:07:13.018031761 +0000 UTC m=+0.424021346 container attach 054171616c707d1c38f8fbbd67de58cce7674766b77dd772346e29f3191709f5 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2-deactivate, RELEASE=main-a278231, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2) 2024-09-14T04:07:13.286 INFO:journalctl@ceph.osd.2.smithi070.stdout:Sep 14 04:07:13 smithi070 podman[132746]: 2024-09-14 04:07:13.117995886 +0000 UTC m=+0.523985446 container died 054171616c707d1c38f8fbbd67de58cce7674766b77dd772346e29f3191709f5 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, RELEASE=main-a278231, CEPH_POINT_RELEASE=) 2024-09-14T04:07:13.707 DEBUG:teuthology.orchestra.run.smithi070:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.2.service' 2024-09-14T04:07:13.746 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T04:07:13.746 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-14T04:07:13.746 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-14T04:07:13.746 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.3 2024-09-14T04:07:14.035 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 04:07:13 smithi070 systemd[1]: Stopping Ceph osd.3 for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T04:07:14.535 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 04:07:14 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3[117637]: 2024-09-14T04:07:14.084+0000 7f4b38d02640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:07:14.536 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 04:07:14 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3[117637]: 2024-09-14T04:07:14.084+0000 7f4b38d02640 -1 osd.3 1069 *** Got signal Terminated *** 2024-09-14T04:07:14.536 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 04:07:14 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3[117637]: 2024-09-14T04:07:14.084+0000 7f4b38d02640 -1 osd.3 1069 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T04:07:16.026 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:15 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[96999]: 2024-09-14T04:07:15.670+0000 7f9072f2d640 -1 osd.5 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:50.888890+0000 front 2024-09-14T04:06:50.888943+0000 (oldest deadline 2024-09-14T04:07:15.588797+0000) 2024-09-14T04:07:16.934 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:16 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[96999]: 2024-09-14T04:07:16.637+0000 7f9072f2d640 -1 osd.5 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:50.888890+0000 front 2024-09-14T04:06:50.888943+0000 (oldest deadline 2024-09-14T04:07:15.588797+0000) 2024-09-14T04:07:17.277 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:16 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:16.933+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.146491+0000 front 2024-09-14T04:06:53.146435+0000 (oldest deadline 2024-09-14T04:07:16.646555+0000) 2024-09-14T04:07:17.909 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:17 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[96999]: 2024-09-14T04:07:17.656+0000 7f9072f2d640 -1 osd.5 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:50.888890+0000 front 2024-09-14T04:06:50.888943+0000 (oldest deadline 2024-09-14T04:07:15.588797+0000) 2024-09-14T04:07:18.276 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:17 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:17.924+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.146491+0000 front 2024-09-14T04:06:53.146435+0000 (oldest deadline 2024-09-14T04:07:16.646555+0000) 2024-09-14T04:07:18.976 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 04:07:18 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4[92016]: 2024-09-14T04:07:18.745+0000 7f6d53a2c640 -1 osd.4 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.160093+0000 front 2024-09-14T04:06:53.160215+0000 (oldest deadline 2024-09-14T04:07:18.460079+0000) 2024-09-14T04:07:18.977 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:18 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[96999]: 2024-09-14T04:07:18.661+0000 7f9072f2d640 -1 osd.5 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:50.888890+0000 front 2024-09-14T04:06:50.888943+0000 (oldest deadline 2024-09-14T04:07:15.588797+0000) 2024-09-14T04:07:19.276 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:18 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:18.974+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.146491+0000 front 2024-09-14T04:06:53.146435+0000 (oldest deadline 2024-09-14T04:07:16.646555+0000) 2024-09-14T04:07:19.775 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 04:07:19 smithi070 podman[132874]: 2024-09-14 04:07:19.486697125 +0000 UTC m=+5.517229376 container died 8efc112edbe957b36f8c9922f10b8f838a422c29e4a1aa44728640497b3cf806 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T04:07:19.966 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:19 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[96999]: 2024-09-14T04:07:19.623+0000 7f9072f2d640 -1 osd.5 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:50.888890+0000 front 2024-09-14T04:06:50.888943+0000 (oldest deadline 2024-09-14T04:07:15.588797+0000) 2024-09-14T04:07:19.966 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 04:07:19 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4[92016]: 2024-09-14T04:07:19.779+0000 7f6d53a2c640 -1 osd.4 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.160093+0000 front 2024-09-14T04:06:53.160215+0000 (oldest deadline 2024-09-14T04:07:18.460079+0000) 2024-09-14T04:07:19.966 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:19 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:19.817+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:54.848265+0000 front 2024-09-14T04:06:54.848305+0000 (oldest deadline 2024-09-14T04:07:19.548373+0000) 2024-09-14T04:07:20.035 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 04:07:19 smithi070 podman[132874]: 2024-09-14 04:07:19.890254812 +0000 UTC m=+5.920787053 container cleanup 8efc112edbe957b36f8c9922f10b8f838a422c29e4a1aa44728640497b3cf806 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, RELEASE=main-a278231, org.label-schema.vendor=CentOS) 2024-09-14T04:07:20.036 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 04:07:19 smithi070 bash[132874]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3 2024-09-14T04:07:20.276 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:19 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:19.965+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.146491+0000 front 2024-09-14T04:06:53.146435+0000 (oldest deadline 2024-09-14T04:07:16.646555+0000) 2024-09-14T04:07:20.458 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 04:07:20 smithi070 podman[132896]: 2024-09-14 04:07:20.133493503 +0000 UTC m=+0.649332447 container remove 8efc112edbe957b36f8c9922f10b8f838a422c29e4a1aa44728640497b3cf806 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-14T04:07:20.779 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 04:07:20 smithi070 podman[132988]: 2024-09-14 04:07:20.360679164 +0000 UTC m=+0.015568570 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:07:20.779 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 04:07:20 smithi070 podman[132988]: 2024-09-14 04:07:20.467646071 +0000 UTC m=+0.122535479 container create 6d817710ad0f63d0678b03e034b72b4a6d810ecc1940f72132bb8b9311deb5df (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=main-a278231) 2024-09-14T04:07:20.779 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 04:07:20 smithi070 podman[132988]: 2024-09-14 04:07:20.686504992 +0000 UTC m=+0.341394399 container init 6d817710ad0f63d0678b03e034b72b4a6d810ecc1940f72132bb8b9311deb5df (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3-deactivate, io.buildah.version=1.37.2, RELEASE=main-a278231, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T04:07:20.779 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 04:07:20 smithi070 podman[132988]: 2024-09-14 04:07:20.691428112 +0000 UTC m=+0.346317527 container start 6d817710ad0f63d0678b03e034b72b4a6d810ecc1940f72132bb8b9311deb5df (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3-deactivate, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=main-a278231, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True) 2024-09-14T04:07:20.779 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 04:07:20 smithi070 podman[132988]: 2024-09-14 04:07:20.777999914 +0000 UTC m=+0.432889324 container attach 6d817710ad0f63d0678b03e034b72b4a6d810ecc1940f72132bb8b9311deb5df (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3-deactivate, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=main-a278231, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-14T04:07:20.982 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 04:07:20 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4[92016]: 2024-09-14T04:07:20.824+0000 7f6d53a2c640 -1 osd.4 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.160093+0000 front 2024-09-14T04:06:53.160215+0000 (oldest deadline 2024-09-14T04:07:18.460079+0000) 2024-09-14T04:07:20.982 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:20 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:20.823+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:54.848265+0000 front 2024-09-14T04:06:54.848305+0000 (oldest deadline 2024-09-14T04:07:19.548373+0000) 2024-09-14T04:07:20.983 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:20 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[96999]: 2024-09-14T04:07:20.575+0000 7f9072f2d640 -1 osd.5 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:50.888890+0000 front 2024-09-14T04:06:50.888943+0000 (oldest deadline 2024-09-14T04:07:15.588797+0000) 2024-09-14T04:07:21.035 INFO:journalctl@ceph.osd.3.smithi070.stdout:Sep 14 04:07:20 smithi070 podman[132988]: 2024-09-14 04:07:20.874870151 +0000 UTC m=+0.529759565 container died 6d817710ad0f63d0678b03e034b72b4a6d810ecc1940f72132bb8b9311deb5df (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=main-a278231, CEPH_POINT_RELEASE=) 2024-09-14T04:07:21.276 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:20 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:20.981+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.146491+0000 front 2024-09-14T04:06:53.146435+0000 (oldest deadline 2024-09-14T04:07:16.646555+0000) 2024-09-14T04:07:21.468 DEBUG:teuthology.orchestra.run.smithi070:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.3.service' 2024-09-14T04:07:21.506 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T04:07:21.506 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-14T04:07:21.506 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-14T04:07:21.507 DEBUG:teuthology.orchestra.run.smithi094:> sudo systemctl stop ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.4 2024-09-14T04:07:21.788 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 04:07:21 smithi094 systemd[1]: Stopping Ceph osd.4 for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T04:07:21.788 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:21 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[96999]: 2024-09-14T04:07:21.590+0000 7f9072f2d640 -1 osd.5 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:50.888890+0000 front 2024-09-14T04:06:50.888943+0000 (oldest deadline 2024-09-14T04:07:15.588797+0000) 2024-09-14T04:07:22.276 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:22 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:22.023+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.146491+0000 front 2024-09-14T04:06:53.146435+0000 (oldest deadline 2024-09-14T04:07:16.646555+0000) 2024-09-14T04:07:22.277 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:21 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:21.870+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:54.848265+0000 front 2024-09-14T04:06:54.848305+0000 (oldest deadline 2024-09-14T04:07:19.548373+0000) 2024-09-14T04:07:22.277 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 04:07:21 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4[92016]: 2024-09-14T04:07:21.786+0000 7f6d53a2c640 -1 osd.4 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.160093+0000 front 2024-09-14T04:06:53.160215+0000 (oldest deadline 2024-09-14T04:07:18.460079+0000) 2024-09-14T04:07:22.277 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 04:07:21 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4[92016]: 2024-09-14T04:07:21.873+0000 7f6d5b41f640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:07:22.278 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 04:07:21 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4[92016]: 2024-09-14T04:07:21.873+0000 7f6d5b41f640 -1 osd.4 1069 *** Got signal Terminated *** 2024-09-14T04:07:22.278 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 04:07:21 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4[92016]: 2024-09-14T04:07:21.873+0000 7f6d5b41f640 -1 osd.4 1069 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T04:07:23.026 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 04:07:22 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4[92016]: 2024-09-14T04:07:22.774+0000 7f6d53a2c640 -1 osd.4 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.160093+0000 front 2024-09-14T04:06:53.160215+0000 (oldest deadline 2024-09-14T04:07:18.460079+0000) 2024-09-14T04:07:23.027 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:22 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[96999]: 2024-09-14T04:07:22.613+0000 7f9072f2d640 -1 osd.5 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:50.888890+0000 front 2024-09-14T04:06:50.888943+0000 (oldest deadline 2024-09-14T04:07:15.588797+0000) 2024-09-14T04:07:23.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:22 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:22.860+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:54.848265+0000 front 2024-09-14T04:06:54.848305+0000 (oldest deadline 2024-09-14T04:07:19.548373+0000) 2024-09-14T04:07:23.526 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:23 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:23.043+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.146491+0000 front 2024-09-14T04:06:53.146435+0000 (oldest deadline 2024-09-14T04:07:16.646555+0000) 2024-09-14T04:07:24.020 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 04:07:23 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4[92016]: 2024-09-14T04:07:23.757+0000 7f6d53a2c640 -1 osd.4 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.160093+0000 front 2024-09-14T04:06:53.160215+0000 (oldest deadline 2024-09-14T04:07:18.460079+0000) 2024-09-14T04:07:24.021 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:23 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[96999]: 2024-09-14T04:07:23.654+0000 7f9072f2d640 -1 osd.5 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:50.888890+0000 front 2024-09-14T04:06:50.888943+0000 (oldest deadline 2024-09-14T04:07:15.588797+0000) 2024-09-14T04:07:24.021 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:23 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:23.839+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:54.848265+0000 front 2024-09-14T04:06:54.848305+0000 (oldest deadline 2024-09-14T04:07:19.548373+0000) 2024-09-14T04:07:24.276 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:24 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:24.019+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.146491+0000 front 2024-09-14T04:06:53.146435+0000 (oldest deadline 2024-09-14T04:07:16.646555+0000) 2024-09-14T04:07:24.277 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:24 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:24.019+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.547663+0000 front 2024-09-14T04:06:59.547746+0000 (oldest deadline 2024-09-14T04:07:23.647618+0000) 2024-09-14T04:07:25.026 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:24 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[96999]: 2024-09-14T04:07:24.643+0000 7f9072f2d640 -1 osd.5 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:50.888890+0000 front 2024-09-14T04:06:50.888943+0000 (oldest deadline 2024-09-14T04:07:15.588797+0000) 2024-09-14T04:07:25.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:24 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:24.790+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:54.848265+0000 front 2024-09-14T04:06:54.848305+0000 (oldest deadline 2024-09-14T04:07:19.548373+0000) 2024-09-14T04:07:25.027 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 04:07:24 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4[92016]: 2024-09-14T04:07:24.739+0000 7f6d53a2c640 -1 osd.4 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.160093+0000 front 2024-09-14T04:06:53.160215+0000 (oldest deadline 2024-09-14T04:07:18.460079+0000) 2024-09-14T04:07:25.027 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 04:07:24 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4[92016]: 2024-09-14T04:07:24.739+0000 7f6d53a2c640 -1 osd.4 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:58.460407+0000 front 2024-09-14T04:06:58.460582+0000 (oldest deadline 2024-09-14T04:07:24.360473+0000) 2024-09-14T04:07:25.527 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:25.046+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.146491+0000 front 2024-09-14T04:06:53.146435+0000 (oldest deadline 2024-09-14T04:07:16.646555+0000) 2024-09-14T04:07:25.527 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:25.046+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.547663+0000 front 2024-09-14T04:06:59.547746+0000 (oldest deadline 2024-09-14T04:07:23.647618+0000) 2024-09-14T04:07:26.026 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[96999]: 2024-09-14T04:07:25.685+0000 7f9072f2d640 -1 osd.5 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:50.888890+0000 front 2024-09-14T04:06:50.888943+0000 (oldest deadline 2024-09-14T04:07:15.588797+0000) 2024-09-14T04:07:26.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:25.750+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:54.848265+0000 front 2024-09-14T04:06:54.848305+0000 (oldest deadline 2024-09-14T04:07:19.548373+0000) 2024-09-14T04:07:26.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:25.750+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.548880+0000 front 2024-09-14T04:06:59.548862+0000 (oldest deadline 2024-09-14T04:07:24.848734+0000) 2024-09-14T04:07:26.027 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 04:07:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4[92016]: 2024-09-14T04:07:25.713+0000 7f6d53a2c640 -1 osd.4 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.160093+0000 front 2024-09-14T04:06:53.160215+0000 (oldest deadline 2024-09-14T04:07:18.460079+0000) 2024-09-14T04:07:26.028 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 04:07:25 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4[92016]: 2024-09-14T04:07:25.713+0000 7f6d53a2c640 -1 osd.4 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:58.460407+0000 front 2024-09-14T04:06:58.460582+0000 (oldest deadline 2024-09-14T04:07:24.360473+0000) 2024-09-14T04:07:26.527 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:26 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:26.057+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.146491+0000 front 2024-09-14T04:06:53.146435+0000 (oldest deadline 2024-09-14T04:07:16.646555+0000) 2024-09-14T04:07:26.527 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:26 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:26.057+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.547663+0000 front 2024-09-14T04:06:59.547746+0000 (oldest deadline 2024-09-14T04:07:23.647618+0000) 2024-09-14T04:07:27.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:26 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:26.727+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:54.848265+0000 front 2024-09-14T04:06:54.848305+0000 (oldest deadline 2024-09-14T04:07:19.548373+0000) 2024-09-14T04:07:27.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:26 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:26.727+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.548880+0000 front 2024-09-14T04:06:59.548862+0000 (oldest deadline 2024-09-14T04:07:24.848734+0000) 2024-09-14T04:07:27.027 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 04:07:26 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4[92016]: 2024-09-14T04:07:26.710+0000 7f6d53a2c640 -1 osd.4 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.160093+0000 front 2024-09-14T04:06:53.160215+0000 (oldest deadline 2024-09-14T04:07:18.460079+0000) 2024-09-14T04:07:27.028 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 04:07:26 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4[92016]: 2024-09-14T04:07:26.710+0000 7f6d53a2c640 -1 osd.4 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:58.460407+0000 front 2024-09-14T04:06:58.460582+0000 (oldest deadline 2024-09-14T04:07:24.360473+0000) 2024-09-14T04:07:27.028 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:26 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[96999]: 2024-09-14T04:07:26.677+0000 7f9072f2d640 -1 osd.5 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:50.888890+0000 front 2024-09-14T04:06:50.888943+0000 (oldest deadline 2024-09-14T04:07:15.588797+0000) 2024-09-14T04:07:27.028 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:26 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[96999]: 2024-09-14T04:07:26.677+0000 7f9072f2d640 -1 osd.5 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:07:02.490449+0000 front 2024-09-14T04:07:02.490340+0000 (oldest deadline 2024-09-14T04:07:25.990337+0000) 2024-09-14T04:07:27.419 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 04:07:27 smithi094 podman[119083]: 2024-09-14 04:07:27.265346986 +0000 UTC m=+5.559909430 container died 09f583ed6c7702dd334d1ec70d731d1ecc9590be5d663b896b6b21c81a35f481 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=main-a278231) 2024-09-14T04:07:27.419 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:27 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:27.042+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.146491+0000 front 2024-09-14T04:06:53.146435+0000 (oldest deadline 2024-09-14T04:07:16.646555+0000) 2024-09-14T04:07:27.419 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:27 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:27.042+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.547663+0000 front 2024-09-14T04:06:59.547746+0000 (oldest deadline 2024-09-14T04:07:23.647618+0000) 2024-09-14T04:07:27.763 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 04:07:27 smithi094 podman[119083]: 2024-09-14 04:07:27.622011394 +0000 UTC m=+5.916573835 container cleanup 09f583ed6c7702dd334d1ec70d731d1ecc9590be5d663b896b6b21c81a35f481 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4, RELEASE=main-a278231, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-14T04:07:27.764 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 04:07:27 smithi094 bash[119083]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4 2024-09-14T04:07:27.764 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:27 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[96999]: 2024-09-14T04:07:27.659+0000 7f9072f2d640 -1 osd.5 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:50.888890+0000 front 2024-09-14T04:06:50.888943+0000 (oldest deadline 2024-09-14T04:07:15.588797+0000) 2024-09-14T04:07:27.764 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:27 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[96999]: 2024-09-14T04:07:27.659+0000 7f9072f2d640 -1 osd.5 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:07:02.490449+0000 front 2024-09-14T04:07:02.490340+0000 (oldest deadline 2024-09-14T04:07:25.990337+0000) 2024-09-14T04:07:28.026 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 04:07:27 smithi094 podman[119106]: 2024-09-14 04:07:27.857193074 +0000 UTC m=+0.589701388 container remove 09f583ed6c7702dd334d1ec70d731d1ecc9590be5d663b896b6b21c81a35f481 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=main-a278231, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T04:07:28.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:27 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:27.762+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:54.848265+0000 front 2024-09-14T04:06:54.848305+0000 (oldest deadline 2024-09-14T04:07:19.548373+0000) 2024-09-14T04:07:28.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:27 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:27.762+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.548880+0000 front 2024-09-14T04:06:59.548862+0000 (oldest deadline 2024-09-14T04:07:24.848734+0000) 2024-09-14T04:07:28.402 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:28 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:28.076+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.146491+0000 front 2024-09-14T04:06:53.146435+0000 (oldest deadline 2024-09-14T04:07:16.646555+0000) 2024-09-14T04:07:28.402 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:28 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:28.076+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.547663+0000 front 2024-09-14T04:06:59.547746+0000 (oldest deadline 2024-09-14T04:07:23.647618+0000) 2024-09-14T04:07:28.402 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 04:07:28 smithi094 podman[119197]: 2024-09-14 04:07:28.184777499 +0000 UTC m=+0.015898919 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:07:28.402 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 04:07:28 smithi094 podman[119197]: 2024-09-14 04:07:28.299840354 +0000 UTC m=+0.130961779 container create 071e279ca93682872b269becb32d63e4d086924a96a3ae89e8601e7fe3fb2863 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4-deactivate, RELEASE=main-a278231, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-14T04:07:28.692 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 04:07:28 smithi094 podman[119197]: 2024-09-14 04:07:28.502754325 +0000 UTC m=+0.333875742 container init 071e279ca93682872b269becb32d63e4d086924a96a3ae89e8601e7fe3fb2863 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, io.buildah.version=1.37.2) 2024-09-14T04:07:28.692 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 04:07:28 smithi094 podman[119197]: 2024-09-14 04:07:28.507577343 +0000 UTC m=+0.338698763 container start 071e279ca93682872b269becb32d63e4d086924a96a3ae89e8601e7fe3fb2863 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, RELEASE=main-a278231, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-14T04:07:28.692 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 04:07:28 smithi094 podman[119197]: 2024-09-14 04:07:28.585779081 +0000 UTC m=+0.416900512 container attach 071e279ca93682872b269becb32d63e4d086924a96a3ae89e8601e7fe3fb2863 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=main-a278231, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-14T04:07:28.981 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:28 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[96999]: 2024-09-14T04:07:28.704+0000 7f9072f2d640 -1 osd.5 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:50.888890+0000 front 2024-09-14T04:06:50.888943+0000 (oldest deadline 2024-09-14T04:07:15.588797+0000) 2024-09-14T04:07:28.981 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:28 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[96999]: 2024-09-14T04:07:28.704+0000 7f9072f2d640 -1 osd.5 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:07:02.490449+0000 front 2024-09-14T04:07:02.490340+0000 (oldest deadline 2024-09-14T04:07:25.990337+0000) 2024-09-14T04:07:28.982 INFO:journalctl@ceph.osd.4.smithi094.stdout:Sep 14 04:07:28 smithi094 podman[119197]: 2024-09-14 04:07:28.691485927 +0000 UTC m=+0.522607352 container died 071e279ca93682872b269becb32d63e4d086924a96a3ae89e8601e7fe3fb2863 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, RELEASE=main-a278231, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T04:07:28.982 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:28 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:28.773+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:54.848265+0000 front 2024-09-14T04:06:54.848305+0000 (oldest deadline 2024-09-14T04:07:19.548373+0000) 2024-09-14T04:07:28.982 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:28 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:28.773+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.548880+0000 front 2024-09-14T04:06:59.548862+0000 (oldest deadline 2024-09-14T04:07:24.848734+0000) 2024-09-14T04:07:29.274 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:29 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:29.075+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.146491+0000 front 2024-09-14T04:06:53.146435+0000 (oldest deadline 2024-09-14T04:07:16.646555+0000) 2024-09-14T04:07:29.274 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:29 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:29.075+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.547663+0000 front 2024-09-14T04:06:59.547746+0000 (oldest deadline 2024-09-14T04:07:23.647618+0000) 2024-09-14T04:07:29.292 DEBUG:teuthology.orchestra.run.smithi094:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.4.service' 2024-09-14T04:07:29.332 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T04:07:29.333 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-14T04:07:29.333 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-14T04:07:29.334 DEBUG:teuthology.orchestra.run.smithi094:> sudo systemctl stop ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.5 2024-09-14T04:07:29.527 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:29 smithi094 systemd[1]: Stopping Ceph osd.5 for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T04:07:30.028 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:29 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:29.739+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:54.848265+0000 front 2024-09-14T04:06:54.848305+0000 (oldest deadline 2024-09-14T04:07:19.548373+0000) 2024-09-14T04:07:30.029 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:29 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:29.739+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.548880+0000 front 2024-09-14T04:06:59.548862+0000 (oldest deadline 2024-09-14T04:07:24.848734+0000) 2024-09-14T04:07:30.029 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:29 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[96999]: 2024-09-14T04:07:29.643+0000 7f907a920640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:07:30.029 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:29 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[96999]: 2024-09-14T04:07:29.643+0000 7f907a920640 -1 osd.5 1069 *** Got signal Terminated *** 2024-09-14T04:07:30.029 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:29 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[96999]: 2024-09-14T04:07:29.643+0000 7f907a920640 -1 osd.5 1069 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T04:07:30.030 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:29 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[96999]: 2024-09-14T04:07:29.675+0000 7f9072f2d640 -1 osd.5 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:50.888890+0000 front 2024-09-14T04:06:50.888943+0000 (oldest deadline 2024-09-14T04:07:15.588797+0000) 2024-09-14T04:07:30.030 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:29 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[96999]: 2024-09-14T04:07:29.675+0000 7f9072f2d640 -1 osd.5 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:07:02.490449+0000 front 2024-09-14T04:07:02.490340+0000 (oldest deadline 2024-09-14T04:07:25.990337+0000) 2024-09-14T04:07:30.526 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:30.041+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.146491+0000 front 2024-09-14T04:06:53.146435+0000 (oldest deadline 2024-09-14T04:07:16.646555+0000) 2024-09-14T04:07:30.527 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:30.041+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.547663+0000 front 2024-09-14T04:06:59.547746+0000 (oldest deadline 2024-09-14T04:07:23.647618+0000) 2024-09-14T04:07:31.027 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[96999]: 2024-09-14T04:07:30.671+0000 7f9072f2d640 -1 osd.5 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:50.888890+0000 front 2024-09-14T04:06:50.888943+0000 (oldest deadline 2024-09-14T04:07:15.588797+0000) 2024-09-14T04:07:31.027 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[96999]: 2024-09-14T04:07:30.671+0000 7f9072f2d640 -1 osd.5 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:07:02.490449+0000 front 2024-09-14T04:07:02.490340+0000 (oldest deadline 2024-09-14T04:07:25.990337+0000) 2024-09-14T04:07:31.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:30.729+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:54.848265+0000 front 2024-09-14T04:06:54.848305+0000 (oldest deadline 2024-09-14T04:07:19.548373+0000) 2024-09-14T04:07:31.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:30 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:30.729+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.548880+0000 front 2024-09-14T04:06:59.548862+0000 (oldest deadline 2024-09-14T04:07:24.848734+0000) 2024-09-14T04:07:31.526 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:31 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:31.087+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.146491+0000 front 2024-09-14T04:06:53.146435+0000 (oldest deadline 2024-09-14T04:07:16.646555+0000) 2024-09-14T04:07:31.527 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:31 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:31.087+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.547663+0000 front 2024-09-14T04:06:59.547746+0000 (oldest deadline 2024-09-14T04:07:23.647618+0000) 2024-09-14T04:07:32.026 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:31 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:31.736+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:54.848265+0000 front 2024-09-14T04:06:54.848305+0000 (oldest deadline 2024-09-14T04:07:19.548373+0000) 2024-09-14T04:07:32.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:31 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:31.736+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.548880+0000 front 2024-09-14T04:06:59.548862+0000 (oldest deadline 2024-09-14T04:07:24.848734+0000) 2024-09-14T04:07:32.027 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:31 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[96999]: 2024-09-14T04:07:31.718+0000 7f9072f2d640 -1 osd.5 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:50.888890+0000 front 2024-09-14T04:06:50.888943+0000 (oldest deadline 2024-09-14T04:07:15.588797+0000) 2024-09-14T04:07:32.027 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:31 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[96999]: 2024-09-14T04:07:31.718+0000 7f9072f2d640 -1 osd.5 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:07:02.490449+0000 front 2024-09-14T04:07:02.490340+0000 (oldest deadline 2024-09-14T04:07:25.990337+0000) 2024-09-14T04:07:32.527 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:32 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:32.097+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.146491+0000 front 2024-09-14T04:06:53.146435+0000 (oldest deadline 2024-09-14T04:07:16.646555+0000) 2024-09-14T04:07:32.527 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:32 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:32.097+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.547663+0000 front 2024-09-14T04:06:59.547746+0000 (oldest deadline 2024-09-14T04:07:23.647618+0000) 2024-09-14T04:07:33.026 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:32 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[96999]: 2024-09-14T04:07:32.766+0000 7f9072f2d640 -1 osd.5 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:50.888890+0000 front 2024-09-14T04:06:50.888943+0000 (oldest deadline 2024-09-14T04:07:15.588797+0000) 2024-09-14T04:07:33.027 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:32 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[96999]: 2024-09-14T04:07:32.766+0000 7f9072f2d640 -1 osd.5 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:07:02.490449+0000 front 2024-09-14T04:07:02.490340+0000 (oldest deadline 2024-09-14T04:07:25.990337+0000) 2024-09-14T04:07:33.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:32 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:32.753+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:54.848265+0000 front 2024-09-14T04:06:54.848305+0000 (oldest deadline 2024-09-14T04:07:19.548373+0000) 2024-09-14T04:07:33.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:32 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:32.754+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.548880+0000 front 2024-09-14T04:06:59.548862+0000 (oldest deadline 2024-09-14T04:07:24.848734+0000) 2024-09-14T04:07:33.526 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:33.072+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.146491+0000 front 2024-09-14T04:06:53.146435+0000 (oldest deadline 2024-09-14T04:07:16.646555+0000) 2024-09-14T04:07:33.527 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:33.072+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.547663+0000 front 2024-09-14T04:06:59.547746+0000 (oldest deadline 2024-09-14T04:07:23.647618+0000) 2024-09-14T04:07:34.026 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:33.798+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:54.848265+0000 front 2024-09-14T04:06:54.848305+0000 (oldest deadline 2024-09-14T04:07:19.548373+0000) 2024-09-14T04:07:34.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:33.798+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.548880+0000 front 2024-09-14T04:06:59.548862+0000 (oldest deadline 2024-09-14T04:07:24.848734+0000) 2024-09-14T04:07:34.027 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[96999]: 2024-09-14T04:07:33.734+0000 7f9072f2d640 -1 osd.5 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:50.888890+0000 front 2024-09-14T04:06:50.888943+0000 (oldest deadline 2024-09-14T04:07:15.588797+0000) 2024-09-14T04:07:34.028 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:33 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5[96999]: 2024-09-14T04:07:33.734+0000 7f9072f2d640 -1 osd.5 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:07:02.490449+0000 front 2024-09-14T04:07:02.490340+0000 (oldest deadline 2024-09-14T04:07:25.990337+0000) 2024-09-14T04:07:34.528 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:34 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:34.025+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.146491+0000 front 2024-09-14T04:06:53.146435+0000 (oldest deadline 2024-09-14T04:07:16.646555+0000) 2024-09-14T04:07:34.529 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:34 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:34.025+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.547663+0000 front 2024-09-14T04:06:59.547746+0000 (oldest deadline 2024-09-14T04:07:23.647618+0000) 2024-09-14T04:07:34.529 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:34 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:34.025+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-09-14T04:07:08.248940+0000 front 2024-09-14T04:07:08.248833+0000 (oldest deadline 2024-09-14T04:07:33.548881+0000) 2024-09-14T04:07:35.077 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:34 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:34.804+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:54.848265+0000 front 2024-09-14T04:06:54.848305+0000 (oldest deadline 2024-09-14T04:07:19.548373+0000) 2024-09-14T04:07:35.077 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:34 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:34.804+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.548880+0000 front 2024-09-14T04:06:59.548862+0000 (oldest deadline 2024-09-14T04:07:24.848734+0000) 2024-09-14T04:07:35.077 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:35 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:35.034+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.146491+0000 front 2024-09-14T04:06:53.146435+0000 (oldest deadline 2024-09-14T04:07:16.646555+0000) 2024-09-14T04:07:35.077 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:35 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:35.034+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.547663+0000 front 2024-09-14T04:06:59.547746+0000 (oldest deadline 2024-09-14T04:07:23.647618+0000) 2024-09-14T04:07:35.077 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:35 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:35.034+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-09-14T04:07:08.248940+0000 front 2024-09-14T04:07:08.248833+0000 (oldest deadline 2024-09-14T04:07:33.548881+0000) 2024-09-14T04:07:35.356 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:35 smithi094 podman[119325]: 2024-09-14 04:07:35.076132782 +0000 UTC m=+5.526202945 container died 162396f2e7b684b77df919d2a9ad5a2e79cbe98d24eaa87ab5e92a7093147ee4 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=main-a278231, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-14T04:07:35.703 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:35 smithi094 podman[119325]: 2024-09-14 04:07:35.443239951 +0000 UTC m=+5.893310114 container cleanup 162396f2e7b684b77df919d2a9ad5a2e79cbe98d24eaa87ab5e92a7093147ee4 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , RELEASE=main-a278231, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T04:07:35.703 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:35 smithi094 bash[119325]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5 2024-09-14T04:07:36.027 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:35 smithi094 podman[119348]: 2024-09-14 04:07:35.702912812 +0000 UTC m=+0.621404447 container remove 162396f2e7b684b77df919d2a9ad5a2e79cbe98d24eaa87ab5e92a7093147ee4 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=main-a278231, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T04:07:36.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:35 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:35.826+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:54.848265+0000 front 2024-09-14T04:06:54.848305+0000 (oldest deadline 2024-09-14T04:07:19.548373+0000) 2024-09-14T04:07:36.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:35 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:35.826+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.548880+0000 front 2024-09-14T04:06:59.548862+0000 (oldest deadline 2024-09-14T04:07:24.848734+0000) 2024-09-14T04:07:36.390 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:36 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:36.059+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.146491+0000 front 2024-09-14T04:06:53.146435+0000 (oldest deadline 2024-09-14T04:07:16.646555+0000) 2024-09-14T04:07:36.390 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:36 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:36.059+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.547663+0000 front 2024-09-14T04:06:59.547746+0000 (oldest deadline 2024-09-14T04:07:23.647618+0000) 2024-09-14T04:07:36.391 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:36 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:36.059+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-09-14T04:07:08.248940+0000 front 2024-09-14T04:07:08.248833+0000 (oldest deadline 2024-09-14T04:07:33.548881+0000) 2024-09-14T04:07:36.391 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:36 smithi094 podman[119444]: 2024-09-14 04:07:35.975536449 +0000 UTC m=+0.015794711 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:07:36.391 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:36 smithi094 podman[119444]: 2024-09-14 04:07:36.078953595 +0000 UTC m=+0.119211857 container create 25dd4eb532910d5e89099691b59d6d27c2e32e922cf6f72bd1e9fa20afa500eb (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-14T04:07:36.391 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:36 smithi094 podman[119444]: 2024-09-14 04:07:36.298197102 +0000 UTC m=+0.338455377 container init 25dd4eb532910d5e89099691b59d6d27c2e32e922cf6f72bd1e9fa20afa500eb (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5-deactivate, RELEASE=main-a278231, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-14T04:07:36.391 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:36 smithi094 podman[119444]: 2024-09-14 04:07:36.303157154 +0000 UTC m=+0.343415416 container start 25dd4eb532910d5e89099691b59d6d27c2e32e922cf6f72bd1e9fa20afa500eb (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-14T04:07:36.392 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:36 smithi094 podman[119444]: 2024-09-14 04:07:36.389760421 +0000 UTC m=+0.430018688 container attach 25dd4eb532910d5e89099691b59d6d27c2e32e922cf6f72bd1e9fa20afa500eb (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-14T04:07:36.777 INFO:journalctl@ceph.osd.5.smithi094.stdout:Sep 14 04:07:36 smithi094 podman[119444]: 2024-09-14 04:07:36.489150233 +0000 UTC m=+0.529408496 container died 25dd4eb532910d5e89099691b59d6d27c2e32e922cf6f72bd1e9fa20afa500eb (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-14T04:07:37.070 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:36 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:36.835+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:54.848265+0000 front 2024-09-14T04:06:54.848305+0000 (oldest deadline 2024-09-14T04:07:19.548373+0000) 2024-09-14T04:07:37.070 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:36 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:36.835+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.548880+0000 front 2024-09-14T04:06:59.548862+0000 (oldest deadline 2024-09-14T04:07:24.848734+0000) 2024-09-14T04:07:37.088 DEBUG:teuthology.orchestra.run.smithi094:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.5.service' 2024-09-14T04:07:37.129 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T04:07:37.129 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-14T04:07:37.130 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-14T04:07:37.130 DEBUG:teuthology.orchestra.run.smithi094:> sudo systemctl stop ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.6 2024-09-14T04:07:37.450 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:37.097+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.146491+0000 front 2024-09-14T04:06:53.146435+0000 (oldest deadline 2024-09-14T04:07:16.646555+0000) 2024-09-14T04:07:37.450 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:37.097+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.547663+0000 front 2024-09-14T04:06:59.547746+0000 (oldest deadline 2024-09-14T04:07:23.647618+0000) 2024-09-14T04:07:37.451 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:37.097+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-09-14T04:07:08.248940+0000 front 2024-09-14T04:07:08.248833+0000 (oldest deadline 2024-09-14T04:07:33.548881+0000) 2024-09-14T04:07:37.451 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:37 smithi094 systemd[1]: Stopping Ceph osd.6 for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T04:07:37.451 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:37.447+0000 7f90cdc18640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:07:37.451 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:37.447+0000 7f90cdc18640 -1 osd.6 1069 *** Got signal Terminated *** 2024-09-14T04:07:37.451 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:37.447+0000 7f90cdc18640 -1 osd.6 1069 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T04:07:38.092 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:37.808+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:54.848265+0000 front 2024-09-14T04:06:54.848305+0000 (oldest deadline 2024-09-14T04:07:19.548373+0000) 2024-09-14T04:07:38.092 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:37.808+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.548880+0000 front 2024-09-14T04:06:59.548862+0000 (oldest deadline 2024-09-14T04:07:24.848734+0000) 2024-09-14T04:07:38.092 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:37 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:37.808+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-09-14T04:07:10.050130+0000 front 2024-09-14T04:07:10.050017+0000 (oldest deadline 2024-09-14T04:07:37.050204+0000) 2024-09-14T04:07:38.526 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:38 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:38.089+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.146491+0000 front 2024-09-14T04:06:53.146435+0000 (oldest deadline 2024-09-14T04:07:16.646555+0000) 2024-09-14T04:07:38.527 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:38 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:38.089+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.547663+0000 front 2024-09-14T04:06:59.547746+0000 (oldest deadline 2024-09-14T04:07:23.647618+0000) 2024-09-14T04:07:38.527 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:38 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:38.089+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-09-14T04:07:08.248940+0000 front 2024-09-14T04:07:08.248833+0000 (oldest deadline 2024-09-14T04:07:33.548881+0000) 2024-09-14T04:07:39.277 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:39 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:39.046+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.146491+0000 front 2024-09-14T04:06:53.146435+0000 (oldest deadline 2024-09-14T04:07:16.646555+0000) 2024-09-14T04:07:39.277 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:39 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:39.046+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.547663+0000 front 2024-09-14T04:06:59.547746+0000 (oldest deadline 2024-09-14T04:07:23.647618+0000) 2024-09-14T04:07:39.277 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:39 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:39.046+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-09-14T04:07:08.248940+0000 front 2024-09-14T04:07:08.248833+0000 (oldest deadline 2024-09-14T04:07:33.548881+0000) 2024-09-14T04:07:39.278 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:38 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:38.854+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:54.848265+0000 front 2024-09-14T04:06:54.848305+0000 (oldest deadline 2024-09-14T04:07:19.548373+0000) 2024-09-14T04:07:39.278 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:38 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:38.854+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.548880+0000 front 2024-09-14T04:06:59.548862+0000 (oldest deadline 2024-09-14T04:07:24.848734+0000) 2024-09-14T04:07:39.278 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:38 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:38.854+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-09-14T04:07:10.050130+0000 front 2024-09-14T04:07:10.050017+0000 (oldest deadline 2024-09-14T04:07:37.050204+0000) 2024-09-14T04:07:40.277 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:40 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:40.028+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.146491+0000 front 2024-09-14T04:06:53.146435+0000 (oldest deadline 2024-09-14T04:07:16.646555+0000) 2024-09-14T04:07:40.277 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:40 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:40.028+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.547663+0000 front 2024-09-14T04:06:59.547746+0000 (oldest deadline 2024-09-14T04:07:23.647618+0000) 2024-09-14T04:07:40.277 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:40 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:40.028+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-09-14T04:07:08.248940+0000 front 2024-09-14T04:07:08.248833+0000 (oldest deadline 2024-09-14T04:07:33.548881+0000) 2024-09-14T04:07:40.277 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:39 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:39.808+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:54.848265+0000 front 2024-09-14T04:06:54.848305+0000 (oldest deadline 2024-09-14T04:07:19.548373+0000) 2024-09-14T04:07:40.277 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:39 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:39.808+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.548880+0000 front 2024-09-14T04:06:59.548862+0000 (oldest deadline 2024-09-14T04:07:24.848734+0000) 2024-09-14T04:07:40.278 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:39 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:39.808+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-09-14T04:07:10.050130+0000 front 2024-09-14T04:07:10.050017+0000 (oldest deadline 2024-09-14T04:07:37.050204+0000) 2024-09-14T04:07:41.277 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:40 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:40.990+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.146491+0000 front 2024-09-14T04:06:53.146435+0000 (oldest deadline 2024-09-14T04:07:16.646555+0000) 2024-09-14T04:07:41.277 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:40 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:40.990+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.547663+0000 front 2024-09-14T04:06:59.547746+0000 (oldest deadline 2024-09-14T04:07:23.647618+0000) 2024-09-14T04:07:41.277 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:40 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:40.990+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-09-14T04:07:08.248940+0000 front 2024-09-14T04:07:08.248833+0000 (oldest deadline 2024-09-14T04:07:33.548881+0000) 2024-09-14T04:07:41.277 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:40 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:40.806+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:54.848265+0000 front 2024-09-14T04:06:54.848305+0000 (oldest deadline 2024-09-14T04:07:19.548373+0000) 2024-09-14T04:07:41.277 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:40 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:40.806+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.548880+0000 front 2024-09-14T04:06:59.548862+0000 (oldest deadline 2024-09-14T04:07:24.848734+0000) 2024-09-14T04:07:41.278 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:40 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:40.806+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-09-14T04:07:10.050130+0000 front 2024-09-14T04:07:10.050017+0000 (oldest deadline 2024-09-14T04:07:37.050204+0000) 2024-09-14T04:07:41.957 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 04:07:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=sqlstore.transactions t=2024-09-14T04:07:41.591122609Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-14T04:07:41.957 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 04:07:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=cleanup t=2024-09-14T04:07:41.59504972Z level=info msg="Completed cleanup jobs" duration=14.68666ms 2024-09-14T04:07:41.957 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 04:07:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=grafana.update.checker t=2024-09-14T04:07:41.732424472Z level=info msg="Update check succeeded" duration=60.53313ms 2024-09-14T04:07:41.957 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 04:07:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=plugins.update.checker t=2024-09-14T04:07:41.735633473Z level=info msg="Update check succeeded" duration=66.328233ms 2024-09-14T04:07:41.958 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:41.819+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:54.848265+0000 front 2024-09-14T04:06:54.848305+0000 (oldest deadline 2024-09-14T04:07:19.548373+0000) 2024-09-14T04:07:41.958 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:41.819+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.548880+0000 front 2024-09-14T04:06:59.548862+0000 (oldest deadline 2024-09-14T04:07:24.848734+0000) 2024-09-14T04:07:41.958 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:41.819+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-09-14T04:07:10.050130+0000 front 2024-09-14T04:07:10.050017+0000 (oldest deadline 2024-09-14T04:07:37.050204+0000) 2024-09-14T04:07:42.277 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:41.955+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:53.146491+0000 front 2024-09-14T04:06:53.146435+0000 (oldest deadline 2024-09-14T04:07:16.646555+0000) 2024-09-14T04:07:42.277 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:41.955+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.547663+0000 front 2024-09-14T04:06:59.547746+0000 (oldest deadline 2024-09-14T04:07:23.647618+0000) 2024-09-14T04:07:42.277 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:41.955+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-09-14T04:07:08.248940+0000 front 2024-09-14T04:07:08.248833+0000 (oldest deadline 2024-09-14T04:07:33.548881+0000) 2024-09-14T04:07:42.277 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:41 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6[101551]: 2024-09-14T04:07:41.955+0000 7f90ca22d640 -1 osd.6 1069 heartbeat_check: no reply from 172.21.15.70:6830 osd.3 since back 2024-09-14T04:07:17.549947+0000 front 2024-09-14T04:07:17.549937+0000 (oldest deadline 2024-09-14T04:07:41.049924+0000) 2024-09-14T04:07:43.165 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:42 smithi094 podman[119571]: 2024-09-14 04:07:42.903048299 +0000 UTC m=+5.554569189 container died cbdc02cdb8e444c0e3bdaac1b66cca04fbfa0b1685c13caaebaf63506e2dc412 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-14T04:07:43.165 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:42 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:42.793+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:54.848265+0000 front 2024-09-14T04:06:54.848305+0000 (oldest deadline 2024-09-14T04:07:19.548373+0000) 2024-09-14T04:07:43.166 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:42 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:42.793+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.548880+0000 front 2024-09-14T04:06:59.548862+0000 (oldest deadline 2024-09-14T04:07:24.848734+0000) 2024-09-14T04:07:43.166 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:42 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:42.793+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-09-14T04:07:10.050130+0000 front 2024-09-14T04:07:10.050017+0000 (oldest deadline 2024-09-14T04:07:37.050204+0000) 2024-09-14T04:07:43.166 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:42 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:42.793+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6830 osd.3 since back 2024-09-14T04:07:18.751123+0000 front 2024-09-14T04:07:18.751169+0000 (oldest deadline 2024-09-14T04:07:42.251014+0000) 2024-09-14T04:07:43.526 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:43 smithi094 podman[119571]: 2024-09-14 04:07:43.255149519 +0000 UTC m=+5.906670396 container cleanup cbdc02cdb8e444c0e3bdaac1b66cca04fbfa0b1685c13caaebaf63506e2dc412 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=main-a278231, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0) 2024-09-14T04:07:43.527 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:43 smithi094 bash[119571]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6 2024-09-14T04:07:43.849 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:43 smithi094 podman[119591]: 2024-09-14 04:07:43.540074058 +0000 UTC m=+0.633618061 container remove cbdc02cdb8e444c0e3bdaac1b66cca04fbfa0b1685c13caaebaf63506e2dc412 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, ceph=True) 2024-09-14T04:07:43.849 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:43 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:43.751+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:54.848265+0000 front 2024-09-14T04:06:54.848305+0000 (oldest deadline 2024-09-14T04:07:19.548373+0000) 2024-09-14T04:07:43.850 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:43 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:43.751+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.548880+0000 front 2024-09-14T04:06:59.548862+0000 (oldest deadline 2024-09-14T04:07:24.848734+0000) 2024-09-14T04:07:43.850 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:43 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:43.751+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-09-14T04:07:10.050130+0000 front 2024-09-14T04:07:10.050017+0000 (oldest deadline 2024-09-14T04:07:37.050204+0000) 2024-09-14T04:07:43.850 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:43 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:43.751+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6830 osd.3 since back 2024-09-14T04:07:18.751123+0000 front 2024-09-14T04:07:18.751169+0000 (oldest deadline 2024-09-14T04:07:42.251014+0000) 2024-09-14T04:07:44.100 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:43 smithi094 podman[119685]: 2024-09-14 04:07:43.75149886 +0000 UTC m=+0.015752996 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:07:44.100 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:43 smithi094 podman[119685]: 2024-09-14 04:07:43.899952124 +0000 UTC m=+0.164206295 container create c53bb401d61d8cba471cbb6d0c644cce5c57a448d5701903e619c8ee8cef8b01 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6-deactivate, ceph=True, RELEASE=main-a278231, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T04:07:44.100 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:44 smithi094 podman[119685]: 2024-09-14 04:07:44.094009385 +0000 UTC m=+0.358263529 container init c53bb401d61d8cba471cbb6d0c644cce5c57a448d5701903e619c8ee8cef8b01 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=main-a278231, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-14T04:07:44.100 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:44 smithi094 podman[119685]: 2024-09-14 04:07:44.099156553 +0000 UTC m=+0.363410682 container start c53bb401d61d8cba471cbb6d0c644cce5c57a448d5701903e619c8ee8cef8b01 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6-deactivate, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-14T04:07:44.527 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:44 smithi094 podman[119685]: 2024-09-14 04:07:44.185472849 +0000 UTC m=+0.449726988 container attach c53bb401d61d8cba471cbb6d0c644cce5c57a448d5701903e619c8ee8cef8b01 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=main-a278231, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-14T04:07:44.527 INFO:journalctl@ceph.osd.6.smithi094.stdout:Sep 14 04:07:44 smithi094 podman[119685]: 2024-09-14 04:07:44.296175454 +0000 UTC m=+0.560429585 container died c53bb401d61d8cba471cbb6d0c644cce5c57a448d5701903e619c8ee8cef8b01 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=main-a278231) 2024-09-14T04:07:44.907 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:44 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:44.714+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:54.848265+0000 front 2024-09-14T04:06:54.848305+0000 (oldest deadline 2024-09-14T04:07:19.548373+0000) 2024-09-14T04:07:44.907 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:44 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:44.714+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.548880+0000 front 2024-09-14T04:06:59.548862+0000 (oldest deadline 2024-09-14T04:07:24.848734+0000) 2024-09-14T04:07:44.907 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:44 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:44.714+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-09-14T04:07:10.050130+0000 front 2024-09-14T04:07:10.050017+0000 (oldest deadline 2024-09-14T04:07:37.050204+0000) 2024-09-14T04:07:44.907 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:44 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:44.714+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6830 osd.3 since back 2024-09-14T04:07:18.751123+0000 front 2024-09-14T04:07:18.751169+0000 (oldest deadline 2024-09-14T04:07:42.251014+0000) 2024-09-14T04:07:44.938 DEBUG:teuthology.orchestra.run.smithi094:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.6.service' 2024-09-14T04:07:44.976 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T04:07:44.976 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-14T04:07:44.977 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-14T04:07:44.977 DEBUG:teuthology.orchestra.run.smithi094:> sudo systemctl stop ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.7 2024-09-14T04:07:45.276 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:45 smithi094 systemd[1]: Stopping Ceph osd.7 for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T04:07:45.753 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:45 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:45.319+0000 7f6848b6a640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:07:45.753 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:45 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:45.319+0000 7f6848b6a640 -1 osd.7 1069 *** Got signal Terminated *** 2024-09-14T04:07:45.753 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:45 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:45.319+0000 7f6848b6a640 -1 osd.7 1069 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T04:07:45.754 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:45 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:45.751+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:54.848265+0000 front 2024-09-14T04:06:54.848305+0000 (oldest deadline 2024-09-14T04:07:19.548373+0000) 2024-09-14T04:07:45.754 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:45 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:45.751+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.548880+0000 front 2024-09-14T04:06:59.548862+0000 (oldest deadline 2024-09-14T04:07:24.848734+0000) 2024-09-14T04:07:45.754 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:45 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:45.751+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-09-14T04:07:10.050130+0000 front 2024-09-14T04:07:10.050017+0000 (oldest deadline 2024-09-14T04:07:37.050204+0000) 2024-09-14T04:07:45.754 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:45 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:45.751+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6830 osd.3 since back 2024-09-14T04:07:18.751123+0000 front 2024-09-14T04:07:18.751169+0000 (oldest deadline 2024-09-14T04:07:42.251014+0000) 2024-09-14T04:07:47.026 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:46 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:46.734+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:54.848265+0000 front 2024-09-14T04:06:54.848305+0000 (oldest deadline 2024-09-14T04:07:19.548373+0000) 2024-09-14T04:07:47.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:46 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:46.734+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.548880+0000 front 2024-09-14T04:06:59.548862+0000 (oldest deadline 2024-09-14T04:07:24.848734+0000) 2024-09-14T04:07:47.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:46 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:46.734+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-09-14T04:07:10.050130+0000 front 2024-09-14T04:07:10.050017+0000 (oldest deadline 2024-09-14T04:07:37.050204+0000) 2024-09-14T04:07:47.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:46 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:46.734+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6830 osd.3 since back 2024-09-14T04:07:18.751123+0000 front 2024-09-14T04:07:18.751169+0000 (oldest deadline 2024-09-14T04:07:42.251014+0000) 2024-09-14T04:07:47.776 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:47 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:47.703+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:54.848265+0000 front 2024-09-14T04:06:54.848305+0000 (oldest deadline 2024-09-14T04:07:19.548373+0000) 2024-09-14T04:07:47.777 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:47 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:47.703+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.548880+0000 front 2024-09-14T04:06:59.548862+0000 (oldest deadline 2024-09-14T04:07:24.848734+0000) 2024-09-14T04:07:47.777 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:47 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:47.703+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-09-14T04:07:10.050130+0000 front 2024-09-14T04:07:10.050017+0000 (oldest deadline 2024-09-14T04:07:37.050204+0000) 2024-09-14T04:07:47.777 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:47 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:47.703+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6830 osd.3 since back 2024-09-14T04:07:18.751123+0000 front 2024-09-14T04:07:18.751169+0000 (oldest deadline 2024-09-14T04:07:42.251014+0000) 2024-09-14T04:07:49.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:48 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:48.746+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:54.848265+0000 front 2024-09-14T04:06:54.848305+0000 (oldest deadline 2024-09-14T04:07:19.548373+0000) 2024-09-14T04:07:49.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:48 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:48.746+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.548880+0000 front 2024-09-14T04:06:59.548862+0000 (oldest deadline 2024-09-14T04:07:24.848734+0000) 2024-09-14T04:07:49.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:48 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:48.746+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-09-14T04:07:10.050130+0000 front 2024-09-14T04:07:10.050017+0000 (oldest deadline 2024-09-14T04:07:37.050204+0000) 2024-09-14T04:07:49.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:48 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:48.746+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6830 osd.3 since back 2024-09-14T04:07:18.751123+0000 front 2024-09-14T04:07:18.751169+0000 (oldest deadline 2024-09-14T04:07:42.251014+0000) 2024-09-14T04:07:50.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:49 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:49.736+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6806 osd.0 since back 2024-09-14T04:06:54.848265+0000 front 2024-09-14T04:06:54.848305+0000 (oldest deadline 2024-09-14T04:07:19.548373+0000) 2024-09-14T04:07:50.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:49 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:49.736+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-09-14T04:06:59.548880+0000 front 2024-09-14T04:06:59.548862+0000 (oldest deadline 2024-09-14T04:07:24.848734+0000) 2024-09-14T04:07:50.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:49 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:49.736+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-09-14T04:07:10.050130+0000 front 2024-09-14T04:07:10.050017+0000 (oldest deadline 2024-09-14T04:07:37.050204+0000) 2024-09-14T04:07:50.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:49 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7[106362]: 2024-09-14T04:07:49.736+0000 7f6841177640 -1 osd.7 1069 heartbeat_check: no reply from 172.21.15.70:6830 osd.3 since back 2024-09-14T04:07:18.751123+0000 front 2024-09-14T04:07:18.751169+0000 (oldest deadline 2024-09-14T04:07:42.251014+0000) 2024-09-14T04:07:51.095 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:50 smithi094 podman[119811]: 2024-09-14 04:07:50.804137666 +0000 UTC m=+5.605650489 container died ea9896fb7dbc3942967a81594535cb6cb3f19e645e4c71b9a49485678879a51d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-14T04:07:51.419 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:51 smithi094 podman[119811]: 2024-09-14 04:07:51.16740971 +0000 UTC m=+5.968922534 container cleanup ea9896fb7dbc3942967a81594535cb6cb3f19e645e4c71b9a49485678879a51d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=main-a278231, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-14T04:07:51.420 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:51 smithi094 bash[119811]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7 2024-09-14T04:07:51.728 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:51 smithi094 podman[119843]: 2024-09-14 04:07:51.418745585 +0000 UTC m=+0.617796397 container remove ea9896fb7dbc3942967a81594535cb6cb3f19e645e4c71b9a49485678879a51d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-14T04:07:52.026 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:51 smithi094 podman[119934]: 2024-09-14 04:07:51.630214882 +0000 UTC m=+0.016173258 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:07:52.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:51 smithi094 podman[119934]: 2024-09-14 04:07:51.728043887 +0000 UTC m=+0.114002263 container create 8b6c08a9ac032d988a66065c0fdd770816734b2eaa502d7f716e612a5c0dfb6d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-14T04:07:52.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:51 smithi094 podman[119934]: 2024-09-14 04:07:51.938878132 +0000 UTC m=+0.324836509 container init 8b6c08a9ac032d988a66065c0fdd770816734b2eaa502d7f716e612a5c0dfb6d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, GIT_BRANCH=HEAD) 2024-09-14T04:07:52.027 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:51 smithi094 podman[119934]: 2024-09-14 04:07:51.943871894 +0000 UTC m=+0.329830271 container start 8b6c08a9ac032d988a66065c0fdd770816734b2eaa502d7f716e612a5c0dfb6d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS) 2024-09-14T04:07:52.393 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:52 smithi094 podman[119934]: 2024-09-14 04:07:52.030423749 +0000 UTC m=+0.416382126 container attach 8b6c08a9ac032d988a66065c0fdd770816734b2eaa502d7f716e612a5c0dfb6d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7-deactivate, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-14T04:07:52.393 INFO:journalctl@ceph.osd.7.smithi094.stdout:Sep 14 04:07:52 smithi094 podman[119934]: 2024-09-14 04:07:52.122927908 +0000 UTC m=+0.508886289 container died 8b6c08a9ac032d988a66065c0fdd770816734b2eaa502d7f716e612a5c0dfb6d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=main-a278231, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , ceph=True) 2024-09-14T04:07:52.734 DEBUG:teuthology.orchestra.run.smithi094:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@osd.7.service' 2024-09-14T04:07:52.778 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T04:07:52.778 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-14T04:07:52.778 INFO:tasks.cephadm.mds.a:Stopping mds.a... 2024-09-14T04:07:52.779 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ceph-b6380d42-724a-11ef-bceb-c7b262605968@mds.a 2024-09-14T04:07:53.089 INFO:journalctl@ceph.mds.a.smithi070.stdout:Sep 14 04:07:52 smithi070 systemd[1]: Stopping Ceph mds.a for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T04:07:53.535 INFO:journalctl@ceph.mds.a.smithi070.stdout:Sep 14 04:07:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-a[122489]: 2024-09-14T04:07:53.087+0000 7ff7d35ce640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:07:53.536 INFO:journalctl@ceph.mds.a.smithi070.stdout:Sep 14 04:07:53 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-a[122489]: 2024-09-14T04:07:53.088+0000 7ff7d35ce640 -1 mds.a *** got signal Terminated *** 2024-09-14T04:07:57.471 INFO:journalctl@ceph.mds.a.smithi070.stdout:Sep 14 04:07:57 smithi070 podman[133117]: 2024-09-14 04:07:57.095995408 +0000 UTC m=+4.119349294 container died 4fa138982c4f2ae1c100cf405b61fba395e9e5083e6cc5b3190909a8152ec8a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-a, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-14T04:07:57.785 INFO:journalctl@ceph.mds.a.smithi070.stdout:Sep 14 04:07:57 smithi070 podman[133117]: 2024-09-14 04:07:57.568876892 +0000 UTC m=+4.592230712 container cleanup 4fa138982c4f2ae1c100cf405b61fba395e9e5083e6cc5b3190909a8152ec8a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-a, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-14T04:07:57.786 INFO:journalctl@ceph.mds.a.smithi070.stdout:Sep 14 04:07:57 smithi070 bash[133117]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-a 2024-09-14T04:07:58.009 DEBUG:teuthology.orchestra.run.smithi070:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@mds.a.service' 2024-09-14T04:07:58.045 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T04:07:58.090 INFO:tasks.cephadm.mds.a:Stopped mds.a 2024-09-14T04:07:58.090 INFO:tasks.cephadm.mds.b:Stopping mds.b... 2024-09-14T04:07:58.090 DEBUG:teuthology.orchestra.run.smithi094:> sudo systemctl stop ceph-b6380d42-724a-11ef-bceb-c7b262605968@mds.b 2024-09-14T04:07:58.382 INFO:journalctl@ceph.mds.b.smithi094.stdout:Sep 14 04:07:58 smithi094 systemd[1]: Stopping Ceph mds.b for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T04:07:58.776 INFO:journalctl@ceph.mds.b.smithi094.stdout:Sep 14 04:07:58 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-b[110514]: 2024-09-14T04:07:58.380+0000 7fe822c97640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:07:58.777 INFO:journalctl@ceph.mds.b.smithi094.stdout:Sep 14 04:07:58 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-b[110514]: 2024-09-14T04:07:58.380+0000 7fe822c97640 -1 mds.b *** got signal Terminated *** 2024-09-14T04:08:01.147 INFO:journalctl@ceph.mds.b.smithi094.stdout:Sep 14 04:08:00 smithi094 podman[120064]: 2024-09-14 04:08:00.885694971 +0000 UTC m=+2.598244589 container died 9f7842c26184e48971bcffd58a092eee4a7015cf63edf6c5118360985054226d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, RELEASE=main-a278231, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-14T04:08:01.497 INFO:journalctl@ceph.mds.b.smithi094.stdout:Sep 14 04:08:01 smithi094 podman[120064]: 2024-09-14 04:08:01.236969062 +0000 UTC m=+2.949518678 container cleanup 9f7842c26184e48971bcffd58a092eee4a7015cf63edf6c5118360985054226d (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-b, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231) 2024-09-14T04:08:01.497 INFO:journalctl@ceph.mds.b.smithi094.stdout:Sep 14 04:08:01 smithi094 bash[120064]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-mds-b 2024-09-14T04:08:01.672 DEBUG:teuthology.orchestra.run.smithi094:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@mds.b.service' 2024-09-14T04:08:01.706 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T04:08:01.707 INFO:tasks.cephadm.mds.b:Stopped mds.b 2024-09-14T04:08:01.707 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-09-14T04:08:01.707 DEBUG:teuthology.orchestra.run.smithi094:> sudo systemctl stop ceph-b6380d42-724a-11ef-bceb-c7b262605968@prometheus.a 2024-09-14T04:08:02.026 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:08:01 smithi094 systemd[1]: Stopping Ceph prometheus.a for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T04:08:02.310 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:08:02 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:08:02.035Z caller=main.go:964 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-14T04:08:02.310 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:08:02 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:08:02.035Z caller=main.go:988 level=info msg="Stopping scrape discovery manager..." 2024-09-14T04:08:02.310 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:08:02 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:08:02.035Z caller=main.go:1002 level=info msg="Stopping notify discovery manager..." 2024-09-14T04:08:02.310 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:08:02 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:08:02.035Z caller=manager.go:177 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-14T04:08:02.310 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:08:02 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:08:02.035Z caller=main.go:984 level=info msg="Scrape discovery manager stopped" 2024-09-14T04:08:02.310 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:08:02 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:08:02.035Z caller=main.go:998 level=info msg="Notify discovery manager stopped" 2024-09-14T04:08:02.311 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:08:02 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:08:02.035Z caller=manager.go:187 level=info component="rule manager" msg="Rule manager stopped" 2024-09-14T04:08:02.311 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:08:02 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:08:02.035Z caller=main.go:1039 level=info msg="Stopping scrape manager..." 2024-09-14T04:08:02.311 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:08:02 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:08:02.035Z caller=main.go:1031 level=info msg="Scrape manager stopped" 2024-09-14T04:08:02.311 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:08:02 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:08:02.037Z caller=notifier.go:618 level=info component=notifier msg="Stopping notification manager..." 2024-09-14T04:08:02.311 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:08:02 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:08:02.037Z caller=main.go:1261 level=info msg="Notifier manager stopped" 2024-09-14T04:08:02.311 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:08:02 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a[78276]: ts=2024-09-14T04:08:02.037Z caller=main.go:1273 level=info msg="See you next time!" 2024-09-14T04:08:02.311 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:08:02 smithi094 podman[120195]: 2024-09-14 04:08:02.046031718 +0000 UTC m=+0.113241561 container died ce1b353828dc3ceb0980e284aec473d26e5ca7875b6a4b810fc0df86e4912491 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T04:08:02.660 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:08:02 smithi094 podman[120195]: 2024-09-14 04:08:02.399964907 +0000 UTC m=+0.467174746 container cleanup ce1b353828dc3ceb0980e284aec473d26e5ca7875b6a4b810fc0df86e4912491 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T04:08:02.660 INFO:journalctl@ceph.prometheus.a.smithi094.stdout:Sep 14 04:08:02 smithi094 bash[120195]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-prometheus-a 2024-09-14T04:08:02.822 DEBUG:teuthology.orchestra.run.smithi094:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b6380d42-724a-11ef-bceb-c7b262605968@prometheus.a.service' 2024-09-14T04:08:02.854 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T04:08:02.854 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-09-14T04:08:02.855 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid b6380d42-724a-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-14T04:08:04.760 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 04:08:04 smithi070 systemd[1]: Stopping Ceph alertmanager.a for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T04:08:05.035 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 04:08:04 smithi070 ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a[87599]: ts=2024-09-14T04:08:04.759Z caller=main.go:542 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-14T04:08:05.036 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 04:08:04 smithi070 podman[133580]: 2024-09-14 04:08:04.790041362 +0000 UTC m=+0.135897192 container died 45e79e51074c9b37c6986ea0160e7ddc28824a0b8309ad240c45133a9639c7d6 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T04:08:05.316 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 04:08:05 smithi070 podman[133580]: 2024-09-14 04:08:05.123381651 +0000 UTC m=+0.469237480 container cleanup 45e79e51074c9b37c6986ea0160e7ddc28824a0b8309ad240c45133a9639c7d6 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T04:08:05.317 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 04:08:05 smithi070 bash[133580]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a 2024-09-14T04:08:05.683 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 04:08:05 smithi070 podman[133600]: 2024-09-14 04:08:05.316032799 +0000 UTC m=+0.517294929 container remove 45e79e51074c9b37c6986ea0160e7ddc28824a0b8309ad240c45133a9639c7d6 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T04:08:05.683 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 04:08:05 smithi070 podman[133600]: 2024-09-14 04:08:05.441955252 +0000 UTC m=+0.643217386 volume remove 7ff4a966259cd587f7546743d3ee38ff5d11357344d3db734f0fbcab32e5f31a 2024-09-14T04:08:06.036 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 04:08:05 smithi070 systemd[1]: Stopping Ceph node-exporter.a for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T04:08:06.036 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 04:08:05 smithi070 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-14T04:08:06.036 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 04:08:05 smithi070 systemd[1]: Stopped Ceph alertmanager.a for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T04:08:06.036 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Sep 14 04:08:05 smithi070 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@alertmanager.a.service: Consumed 1.799s CPU time. 2024-09-14T04:08:06.454 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 04:08:06 smithi070 podman[133712]: 2024-09-14 04:08:06.158187177 +0000 UTC m=+0.142807920 container died e316760a30b3120bdddb63db6db975934176b86e50fb49f37b7200085fd1dee9 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T04:08:06.771 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 04:08:06 smithi070 podman[133712]: 2024-09-14 04:08:06.553008459 +0000 UTC m=+0.537629201 container cleanup e316760a30b3120bdddb63db6db975934176b86e50fb49f37b7200085fd1dee9 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T04:08:06.772 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 04:08:06 smithi070 bash[133712]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a 2024-09-14T04:08:07.035 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 04:08:06 smithi070 podman[133731]: 2024-09-14 04:08:06.771059584 +0000 UTC m=+0.616831928 container remove e316760a30b3120bdddb63db6db975934176b86e50fb49f37b7200085fd1dee9 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T04:08:07.035 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 04:08:06 smithi070 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-14T04:08:07.036 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 04:08:06 smithi070 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-14T04:08:07.036 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 04:08:06 smithi070 systemd[1]: Stopped Ceph node-exporter.a for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T04:08:07.036 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Sep 14 04:08:06 smithi070 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@node-exporter.a.service: Consumed 3.652s CPU time. 2024-09-14T04:08:07.517 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid b6380d42-724a-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-14T04:08:09.277 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 04:08:09 smithi094 systemd[1]: Stopping Ceph grafana.a for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T04:08:09.700 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 04:08:09 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=server t=2024-09-14T04:08:09.432419212Z level=info msg="Shutdown started" reason="System signal: terminated" 2024-09-14T04:08:09.700 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 04:08:09 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=ticker t=2024-09-14T04:08:09.432812327Z level=info msg=stopped last_tick=2024-09-14T04:08:00Z 2024-09-14T04:08:09.700 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 04:08:09 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=tracing t=2024-09-14T04:08:09.432835927Z level=info msg="Closing tracing" 2024-09-14T04:08:09.700 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 04:08:09 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=grafana-apiserver t=2024-09-14T04:08:09.43285193Z level=info msg="StorageObjectCountTracker pruner is exiting" 2024-09-14T04:08:09.700 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 04:08:09 smithi094 ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a[114801]: logger=sqlstore.transactions t=2024-09-14T04:08:09.443774454Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-14T04:08:09.701 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 04:08:09 smithi094 podman[120647]: 2024-09-14 04:08:09.453293738 +0000 UTC m=+0.130725299 container died b7cb3ebd57f568c8c28045d8be60e740b566b10beda2ce7108ab317872775ebf (image=quay.io/ceph/grafana:10.4.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-14T04:08:09.965 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 04:08:09 smithi094 podman[120647]: 2024-09-14 04:08:09.780343954 +0000 UTC m=+0.457775513 container cleanup b7cb3ebd57f568c8c28045d8be60e740b566b10beda2ce7108ab317872775ebf (image=quay.io/ceph/grafana:10.4.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-14T04:08:09.966 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 04:08:09 smithi094 bash[120647]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a 2024-09-14T04:08:10.276 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 04:08:09 smithi094 podman[120665]: 2024-09-14 04:08:09.965119243 +0000 UTC m=+0.508632944 container remove b7cb3ebd57f568c8c28045d8be60e740b566b10beda2ce7108ab317872775ebf (image=quay.io/ceph/grafana:10.4.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-14T04:08:10.277 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 04:08:10 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@grafana.a.service: Deactivated successfully. 2024-09-14T04:08:10.277 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 04:08:10 smithi094 systemd[1]: Stopped Ceph grafana.a for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T04:08:10.277 INFO:journalctl@ceph.grafana.a.smithi094.stdout:Sep 14 04:08:10 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@grafana.a.service: Consumed 7.117s CPU time. 2024-09-14T04:08:10.624 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 04:08:10 smithi094 systemd[1]: Stopping Ceph node-exporter.b for b6380d42-724a-11ef-bceb-c7b262605968... 2024-09-14T04:08:10.886 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 04:08:10 smithi094 podman[120776]: 2024-09-14 04:08:10.62249726 +0000 UTC m=+0.121465549 container died 767f8a0931752aa06aea4956c4685619a49e9837abc3bd42fb1c88cd9bcfe05f (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T04:08:11.245 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 04:08:10 smithi094 podman[120776]: 2024-09-14 04:08:10.976514939 +0000 UTC m=+0.475483217 container cleanup 767f8a0931752aa06aea4956c4685619a49e9837abc3bd42fb1c88cd9bcfe05f (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T04:08:11.245 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 04:08:10 smithi094 bash[120776]: ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b 2024-09-14T04:08:11.527 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 04:08:11 smithi094 podman[120793]: 2024-09-14 04:08:11.244326312 +0000 UTC m=+0.624127673 container remove 767f8a0931752aa06aea4956c4685619a49e9837abc3bd42fb1c88cd9bcfe05f (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-b6380d42-724a-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T04:08:11.527 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 04:08:11 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-14T04:08:11.527 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 04:08:11 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-14T04:08:11.527 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 04:08:11 smithi094 systemd[1]: Stopped Ceph node-exporter.b for b6380d42-724a-11ef-bceb-c7b262605968. 2024-09-14T04:08:11.527 INFO:journalctl@ceph.node-exporter.b.smithi094.stdout:Sep 14 04:08:11 smithi094 systemd[1]: ceph-b6380d42-724a-11ef-bceb-c7b262605968@node-exporter.b.service: Consumed 3.658s CPU time. 2024-09-14T04:08:12.103 DEBUG:teuthology.orchestra.run.smithi070:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-14T04:08:12.135 DEBUG:teuthology.orchestra.run.smithi094:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-14T04:08:12.162 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-14T04:08:12.164 DEBUG:teuthology.misc:Transferring archived files from smithi070:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904541/remote/smithi070/crash 2024-09-14T04:08:12.165 DEBUG:teuthology.orchestra.run.smithi070:> sudo tar c -f - -C /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/crash -- . 2024-09-14T04:08:12.279 INFO:teuthology.orchestra.run.smithi070.stderr:tar: /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-14T04:08:12.279 INFO:teuthology.orchestra.run.smithi070.stderr:tar: Error is not recoverable: exiting now 2024-09-14T04:08:12.281 DEBUG:teuthology.misc:Transferring archived files from smithi094:/var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904541/remote/smithi094/crash 2024-09-14T04:08:12.282 DEBUG:teuthology.orchestra.run.smithi094:> sudo tar c -f - -C /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/crash -- . 2024-09-14T04:08:12.332 INFO:teuthology.orchestra.run.smithi094.stderr:tar: /var/lib/ceph/b6380d42-724a-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-14T04:08:12.333 INFO:teuthology.orchestra.run.smithi094.stderr:tar: Error is not recoverable: exiting now 2024-09-14T04:08:12.333 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-14T04:08:12.334 DEBUG:teuthology.orchestra.run.smithi070:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(POOL_APP_NOT_ENABLED\)' | egrep -v OSD_DOWN | egrep -v 'mons down' | egrep -v 'mon down' | egrep -v MON_DOWN | egrep -v 'out of quorum' | egrep -v PG_DEGRADED | egrep -v 'Reduced data availability' | egrep -v 'Degraded data redundancy' | egrep -v OSDMAP_FLAGS | egrep -v 'mons down' | egrep -v 'mon down' | egrep -v MON_DOWN | egrep -v 'out of quorum' | egrep -v PG_AVAILABILITY | head -n 1 2024-09-14T04:08:12.372 INFO:teuthology.orchestra.run.smithi070.stdout:2024-09-14T03:56:29.056366+0000 mon.a (mon.0) 843 : cluster [WRN] Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-14T04:08:12.372 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-09-14T04:08:12.373 INFO:tasks.cephadm:Compressing logs... 2024-09-14T04:08:12.373 DEBUG:teuthology.orchestra.run.smithi070:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-14T04:08:12.415 DEBUG:teuthology.orchestra.run.smithi094:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-14T04:08:12.440 INFO:teuthology.orchestra.run.smithi094.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-osd.7.log 2024-09-14T04:08:12.440 INFO:teuthology.orchestra.run.smithi094.stderr:: No such file or directory 2024-09-14T04:08:12.440 INFO:teuthology.orchestra.run.smithi094.stderr:gzip -5 --verbose -- /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-14T04:08:12.440 INFO:teuthology.orchestra.run.smithi094.stderr:/var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-mds.b.log 2024-09-14T04:08:12.446 INFO:teuthology.orchestra.run.smithi070.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-osd.0.log 2024-09-14T04:08:12.447 INFO:teuthology.orchestra.run.smithi070.stderr:: No such file or directory 2024-09-14T04:08:12.447 INFO:teuthology.orchestra.run.smithi070.stderr:gzip -5 --verbose -- /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-osd.1.log 2024-09-14T04:08:12.447 INFO:teuthology.orchestra.run.smithi070.stderr:/var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-14T04:08:12.447 INFO:teuthology.orchestra.run.smithi070.stderr:/var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-14T04:08:12.447 INFO:teuthology.orchestra.run.smithi070.stderr:/var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-mon.c.log 2024-09-14T04:08:12.447 INFO:teuthology.orchestra.run.smithi070.stderr:gzip -5 --verbose -- /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-mds.a.log/var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph.audit.log: 2024-09-14T04:08:12.448 INFO:teuthology.orchestra.run.smithi070.stderr:/var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-mon.c.log: gzip -5 --verbose -- /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-14T04:08:12.448 INFO:teuthology.orchestra.run.smithi070.stderr:/var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-mds.a.log: gzip -5 --verbose -- /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-mon.a.log 2024-09-14T04:08:12.448 INFO:teuthology.orchestra.run.smithi070.stderr:/var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph.cephadm.log: /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-osd.2.log 2024-09-14T04:08:12.448 INFO:teuthology.orchestra.run.smithi070.stderr: 91.0% -- replaced with /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-14T04:08:12.450 INFO:teuthology.orchestra.run.smithi094.stderr:/var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-osd.4.log 2024-09-14T04:08:12.450 INFO:teuthology.orchestra.run.smithi094.stderr:/var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-mds.b.log: gzip -5 --verbose -- /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-14T04:08:12.451 INFO:teuthology.orchestra.run.smithi094.stderr:gzip -5 --verbose -- /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-14T04:08:12.451 INFO:teuthology.orchestra.run.smithi094.stderr:gzip -5 --verbose -- /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-osd.6.log 2024-09-14T04:08:12.451 INFO:teuthology.orchestra.run.smithi094.stderr:/var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-osd.4.log: 83.1% -- replaced with /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-mds.b.log.gz 2024-09-14T04:08:12.451 INFO:teuthology.orchestra.run.smithi094.stderr:/var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph.audit.log: /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-osd.5.log 2024-09-14T04:08:12.451 INFO:teuthology.orchestra.run.smithi094.stderr:/var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph.log 2024-09-14T04:08:12.451 INFO:teuthology.orchestra.run.smithi094.stderr:/var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-osd.5.log: 84.2% -- replaced with /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-14T04:08:12.451 INFO:teuthology.orchestra.run.smithi094.stderr:gzip -5 --verbose -- /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-mgr.x.log 2024-09-14T04:08:12.452 INFO:teuthology.orchestra.run.smithi094.stderr:gzip -5 --verbose -- /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-mon.b.log 2024-09-14T04:08:12.452 INFO:teuthology.orchestra.run.smithi094.stderr:/var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-mgr.x.log: /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph.log: 90.5% -- replaced with /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-14T04:08:12.452 INFO:teuthology.orchestra.run.smithi094.stderr: 85.9% -- replaced with /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-14T04:08:12.452 INFO:teuthology.orchestra.run.smithi094.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-14T04:08:12.456 INFO:teuthology.orchestra.run.smithi070.stderr: 94.2% -- replaced with /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-14T04:08:12.456 INFO:teuthology.orchestra.run.smithi070.stderr: 91.8% -- replaced with /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-mds.a.log.gz 2024-09-14T04:08:12.456 INFO:teuthology.orchestra.run.smithi070.stderr:gzip -5 --verbose -- /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-mgr.y.log 2024-09-14T04:08:12.456 INFO:teuthology.orchestra.run.smithi070.stderr:/var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph.log 2024-09-14T04:08:12.456 INFO:teuthology.orchestra.run.smithi070.stderr:/var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-osd.3.log 2024-09-14T04:08:12.463 INFO:teuthology.orchestra.run.smithi070.stderr:/var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph.log: 92.1% -- replaced with /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-14T04:08:12.463 INFO:teuthology.orchestra.run.smithi070.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-14T04:08:12.464 INFO:teuthology.orchestra.run.smithi094.stderr:/var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-mon.b.log: /var/log/ceph/cephadm.log: 91.1% -- replaced with /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-14T04:08:12.465 INFO:teuthology.orchestra.run.smithi070.stderr:/var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-osd.3.log: /var/log/ceph/cephadm.log: 91.1% -- replaced with /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-14T04:08:12.468 INFO:teuthology.orchestra.run.smithi070.stderr: 92.3% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-14T04:08:12.475 INFO:teuthology.orchestra.run.smithi094.stderr: 92.0% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-14T04:08:12.514 INFO:teuthology.orchestra.run.smithi094.stderr: 89.8% -- replaced with /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-mgr.x.log.gz 2024-09-14T04:08:12.900 INFO:teuthology.orchestra.run.smithi070.stderr: 89.3% -- replaced with /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-mgr.y.log.gz 2024-09-14T04:08:12.937 INFO:teuthology.orchestra.run.smithi070.stderr: 91.9% -- replaced with /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-mon.c.log.gz 2024-09-14T04:08:12.949 INFO:teuthology.orchestra.run.smithi094.stderr: 92.1% -- replaced with /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-mon.b.log.gz 2024-09-14T04:08:14.525 INFO:teuthology.orchestra.run.smithi070.stderr: 90.7% -- replaced with /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-mon.a.log.gz 2024-09-14T04:08:16.975 INFO:teuthology.orchestra.run.smithi094.stderr: 94.3% -- replaced with /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-osd.6.log.gz 2024-09-14T04:08:18.306 INFO:teuthology.orchestra.run.smithi094.stderr: 94.1% -- replaced with /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-osd.5.log.gz 2024-09-14T04:08:18.389 INFO:teuthology.orchestra.run.smithi070.stderr: 93.9% -- replaced with /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-osd.0.log.gz 2024-09-14T04:08:18.782 INFO:teuthology.orchestra.run.smithi094.stderr: 94.3% -- replaced with /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-osd.7.log.gz 2024-09-14T04:08:18.892 INFO:teuthology.orchestra.run.smithi094.stderr: 94.0% -- replaced with /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-osd.4.log.gz 2024-09-14T04:08:18.894 INFO:teuthology.orchestra.run.smithi094.stderr: 2024-09-14T04:08:18.894 INFO:teuthology.orchestra.run.smithi094.stderr:real 0m6.464s 2024-09-14T04:08:18.894 INFO:teuthology.orchestra.run.smithi094.stderr:user 0m22.889s 2024-09-14T04:08:18.895 INFO:teuthology.orchestra.run.smithi094.stderr:sys 0m0.915s 2024-09-14T04:08:18.912 INFO:teuthology.orchestra.run.smithi070.stderr: 93.8% -- replaced with /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-osd.2.log.gz 2024-09-14T04:08:20.815 INFO:teuthology.orchestra.run.smithi070.stderr: 94.4% -- replaced with /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-osd.3.log.gz 2024-09-14T04:08:21.743 INFO:teuthology.orchestra.run.smithi070.stderr: 94.3% -- replaced with /var/log/ceph/b6380d42-724a-11ef-bceb-c7b262605968/ceph-osd.1.log.gz 2024-09-14T04:08:21.745 INFO:teuthology.orchestra.run.smithi070.stderr: 2024-09-14T04:08:21.745 INFO:teuthology.orchestra.run.smithi070.stderr:real 0m9.316s 2024-09-14T04:08:21.745 INFO:teuthology.orchestra.run.smithi070.stderr:user 0m31.901s 2024-09-14T04:08:21.745 INFO:teuthology.orchestra.run.smithi070.stderr:sys 0m1.208s 2024-09-14T04:08:21.746 INFO:tasks.cephadm:Archiving logs... 2024-09-14T04:08:21.746 DEBUG:teuthology.misc:Transferring archived files from smithi070:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904541/remote/smithi070/log 2024-09-14T04:08:21.747 DEBUG:teuthology.orchestra.run.smithi070:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-14T04:08:26.141 DEBUG:teuthology.misc:Transferring archived files from smithi094:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904541/remote/smithi094/log 2024-09-14T04:08:26.142 DEBUG:teuthology.orchestra.run.smithi094:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-14T04:08:29.408 INFO:tasks.cephadm:Removing cluster... 2024-09-14T04:08:29.408 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid b6380d42-724a-11ef-bceb-c7b262605968 --force 2024-09-14T04:08:29.924 DEBUG:teuthology.orchestra.run.smithi094:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid b6380d42-724a-11ef-bceb-c7b262605968 --force 2024-09-14T04:08:30.420 INFO:tasks.cephadm:Removing cephadm ... 2024-09-14T04:08:30.421 DEBUG:teuthology.orchestra.run.smithi070:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-14T04:08:30.437 DEBUG:teuthology.orchestra.run.smithi094:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-14T04:08:30.453 INFO:tasks.cephadm:Teardown complete 2024-09-14T04:08:30.453 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-09-14T04:08:30.465 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-09-14T04:08:30.465 DEBUG:teuthology.orchestra.run.smithi070:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-14T04:08:30.480 DEBUG:teuthology.orchestra.run.smithi094:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-14T04:08:30.570 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-14T04:08:30.582 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-14T04:08:30.583 DEBUG:teuthology.orchestra.run.smithi070:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-14T04:08:30.586 DEBUG:teuthology.orchestra.run.smithi094:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-14T04:08:30.599 INFO:teuthology.orchestra.run.smithi070.stderr:bash: line 1: ntpq: command not found 2024-09-14T04:08:30.600 INFO:teuthology.orchestra.run.smithi094.stderr:bash: line 1: ntpq: command not found 2024-09-14T04:08:30.603 INFO:teuthology.orchestra.run.smithi070.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-14T04:08:30.603 INFO:teuthology.orchestra.run.smithi070.stdout:=============================================================================== 2024-09-14T04:08:30.603 INFO:teuthology.orchestra.run.smithi070.stdout:^* hv01.front.sepia.ceph.com 3 6 377 49 +48us[ +66us] +/- 64ms 2024-09-14T04:08:30.604 INFO:teuthology.orchestra.run.smithi070.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 47 -316us[ -316us] +/- 64ms 2024-09-14T04:08:30.604 INFO:teuthology.orchestra.run.smithi070.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 48 +360us[ +360us] +/- 82ms 2024-09-14T04:08:30.604 INFO:teuthology.orchestra.run.smithi070.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T04:08:30.605 INFO:teuthology.orchestra.run.smithi094.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-14T04:08:30.606 INFO:teuthology.orchestra.run.smithi094.stdout:=============================================================================== 2024-09-14T04:08:30.606 INFO:teuthology.orchestra.run.smithi094.stdout:^* hv01.front.sepia.ceph.com 3 6 377 49 +31us[ +42us] +/- 64ms 2024-09-14T04:08:30.606 INFO:teuthology.orchestra.run.smithi094.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 50 -321us[ -310us] +/- 64ms 2024-09-14T04:08:30.606 INFO:teuthology.orchestra.run.smithi094.stdout:^+ hv03.front.sepia.ceph.com 3 7 377 48 +382us[ +382us] +/- 82ms 2024-09-14T04:08:30.606 INFO:teuthology.orchestra.run.smithi094.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T04:08:30.606 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-14T04:08:30.616 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-14T04:08:30.616 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-14T04:08:30.630 DEBUG:teuthology.orchestra.run.smithi070:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-14T04:08:30.734 DEBUG:teuthology.orchestra.run.smithi094:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-14T04:08:30.812 DEBUG:teuthology.orchestra.run.smithi070:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-14T04:08:30.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:30.843 DEBUG:teuthology.orchestra.run.smithi094:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-14T04:08:30.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:30.873 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-14T04:08:30.888 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-14T04:08:30.898 INFO:teuthology.task.internal:Duration was 2161.969970 seconds 2024-09-14T04:08:30.899 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-14T04:08:30.911 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-14T04:08:30.911 DEBUG:teuthology.orchestra.run.smithi070:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-14T04:08:30.914 DEBUG:teuthology.orchestra.run.smithi094:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-14T04:08:30.950 INFO:teuthology.orchestra.run.smithi070.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-14T04:08:30.952 INFO:teuthology.orchestra.run.smithi094.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-14T04:08:31.326 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-14T04:08:31.326 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi070.front.sepia.ceph.com 2024-09-14T04:08:31.327 DEBUG:teuthology.orchestra.run.smithi070:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-14T04:08:31.387 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi094.front.sepia.ceph.com 2024-09-14T04:08:31.387 DEBUG:teuthology.orchestra.run.smithi094:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-14T04:08:31.416 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-14T04:08:31.416 DEBUG:teuthology.orchestra.run.smithi070:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-14T04:08:31.429 DEBUG:teuthology.orchestra.run.smithi094:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-14T04:08:31.483 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-14T04:08:31.484 DEBUG:teuthology.orchestra.run.smithi070:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-14T04:08:31.497 DEBUG:teuthology.orchestra.run.smithi094:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-14T04:08:31.966 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-14T04:08:31.979 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-14T04:08:31.979 DEBUG:teuthology.orchestra.run.smithi070:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-14T04:08:32.005 DEBUG:teuthology.orchestra.run.smithi094:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-14T04:08:32.030 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-14T04:08:32.041 DEBUG:teuthology.orchestra.run.smithi070:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-14T04:08:32.048 DEBUG:teuthology.orchestra.run.smithi094:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-14T04:08:32.068 INFO:teuthology.orchestra.run.smithi070.stdout:kernel.core_pattern = core 2024-09-14T04:08:32.093 INFO:teuthology.orchestra.run.smithi094.stdout:kernel.core_pattern = core 2024-09-14T04:08:32.118 DEBUG:teuthology.orchestra.run.smithi070:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-14T04:08:32.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:32.137 DEBUG:teuthology.orchestra.run.smithi094:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-14T04:08:32.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:32.173 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-14T04:08:32.189 INFO:teuthology.task.internal:Transferring archived files... 2024-09-14T04:08:32.190 DEBUG:teuthology.misc:Transferring archived files from smithi070:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904541/remote/smithi070 2024-09-14T04:08:32.191 DEBUG:teuthology.orchestra.run.smithi070:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-14T04:08:32.238 DEBUG:teuthology.misc:Transferring archived files from smithi094:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904541/remote/smithi094 2024-09-14T04:08:32.238 DEBUG:teuthology.orchestra.run.smithi094:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-14T04:08:32.278 INFO:teuthology.task.internal:Removing archive directory... 2024-09-14T04:08:32.278 DEBUG:teuthology.orchestra.run.smithi070:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-14T04:08:32.281 DEBUG:teuthology.orchestra.run.smithi094:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-14T04:08:32.325 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-14T04:08:32.337 INFO:teuthology.task.internal:Not uploading archives. 2024-09-14T04:08:32.337 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-14T04:08:32.347 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-14T04:08:32.348 DEBUG:teuthology.orchestra.run.smithi070:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-14T04:08:32.351 DEBUG:teuthology.orchestra.run.smithi094:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-14T04:08:32.363 INFO:teuthology.orchestra.run.smithi070.stdout: 265613 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 14 04:08 /home/ubuntu/cephtest 2024-09-14T04:08:32.364 INFO:teuthology.orchestra.run.smithi070.stdout: 524358 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 14 04:02 /home/ubuntu/cephtest/mnt.0 2024-09-14T04:08:32.364 INFO:teuthology.orchestra.run.smithi070.stdout: 524363 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 14 04:06 /home/ubuntu/cephtest/mnt.0/client.0 2024-09-14T04:08:32.364 INFO:teuthology.orchestra.run.smithi070.stdout: 669579 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 14 04:06 /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-14T04:08:32.365 INFO:teuthology.orchestra.run.smithi070.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2024-09-14T04:08:32.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:08:32.368 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 160, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.10/contextlib.py", line 142, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi070 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-09-14T04:08:32.382 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-14T04:08:32.395 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-14T04:08:32.415 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi070 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-09-14T04:08:32.420 INFO:teuthology.run:Summary data: description: upgrade/quincy-x/parallel/{0-random-distro$/{centos_9.stream_runc} 0-start 1-tasks mon_election/connectivity upgrade-sequence workload/{ec-rados-default rados_api rados_loadgenbig rbd_import_export test_rbd_api test_rbd_python}} duration: 2161.969969511032 failure_reason: 'Command failed (workunit test cls/test_cls_rbd.sh) on smithi070 with status 1: ''mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh''' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=175b3f66ed9049f686ef4fb3001af135 status: fail success: false 2024-09-14T04:08:32.420 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-14T04:08:32.427 INFO:teuthology.orchestra.run.smithi094.stdout: 265861 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 14 04:08 /home/ubuntu/cephtest 2024-09-14T04:08:32.517 INFO:teuthology.run:FAIL